diff --git a/contentcuration/contentcuration/frontend/shared/utils/__tests__/zipFile.spec.js b/contentcuration/contentcuration/frontend/shared/utils/__tests__/zipFile.spec.js index 9b147fe9c9..f391357773 100644 --- a/contentcuration/contentcuration/frontend/shared/utils/__tests__/zipFile.spec.js +++ b/contentcuration/contentcuration/frontend/shared/utils/__tests__/zipFile.spec.js @@ -47,23 +47,38 @@ const TEST_CASES = { }, }; -function getMD5(data) { - return createHash('md5').update(Buffer.from(data)).digest('hex'); +function readBytes(blob) { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(new Uint8Array(reader.result)); + reader.onerror = reject; + reader.readAsArrayBuffer(blob); + }); +} + +async function getMD5(blob) { + return createHash('md5') + .update(await readBytes(blob)) + .digest('hex'); +} + +function zipFiles(files, paths = Object.keys(files)) { + return createPredictableZip(paths, async path => files[path]); } describe('createPredictableZip', () => { for (const [name, testCase] of Object.entries(TEST_CASES)) { it(`creates expected zip for ${name} case`, async () => { - const zipData = await createPredictableZip(testCase.files, testCase.options); - expect(getMD5(zipData)).toBe(testCase.expectedMD5); + const zipData = await zipFiles(testCase.files); + expect(await getMD5(zipData)).toBe(testCase.expectedMD5); }); } it('creates consistent hashes regardless of input order', async () => { - const reversed = { ...TEST_CASES.reversed.files }; - const zipData1 = await createPredictableZip(TEST_CASES.reversed.files); - const zipData2 = await createPredictableZip(reversed); - expect(getMD5(zipData1)).toBe(getMD5(zipData2)); + const files = TEST_CASES.reversed.files; + const zipData1 = await zipFiles(files); + const zipData2 = await zipFiles(files, Object.keys(files).reverse()); + expect(await getMD5(zipData1)).toBe(await getMD5(zipData2)); }); }); diff --git a/contentcuration/contentcuration/frontend/shared/utils/zipFile.js b/contentcuration/contentcuration/frontend/shared/utils/zipFile.js index f756e299d3..327007d1bc 100644 --- a/contentcuration/contentcuration/frontend/shared/utils/zipFile.js +++ b/contentcuration/contentcuration/frontend/shared/utils/zipFile.js @@ -156,11 +156,11 @@ function calculateCrc32(data) { } /** - * Creates a predictable ZIP file from the given files. + * Creates a predictable ZIP file, reading and compressing one file at a time. * - * @param {Object.} files - An object where the keys are file paths and - * the values are file contents. - * @returns {Promise} - A promise that resolves to a Buffer containing the ZIP file data. + * @param {string[]} paths - Paths of the files to include in the zip. + * @param {function(string): Promise} readFile - Returns the contents of a path. + * @returns {Promise} - A promise that resolves to a Blob containing the ZIP file data. * * @example * const files = { @@ -168,19 +168,19 @@ function calculateCrc32(data) { * 'file2.bin': new Uint8Array([0x00, 0x01, 0x02, 0x03]), * 'folder/nested.txt': 'Nested content', * }; - * createPredictableZip(files).then(zipBuffer => { - * // Do something with the zipBuffer + * createPredictableZip(Object.keys(files), async path => files[path]).then(zipBlob => { + * // Do something with the zipBlob * }); */ -export async function createPredictableZip(files) { - const sortedPaths = Object.keys(files).sort(); - const entries = []; +export async function createPredictableZip(paths, readFile) { + const sortedPaths = [...paths].sort(); + const parts = []; let offset = 0; const centralDir = []; let centralDirSize = 0; for (const path of sortedPaths) { - const content = files[path]; + const content = await readFile(path); // Handle binary data more carefully const rawData = typeof content === 'string' @@ -207,24 +207,14 @@ export async function createPredictableZip(files) { checksum, ); - const entry = { - header, - compressed: new Uint8Array(compressed), - path: filenameCleaned, - crc32: checksum, - compressedSize: compressed.length, - uncompressedSize: rawData.length, - offset, - }; - - entries.push(entry); + parts.push(header, compressed); const dirEntry = createCentralDirectoryEntry( - entry.path, - entry.compressedSize, - entry.uncompressedSize, - entry.offset, - entry.crc32, + filenameCleaned, + compressed.length, + rawData.length, + offset, + checksum, ); centralDir.push(dirEntry); centralDirSize += dirEntry.length; @@ -232,13 +222,9 @@ export async function createPredictableZip(files) { offset += header.length + compressed.length; } - const endOfCentralDir = createEndOfCentralDirectory(entries.length, centralDirSize, offset); + const endOfCentralDir = createEndOfCentralDirectory(centralDir.length, centralDirSize, offset); - return new Uint8Array([ - ...entries.map(e => [...e.header, ...e.compressed]).flat(), - ...centralDir.map(dir => [...dir]).flat(), - ...endOfCentralDir, - ]); + return new Blob([...parts, ...centralDir, endOfCentralDir]); } /** diff --git a/contentcuration/contentcuration/frontend/shared/vuex/file/__tests__/clean.spec.js b/contentcuration/contentcuration/frontend/shared/vuex/file/__tests__/clean.spec.js index 05513c9ed4..85a8763fe1 100644 --- a/contentcuration/contentcuration/frontend/shared/vuex/file/__tests__/clean.spec.js +++ b/contentcuration/contentcuration/frontend/shared/vuex/file/__tests__/clean.spec.js @@ -1,6 +1,16 @@ +import { randomFillSync } from 'crypto'; import JSZip from 'jszip'; import { cleanFile } from '../clean'; +function readBytes(blob) { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(new Uint8Array(reader.result)); + reader.onerror = reject; + reader.readAsArrayBuffer(blob); + }); +} + describe('cleanFile', () => { let mockZip; let originalCreateElement; @@ -19,10 +29,14 @@ describe('cleanFile', () => { // Helper function to create a zip file with given contents async function createTestZip(files, options = {}) { + const zip = new JSZip(); for (const [path, content] of Object.entries(files)) { - mockZip.file(path, content); + zip.file(path, content, { date: options.date }); } - const zipContent = await mockZip.generateAsync({ type: 'blob' }); + const zipContent = await zip.generateAsync({ + type: 'blob', + compression: options.compression || 'STORE', + }); return new File([zipContent], options.filename || 'test.zip', { type: 'application/zip', lastModified: options.lastModified || Date.now(), @@ -85,6 +99,61 @@ describe('cleanFile', () => { expect(cleanedPaths).toContain('index.html'); expect(cleanedPaths).toContain('assets/style.css'); }); + + it('should preserve file contents', async () => { + const binary = new Uint8Array([0x00, 0xff, 0x10, 0x80, 0x7f]); + const file = await createTestZip({ + 'dist/index.html': 'ünïcödé', + 'dist/model/weights.bin': binary, + }); + const cleanedZip = await JSZip.loadAsync(await cleanFile(file)); + + expect(await cleanedZip.file('index.html').async('string')).toBe('ünïcödé'); + expect(await cleanedZip.file('model/weights.bin').async('uint8array')).toEqual(binary); + }); + + it('should produce identical bytes for the same files zipped differently', async () => { + const files = { + 'index.html': '', + 'js/main.js': "console.log('hello')", + }; + const nestedReversed = { + 'build/js/main.js': files['js/main.js'], + 'build/index.html': files['index.html'], + }; + const first = await cleanFile(await createTestZip(files, { date: new Date(2020, 1, 1) })); + const second = await cleanFile( + await createTestZip(nestedReversed, { + compression: 'DEFLATE', + date: new Date(2024, 5, 5), + }), + ); + + expect(await readBytes(second)).toEqual(await readBytes(first)); + }); + + it('should keep the original file name, type and modification time', async () => { + const file = await createTestZip( + { 'dist/index.html': '' }, + { filename: 'course.zip', lastModified: 1234567890 }, + ); + const cleanedFile = await cleanFile(file); + + expect(cleanedFile.name).toBe('course.zip'); + expect(cleanedFile.type).toBe('application/zip'); + expect(cleanedFile.lastModified).toBe(1234567890); + }); + + it('should clean zips of incompressible content larger than 100 MB', async () => { + const size = 110 * 1024 * 1024; + const file = await createTestZip({ + 'index.html': '', + 'model/weights.bin': randomFillSync(new Uint8Array(size)), + }); + const cleanedFile = await cleanFile(file); + + expect(cleanedFile.size).toBeGreaterThan(size); + }, 120000); }); describe('Error handling', () => { diff --git a/contentcuration/contentcuration/frontend/shared/vuex/file/clean.js b/contentcuration/contentcuration/frontend/shared/vuex/file/clean.js index a774cbdda7..7dd0cb6900 100644 --- a/contentcuration/contentcuration/frontend/shared/vuex/file/clean.js +++ b/contentcuration/contentcuration/frontend/shared/vuex/file/clean.js @@ -3,28 +3,6 @@ import { inferPreset } from './utils'; import { createPredictableZip, findCommonRoot } from 'shared/utils/zipFile'; import { FormatPresetsNames } from 'shared/leUtils/FormatPresets'; -/** - * Creates a new files object with common root directory removed - * @param {Object} files - JSZip files object - * @param {string} commonRoot - Common root path to remove - * @returns {Object} New files object with paths remapped - */ -async function remapFiles(files, commonRoot) { - const cleanedFiles = {}; - commonRoot = commonRoot === '' ? commonRoot : commonRoot + '/'; - const commonRootLength = commonRoot.length; - - for (const [path, file] of Object.entries(files)) { - if (!file.dir) { - // Skip directory entries - const newPath = path.slice(commonRootLength); - cleanedFiles[newPath] = await file.async('uint8array'); - } - } - - return cleanedFiles; -} - /** * Cleans an HTML5 zip file by removing unnecessary directory nesting * @param {File} file - The HTML5 zip file to clean @@ -37,13 +15,18 @@ export async function cleanHTML5Zip(file) { // Find and remove common root directory const commonRoot = findCommonRoot(zipContent.files); - const cleanedFiles = await remapFiles(zipContent.files, commonRoot); + const rootPrefix = commonRoot === '' ? '' : commonRoot + '/'; + const cleanedPaths = Object.entries(zipContent.files) + .filter(([, entry]) => !entry.dir) + .map(([path]) => path.slice(rootPrefix.length)); // Create new predictable zip with cleaned files - const cleanedZipBuffer = await createPredictableZip(cleanedFiles); + const cleanedZip = await createPredictableZip(cleanedPaths, path => + zipContent.files[rootPrefix + path].async('uint8array'), + ); // Create new File object with original metadata - const cleanedFile = new File([cleanedZipBuffer], file.name, { + const cleanedFile = new File([cleanedZip], file.name, { type: file.type, lastModified: file.lastModified, });