Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -47,23 +47,38 @@ const TEST_CASES = {
},
};

function getMD5(data) {
return createHash('md5').update(Buffer.from(data)).digest('hex');
function readBytes(blob) {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(new Uint8Array(reader.result));
reader.onerror = reject;
reader.readAsArrayBuffer(blob);
});
}

async function getMD5(blob) {
return createHash('md5')
.update(await readBytes(blob))
.digest('hex');
}

function zipFiles(files, paths = Object.keys(files)) {
return createPredictableZip(paths, async path => files[path]);
}

describe('createPredictableZip', () => {
for (const [name, testCase] of Object.entries(TEST_CASES)) {
it(`creates expected zip for ${name} case`, async () => {
const zipData = await createPredictableZip(testCase.files, testCase.options);
expect(getMD5(zipData)).toBe(testCase.expectedMD5);
const zipData = await zipFiles(testCase.files);
expect(await getMD5(zipData)).toBe(testCase.expectedMD5);
});
}

it('creates consistent hashes regardless of input order', async () => {
const reversed = { ...TEST_CASES.reversed.files };
const zipData1 = await createPredictableZip(TEST_CASES.reversed.files);
const zipData2 = await createPredictableZip(reversed);
expect(getMD5(zipData1)).toBe(getMD5(zipData2));
const files = TEST_CASES.reversed.files;
const zipData1 = await zipFiles(files);
const zipData2 = await zipFiles(files, Object.keys(files).reverse());
expect(await getMD5(zipData1)).toBe(await getMD5(zipData2));
});
});

Expand Down
50 changes: 18 additions & 32 deletions contentcuration/contentcuration/frontend/shared/utils/zipFile.js
Original file line number Diff line number Diff line change
Expand Up @@ -156,31 +156,31 @@ function calculateCrc32(data) {
}

/**
* Creates a predictable ZIP file from the given files.
* Creates a predictable ZIP file, reading and compressing one file at a time.
*
* @param {Object.<string, string|Uint8Array>} files - An object where the keys are file paths and
* the values are file contents.
* @returns {Promise<Buffer>} - A promise that resolves to a Buffer containing the ZIP file data.
* @param {string[]} paths - Paths of the files to include in the zip.
* @param {function(string): Promise<string|Uint8Array>} readFile - Returns the contents of a path.
* @returns {Promise<Blob>} - A promise that resolves to a Blob containing the ZIP file data.
*
* @example
* const files = {
* 'file1.txt': 'Hello, world!',
* 'file2.bin': new Uint8Array([0x00, 0x01, 0x02, 0x03]),
* 'folder/nested.txt': 'Nested content',
* };
* createPredictableZip(files).then(zipBuffer => {
* // Do something with the zipBuffer
* createPredictableZip(Object.keys(files), async path => files[path]).then(zipBlob => {
* // Do something with the zipBlob
* });
*/
export async function createPredictableZip(files) {
const sortedPaths = Object.keys(files).sort();
const entries = [];
export async function createPredictableZip(paths, readFile) {
const sortedPaths = [...paths].sort();
const parts = [];
let offset = 0;
const centralDir = [];
let centralDirSize = 0;

for (const path of sortedPaths) {
const content = files[path];
const content = await readFile(path);
// Handle binary data more carefully
const rawData =
typeof content === 'string'
Expand All @@ -207,38 +207,24 @@ export async function createPredictableZip(files) {
checksum,
);

const entry = {
header,
compressed: new Uint8Array(compressed),
path: filenameCleaned,
crc32: checksum,
compressedSize: compressed.length,
uncompressedSize: rawData.length,
offset,
};

entries.push(entry);
parts.push(header, compressed);

const dirEntry = createCentralDirectoryEntry(
entry.path,
entry.compressedSize,
entry.uncompressedSize,
entry.offset,
entry.crc32,
filenameCleaned,
compressed.length,
rawData.length,
offset,
checksum,
);
centralDir.push(dirEntry);
centralDirSize += dirEntry.length;

offset += header.length + compressed.length;
}

const endOfCentralDir = createEndOfCentralDirectory(entries.length, centralDirSize, offset);
const endOfCentralDir = createEndOfCentralDirectory(centralDir.length, centralDirSize, offset);

return new Uint8Array([
...entries.map(e => [...e.header, ...e.compressed]).flat(),
...centralDir.map(dir => [...dir]).flat(),
...endOfCentralDir,
]);
return new Blob([...parts, ...centralDir, endOfCentralDir]);
}

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,16 @@
import { randomFillSync } from 'crypto';
import JSZip from 'jszip';
import { cleanFile } from '../clean';

function readBytes(blob) {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(new Uint8Array(reader.result));
reader.onerror = reject;
reader.readAsArrayBuffer(blob);
});
}

describe('cleanFile', () => {
let mockZip;
let originalCreateElement;
Expand All @@ -19,10 +29,14 @@ describe('cleanFile', () => {

// Helper function to create a zip file with given contents
async function createTestZip(files, options = {}) {
const zip = new JSZip();
for (const [path, content] of Object.entries(files)) {
mockZip.file(path, content);
zip.file(path, content, { date: options.date });
}
const zipContent = await mockZip.generateAsync({ type: 'blob' });
const zipContent = await zip.generateAsync({
type: 'blob',
compression: options.compression || 'STORE',
});
return new File([zipContent], options.filename || 'test.zip', {
type: 'application/zip',
lastModified: options.lastModified || Date.now(),
Expand Down Expand Up @@ -85,6 +99,61 @@ describe('cleanFile', () => {
expect(cleanedPaths).toContain('index.html');
expect(cleanedPaths).toContain('assets/style.css');
});

it('should preserve file contents', async () => {
const binary = new Uint8Array([0x00, 0xff, 0x10, 0x80, 0x7f]);
const file = await createTestZip({
'dist/index.html': '<html>ünïcödé</html>',
'dist/model/weights.bin': binary,
});
const cleanedZip = await JSZip.loadAsync(await cleanFile(file));

expect(await cleanedZip.file('index.html').async('string')).toBe('<html>ünïcödé</html>');
expect(await cleanedZip.file('model/weights.bin').async('uint8array')).toEqual(binary);
});

it('should produce identical bytes for the same files zipped differently', async () => {
const files = {
'index.html': '<html></html>',
'js/main.js': "console.log('hello')",
};
const nestedReversed = {
'build/js/main.js': files['js/main.js'],
'build/index.html': files['index.html'],
};
const first = await cleanFile(await createTestZip(files, { date: new Date(2020, 1, 1) }));
const second = await cleanFile(
await createTestZip(nestedReversed, {
compression: 'DEFLATE',
date: new Date(2024, 5, 5),
}),
);

expect(await readBytes(second)).toEqual(await readBytes(first));
});

it('should keep the original file name, type and modification time', async () => {
const file = await createTestZip(
{ 'dist/index.html': '<html></html>' },
{ filename: 'course.zip', lastModified: 1234567890 },
);
const cleanedFile = await cleanFile(file);

expect(cleanedFile.name).toBe('course.zip');
expect(cleanedFile.type).toBe('application/zip');
expect(cleanedFile.lastModified).toBe(1234567890);
});

it('should clean zips of incompressible content larger than 100 MB', async () => {
const size = 110 * 1024 * 1024;
const file = await createTestZip({
'index.html': '<html></html>',
'model/weights.bin': randomFillSync(new Uint8Array(size)),
});
const cleanedFile = await cleanFile(file);

expect(cleanedFile.size).toBeGreaterThan(size);
}, 120000);
});

describe('Error handling', () => {
Expand Down
33 changes: 8 additions & 25 deletions contentcuration/contentcuration/frontend/shared/vuex/file/clean.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,28 +3,6 @@ import { inferPreset } from './utils';
import { createPredictableZip, findCommonRoot } from 'shared/utils/zipFile';
import { FormatPresetsNames } from 'shared/leUtils/FormatPresets';

/**
* Creates a new files object with common root directory removed
* @param {Object} files - JSZip files object
* @param {string} commonRoot - Common root path to remove
* @returns {Object} New files object with paths remapped
*/
async function remapFiles(files, commonRoot) {
const cleanedFiles = {};
commonRoot = commonRoot === '' ? commonRoot : commonRoot + '/';
const commonRootLength = commonRoot.length;

for (const [path, file] of Object.entries(files)) {
if (!file.dir) {
// Skip directory entries
const newPath = path.slice(commonRootLength);
cleanedFiles[newPath] = await file.async('uint8array');
}
}

return cleanedFiles;
}

/**
* Cleans an HTML5 zip file by removing unnecessary directory nesting
* @param {File} file - The HTML5 zip file to clean
Expand All @@ -37,13 +15,18 @@ export async function cleanHTML5Zip(file) {

// Find and remove common root directory
const commonRoot = findCommonRoot(zipContent.files);
const cleanedFiles = await remapFiles(zipContent.files, commonRoot);
const rootPrefix = commonRoot === '' ? '' : commonRoot + '/';
const cleanedPaths = Object.entries(zipContent.files)
.filter(([, entry]) => !entry.dir)
.map(([path]) => path.slice(rootPrefix.length));

// Create new predictable zip with cleaned files
const cleanedZipBuffer = await createPredictableZip(cleanedFiles);
const cleanedZip = await createPredictableZip(cleanedPaths, path =>
zipContent.files[rootPrefix + path].async('uint8array'),
);

// Create new File object with original metadata
const cleanedFile = new File([cleanedZipBuffer], file.name, {
const cleanedFile = new File([cleanedZip], file.name, {
type: file.type,
lastModified: file.lastModified,
});
Expand Down
Loading