From f81a23fd600c593499049f021bdaa46e2245673c Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Wed, 4 Nov 2026 23:09:16 +0100 Subject: [PATCH 1/3] An npm package that gives Playwright tests a generated set, and an example of what it finds npm/testing-files-generator plans a set with tfg while a spec file loads, writing nothing, so every file becomes a test with a title, tags and the sentence that says what the file is for. The first test that asks for a path writes the set, once for every worker: each writes a copy and the first to move it to its name wins, the others take it and remove theirs through tfg cleanup. Later runs take the same files, keyed by the arguments as well as the recipe hash, which leaves out the seed and --clean. check compares what the system did with the manifest and throws with that sentence. Unspecified files are skipped, never passed. No dependencies, no install script, MPL-2.0. Not published yet. examples/playwright is an upload form with four mistakes. The generated set finds all four through the page and the API, the tests written with Buffer.alloc find none, and with the mistakes fixed nothing fails. CI runs the package's tests on three systems and the example on Linux. Guards hold the package to the platforms the release builds, the closed lists and fields of the manifest, and the punctuation of the text it shows. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 60 +++ examples/playwright/.gitignore | 3 + examples/playwright/README.md | 40 ++ examples/playwright/app/index.html | 13 + examples/playwright/app/page.js | 22 + examples/playwright/app/server.mjs | 108 ++++ examples/playwright/check.mjs | 100 ++++ examples/playwright/package-lock.json | 469 ++++++++++++++++++ examples/playwright/package.json | 17 + examples/playwright/playwright.config.ts | 26 + examples/playwright/shared.ts | 5 + examples/playwright/tests/typical.spec.ts | 41 ++ examples/playwright/tests/upload.spec.ts | 45 ++ examples/playwright/tsconfig.json | 12 + internal/guard/ascii_test.go | 5 + internal/guard/mutation/entries.py | 51 ++ internal/guard/npmpackage_test.go | 365 ++++++++++++++ npm/testing-files-generator/LICENSE | 373 ++++++++++++++ npm/testing-files-generator/README.md | 110 ++++ npm/testing-files-generator/bin/tfg.js | 52 ++ npm/testing-files-generator/index.d.ts | 129 +++++ npm/testing-files-generator/index.js | 12 + npm/testing-files-generator/lib/binary.js | 56 +++ npm/testing-files-generator/lib/fileset.js | 379 ++++++++++++++ npm/testing-files-generator/lib/run.js | 70 +++ npm/testing-files-generator/lib/testfile.js | 129 +++++ npm/testing-files-generator/package.json | 51 ++ .../test/binary.test.js | 41 ++ .../test/fileset.test.js | 178 +++++++ npm/testing-files-generator/test/helpers.js | 58 +++ .../test/testfile.test.js | 59 +++ 31 files changed, 3079 insertions(+) create mode 100644 examples/playwright/.gitignore create mode 100644 examples/playwright/README.md create mode 100644 examples/playwright/app/index.html create mode 100644 examples/playwright/app/page.js create mode 100644 examples/playwright/app/server.mjs create mode 100644 examples/playwright/check.mjs create mode 100644 examples/playwright/package-lock.json create mode 100644 examples/playwright/package.json create mode 100644 examples/playwright/playwright.config.ts create mode 100644 examples/playwright/shared.ts create mode 100644 examples/playwright/tests/typical.spec.ts create mode 100644 examples/playwright/tests/upload.spec.ts create mode 100644 examples/playwright/tsconfig.json create mode 100644 internal/guard/npmpackage_test.go create mode 100644 npm/testing-files-generator/LICENSE create mode 100644 npm/testing-files-generator/README.md create mode 100644 npm/testing-files-generator/bin/tfg.js create mode 100644 npm/testing-files-generator/index.d.ts create mode 100644 npm/testing-files-generator/index.js create mode 100644 npm/testing-files-generator/lib/binary.js create mode 100644 npm/testing-files-generator/lib/fileset.js create mode 100644 npm/testing-files-generator/lib/run.js create mode 100644 npm/testing-files-generator/lib/testfile.js create mode 100644 npm/testing-files-generator/package.json create mode 100644 npm/testing-files-generator/test/binary.test.js create mode 100644 npm/testing-files-generator/test/fileset.test.js create mode 100644 npm/testing-files-generator/test/helpers.js create mode 100644 npm/testing-files-generator/test/testfile.test.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47a5ee43..dae8548b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1482,6 +1482,66 @@ jobs: run: go test -tags "$(cat .github/build-tags)" ./internal/guard/ -count=1 -run 'ReferenceTool' -v shell: bash + npm: + name: npm package on ${{ matrix.os }} + # The npm package (npm/testing-files-generator) runs tfg and reads what it + # writes from JavaScript, so nothing in the Go suite runs it. Its own tests + # run here against the binary this commit builds, on the three systems a + # release has binaries for - Windows above all, because the rename that + # settles a race between test workers is refused there in ways it is not + # elsewhere. Node 22 is the oldest the package declares and Playwright + # supports, so it is the one tested. + # + # The Playwright example runs on Linux only. It is the package's acceptance + # test: an upload form with four mistakes, where the generated set has to + # find all four, the tests written with Buffer.alloc have to find none, and + # with the mistakes fixed nothing may fail - that last run is the control + # without which a red test could come from the tests. One browser on one + # system is enough to show that, and the package itself is tested on all + # three. docs/PLAYWRIGHT-2026-11-04.md. + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + env: + CGO_ENABLED: "0" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: false + cache: true + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: build tfg where the package's tests and the example look for it + run: go build -tags "$(cat .github/build-tags)" -o "tfg$(go env GOEXE)" ./cmd/tfg + shell: bash + + - name: the package's own tests + working-directory: npm/testing-files-generator + run: node --test "test/*.test.js" + shell: bash + + - name: the Playwright example finds the four mistakes, and nothing once they are fixed + if: runner.os == 'Linux' + working-directory: examples/playwright + run: | + set -euo pipefail + npm ci --no-audit --no-fund + npx playwright install --with-deps chromium + npx tsc --noEmit + node check.mjs + shell: bash + fuzz: name: fuzzing with a time budget # Weekly and by hand, never on a push. Fuzzing searches without end, so a diff --git a/examples/playwright/.gitignore b/examples/playwright/.gitignore new file mode 100644 index 00000000..3b28d5e2 --- /dev/null +++ b/examples/playwright/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +results/ +test-results/ diff --git a/examples/playwright/README.md b/examples/playwright/README.md new file mode 100644 index 00000000..426d7b51 --- /dev/null +++ b/examples/playwright/README.md @@ -0,0 +1,40 @@ +# An upload form, the tests it usually gets, and the tests a generated set gives it + +`app/server.mjs` is an upload form with four mistakes, each one a mistake real upload forms have: + +1. The server never checks the size. Only the page does. +2. The content and the extension are each checked against the allowed types, and never against each other, so a PDF named `.jpg` goes through. +3. An allowed extension anywhere in the name counts, so `invoice.jpg.exe` is taken for a picture. +4. The page refuses a file of exactly the limit, which is the largest size allowed. + +`tests/typical.spec.ts` is what such a form usually gets: a real picture that should go through, and files made in the test with `Buffer.alloc` that should not. Every one of those tests passes. The file over the limit is zeros rather than a PDF, so the server turns it away for what it holds, and the test meant to prove the size limit passes on a server that has none. + +`tests/upload.spec.ts` sends the `upload-validation` set from [testing-files-generator](../../npm/testing-files-generator) through the page and straight to the server. It finds all four mistakes, and each failure says what it means. + +## Run it + +From the root of the repository, build tfg first. The example finds it there. + +```bash +go build -tags "$(cat .github/build-tags)" -o tfg.exe ./cmd/tfg +``` + +```bash +cd examples/playwright && npm install && npx playwright install chromium +``` + +```bash +npx playwright test +``` + +Eleven tests fail, five through the page and six through the API, and every failure names its file and the mistake it shows: + +``` +1mb_over_1b.jpg was taken, and it should have been turned away (size_limit). +A JPEG file 1 B over the limit of 1mb. Your system should turn it away - taking it means +the limit is not enforced, or enforced only in the browser. +``` + +Ten are skipped, because what a form does with an empty file or a name full of emoji is its own policy, and the set says so rather than pass them. + +`FIXED=1 npx playwright test` corrects all four mistakes, and nothing fails. `npm run check` runs both and says whether the example still shows what it is for. The repository's CI runs that. diff --git a/examples/playwright/app/index.html b/examples/playwright/app/index.html new file mode 100644 index 00000000..fe0390c4 --- /dev/null +++ b/examples/playwright/app/index.html @@ -0,0 +1,13 @@ + + +Upload + +
+ + + +
+

+ + + diff --git a/examples/playwright/app/page.js b/examples/playwright/app/page.js new file mode 100644 index 00000000..44dcacdf --- /dev/null +++ b/examples/playwright/app/page.js @@ -0,0 +1,22 @@ +// The form's own check, in the browser, before anything is sent. + +const limit = Number(document.body.dataset.limit); +const fixed = document.body.dataset.fixed === '1'; + +document.getElementById('upload').addEventListener('submit', async (event) => { + event.preventDefault(); + const status = document.getElementById('status'); + const file = document.getElementById('file').files[0]; + // Mistake 4: a file of exactly the limit is refused. The limit is the + // largest size allowed, so the comparison should be >. + const tooLarge = fixed ? file.size > limit : file.size >= limit; + if (tooLarge) { + status.textContent = 'Refused (size_limit)'; + return; + } + const body = new FormData(); + body.append('file', file); + const response = await fetch('/upload', { method: 'POST', body }); + const answer = await response.json(); + status.textContent = response.ok ? 'Uploaded' : `Refused (${answer.reason})`; +}); diff --git a/examples/playwright/app/server.mjs b/examples/playwright/app/server.mjs new file mode 100644 index 00000000..e83c0e6d --- /dev/null +++ b/examples/playwright/app/server.mjs @@ -0,0 +1,108 @@ +// An upload form with four mistakes made on purpose, each one a mistake real +// upload forms have. playwright.config.ts starts it. FIXED=1 corrects all four, +// which is how check.mjs shows that the red tests come from the mistakes and +// not from the tests. +// +// It uses nothing outside Node, so the example has no server dependency. + +import { readFileSync } from 'node:fs'; +import http from 'node:http'; +import { Readable } from 'node:stream'; + +const limit = Number(process.env.UPLOAD_LIMIT_BYTES); +const port = Number(process.env.PORT); +const fixed = process.env.FIXED === '1'; +const allowed = ['jpg', 'png', 'pdf']; + +if (!Number.isInteger(limit) || !Number.isInteger(port)) { + throw new Error('Set UPLOAD_LIMIT_BYTES and PORT. playwright.config.ts sets both.'); +} + +// What a file holds, read from its first bytes rather than from its name. +function sniff(bytes) { + if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) { + return 'jpg'; + } + if (bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) { + return 'png'; + } + if (bytes.subarray(0, 5).toString('latin1') === '%PDF-') { + return 'pdf'; + } + return ''; +} + +function extensionOf(name) { + const lower = name.toLowerCase(); + if (fixed) { + const dot = lower.lastIndexOf('.'); + return dot > 0 ? lower.slice(dot + 1) : ''; + } + // Mistake 3: an allowed extension anywhere in the name counts, so + // invoice.jpg.exe is taken for a picture. + return allowed.find((ext) => lower.includes(`.${ext}`)) ?? ''; +} + +function verdict(name, bytes) { + // Mistake 1: the server never looks at the size. The page does, so a test + // through the page sees a limit that a request straight to the server does not. + if (fixed && bytes.length > limit) { + return { status: 413, reason: 'size_limit' }; + } + const extension = extensionOf(name); + if (!allowed.includes(extension)) { + return { status: 422, reason: 'extension_rule' }; + } + const content = sniff(bytes); + if (!allowed.includes(content)) { + return { status: 415, reason: 'content_malformed' }; + } + // Mistake 2: the content and the extension are each checked against the + // allowed list and never against each other, so a PDF named .jpg goes through. + if (fixed && content !== extension) { + return { status: 415, reason: 'mime_mismatch' }; + } + return { status: 201 }; +} + +// The page carries the limit and the mode, and page.js reads them from it. +const page = readFileSync(new URL('index.html', import.meta.url), 'utf8') + .replace('{{limit}}', String(limit)) + .replace('{{fixed}}', fixed ? '1' : ''); +const script = readFileSync(new URL('page.js', import.meta.url), 'utf8'); + +async function upload(req, res) { + const request = new Request(`http://localhost${req.url}`, { + method: 'POST', headers: req.headers, body: Readable.toWeb(req), duplex: 'half', + }); + const file = (await request.formData()).get('file'); + if (!file || typeof file === 'string') { + return answer(res, 400, { reason: 'none' }); + } + const { status, reason } = verdict(file.name, Buffer.from(await file.arrayBuffer())); + return answer(res, status, reason ? { reason } : { stored: file.name }); +} + +function answer(res, status, body) { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(JSON.stringify(body)); +} + +http.createServer((req, res) => { + if (req.method === 'GET' && req.url === '/') { + res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }); + res.end(page); + return; + } + if (req.method === 'GET' && req.url === '/page.js') { + res.writeHead(200, { 'content-type': 'text/javascript; charset=utf-8' }); + res.end(script); + return; + } + if (req.method === 'POST' && req.url === '/upload') { + upload(req, res).catch((err) => answer(res, 500, { reason: 'none', error: String(err) })); + return; + } + res.writeHead(404); + res.end(); +}).listen(port, () => console.log(`upload form on http://localhost:${port}${fixed ? ', mistakes fixed' : ''}`)); diff --git a/examples/playwright/check.mjs b/examples/playwright/check.mjs new file mode 100644 index 00000000..e40f9830 --- /dev/null +++ b/examples/playwright/check.mjs @@ -0,0 +1,100 @@ +// Runs the example twice and says whether it shows what it is for. +// +// With the four mistakes, the tests from the generated set fail on exactly the +// files each mistake lets through or turns away, and every typical test passes. +// With FIXED=1, nothing fails. The second run is the control: without it, red +// tests could come from the tests themselves rather than from the app. + +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; + +// Playwright's own command line, run by this Node rather than through npx and +// a shell, which Windows would need for npx. +const playwright = createRequire(import.meta.url).resolve('@playwright/test/cli'); + +// Each mistake in app/server.mjs, and the tests it turns red. +const mistakes = { + 'through the page': { + '1mb_at_limit.jpg': 4, + 'pdf_as_jpg.jpg': 2, + 'png_as_pdf.pdf': 2, + 'jpg_as_png.png': 2, + 'invoice.jpg.exe': 3, + }, + 'through the API': { + '1mb_over_1b.jpg': 1, + '1mb_far_over_2x.jpg': 1, + 'pdf_as_jpg.jpg': 2, + 'png_as_pdf.pdf': 2, + 'jpg_as_png.png': 2, + 'invoice.jpg.exe': 3, + }, +}; + +function run(fixed) { + spawnSync(process.execPath, [playwright, 'test', '--reporter=json'], { + stdio: ['ignore', 'ignore', 'inherit'], + env: { ...process.env, FIXED: fixed ? '1' : '', PLAYWRIGHT_JSON_OUTPUT_NAME: 'results/check.json' }, + }); + const report = JSON.parse(readFileSync('results/check.json', 'utf8')); + const tests = []; + const walk = (suite, titles) => { + for (const spec of suite.specs ?? []) { + for (const one of spec.tests) { + tests.push({ file: spec.file, describe: titles.at(-1) ?? '', title: spec.title, status: one.status }); + } + } + for (const child of suite.suites ?? []) { + walk(child, [...titles, child.title]); + } + }; + for (const suite of report.suites) { + walk(suite, []); + } + if (report.errors?.length) { + throw new Error(`The run did not get as far as its tests: ${report.errors.map((e) => e.message).join('\n')}`); + } + return tests; +} + +const problems = []; +const say = (problem) => problems.push(problem); + +for (const fixed of [false, true]) { + const mode = fixed ? 'with the mistakes fixed' : 'with the four mistakes'; + const tests = run(fixed); + const ran = tests.filter((t) => t.status !== 'skipped'); + const generated = tests.filter((t) => t.file.endsWith('upload.spec.ts')); + if (ran.length === 0 || generated.every((t) => t.status === 'skipped')) { + say(`${mode}: no test from the generated set ran`); + } + for (const t of tests) { + const name = t.title.split(' - expected ')[0]; + const mistake = mistakes[t.describe]?.[name]; + const shouldFail = !fixed && mistake !== undefined && t.file.endsWith('upload.spec.ts'); + const failed = t.status === 'unexpected'; + if (shouldFail && !failed) { + say(`${mode}: ${t.describe} - ${name} should be red for mistake ${mistake} and is ${t.status}`); + } + if (!shouldFail && failed) { + say(`${mode}: ${t.describe ? `${t.describe} - ` : ''}${t.title} failed, and nothing in the app should make it`); + } + } + const red = tests.filter((t) => t.status === 'unexpected').length; + const skipped = tests.filter((t) => t.status === 'skipped').length; + console.log(`${mode}: ${red} red, ${tests.length - red - skipped} green, ${skipped} skipped as unspecified`); +} + +const found = new Set(Object.values(mistakes).flatMap((files) => Object.values(files))); +for (const n of [1, 2, 3, 4]) { + if (!found.has(n)) { + say(`mistake ${n} has no test that finds it`); + } +} + +if (problems.length > 0) { + console.error(`The example does not show what it is for:\n- ${problems.join('\n- ')}`); + process.exit(1); +} +console.log('The generated set finds all four mistakes, the typical tests find none, and with the mistakes fixed nothing is red.'); diff --git a/examples/playwright/package-lock.json b/examples/playwright/package-lock.json new file mode 100644 index 00000000..d6820d11 --- /dev/null +++ b/examples/playwright/package-lock.json @@ -0,0 +1,469 @@ +{ + "name": "testing-files-generator-playwright-example", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "testing-files-generator-playwright-example", + "license": "MPL-2.0", + "devDependencies": { + "@playwright/test": "1.64.1", + "@types/node": "26.6.6", + "testing-files-generator": "file:../../npm/testing-files-generator", + "typescript": "7.0.2" + } + }, + "../../npm/testing-files-generator": { + "version": "0.0.0-development", + "dev": true, + "license": "MPL-2.0", + "bin": { + "tfg": "bin/tfg.js" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@playwright/test": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.64.1.tgz", + "integrity": "sha512-IaYGml/nSOiZWROfPtuIL7neuu9qDOZ3TNvNljUIrhDMxiGTgPjzj/g9p9RxpS2tSepPDRY+UwAaEeiKtUvIAg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.64.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@types/node": { + "version": "26.6.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.6.tgz", + "integrity": "sha512-MzQj+8Kx3XI40OmUAE6Qoh58ZMNstNNMwn2nfd3qWsfe7nDNSqJNzes4njhz6AHwrSHOffSLwXU6yb474Q4aog==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/@typescript/typescript-aix-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", + "integrity": "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.0.2.tgz", + "integrity": "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.0.2.tgz", + "integrity": "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-arm64/-/typescript-freebsd-arm64-7.0.2.tgz", + "integrity": "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-x64/-/typescript-freebsd-x64-7.0.2.tgz", + "integrity": "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.0.2.tgz", + "integrity": "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.0.2.tgz", + "integrity": "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-loong64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-loong64/-/typescript-linux-loong64-7.0.2.tgz", + "integrity": "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-mips64el": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-mips64el/-/typescript-linux-mips64el-7.0.2.tgz", + "integrity": "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-ppc64/-/typescript-linux-ppc64-7.0.2.tgz", + "integrity": "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-riscv64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-riscv64/-/typescript-linux-riscv64-7.0.2.tgz", + "integrity": "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-s390x": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-s390x/-/typescript-linux-s390x-7.0.2.tgz", + "integrity": "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.0.2.tgz", + "integrity": "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-arm64/-/typescript-netbsd-arm64-7.0.2.tgz", + "integrity": "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-x64/-/typescript-netbsd-x64-7.0.2.tgz", + "integrity": "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-arm64/-/typescript-openbsd-arm64-7.0.2.tgz", + "integrity": "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-x64/-/typescript-openbsd-x64-7.0.2.tgz", + "integrity": "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-sunos-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-sunos-x64/-/typescript-sunos-x64-7.0.2.tgz", + "integrity": "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.0.2.tgz", + "integrity": "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.0.2.tgz", + "integrity": "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/playwright": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.64.1.tgz", + "integrity": "sha512-HCrVhHr5QQuwYoswEUGTb7gGp522yflbv/ST8nF4sFWGuLjQCt17Q/EubpGGSNKYn1tsTCPZxqHhHCfVURu7UA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.64.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.64.1.tgz", + "integrity": "sha512-BC4s8y6SEVA6Q1rSxZTto8hI1KUI3xXlHGZgvGqP4IqPdI/a6FX9H29IDNz7vKo9l21b6LfYosScPL6M3xxyDQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/testing-files-generator": { + "resolved": "../../npm/testing-files-generator", + "link": true + }, + "node_modules/typescript": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", + "integrity": "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc" + }, + "engines": { + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-aix-ppc64": "7.0.2", + "@typescript/typescript-darwin-arm64": "7.0.2", + "@typescript/typescript-darwin-x64": "7.0.2", + "@typescript/typescript-freebsd-arm64": "7.0.2", + "@typescript/typescript-freebsd-x64": "7.0.2", + "@typescript/typescript-linux-arm": "7.0.2", + "@typescript/typescript-linux-arm64": "7.0.2", + "@typescript/typescript-linux-loong64": "7.0.2", + "@typescript/typescript-linux-mips64el": "7.0.2", + "@typescript/typescript-linux-ppc64": "7.0.2", + "@typescript/typescript-linux-riscv64": "7.0.2", + "@typescript/typescript-linux-s390x": "7.0.2", + "@typescript/typescript-linux-x64": "7.0.2", + "@typescript/typescript-netbsd-arm64": "7.0.2", + "@typescript/typescript-netbsd-x64": "7.0.2", + "@typescript/typescript-openbsd-arm64": "7.0.2", + "@typescript/typescript-openbsd-x64": "7.0.2", + "@typescript/typescript-sunos-x64": "7.0.2", + "@typescript/typescript-win32-arm64": "7.0.2", + "@typescript/typescript-win32-x64": "7.0.2" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/examples/playwright/package.json b/examples/playwright/package.json new file mode 100644 index 00000000..af04827c --- /dev/null +++ b/examples/playwright/package.json @@ -0,0 +1,17 @@ +{ + "name": "testing-files-generator-playwright-example", + "private": true, + "description": "An upload form with four mistakes, the tests it usually gets, which pass, and tests from a generated set, which find all four.", + "license": "MPL-2.0", + "scripts": { + "test": "playwright test", + "check": "node check.mjs", + "types": "tsc --noEmit" + }, + "devDependencies": { + "@playwright/test": "1.64.1", + "@types/node": "26.6.6", + "testing-files-generator": "file:../../npm/testing-files-generator", + "typescript": "7.0.2" + } +} diff --git a/examples/playwright/playwright.config.ts b/examples/playwright/playwright.config.ts new file mode 100644 index 00000000..c1388dbc --- /dev/null +++ b/examples/playwright/playwright.config.ts @@ -0,0 +1,26 @@ +import { defineConfig } from '@playwright/test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import { limitBytes, port } from './shared'; + +// Inside the repository the package is linked rather than installed, so its +// binary comes from the repository's own build. A copy of this example outside +// the repository installs testing-files-generator from npm and needs none of this. +const built = path.join(__dirname, '..', '..', process.platform === 'win32' ? 'tfg.exe' : 'tfg'); +if (!process.env.TFG_BINARY && fs.existsSync(built)) { + process.env.TFG_BINARY = built; +} + +export default defineConfig({ + testDir: 'tests', + fullyParallel: true, + reporter: [['list'], ['json', { outputFile: 'results/report.json' }], ['html', { open: 'never', outputFolder: 'results/html' }]], + use: { baseURL: `http://localhost:${port}` }, + webServer: { + command: 'node app/server.mjs', + url: `http://localhost:${port}/`, + reuseExistingServer: false, + env: { UPLOAD_LIMIT_BYTES: String(limitBytes), PORT: String(port), FIXED: process.env.FIXED ?? '' }, + }, +}); diff --git a/examples/playwright/shared.ts b/examples/playwright/shared.ts new file mode 100644 index 00000000..5be9e232 --- /dev/null +++ b/examples/playwright/shared.ts @@ -0,0 +1,5 @@ +// The one limit the form declares, read by the app, the tests made in code and +// the generated set alike. Units count in 1024s, as they do in tfg. +export const limit = '1mb'; +export const limitBytes = 1024 * 1024; +export const port = 3210; diff --git a/examples/playwright/tests/typical.spec.ts b/examples/playwright/tests/typical.spec.ts new file mode 100644 index 00000000..22221360 --- /dev/null +++ b/examples/playwright/tests/typical.spec.ts @@ -0,0 +1,41 @@ +import { test, expect } from '@playwright/test'; + +import { limitBytes } from '../shared'; + +// The tests an upload form usually gets: one real picture that should go +// through, and files made in the test that should not. Against this app every +// one of them passes, while the app has the four mistakes upload.spec.ts finds. +// +// The file over the limit is the reason. Buffer.alloc gives zeros, not a PDF, +// so the server turns it away for what it holds - and the test that was meant +// to prove a size limit passes on a server that has none. + +// A picture of one pixel, the kind of fixture kept as text in a test. +const photo = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==', 'base64'); +const tooLarge = { name: 'big.pdf', mimeType: 'application/pdf', buffer: Buffer.alloc(limitBytes + 1) }; + +test('a photo is uploaded', async ({ page }) => { + await page.goto('/'); + await page.getByLabel('File').setInputFiles({ name: 'photo.png', mimeType: 'image/png', buffer: photo }); + await page.getByRole('button', { name: 'Upload' }).click(); + await expect(page.getByRole('status')).toHaveText('Uploaded'); +}); + +test('a file over the limit is refused in the page', async ({ page }) => { + await page.goto('/'); + await page.getByLabel('File').setInputFiles(tooLarge); + await page.getByRole('button', { name: 'Upload' }).click(); + await expect(page.getByRole('status')).toHaveText(/^Refused/); +}); + +test('a file over the limit is refused by the server', async ({ request }) => { + const response = await request.post('/upload', { multipart: { file: tooLarge } }); + expect(response.ok()).toBe(false); +}); + +test('an executable is refused', async ({ request }) => { + const response = await request.post('/upload', { + multipart: { file: { name: 'setup.exe', mimeType: 'application/octet-stream', buffer: Buffer.from('MZ') } }, + }); + expect(response.ok()).toBe(false); +}); diff --git a/examples/playwright/tests/upload.spec.ts b/examples/playwright/tests/upload.spec.ts new file mode 100644 index 00000000..7ad4e8f1 --- /dev/null +++ b/examples/playwright/tests/upload.spec.ts @@ -0,0 +1,45 @@ +import { test, expect } from '@playwright/test'; +import { createReadStream } from 'node:fs'; +import { fileSet, type Observed } from 'testing-files-generator'; + +import { limit } from '../shared'; + +// The set the upload-validation preset builds around the form's limit: either +// side of it, types the form takes and turns away, content under the wrong +// name, names that are hard to store. The form takes one file at a time, so +// the mass upload is left out. +const uploads = fileSet({ preset: 'upload-validation', options: { limit, bulk: 0 } }); + +test.beforeAll(() => uploads.write()); + +test.describe('through the page', () => { + for (const file of uploads.files) { + test(file.title, file.details, async ({ page }) => { + test.skip(file.unspecified, file.whyUnspecified); + await page.goto('/'); + await page.getByLabel('File').setInputFiles(await file.path()); + await page.getByRole('button', { name: 'Upload' }).click(); + const status = page.getByRole('status'); + await expect(status).not.toBeEmpty(); + file.check(shown(await status.textContent())); + }); + } +}); + +// The same files sent to the server directly, the way anything other than this +// page can send them. +test.describe('through the API', () => { + for (const file of uploads.files) { + test(file.title, file.details, async ({ request }) => { + test.skip(file.unspecified, file.whyUnspecified); + const response = await request.post('/upload', { multipart: { file: createReadStream(await file.path()) } }); + const answer = await response.json(); + file.check(response.ok() ? 'accept' : { outcome: 'reject', reason: answer.reason }); + }); + } +}); + +function shown(text: string | null): Observed { + const refused = /^Refused \((\w+)\)$/.exec(text ?? ''); + return refused ? { outcome: 'reject', reason: refused[1] } : 'accept'; +} diff --git a/examples/playwright/tsconfig.json b/examples/playwright/tsconfig.json new file mode 100644 index 00000000..d82f15ef --- /dev/null +++ b/examples/playwright/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "skipLibCheck": false, + "types": ["node"] + }, + "include": ["playwright.config.ts", "shared.ts", "tests/**/*.ts"] +} diff --git a/internal/guard/ascii_test.go b/internal/guard/ascii_test.go index dd62a961..1de7d63d 100644 --- a/internal/guard/ascii_test.go +++ b/internal/guard/ascii_test.go @@ -65,6 +65,11 @@ var englishFiles = []string{ // the copy. Repository text somebody reads to learn what was changed in // a module they thought they knew - the same criterion again. "third_party/go-gl-gl/PATCH.md", + // The npm package's page and its example's, added 2026-11-04 with them. + // The first is what npm shows a person deciding whether to install it, + // the second what they read before running it - the same criterion again. + "npm/testing-files-generator/README.md", + "examples/playwright/README.md", } // symbolsAllowed names the files that may hold a symbol such as an emoji. diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index 8147e36c..cfdebf10 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -16844,4 +16844,55 @@ "\t\tif code != ExitOK {", "TestGenerateSendsItsManifestAPartAtATime", ), + # --- the npm package is held to the release and the manifest it reads + # (2026-11-04, docs/PLAYWRIGHT-2026-11-04.md) --- + ( + "the npm package forgets a platform the release builds a binary for", + "npm/testing-files-generator/lib/binary.js", + " 'linux arm64': '@testing-files-generator/linux-arm64',\n", + "", + "TestThePackageKnowsEveryPlatformTheReleaseBuilds", + ), + ( + "the npm package looks for a binary in a package the release does not publish", + "npm/testing-files-generator/lib/binary.js", + "'@testing-files-generator/win32-arm64'", + "'@testing-files-generator/windows-arm64'", + "TestThePackageKnowsEveryPlatformTheReleaseBuilds", + ), + ( + "the npm package's types lose a reason the program declares", + "npm/testing-files-generator/index.d.ts", + " | 'duplicate'\n", + "", + "TestThePackageHoldsTheManifestsClosedLists", + ), + ( + "the npm package refuses a manifest that expects sanitize", + "npm/testing-files-generator/lib/testfile.js", + "'sanitize', 'unspecified']", + "'unspecified']", + "TestThePackageHoldsTheManifestsClosedLists", + ), + ( + "the npm package's types promise a field the manifest does not write", + "npm/testing-files-generator/index.d.ts", + " target_id: string;\n", + " target: string;\n", + "TestThePackageTypesOnlyPromiseFieldsTheManifestHas", + ), + ( + "an error the npm package throws into a test picks up a semicolon", + "npm/testing-files-generator/lib/binary.js", + "or unset it to use", + "; unset it to use", + "TestTheTextOfThePackageIsAsciiWithPlainPunctuation", + ), + ( + "the example's page says something with a semicolon", + "examples/playwright/app/page.js", + "'Uploaded'", + "'Uploaded; done'", + "TestTheTextOfThePackageIsAsciiWithPlainPunctuation", + ), ] diff --git a/internal/guard/npmpackage_test.go b/internal/guard/npmpackage_test.go new file mode 100644 index 00000000..efba3e01 --- /dev/null +++ b/internal/guard/npmpackage_test.go @@ -0,0 +1,365 @@ +package guard + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "sort" + "strings" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/manifest" + "github.com/donislawdev/TestingFilesGenerator/internal/recipe" +) + +// The npm package (npm/testing-files-generator) runs this program and reads +// what it writes, from JavaScript. Nothing in Go calls it, so nothing in Go +// notices when the two drift apart - a platform the release builds and the +// package cannot find, an outcome the program declares and the package refuses, +// a manifest field the package's types promise and the manifest does not have. +// These guards read the package's files the way the release and the manifest +// are read here, and hold one to the other. docs/PLAYWRIGHT-2026-11-04.md. + +const npmPackage = "npm/testing-files-generator" + +func npmFile(t *testing.T, rel string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(repoRoot(t), filepath.FromSlash(rel))) + if err != nil { + t.Fatalf("reading %s: %v", rel, err) + } + return string(b) +} + +// releaseTargets is the list of GOOS/GOARCH pairs the release loop builds a +// command line for. +func releaseTargets(t *testing.T) []string { + t.Helper() + body := npmFile(t, ".github/workflows/release.yml") + loop := regexp.MustCompile(`(?s)for target in \\\n(.*?)\n\s*do\n`).FindStringSubmatch(body) + if loop == nil { + t.Fatal("the release workflow has no loop of command line targets - this guard no longer reads it") + } + targets := regexp.MustCompile(`[a-z0-9]+/[a-z0-9]+`).FindAllString(loop[1], -1) + if len(targets) == 0 { + t.Fatal("the release loop names no targets this guard can read") + } + sort.Strings(targets) + return targets +} + +func TestThePackageKnowsEveryPlatformTheReleaseBuilds(t *testing.T) { + node := map[string]string{"windows": "win32", "linux": "linux", "darwin": "darwin", "amd64": "x64", "arm64": "arm64"} + var want []string + for _, target := range releaseTargets(t) { + goos, goarch, _ := strings.Cut(target, "/") + if node[goos] == "" || node[goarch] == "" { + t.Fatalf("the release builds %s, and this guard does not know what Node calls it", target) + } + want = append(want, node[goos]+" "+node[goarch]) + } + + body := npmFile(t, npmPackage+"/lib/binary.js") + entries := regexp.MustCompile(`'([a-z0-9]+ [a-z0-9]+)': '(@testing-files-generator/[a-z0-9-]+)'`).FindAllStringSubmatch(body, -1) + var got []string + for _, e := range entries { + got = append(got, e[1]) + if name := "@testing-files-generator/" + strings.ReplaceAll(e[1], " ", "-"); e[2] != name { + t.Errorf("the binary for %s comes in %s, and the name the release publishes for it is %s", e[1], e[2], name) + } + } + sort.Strings(got) + sort.Strings(want) + if !reflect.DeepEqual(got, want) { + t.Errorf("the package looks for a binary on %v, and the release builds one for %v", got, want) + } +} + +// unionOf reads the members of a TypeScript union of string literals. +func unionOf(t *testing.T, body, name string) []string { + t.Helper() + decl := regexp.MustCompile(`(?s)export type ` + name + ` =(.*?);`).FindStringSubmatch(body) + if decl == nil { + t.Fatalf("index.d.ts has no type %s", name) + } + var out []string + for _, m := range regexp.MustCompile(`'([a-z_]+)'`).FindAllStringSubmatch(decl[1], -1) { + out = append(out, m[1]) + } + sort.Strings(out) + return out +} + +func TestThePackageHoldsTheManifestsClosedLists(t *testing.T) { + types := npmFile(t, npmPackage+"/index.d.ts") + if got, want := unionOf(t, types, "Outcome"), recipe.Outcomes(); !reflect.DeepEqual(got, want) { + t.Errorf("index.d.ts says an outcome is one of %v, and the program declares %v", got, want) + } + if got, want := unionOf(t, types, "Reason"), recipe.Reasons(); !reflect.DeepEqual(got, want) { + t.Errorf("index.d.ts says a reason is one of %v, and the program declares %v", got, want) + } + + runtime := regexp.MustCompile(`const outcomes = \[([^\]]*)\]`).FindStringSubmatch(npmFile(t, npmPackage+"/lib/testfile.js")) + if runtime == nil { + t.Fatal("lib/testfile.js has no list of outcomes this guard can read") + } + var got []string + for _, m := range regexp.MustCompile(`'([a-z_]+)'`).FindAllStringSubmatch(runtime[1], -1) { + got = append(got, m[1]) + } + sort.Strings(got) + if want := recipe.Outcomes(); !reflect.DeepEqual(got, want) { + t.Errorf("the package accepts the outcomes %v, and the program declares %v", got, want) + } +} + +// jsonFields is the set of names a manifest type writes. +func jsonFields(typ reflect.Type) map[string]bool { + out := map[string]bool{} + for i := 0; i < typ.NumField(); i++ { + name, _, _ := strings.Cut(typ.Field(i).Tag.Get("json"), ",") + if name != "" && name != "-" { + out[name] = true + } + } + return out +} + +// typeScriptFields reads the field names of one block of index.d.ts - an +// interface, or an object written inside one - leaving out the index +// signature that lets a manifest carry more than the package reads. +func typeScriptFields(block string) []string { + var out []string + for _, m := range regexp.MustCompile(`(?m)(?:^\s*|[{;]\s*)([a-z_]+)\??:`).FindAllStringSubmatch(block, -1) { + out = append(out, m[1]) + } + return out +} + +func TestThePackageTypesOnlyPromiseFieldsTheManifestHas(t *testing.T) { + types := npmFile(t, npmPackage+"/index.d.ts") + iface := func(name string) string { + m := regexp.MustCompile(`(?s)export interface ` + name + ` \{\n(.*?)\n\}`).FindStringSubmatch(types) + if m == nil { + t.Fatalf("index.d.ts has no interface %s", name) + } + return m[1] + } + inline := func(block, name string) string { + m := regexp.MustCompile(`(?m)^\s*` + name + `: \{(.*)\};$`).FindStringSubmatch(block) + if m == nil { + t.Fatalf("the Manifest interface in index.d.ts has no %s block on one line", name) + } + return m[1] + } + whole := iface("Manifest") + top := regexp.MustCompile(`(?m)^\s*\w+: \{.*\};$`).ReplaceAllString(whole, "") + cases := []struct { + where string + block string + typ reflect.Type + }{ + {"ManifestFile", iface("ManifestFile"), reflect.TypeOf(manifest.File{})}, + {"Expected", iface("Expected"), reflect.TypeOf(manifest.Expected{})}, + {"Manifest", top + "\ntool: x;\nrun: x;\nsummary: x;", reflect.TypeOf(manifest.Manifest{})}, + {"Manifest.tool", inline(whole, "tool"), reflect.TypeOf(manifest.Tool{})}, + {"Manifest.run", inline(whole, "run"), reflect.TypeOf(manifest.Run{})}, + {"Manifest.summary", inline(whole, "summary"), reflect.TypeOf(manifest.Summary{})}, + } + for _, c := range cases { + have := jsonFields(c.typ) + fields := typeScriptFields(c.block) + if len(fields) == 0 { + t.Errorf("%s in index.d.ts declares no field this guard can read", c.where) + } + for _, f := range fields { + if !have[f] { + t.Errorf("%s in index.d.ts promises %s, and the manifest's %s writes no such field", c.where, f, c.typ.Name()) + } + } + } +} + +// The text of the package and of its example is what a person reads - an +// error thrown into their test, a comment their editor shows, a line the +// example prints. It is repository text, so it is English, ASCII and rule 13. +var npmTextRoots = []string{npmPackage + "/", "examples/playwright/"} + +// npmLiterals are the files whose strings are sentences somebody reads. In +// the rest a string is a test's data, a script for a child process or a +// header value, and judging it as prose would be crying wolf - the same line +// goFileFaults draws for Go. +var npmLiterals = map[string]bool{ + npmPackage + "/index.js": true, + npmPackage + "/bin/tfg.js": true, + npmPackage + "/lib/binary.js": true, + npmPackage + "/lib/fileset.js": true, + npmPackage + "/lib/run.js": true, + npmPackage + "/lib/testfile.js": true, + "examples/playwright/app/page.js": true, + "examples/playwright/check.mjs": true, +} + +func npmTextFiles(t *testing.T) []string { + t.Helper() + var out []string + for _, name := range strings.Fields(gitOutput(t, "ls-files", "--cached", "--others", "--exclude-standard")) { + for _, root := range npmTextRoots { + if strings.HasPrefix(name, root) { + out = append(out, name) + } + } + } + return out +} + +func TestTheTextOfThePackageIsAsciiWithPlainPunctuation(t *testing.T) { + files := npmTextFiles(t) + sources, literals := 0, 0 + for _, name := range files { + body := npmFile(t, name) + for n, line := range strings.Split(body, "\n") { + if i := strings.IndexFunc(line, func(r rune) bool { return r > 127 }); i >= 0 { + t.Errorf("%s:%d holds %q - text in the repository is English", name, n+1, []rune(line[i:])[0]) + } + } + switch filepath.Ext(name) { + case ".js", ".mjs", ".ts": + default: + continue + } + sources++ + prose, err := jsProse(body, npmLiterals[name]) + if err != nil { + t.Errorf("%s cannot be read for its prose: %v", name, err) + continue + } + if npmLiterals[name] { + literals++ + } + for _, p := range prose { + for _, fault := range proseFaults(p.text, false) { + t.Errorf("%s:%d holds %s", name, p.line, fault) + } + } + } + if sources == 0 || literals < 6 { + t.Fatalf("read %d sources and the strings of %d - the package is not where this guard looks", sources, literals) + } +} + +type jsText struct { + line int + text string +} + +// jsProse returns what a person reads in a JavaScript or TypeScript file: its +// comments, and with literals the words of its strings, the code inside a +// template's ${} left out. It reads only as much of the language as that +// takes, and says so rather than guess when a file ends inside a string. +func jsProse(src string, literals bool) ([]jsText, error) { + var out []jsText + line := 1 + add := func(at int, text string) { + if strings.TrimSpace(text) != "" { + out = append(out, jsText{at, text}) + } + } + // last is the last character of code that is not space, which tells a + // regular expression from a division. + last := byte('\n') + for i := 0; i < len(src); i++ { + c := src[i] + switch { + case c == '\n': + line++ + case c == '/' && i+1 < len(src) && src[i+1] == '/': + end := strings.IndexByte(src[i:], '\n') + if end < 0 { + end = len(src) - i + } + add(line, src[i+2:i+end]) + i += end - 1 + continue + case c == '/' && i+1 < len(src) && src[i+1] == '*': + end := strings.Index(src[i+2:], "*/") + if end < 0 { + return nil, errorAt(line, "a comment that never ends") + } + body := src[i+2 : i+2+end] + add(line, strings.NewReplacer("\n *", "\n", "\n*", "\n").Replace(body)) + line += strings.Count(body, "\n") + i += end + 3 + continue + case c == '/' && strings.IndexByte("(,=:[!&|?{};+\n", last) >= 0: + j := i + 1 + class := false + for ; j < len(src) && (src[j] != '/' || class); j++ { + switch src[j] { + case '\\': + j++ + case '[': + class = true + case ']': + class = false + case '\n': + return nil, errorAt(line, "a regular expression that never ends") + } + } + i = j + last = '/' + continue + case c == '\'' || c == '"' || c == '`': + start, j := line, i+1 + var text strings.Builder + for ; j < len(src) && src[j] != c; j++ { + switch { + case src[j] == '\\': + j++ + case c == '`' && src[j] == '$' && j+1 < len(src) && src[j+1] == '{': + // The code of an interpolation is skipped to its closing + // brace. A brace inside a string in there would end it + // early, and the package writes none. + j++ + for open := 1; open > 0 && j+1 < len(src); { + j++ + switch src[j] { + case '{': + open++ + case '}': + open-- + } + } + text.WriteByte(' ') + case src[j] == '\n': + if c != '`' { + return nil, errorAt(line, "a string that never ends on its line") + } + line++ + text.WriteByte('\n') + default: + text.WriteByte(src[j]) + } + } + if j >= len(src) { + return nil, errorAt(start, "a string that never ends") + } + if literals { + add(start, text.String()) + } + i = j + last = c + continue + } + if c != ' ' && c != '\t' && c != '\r' { + last = c + } + } + return out, nil +} + +func errorAt(line int, what string) error { + return fmt.Errorf("line %d starts %s", line, what) +} diff --git a/npm/testing-files-generator/LICENSE b/npm/testing-files-generator/LICENSE new file mode 100644 index 00000000..d0a1fa14 --- /dev/null +++ b/npm/testing-files-generator/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at https://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/npm/testing-files-generator/README.md b/npm/testing-files-generator/README.md new file mode 100644 index 00000000..d3850bc7 --- /dev/null +++ b/npm/testing-files-generator/README.md @@ -0,0 +1,110 @@ +# testing-files-generator + +Real files of exact sizes and formats for your tests, and what the system under test should do with each one. + +A PDF of exactly 1 MiB and one byte is a real PDF, so a server that looks inside it can only turn it away for its size. A file made with `Buffer.alloc` is zeros, so the same server turns it away for what it holds, and a test meant to prove a size limit passes on a server that has none. [The example](../../examples/playwright) shows exactly that. + +## In a Playwright test + +```ts +import { test, expect } from '@playwright/test'; +import { fileSet } from 'testing-files-generator'; + +const uploads = fileSet({ preset: 'upload-validation', options: { limit: '2mb' } }); + +for (const file of uploads.except({ group: 'bulk-upload' }).files) { + test(file.title, file.details, async ({ page }) => { + test.skip(file.unspecified, file.whyUnspecified); + await page.goto('/upload'); + await page.getByLabel('File').setInputFiles(await file.path()); + await page.getByRole('button', { name: 'Upload' }).click(); + const status = page.getByRole('status'); + await expect(status).not.toBeEmpty(); + file.check((await status.textContent())?.includes('Uploaded') ? 'accept' : 'reject'); + }); +} +``` + +You write the two things only you know: how a file is uploaded in your app, and how it shows that a file was taken or turned away. The set brings the rest. + +- **One test for each file.** The title names the file and what it should meet, such as `1mb_over_1b.jpg - expected reject (size_limit)`. The report shows what the file is for, and tags such as `@reject` and `@size-limit` narrow a run with `--grep`. +- **A failure says what it means.** `check` throws with the sentence the set carries for that file: + + ``` + 1mb_over_1b.jpg was taken, and it should have been turned away (size_limit). + A JPEG file 1 B over the limit of 1mb. Your system should turn it away - taking it means + the limit is not enforced, or enforced only in the browser. + ``` + +- **Files whose outcome is your policy are skipped, never passed.** An empty file or a name with an emoji in it has no right answer until you decide one. `whyUnspecified` says so in the report, and `check` refuses to pass such a file. +- **The same files can go straight to your server**, past the page and its checks: + + ```ts + import { createReadStream } from 'node:fs'; + + const response = await request.post('/upload', { multipart: { file: createReadStream(await file.path()) } }); + file.check(response.ok() ? 'accept' : 'reject'); + ``` + + `check` also takes `{ outcome: 'reject', reason: 'size_limit' }`. A reason is compared only for a refusal, because it names the rule a file is about - a file a byte under a limit is expected to be accepted with `size_limit` as the rule in play. + +## The sets + +`fileSet` takes a preset, a recipe file or plain flags of `tfg generate`: + +```ts +fileSet({ preset: 'size-boundaries', options: { limit: '10mb', format: 'pdf' } }); +fileSet({ recipe: 'fixtures/uploads.yaml' }); +fileSet({ options: { format: 'png', size: '5mb', count: 3, seed: 7, expected: 'accept' } }); +``` + +Files made from flags expect nothing unless `expected` says what, so without it every one of them is skipped. + +| preset | the question it answers | +|---|---| +| `upload-validation` | Does my upload form take what it should and turn the rest away? | +| `size-boundaries` | Is a size limit enforced exactly where it is declared? | +| `filename-handling` | Will my system store, show and give back a file name it did not expect? | +| `tabular-import` | Does my table import survive what real tools export? | +| `text-encoding` | Does my reader know which encoding a file is in, or is it guessing? | +| `empty-and-minimal` | Does a file that is valid and as small as the format allows get through? | + +`npx tfg preset show ` lists what a preset takes, and `npx tfg formats` lists the formats. + +`only` and `except` take part of a set by `group`, `target` or `outcome`, and `paths()` gives the files of a part together, for a form that takes several at once: + +```ts +const together = await uploads.only({ group: 'bulk-upload' }).paths(); +await page.getByLabel('Files').setInputFiles(together); +``` + +## When the files are written + +1. **While Playwright loads the spec file, nothing is written.** `fileSet` asks for the plan, the list of files and what each should meet, and Playwright declares the tests from it. Listing the tests with `--list` writes nothing either. +2. **The first test that asks for a path writes the whole set**, once, however many workers ask at the same moment. The default `upload-validation` set, 71 files and 116 MB, was written in under a second on the machine it was measured on. +3. **Every later run takes the same files.** The same options give the same bytes, so a set is written once and kept, in `node_modules/.cache/testing-files-generator` or the directory `TFG_CACHE_DIR` names. + +A large set is better written before the tests start, so its time does not count against the first test: + +```ts +test.beforeAll(async () => { + test.setTimeout(120_000); + await uploads.write(); +}); +``` + +Nothing is ever removed on its own. Delete the cache directory whenever you like. A run stopped part way leaves an unfinished set, which nothing uses, and the next run names it so it does not sit there unseen. + +## Where tfg comes from + +The package installs the `tfg` program for your system: Windows on x64 and Arm, Linux on x64 and Arm, macOS on Apple silicon. `npx tfg` runs it, with the same arguments, output and exit codes as tfg run directly. + +To use a tfg installed another way, set `TFG_BINARY` to its path, or pass `binary` to `fileSet`. On a system with no binary of its own, such as an Intel Mac, the container image `ghcr.io/donislawdev/tfg` runs the same program. + +## Not only Playwright + +`fileSet` gives plain data and plain promises, and imports nothing from Playwright. The same loop works in Vitest, Jest or a script, with `file.title` as a test name. + +## Licences + +This package is under the [Mozilla Public License 2.0](LICENSE). The `tfg` program it runs is under the GNU General Public License 3.0 and runs as a separate program. The files it generates are yours: they are output of the program and not derived works of it, so you can commit them and ship them inside a closed source product with no obligation of any kind. diff --git a/npm/testing-files-generator/bin/tfg.js b/npm/testing-files-generator/bin/tfg.js new file mode 100644 index 00000000..09847900 --- /dev/null +++ b/npm/testing-files-generator/bin/tfg.js @@ -0,0 +1,52 @@ +#!/usr/bin/env node +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +'use strict'; + +// npx tfg runs the binary this package installed, with the same arguments, the +// same streams and the same exit code, so a script cannot tell it from tfg run +// directly. + +const { spawn } = require('node:child_process'); +const { binaryPath } = require('../lib/binary.js'); + +// RUNTIME in docs/CLI.md section 3 - the tool could not do its work, and the +// arguments were never looked at. +const cannotRun = 1; + +let binary; +try { + binary = binaryPath(); +} catch (err) { + process.stderr.write(`tfg: ${err.message}\n`); + process.exit(cannotRun); +} + +const child = spawn(binary, process.argv.slice(2), { stdio: 'inherit', windowsHide: false }); + +// A Ctrl+C in a terminal reaches tfg as well as this process. This one waits, +// so tfg can stop the way it stops on its own and its exit code is the one a +// script sees. A signal sent to this process alone is passed on. +for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) { + process.on(signal, () => { + if (signal !== 'SIGINT') { + child.kill(signal); + } + }); +} + +child.on('error', (err) => { + process.stderr.write(`tfg: tfg could not be started from ${binary}: ${err.message}\n`); + process.exit(cannotRun); +}); + +child.on('close', (status, signal) => { + if (signal) { + process.removeAllListeners(signal); + process.kill(process.pid, signal); + return; + } + process.exit(status ?? cannotRun); +}); diff --git a/npm/testing-files-generator/index.d.ts b/npm/testing-files-generator/index.d.ts new file mode 100644 index 00000000..fd257434 --- /dev/null +++ b/npm/testing-files-generator/index.d.ts @@ -0,0 +1,129 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +/** What a system under test is expected to do with a file. */ +export type Outcome = 'accept' | 'reject' | 'sanitize' | 'unspecified'; + +/** The rule a file is about. It names the rule and not what the system did, so a file a byte under a limit is expected to be accepted with size_limit. */ +export type Reason = + | 'content_malformed' + | 'count_limit' + | 'dimensions_limit' + | 'duplicate' + | 'encoding_invalid' + | 'extension_rule' + | 'filename_invalid' + | 'filename_too_long' + | 'filename_traversal' + | 'malware_signature' + | 'mime_mismatch' + | 'nesting_depth' + | 'none' + | 'size_limit' + | 'size_zero'; + +export interface Expected { + outcome: Outcome; + reason?: Reason; + detail?: string; + confidence?: string; +} + +/** One entry of a manifest, as tfg writes it. */ +export interface ManifestFile { + id: string; + path: string; + name: string; + materialized: boolean; + target_id: string; + bytes: number; + format: string; + group?: string; + purpose?: string; + expected: Expected; + [field: string]: unknown; +} + +/** A manifest, as tfg writes it. For a plan nothing is written yet and every file has materialized false. */ +export interface Manifest { + manifest_version: string; + tool: { name: string; version: string; generators: Record; [field: string]: unknown }; + run: { id: string; seed: number; complete: boolean; recipe_hash?: string; [field: string]: unknown }; + summary: { file_count: number; total_bytes: number; materialized: number; [field: string]: unknown }; + files: ManifestFile[]; + [field: string]: unknown; +} + +export type OptionValue = string | number | boolean | ReadonlyArray; + +export interface FileSetSpec { + /** A preset, as tfg preset list names it. */ + preset?: string; + /** A recipe file. */ + recipe?: string; + /** Flags of tfg generate and of the preset, without the dashes: { limit: '2mb', seed: 7, clean: true }. */ + options?: Record; + /** Where sets are written. TFG_CACHE_DIR, or node_modules/.cache/testing-files-generator, or the system's temporary directory. */ + cacheDir?: string; + /** The tfg binary to run. TFG_BINARY, or the one this package installed. */ + binary?: string; +} + +export interface Filter { + group?: string | readonly string[]; + target?: string | readonly string[]; + outcome?: Outcome | readonly Outcome[]; +} + +export type Observed = + | Exclude + | { outcome: Exclude; reason?: string }; + +/** The shape test(title, details, body) takes in Playwright. */ +export interface TestDetails { + tag: string[]; + annotation: Array<{ type: string; description: string }>; +} + +export interface TestFile { + readonly entry: ManifestFile; + readonly relativePath: string; + readonly name: string; + readonly format: string; + readonly bytes: number; + readonly group: string; + readonly target: string; + readonly purpose: string; + readonly expected: Expected; + readonly unspecified: boolean; + readonly whyUnspecified: string; + /** The file's path and what it is expected to meet. Unique within a set. */ + readonly title: string; + readonly details: TestDetails; + /** Writes the set the first time any of its files is asked for, and gives this file's absolute path. */ + path(): Promise; + /** Throws when what the system did differs from what is expected, with the sentence that says what the difference means. */ + check(observed: Observed): void; +} + +export interface FileSet { + readonly files: readonly TestFile[]; + readonly plan: Manifest; + only(filter: Filter): FileSet; + except(filter: Filter): FileSet; + /** Writes the whole set once per process and gives its directory. */ + write(): Promise; + paths(): Promise; +} + +/** Plans a set of files, writing nothing. The files are written the first time one of them is asked for. */ +export function fileSet(spec: FileSetSpec): FileSet; + +/** The tfg binary this package runs. */ +export function binaryPath(): string; + +export class TfgError extends Error { + readonly status: number | string; + readonly stderr: string; +} diff --git a/npm/testing-files-generator/index.js b/npm/testing-files-generator/index.js new file mode 100644 index 00000000..76530289 --- /dev/null +++ b/npm/testing-files-generator/index.js @@ -0,0 +1,12 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +'use strict'; + +// Each export is assigned on its own line, the form Node reads the names of +// when an ES module imports this file, so `import { fileSet }` works as well as +// require does. +exports.fileSet = require('./lib/fileset.js').fileSet; +exports.binaryPath = require('./lib/binary.js').binaryPath; +exports.TfgError = require('./lib/run.js').TfgError; diff --git a/npm/testing-files-generator/lib/binary.js b/npm/testing-files-generator/lib/binary.js new file mode 100644 index 00000000..48cb522e --- /dev/null +++ b/npm/testing-files-generator/lib/binary.js @@ -0,0 +1,56 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +'use strict'; + +const fs = require('node:fs'); + +// The systems a release builds a command line for, as Node names them, and the +// package that carries each binary. A guard holds this to the list of targets +// in .github/workflows/release.yml, because a platform here with nothing behind +// it installs a package that cannot run, and a platform there missing here is a +// binary nobody can reach. Intel Macs are not on either list, by the owner's +// decision of 2026-08-20. +const platforms = { + 'win32 x64': '@testing-files-generator/win32-x64', + 'win32 arm64': '@testing-files-generator/win32-arm64', + 'linux x64': '@testing-files-generator/linux-x64', + 'linux arm64': '@testing-files-generator/linux-arm64', + 'darwin arm64': '@testing-files-generator/darwin-arm64', +}; + +// binaryPath finds the tfg binary this package runs. +// +// TFG_BINARY comes first and is taken as given - a tfg installed another way, +// or one built from the repository. PATH is never searched: a different tfg +// found there would answer with its own version and say nothing about it. +// Everything is passed in so the tests can ask about a system they are not on. +function binaryPath({ env = process.env, platform = process.platform, arch = process.arch, resolve = require.resolve } = {}) { + const given = env.TFG_BINARY; + if (given) { + if (!fs.existsSync(given)) { + throw new Error(`TFG_BINARY is set to ${given} and there is no file there. ` + + 'Point it at a tfg binary, or unset it to use the one this package installs.'); + } + return given; + } + const pkg = platforms[`${platform} ${arch}`]; + const known = Object.keys(platforms).join(', '); + if (!pkg) { + throw new Error(`testing-files-generator has no binary for ${platform} ${arch}. ` + + `It comes for ${known}. ` + + 'On another system, run tfg from its container image ghcr.io/donislawdev/tfg, ' + + 'or install it another way and set TFG_BINARY to its path.'); + } + const exe = platform === 'win32' ? 'tfg.exe' : 'tfg'; + try { + return resolve(`${pkg}/${exe}`); + } catch { + throw new Error(`The binary for ${platform} ${arch} comes in the package ${pkg}, and it is not installed. ` + + 'npm leaves it out when it installs with --omit=optional or --no-optional. ' + + 'Install again without that option, or set TFG_BINARY to the path of a tfg binary.'); + } +} + +module.exports = { binaryPath, platforms }; diff --git a/npm/testing-files-generator/lib/fileset.js b/npm/testing-files-generator/lib/fileset.js new file mode 100644 index 00000000..095580db --- /dev/null +++ b/npm/testing-files-generator/lib/fileset.js @@ -0,0 +1,379 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +'use strict'; + +const { spawnSync } = require('node:child_process'); +const crypto = require('node:crypto'); +const { EventEmitter } = require('node:events'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +const { binaryPath } = require('./binary.js'); +const { run, runToFile } = require('./run.js'); +const { TestFile, outcomes } = require('./testfile.js'); + +// The manifest major this package reads. Another major is refused rather than +// read as if it were this one - docs/MANIFEST.md, the version of the contract. +const manifestMajor = '1'; + +// Flags this package sets itself. An option of the same name would fight it. +const reserved = { + out: 'this package writes every set into a directory of its own', + json: 'this package reads the manifest from the directory it writes', + 'dry-run': 'this package plans each set with a dry run and then writes it', + preset: 'a preset is named by the preset property beside the options', +}; + +// Everything this package leaves while it works starts with this, followed by +// the process id, so a later run can tell a set being written now from one a +// stopped run left behind. +const unfinished = '.tmp-'; + +// A rename the system refuses for a moment. On Windows a scanner holding a new +// file open is enough, so these are tried again before anything is said. +const busy = new Set(['EPERM', 'EACCES', 'EBUSY']); +const renameTries = 10; + +// sources keeps one plan per set and process. A spec file is loaded once in the +// runner and again in every worker, and two spec files can ask for one set. +const sources = new Map(); +let noted = false; + +// events says what a write did - wrote a copy, moved it to its name, or took +// one another process moved first. Not part of the package's interface: the +// tests listen, because a race that never happened passes as well as one won. +const events = new EventEmitter(); + +function fileSet(spec = {}) { + if (typeof spec !== 'object' || spec === null) { + throw new TypeError('fileSet takes an object, for example { preset: \'upload-validation\', options: { limit: \'2mb\' } }.'); + } + const binary = spec.binary ?? binaryPath(); + const args = argsOf(spec); + const cacheDir = path.resolve(spec.cacheDir ?? defaultCacheDir()); + const recipe = spec.recipe ? fs.readFileSync(path.resolve(spec.recipe), 'utf8') : ''; + const key = digest([identity(binary), args, recipe, cacheDir]); + let source = sources.get(key); + if (!source) { + source = new Source(binary, args, cacheDir, labelOf(spec)); + sources.set(key, source); + } + return new FileSet(source, source.files); +} + +function argsOf(spec) { + const args = ['generate']; + if (spec.recipe) { + args.push(path.resolve(spec.recipe)); + } + if (spec.preset) { + args.push(`--preset=${spec.preset}`); + } + for (const [key, value] of Object.entries(spec.options ?? {})) { + if (Object.hasOwn(reserved, key)) { + throw new Error(`The option ${key} cannot be set here - ${reserved[key]}.`); + } + if (!/^[a-z][a-z0-9-]*$/.test(key)) { + throw new Error(`${JSON.stringify(key)} is not a flag tfg has. Options are flag names without the dashes, such as limit or far-over.`); + } + for (const one of Array.isArray(value) ? value : [value]) { + if (one === true) { + args.push(`--${key}`); + } else if (typeof one === 'string' || typeof one === 'number') { + args.push(`--${key}=${one}`); + } else if (one !== false && one !== undefined && one !== null) { + throw new TypeError(`The option ${key} takes a string, a number, true or a list of them. It was given a ${typeof one}.`); + } + } + } + return args; +} + +function defaultCacheDir() { + if (process.env.TFG_CACHE_DIR) { + return process.env.TFG_CACHE_DIR; + } + const modules = path.join(process.cwd(), 'node_modules'); + if (fs.existsSync(modules)) { + return path.join(modules, '.cache', 'testing-files-generator'); + } + return path.join(os.tmpdir(), 'testing-files-generator'); +} + +function labelOf(spec) { + const name = spec.preset ?? (spec.recipe ? path.basename(spec.recipe).replace(/\.[^.]*$/, '') : 'flags'); + return name.replace(/[^A-Za-z0-9_-]+/g, '-').slice(0, 40); +} + +// identity is what tells two binaries apart without running them, so a new +// build or another version plans again rather than reusing an old plan. +function identity(binary) { + const stat = fs.statSync(binary); + return [fs.realpathSync(binary), stat.size, stat.mtimeMs]; +} + +function digest(value) { + return crypto.createHash('sha256').update(JSON.stringify(value)).digest('hex'); +} + +function scratch(cacheDir) { + return path.join(cacheDir, `${unfinished}${process.pid}-${crypto.randomBytes(4).toString('hex')}`); +} + +// Source is one set: its plan, where it is written, and the one write a process +// makes of it however many files ask. +class Source { + constructor(binary, args, cacheDir, label) { + this.binary = binary; + this.args = args; + this.cacheDir = cacheDir; + fs.mkdirSync(cacheDir, { recursive: true }); + noteUnfinished(cacheDir); + this.plan = this._planned(); + // The recipe hash leaves out the seed and --clean, measured 2026-11-04, so + // the arguments are part of the name and two seeds never share a set. + const run = this.plan.run; + this.dir = path.join(cacheDir, `${label}-${digest([args, run.recipe_hash, run.seed, this.plan.tool]).slice(0, 20)}`); + this.files = this.plan.files.map((entry) => new TestFile(this, entry)); + this._written = undefined; + } + + // _planned asks tfg what the set holds without writing any of it. The dry run + // names a directory that is never made, because a dry run into a directory + // holding a manifest is refused like a real one. + _planned() { + const out = `${scratch(this.cacheDir)}.json`; + let plan; + try { + runToFile(this.binary, [...this.args, `--out=${scratch(this.cacheDir)}`, '--dry-run', '--json'], out); + plan = JSON.parse(fs.readFileSync(out, 'utf8')); + } finally { + fs.rmSync(out, { force: true }); + } + readable(plan); + return plan; + } + + write() { + if (!this._written) { + this._written = this._write().catch((err) => { + this._written = undefined; + throw err; + }); + } + return this._written; + } + + async _write() { + if (fs.existsSync(this.dir)) { + const problem = mismatch(this.dir, this.plan); + if (problem) { + throw stale(this.dir, problem); + } + return this.dir; + } + const tmp = scratch(this.cacheDir); + fs.mkdirSync(tmp); + try { + await run(this.binary, [...this.args, `--out=${tmp}`]); + const problem = mismatch(tmp, this.plan); + if (problem) { + throw new Error(`tfg wrote a set that does not match its own plan - ${problem}. This is a fault in tfg or in this package. Please report it.`); + } + } catch (err) { + discard(this.binary, tmp); + throw err; + } + events.emit('wrote', tmp); + return this._settle(tmp); + } + + // _settle moves a finished set to its name. Several workers can write the + // same set at once, and the first to move it wins - the others find the + // winner whole, take it, and remove their own copy through its manifest. + async _settle(tmp) { + for (let attempt = 1; ; attempt++) { + try { + fs.renameSync(tmp, this.dir); + events.emit('moved', this.dir); + return this.dir; + } catch (err) { + if (fs.existsSync(this.dir)) { + const problem = mismatch(this.dir, this.plan); + discard(this.binary, tmp); + if (problem) { + throw stale(this.dir, problem); + } + events.emit('took', this.dir); + return this.dir; + } + if (!busy.has(err.code) || attempt === renameTries) { + discard(this.binary, tmp); + throw new Error(`The set written in ${tmp} could not be moved to ${this.dir} - ${err.message}`); + } + await new Promise((resolve) => setTimeout(resolve, attempt * 100)); + } + } + } +} + +// readable refuses a manifest this package cannot read for what it says. +function readable(plan) { + const version = String(plan?.manifest_version ?? ''); + if (version.split('.')[0] !== manifestMajor) { + throw new Error(`tfg wrote a manifest of version ${version || 'none'}, and this package reads version ${manifestMajor}. ` + + 'Install the version of testing-files-generator that comes with that tfg, or point TFG_BINARY at a tfg of the same version as this package.'); + } + for (const entry of plan.files) { + if (!outcomes.includes(entry.expected?.outcome)) { + throw new Error(`${entry.path} expects ${JSON.stringify(entry.expected?.outcome)}, which this package does not know. ` + + `It knows ${outcomes.join(', ')}. Use the version of this package that comes with the tfg it runs.`); + } + } +} + +// mismatch says how a written set differs from its plan, or nothing when it +// does not. Sizes are compared and contents are not: a set is checked on every +// run, and hashing all of it each time would cost more than writing it. +// +// Every file has to be on the disk. A run that ends with code nought writes +// every file it plans, and one that cannot ends with the partial code, which +// run turns into an error. That holds until a run can describe a file without +// writing it on purpose - the archive and manifest-only output of D10. +function mismatch(dir, plan) { + let written; + try { + written = JSON.parse(fs.readFileSync(path.join(dir, 'manifest.json'), 'utf8')); + } catch (err) { + return `its manifest cannot be read (${err.message})`; + } + if (written.run?.recipe_hash !== plan.run.recipe_hash || written.run?.seed !== plan.run.seed) { + return 'it was written from another recipe or seed'; + } + if (written.run?.complete !== true) { + return 'the run that wrote it did not finish'; + } + if (!Array.isArray(written.files) || written.files.length !== plan.files.length) { + return `it holds ${written.files?.length ?? 'no'} files where the plan has ${plan.files.length}`; + } + for (let i = 0; i < plan.files.length; i++) { + const want = plan.files[i]; + const got = written.files[i]; + if (got.path !== want.path || got.bytes !== want.bytes) { + return `its file ${i + 1} is ${got.path} of ${got.bytes} B where the plan has ${want.path} of ${want.bytes} B`; + } + if (!got.materialized) { + return `${got.path} was not written`; + } + let stat; + try { + stat = fs.statSync(path.join(dir, got.path)); + } catch { + return `${got.path} is missing`; + } + if (stat.size !== got.bytes) { + return `${got.path} is ${stat.size} B and its manifest says ${got.bytes} B`; + } + } + return ''; +} + +function stale(dir, problem) { + return new Error(`${dir} is where this set belongs, and it does not hold it - ${problem}. ` + + 'Nothing here removes that directory, because a person may have changed it on purpose. Delete it and run again.'); +} + +// discard removes a set this process wrote and will not use. The files go +// through tfg cleanup, which removes only what the manifest lists, and then +// the manifest and the directories it leaves empty. Whatever is left stays, +// and a later run says so. +function discard(binary, dir) { + const manifest = path.join(dir, 'manifest.json'); + if (fs.existsSync(manifest)) { + const done = spawnSync(binary, ['cleanup', manifest, '--yes'], { stdio: 'ignore', windowsHide: true }); + if (done.status !== 0) { + return; + } + fs.rmSync(manifest, { force: true }); + fs.rmSync(path.join(dir, 'manifest.instructions.md'), { force: true }); + } + removeEmpty(dir); +} + +function removeEmpty(dir) { + let entries; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + if (entry.isDirectory()) { + removeEmpty(path.join(dir, entry.name)); + } + } + try { + fs.rmdirSync(dir); + } catch { + // Not empty: left where it is. + } +} + +// noteUnfinished says, once per process and only outside a test worker, which +// sets a stopped run left behind. Nothing reads them and nothing removes them +// on its own, so without this they would fill the disk in silence. +function noteUnfinished(cacheDir) { + if (noted || process.env.TEST_WORKER_INDEX !== undefined) { + return; + } + noted = true; + const left = fs.readdirSync(cacheDir).filter((name) => { + const pid = Number(name.slice(unfinished.length).split('-')[0]); + return name.startsWith(unfinished) && Number.isInteger(pid) && pid > 0 && !alive(pid); + }); + if (left.length > 0) { + console.warn(`testing-files-generator: ${left.length} unfinished ${left.length === 1 ? 'set was' : 'sets were'} left in ${cacheDir} ` + + `by a run that stopped part way - ${left.join(', ')}. Nothing uses them and they can be deleted.`); + } +} + +function alive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (err) { + return err.code === 'EPERM'; + } +} + +// FileSet is a set, or a part of one. Every part shares the set's one write. +class FileSet { + constructor(source, files) { + this._source = source; + this.files = files; + } + + get plan() { return this._source.plan; } + + only(filter) { return new FileSet(this._source, this.files.filter((file) => matches(file, filter))); } + + except(filter) { return new FileSet(this._source, this.files.filter((file) => !matches(file, filter))); } + + // write writes the whole set, once per process, and gives its directory. + write() { return this._source.write(); } + + async paths() { + const dir = await this._source.write(); + return this.files.map((file) => path.join(dir, file.relativePath)); + } +} + +function matches(file, filter = {}) { + const has = (wanted, value) => wanted === undefined || [].concat(wanted).includes(value); + return has(filter.group, file.group) && has(filter.target, file.target) && has(filter.outcome, file.expected.outcome); +} + +module.exports = { events, fileSet, readable }; diff --git a/npm/testing-files-generator/lib/run.js b/npm/testing-files-generator/lib/run.js new file mode 100644 index 00000000..175b6b81 --- /dev/null +++ b/npm/testing-files-generator/lib/run.js @@ -0,0 +1,70 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +'use strict'; + +const { spawn, spawnSync } = require('node:child_process'); +const fs = require('node:fs'); + +// How much of what tfg says on its error stream is kept for a message. Its +// refusals are a few lines, and a run of many files can print a note for each. +const keptError = 64 * 1024; + +// TfgError is tfg ending with a code other than nought. The message carries +// what tfg said, because its refusals already name the value and the way out. +class TfgError extends Error { + constructor(args, status, stderr) { + const said = stderr.trim() || 'tfg said nothing about why.'; + super(`tfg ${args.join(' ')} ended with code ${status}.\n${said}`); + this.name = 'TfgError'; + this.status = status; + this.stderr = stderr; + } +} + +function notStarted(binary, err) { + return new Error(`tfg could not be started from ${binary}: ${err.message}`); +} + +// runToFile runs tfg to the end, writing its standard output to a file. A file +// and not memory, because the manifest of a large set outgrows the buffer Node +// gives a child process by default. +function runToFile(binary, args, outFile) { + const fd = fs.openSync(outFile, 'w'); + let done; + try { + done = spawnSync(binary, args, { stdio: ['ignore', fd, 'pipe'], windowsHide: true, maxBuffer: keptError * 16 }); + } finally { + fs.closeSync(fd); + } + if (done.error) { + throw notStarted(binary, done.error); + } + if (done.status !== 0) { + throw new TfgError(args, done.status, done.stderr.toString('utf8').slice(-keptError)); + } +} + +// run runs tfg without holding the event loop, so a test runner keeps its own +// timers and messages going while files are written. +function run(binary, args) { + return new Promise((resolve, reject) => { + const child = spawn(binary, args, { stdio: ['ignore', 'ignore', 'pipe'], windowsHide: true }); + let stderr = ''; + child.stderr.setEncoding('utf8'); + child.stderr.on('data', (chunk) => { + stderr = (stderr + chunk).slice(-keptError); + }); + child.on('error', (err) => reject(notStarted(binary, err))); + child.on('close', (status, signal) => { + if (status === 0) { + resolve(); + return; + } + reject(new TfgError(args, status ?? signal, stderr)); + }); + }); +} + +module.exports = { TfgError, run, runToFile }; diff --git a/npm/testing-files-generator/lib/testfile.js b/npm/testing-files-generator/lib/testfile.js new file mode 100644 index 00000000..5ca45136 --- /dev/null +++ b/npm/testing-files-generator/lib/testfile.js @@ -0,0 +1,129 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +'use strict'; + +const path = require('node:path'); +const { inspect } = require('node:util'); + +// The closed list of outcomes a manifest declares, from docs/MANIFEST.md. A +// guard holds this copy, and the one in index.d.ts, to recipe.Outcomes in the +// program, because two copies of a closed list is how they drift. +const outcomes = ['accept', 'reject', 'sanitize', 'unspecified']; + +// How each outcome reads in a sentence about what a system did with a file. +const done = { + accept: 'taken', + reject: 'turned away', + sanitize: 'taken and cleaned', +}; + +// A tag is a word Playwright filters on, so anything else in a group's name +// becomes a hyphen rather than breaking the filter. +function tag(word) { + return `@${String(word).replace(/[^A-Za-z0-9_-]+/g, '-')}`; +} + +function expectedText(expected) { + return expected.reason ? `${expected.outcome} (${expected.reason})` : expected.outcome; +} + +// TestFile is one file of a set, with what the system under test should do +// with it. It is what a test declares itself from: a title, the details a test +// takes, the path once the set is written, and a check against the manifest. +class TestFile { + constructor(source, entry) { + this._source = source; + this.entry = entry; + } + + get relativePath() { return this.entry.path; } + get name() { return this.entry.name; } + get format() { return this.entry.format; } + get bytes() { return this.entry.bytes; } + get group() { return this.entry.group ?? ''; } + get target() { return this.entry.target_id; } + get purpose() { return this.entry.purpose ?? ''; } + get expected() { return this.entry.expected; } + get unspecified() { return this.entry.expected.outcome === 'unspecified'; } + + // The path, which is unique in a set and is what a test title has to be. It + // equals the name today, because a name cannot hold a separator, and the + // path stays the right one if a set ever has directories. + get title() { + return `${this.entry.path} - expected ${expectedText(this.entry.expected)}`; + } + + // details has the shape test(title, details, body) takes, so a report shows + // what each file is for and a run can be narrowed by its tags. + get details() { + const tags = [tag(this.entry.expected.outcome)]; + if (this.group) { + tags.push(tag(this.group)); + } + const annotation = [{ type: 'expected', description: expectedText(this.entry.expected) }]; + if (this.purpose) { + annotation.push({ type: 'purpose', description: this.purpose }); + } + return { tag: tags, annotation }; + } + + get whyUnspecified() { + const why = `${this.entry.path} has no expected outcome - what your system does with it is its own policy.`; + return this.purpose ? `${why} ${this.purpose}` : why; + } + + // path writes the set the first time any of its files is asked for, and + // gives the absolute path of this one. + async path() { + const dir = await this._source.write(); + return path.join(dir, this.entry.path); + } + + // check compares what the system did with what the manifest expects, and + // throws with the file's purpose when they differ - that sentence says what + // the difference means. A reason is compared only for a refusal: a reason + // names the rule a file is about, so a file a byte under a limit is accepted + // with size_limit and there is nothing to compare it with. + check(observed) { + try { + this._compare(observed); + } catch (err) { + // The stack starts at whoever called check, so a test runner shows the + // line of the test rather than a line of this package. + Error.captureStackTrace(err, TestFile.prototype.check); + throw err; + } + } + + _compare(observed) { + const want = this.entry.expected; + if (want.outcome === 'unspecified') { + throw new Error(`${this.entry.path} has no expected outcome, so there is nothing to check it against. ` + + 'Skip it with test.skip(file.unspecified, file.whyUnspecified), or decide what your system should do with it and check that.'); + } + const got = typeof observed === 'string' ? { outcome: observed } : observed; + if (!got || typeof got !== 'object' || !Object.hasOwn(done, got.outcome)) { + throw new TypeError('check takes accept, reject or sanitize, or an object with an outcome and a reason. ' + + `It was given ${inspect(observed)}.`); + } + if (got.outcome !== want.outcome) { + throw mismatch(this, `${this.entry.path} was ${done[got.outcome]}, and it should have been ${done[want.outcome]}` + + (want.reason ? ` (${want.reason}).` : '.'), got); + } + if (want.outcome === 'reject' && want.reason && got.reason && got.reason !== want.reason) { + throw mismatch(this, `${this.entry.path} was turned away for ${got.reason}, and the rule it is about is ${want.reason}.`, got); + } + } +} + +function mismatch(file, sentence, observed) { + const err = new Error(file.purpose ? `${sentence}\n${file.purpose}` : sentence); + err.name = 'ExpectationError'; + err.expected = file.entry.expected; + err.observed = observed; + return err; +} + +module.exports = { TestFile, outcomes }; diff --git a/npm/testing-files-generator/package.json b/npm/testing-files-generator/package.json new file mode 100644 index 00000000..acaea500 --- /dev/null +++ b/npm/testing-files-generator/package.json @@ -0,0 +1,51 @@ +{ + "name": "testing-files-generator", + "version": "0.0.0-development", + "description": "Real files of exact sizes and formats for tests, and what the system under test should do with each one. For Playwright and any other test runner.", + "license": "MPL-2.0", + "author": "DonislawDev", + "homepage": "https://github.com/donislawdev/TestingFilesGenerator/tree/main/npm/testing-files-generator#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/donislawdev/TestingFilesGenerator.git", + "directory": "npm/testing-files-generator" + }, + "bugs": "https://github.com/donislawdev/TestingFilesGenerator/issues", + "keywords": [ + "playwright", + "test files", + "fixtures", + "file upload", + "upload testing", + "test data", + "boundary testing", + "qa" + ], + "type": "commonjs", + "main": "index.js", + "types": "index.d.ts", + "exports": { + ".": { + "types": "./index.d.ts", + "default": "./index.js" + }, + "./package.json": "./package.json" + }, + "bin": { + "tfg": "bin/tfg.js" + }, + "files": [ + "index.js", + "index.d.ts", + "lib/", + "bin/", + "LICENSE", + "README.md" + ], + "engines": { + "node": ">=22" + }, + "scripts": { + "test": "node --test \"test/*.test.js\"" + } +} diff --git a/npm/testing-files-generator/test/binary.test.js b/npm/testing-files-generator/test/binary.test.js new file mode 100644 index 00000000..12948b3d --- /dev/null +++ b/npm/testing-files-generator/test/binary.test.js @@ -0,0 +1,41 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { test } = require('node:test'); + +const { binaryPath, platforms } = require('../lib/binary.js'); + +const nothing = () => { throw new Error('not installed'); }; + +test('TFG_BINARY is taken as given when there is a file there', () => { + assert.equal(binaryPath({ env: { TFG_BINARY: __filename }, resolve: nothing }), __filename); +}); + +test('TFG_BINARY naming nothing is refused with the path it named', () => { + const missing = path.join(__dirname, 'no-such-tfg'); + assert.throws(() => binaryPath({ env: { TFG_BINARY: missing }, resolve: nothing }), + (err) => err.message.includes(missing) && err.message.includes('unset it')); +}); + +test('each platform a release builds resolves to its own package', () => { + for (const [key, pkg] of Object.entries(platforms)) { + const [platform, arch] = key.split(' '); + const exe = platform === 'win32' ? 'tfg.exe' : 'tfg'; + const asked = []; + const found = binaryPath({ env: {}, platform, arch, resolve: (id) => { asked.push(id); return `/resolved/${id}`; } }); + assert.deepEqual(asked, [`${pkg}/${exe}`]); + assert.equal(found, `/resolved/${pkg}/${exe}`); + } +}); + +test('an Intel Mac is told what it can do instead', () => { + assert.throws(() => binaryPath({ env: {}, platform: 'darwin', arch: 'x64', resolve: nothing }), + (err) => err.message.includes('no binary for darwin x64') && err.message.includes('TFG_BINARY') && + err.message.includes('ghcr.io/donislawdev/tfg') && err.message.includes('darwin arm64')); +}); + +test('a platform package left out by --omit=optional is named, with the way back', () => { + assert.throws(() => binaryPath({ env: {}, platform: 'linux', arch: 'x64', resolve: nothing }), + (err) => err.message.includes('@testing-files-generator/linux-x64') && err.message.includes('--omit=optional')); +}); diff --git a/npm/testing-files-generator/test/fileset.test.js b/npm/testing-files-generator/test/fileset.test.js new file mode 100644 index 00000000..93e3ce41 --- /dev/null +++ b/npm/testing-files-generator/test/fileset.test.js @@ -0,0 +1,178 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { test } = require('node:test'); + +const { fileSet, readable } = require('../lib/fileset.js'); +const { TfgError } = require('../lib/run.js'); +const { node, pkg, small, tempDir, tfg } = require('./helpers.js'); + +function plannedIn(cacheDir, spec = small) { + return fileSet({ ...spec, binary: tfg(), cacheDir }); +} + +// The script a fresh process runs to write a set, printing its directory. +function writer(cacheDir, spec = small) { + return `require(${pkg}).fileSet({ ...${JSON.stringify(spec)}, cacheDir: ${JSON.stringify(cacheDir)} })` + + '.write().then((dir) => console.log(dir), (err) => { console.error(err.message); process.exit(3); });'; +} + +function entries(cacheDir) { + return fs.readdirSync(cacheDir).sort(); +} + +test('a plan writes nothing, and says every file with what is expected of it', (t) => { + const cacheDir = tempDir(t); + const set = plannedIn(cacheDir); + assert.deepEqual(entries(cacheDir), [], 'a plan left something in the cache'); + assert.equal(set.files.length, set.plan.summary.file_count); + assert.ok(set.files.length >= 20, `the set this test asks for holds ${set.files.length} files`); + assert.equal(new Set(set.files.map((f) => f.title)).size, set.files.length, 'two files share a title'); + assert.equal(set.only({ outcome: 'unspecified' }).files.length, set.plan.summary.by_expected.unspecified); + assert.equal(set.only({ group: 'bulk-upload' }).files.length, 2); + assert.equal(set.except({ group: 'bulk-upload' }).files.length, set.files.length - 2); +}); + +test('a set is written once, whole, and its files are where their paths say', async (t) => { + const cacheDir = tempDir(t); + const set = plannedIn(cacheDir); + const dir = await set.write(); + assert.equal(await set.write(), dir); + assert.deepEqual(entries(cacheDir), [path.basename(dir)]); + for (const file of set.files) { + assert.equal(fs.statSync(await file.path()).size, file.bytes, file.relativePath); + } +}); + +// racer plans a set, waits until every racer has planned, and only then +// writes, so that all of them find the set unwritten and write it at once. +function racer(cacheDir, barrier, racers) { + const lib = JSON.stringify(path.join(__dirname, '..', 'lib', 'fileset.js')); + const at = JSON.stringify(barrier); + return `const fs = require('node:fs'); +const { fileSet, events } = require(${lib}); +const said = []; +for (const e of ['wrote', 'moved', 'took']) events.on(e, () => said.push(e)); +const set = fileSet({ ...${JSON.stringify(small)}, cacheDir: ${JSON.stringify(cacheDir)} }); +fs.writeFileSync(require('node:path').join(${at}, String(process.pid)), ''); +const ready = () => fs.readdirSync(${at}).length >= ${racers} ? Promise.resolve() : new Promise((r) => setTimeout(r, 5)).then(ready); +ready().then(() => set.write()).then((dir) => console.log(JSON.stringify({ dir, said })), + (err) => { console.error(err.message); process.exit(3); });`; +} + +test('workers writing one set at once end with one copy and nothing left over', async (t) => { + const cacheDir = tempDir(t); + const barrier = tempDir(t); + const racers = 4; + const runs = await Promise.all(Array.from({ length: racers }, () => node(racer(cacheDir, barrier, racers)))); + for (const r of runs) { + assert.equal(r.status, 0, r.stderr); + } + const results = runs.map((r) => JSON.parse(r.stdout)); + const said = results.flatMap((r) => r.said); + const count = (e) => said.filter((s) => s === e).length; + assert.equal(count('wrote'), racers, `not every worker wrote, so there was no race: ${said.join(', ')}`); + assert.equal(count('moved'), 1, said.join(', ')); + assert.equal(count('took'), racers - 1, said.join(', ')); + const dirs = new Set(results.map((r) => r.dir)); + assert.equal(dirs.size, 1, `the workers were given ${[...dirs].join(', ')}`); + assert.deepEqual(entries(cacheDir), [path.basename([...dirs][0])]); + const set = plannedIn(cacheDir); + assert.equal(await set.write(), [...dirs][0]); +}); + +test('a later run takes the set an earlier one wrote', async (t) => { + const cacheDir = tempDir(t); + const first = await node(writer(cacheDir)); + assert.equal(first.status, 0, first.stderr); + const dir = first.stdout.trim(); + const before = fs.statSync(path.join(dir, 'manifest.json')).mtimeMs; + const second = await node(writer(cacheDir)); + assert.equal(second.stdout.trim(), dir); + assert.equal(fs.statSync(path.join(dir, 'manifest.json')).mtimeMs, before, 'the set was written again'); +}); + +test('another seed or --clean is another set, though the recipe hash is the same', async (t) => { + const cacheDir = tempDir(t); + const plain = plannedIn(cacheDir); + const seeded = plannedIn(cacheDir, { ...small, options: { ...small.options, seed: 7 } }); + const clean = plannedIn(cacheDir, { ...small, options: { ...small.options, clean: true } }); + assert.equal(seeded.plan.run.recipe_hash, plain.plan.run.recipe_hash, 'the hash now covers the seed - this test is no longer about anything'); + const dirs = new Set([await plain.write(), await seeded.write(), await clean.write()]); + assert.equal(dirs.size, 3); +}); + +test('a set changed by hand is refused, and left where it is', async (t) => { + const cacheDir = tempDir(t); + const first = await node(writer(cacheDir)); + const dir = first.stdout.trim(); + const gone = path.join(dir, '1mb_at_limit.jpg'); + fs.rmSync(gone); + const second = await node(writer(cacheDir)); + assert.equal(second.status, 3); + assert.match(second.stderr, /does not hold it - 1mb_at_limit\.jpg is missing\. Nothing here removes that directory/); + assert.ok(fs.existsSync(path.join(dir, 'manifest.json')), 'the changed set was removed'); +}); + +test('a file of the set cut short by hand is refused', async (t) => { + const cacheDir = tempDir(t); + const dir = (await node(writer(cacheDir))).stdout.trim(); + fs.truncateSync(path.join(dir, '1mb_at_limit.jpg'), 10); + const again = await node(writer(cacheDir)); + assert.equal(again.status, 3); + assert.match(again.stderr, /1mb_at_limit\.jpg is 10 B and its manifest says 1048576 B/); +}); + +test('a set a stopped run left behind is named once outside the workers, and never used', async (t) => { + const cacheDir = tempDir(t); + const gone = await node(''); + const left = path.join(cacheDir, `.tmp-${gone.pid}-0bad0bad`); + fs.mkdirSync(left); + const plan = `require(${pkg}).fileSet({ ...${JSON.stringify(small)}, cacheDir: ${JSON.stringify(cacheDir)} })`; + const runner = await node(plan, { env: { TEST_WORKER_INDEX: undefined } }); + assert.equal(runner.status, 0, runner.stderr); + const worker = await node(plan, { env: { TEST_WORKER_INDEX: '0' } }); + assert.equal(worker.status, 0, worker.stderr); + assert.equal(worker.stderr, '', 'a worker spoke about the cache'); + assert.match(runner.stderr, new RegExp(`1 unfinished set was left in .*\\.tmp-${gone.pid}-0bad0bad\\. Nothing uses them`)); + assert.ok(fs.existsSync(left), 'the unfinished set was removed'); +}); + +test('what tfg refuses reaches the test with its own words and code', (t) => { + const cacheDir = tempDir(t); + assert.throws(() => plannedIn(cacheDir, { preset: 'upload-validation', options: { limit: '1kb' } }), + (err) => err instanceof TfgError && err.status === 4 && /smallest PDF/.test(err.message)); +}); + +test('the flags this package sets itself cannot be set again', (t) => { + const cacheDir = tempDir(t); + for (const flag of ['out', 'json', 'dry-run', 'preset']) { + assert.throws(() => plannedIn(cacheDir, { preset: 'upload-validation', options: { [flag]: 'x' } }), + new RegExp(`The option ${flag} cannot be set here`)); + } + assert.throws(() => plannedIn(cacheDir, { preset: 'upload-validation', options: { '--limit': '1mb' } }), /not a flag tfg has/); + assert.throws(() => plannedIn(cacheDir, { preset: 'upload-validation', options: { limit: { mb: 1 } } }), TypeError); +}); + +test('a manifest of another major, or an outcome this package does not know, is refused', () => { + const plan = { manifest_version: '1.0', files: [{ path: 'a', expected: { outcome: 'accept' } }] }; + readable(plan); + assert.throws(() => readable({ ...plan, manifest_version: '2.0' }), /manifest of version 2\.0, and this package reads version 1/); + assert.throws(() => readable({ ...plan, files: [{ path: 'a', expected: { outcome: 'quarantine' } }] }), /"quarantine", which this package does not know/); +}); + +test('npx tfg is tfg - the same output and the same exit code', async () => { + const bin = path.join(__dirname, '..', 'bin', 'tfg.js'); + const viaPackage = await node('', { file: bin, args: ['generate', '--format', 'nope', '--dry-run'] }); + const direct = await new Promise((resolve) => { + const child = require('node:child_process').spawn(tfg(), ['generate', '--format', 'nope', '--dry-run']); + let stderr = ''; + child.stderr.on('data', (c) => { stderr += c; }); + child.on('close', (status) => resolve({ status, stderr })); + }); + assert.notEqual(direct.status, 0); + assert.equal(viaPackage.status, direct.status); + assert.equal(viaPackage.stderr, direct.stderr); +}); diff --git a/npm/testing-files-generator/test/helpers.js b/npm/testing-files-generator/test/helpers.js new file mode 100644 index 00000000..75689fe5 --- /dev/null +++ b/npm/testing-files-generator/test/helpers.js @@ -0,0 +1,58 @@ +'use strict'; + +const { spawn } = require('node:child_process'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +const root = path.resolve(__dirname, '..', '..', '..'); +const built = path.join(root, process.platform === 'win32' ? 'tfg.exe' : 'tfg'); + +// tfg is the binary these tests run: TFG_BINARY, or the one the repository's +// build command leaves at its root. Neither is a failure and not a skip - a +// suite that skips without a binary is green while testing nothing. +function tfg() { + const binary = process.env.TFG_BINARY || built; + if (!fs.existsSync(binary)) { + throw new Error(`No tfg binary at ${binary}. Build one from the repository root with ` + + 'go build -tags "$(cat .github/build-tags)" -o tfg.exe ./cmd/tfg, or set TFG_BINARY.'); + } + return binary; +} + +function tempDir(t) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tfg-npm-test-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + return dir; +} + +// node runs a script in a fresh process, with this package and a binary, and +// gives its exit code and both streams. A variable given as undefined is taken +// out of the child's environment rather than set to an empty string. +function node(script, { env = {}, file = '', args = [] } = {}) { + const childEnv = { ...process.env, TFG_BINARY: tfg(), ...env }; + for (const [key, value] of Object.entries(env)) { + if (value === undefined) { + delete childEnv[key]; + } + } + return new Promise((resolve) => { + const child = spawn(process.execPath, file ? [file, ...args] : ['-e', script, ...args], { + env: childEnv, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { stdout += chunk; }); + child.stderr.on('data', (chunk) => { stderr += chunk; }); + child.on('close', (status) => resolve({ status, stdout, stderr, pid: child.pid })); + }); +} + +const pkg = JSON.stringify(path.resolve(__dirname, '..')); + +// A small set that still holds every kind of case: one either side of the +// limit, unspecified files and a group of several files. +const small = { preset: 'upload-validation', options: { limit: '1mb', bulk: 2, 'far-over': 'off' } }; + +module.exports = { node, pkg, small, tempDir, tfg }; diff --git a/npm/testing-files-generator/test/testfile.test.js b/npm/testing-files-generator/test/testfile.test.js new file mode 100644 index 00000000..19fccd75 --- /dev/null +++ b/npm/testing-files-generator/test/testfile.test.js @@ -0,0 +1,59 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { TestFile } = require('../lib/testfile.js'); + +function file(expected, extra = {}) { + return new TestFile({ write: async () => '/set' }, { + path: 'big.jpg', name: 'big.jpg', target_id: 'over', bytes: 9, format: 'jpg', + group: 'size limit', purpose: 'Taking it means the limit is not enforced.', expected, ...extra, + }); +} + +test('a file says what it is expected to meet in its title, its tags and its annotations', () => { + const f = file({ outcome: 'reject', reason: 'size_limit' }); + assert.equal(f.title, 'big.jpg - expected reject (size_limit)'); + assert.deepEqual(f.details.tag, ['@reject', '@size-limit']); + assert.deepEqual(f.details.annotation, [ + { type: 'expected', description: 'reject (size_limit)' }, + { type: 'purpose', description: 'Taking it means the limit is not enforced.' }, + ]); +}); + +test('a difference is thrown with the sentence that says what it means', () => { + const f = file({ outcome: 'reject', reason: 'size_limit' }); + assert.throws(() => f.check('accept'), (err) => err.name === 'ExpectationError' && + err.message === 'big.jpg was taken, and it should have been turned away (size_limit).\nTaking it means the limit is not enforced.' && + err.observed.outcome === 'accept'); + f.check('reject'); + f.check({ outcome: 'reject', reason: 'size_limit' }); + f.check({ outcome: 'reject' }); +}); + +test('a refusal for another rule is a difference, a reason on an accepted file is not', () => { + assert.throws(() => file({ outcome: 'reject', reason: 'size_limit' }).check({ outcome: 'reject', reason: 'mime_mismatch' }), + /turned away for mime_mismatch, and the rule it is about is size_limit/); + file({ outcome: 'accept', reason: 'size_limit' }).check({ outcome: 'accept', reason: 'anything' }); +}); + +test('an unspecified file is never passed by check', () => { + const f = file({ outcome: 'unspecified', reason: 'size_zero' }); + assert.equal(f.unspecified, true); + assert.match(f.whyUnspecified, /^big\.jpg has no expected outcome/); + assert.throws(() => f.check('accept'), /nothing to check it against/); + assert.throws(() => f.check('reject'), /nothing to check it against/); +}); + +test('check refuses what is not an outcome, the names an object inherits included', () => { + const f = file({ outcome: 'accept' }); + for (const bad of ['toString', 'unspecified', 'ACCEPT', undefined, null, 3, {}]) { + assert.throws(() => f.check(bad), TypeError, String(bad)); + } +}); + +test('the path is the set directory joined with the file', async () => { + const path = require('node:path'); + assert.equal(await file({ outcome: 'accept' }).path(), path.join('/set', 'big.jpg')); +}); From 96513b26ae3a6aced3a1b993fdc80b371967c883 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Wed, 4 Nov 2026 23:15:17 +0100 Subject: [PATCH 2/3] The dependency gate lets the repository's own linked npm package through, held to MPL-2.0 by a guard GitHub's graph reports the package linked into examples/playwright twice, by its name and by its path, with no licence, because a linked directory is on no registry. Both names become exceptions, and a guard holds the package's package.json and LICENSE to the MPL-2.0 the exception states. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/dependency_gate.py | 9 +++++++++ internal/guard/mutation/entries.py | 7 +++++++ internal/guard/npmpackage_test.go | 26 ++++++++++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/.github/scripts/dependency_gate.py b/.github/scripts/dependency_gate.py index 7b8c6d08..987f1381 100644 --- a/.github/scripts/dependency_gate.py +++ b/.github/scripts/dependency_gate.py @@ -89,6 +89,15 @@ # in third_party/go-gl-gl/PATCH.md. This names one directory, and a second # copy of something else needs a line of its own here. "./third_party/go-gl-gl": "a copy of github.com/go-gl/gl, MIT, held to the published module plus one patch by a guard", + # This repository's own npm package, linked into its Playwright example + # with file: since 2026-11-04 (#220). The graph reports it twice with no + # licence, once by its name and once by its path, measured on that pull + # request - a linked directory is not on any registry either. Its licence + # is MPL-2.0, in its package.json and its LICENSE, written by this project, + # and TestTheLinkedPackageIsUnderTheLicenceTheGateIsToldItIs holds both to + # that. The two names are one package. + "testing-files-generator": "this repository's npm package, linked into examples/playwright, MPL-2.0 held by a guard", + "../../npm/testing-files-generator": "the same package, as the graph names it by its path", } # GitHub reports license: null for every action, measured on this repository on diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index cfdebf10..2fd25b5f 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -16888,6 +16888,13 @@ "; unset it to use", "TestTheTextOfThePackageIsAsciiWithPlainPunctuation", ), + ( + "the linked npm package changes its licence while the dependency gate still lets it through as MPL-2.0", + "npm/testing-files-generator/package.json", + '"license": "MPL-2.0",', + '"license": "MIT",', + "TestTheLinkedPackageIsUnderTheLicenceTheGateIsToldItIs", + ), ( "the example's page says something with a semicolon", "examples/playwright/app/page.js", diff --git a/internal/guard/npmpackage_test.go b/internal/guard/npmpackage_test.go index efba3e01..82797f91 100644 --- a/internal/guard/npmpackage_test.go +++ b/internal/guard/npmpackage_test.go @@ -1,6 +1,7 @@ package guard import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -77,6 +78,31 @@ func TestThePackageKnowsEveryPlatformTheReleaseBuilds(t *testing.T) { } } +// The dependency gate lets the package through by name, because the graph has +// no licence for a directory linked with file: (#220). That is only true while +// the package is what the gate's exception says it is. +func TestTheLinkedPackageIsUnderTheLicenceTheGateIsToldItIs(t *testing.T) { + var pkg struct { + Name string `json:"name"` + License string `json:"license"` + } + if err := json.Unmarshal([]byte(npmFile(t, npmPackage+"/package.json")), &pkg); err != nil { + t.Fatalf("reading the package's package.json: %v", err) + } + if pkg.Name != "testing-files-generator" || pkg.License != "MPL-2.0" { + t.Errorf("the package is %s under %s, and the dependency gate lets testing-files-generator through as MPL-2.0", pkg.Name, pkg.License) + } + if licence := npmFile(t, npmPackage+"/LICENSE"); !strings.HasPrefix(licence, "Mozilla Public License Version 2.0\n") { + t.Errorf("the package's LICENSE is not the text of the Mozilla Public License 2.0") + } + gate := npmFile(t, ".github/scripts/dependency_gate.py") + for _, name := range []string{`"testing-files-generator": "`, `"../../npm/testing-files-generator": "`} { + if !strings.Contains(gate, name) || !strings.Contains(gate, "MPL-2.0") { + t.Errorf("the dependency gate has no exception %s naming the licence", name) + } + } +} + // unionOf reads the members of a TypeScript union of string literals. func unionOf(t *testing.T, body, name string) []string { t.Helper() From 35d94a7d66699199ee5876fd8ea3599df0acdbf8 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Thu, 5 Nov 2026 08:27:03 +0100 Subject: [PATCH 3/3] A kept set is checked byte for byte before it is used, and tfg's error stream no longer stops it part way A set written by an earlier run, or by another worker, was taken when its files had the sizes the plan says. A test that writes into a file it was given keeps the size and changes the set for every later run. Such a set now goes through tfg verify first and is refused with the files that differ. Checking the default upload set of 115 MB takes about a seventh of the time it takes to write it. The planning run kept tfg's error stream in a buffer of 1 MiB, and a run that said more was stopped and reported as never started. It now goes to a file and only its end is kept. A run stopped by a signal is named by the signal. A binary or recipe the options name and that is not there is said so, with where a relative recipe path is taken from. The advice about --omit=optional is given only when the platform package is missing. The example's check no longer reads a report an earlier run left, and the build commands name the binary each system looks for. The memory guard on another run's record no longer applies its ceiling under the race detector, which allocates on its own account. Measured on one machine over nine runs, its reading moved from 2.35 MB below to 1.72 MB above the other record, against a ceiling of 50 kB. Without the detector the reading is 0.68 to 0.82 MB below, and the jobs that run without it keep the ceiling. Co-Authored-By: Claude Opus 5.5 --- examples/playwright/README.md | 2 +- examples/playwright/check.mjs | 19 ++++-- internal/guard/manifestreadlimits_test.go | 9 +++ internal/guard/raceflag_test.go | 10 +-- npm/testing-files-generator/README.md | 6 +- npm/testing-files-generator/index.d.ts | 2 +- npm/testing-files-generator/lib/binary.js | 9 ++- npm/testing-files-generator/lib/fileset.js | 63 ++++++++++++++----- npm/testing-files-generator/lib/run.js | 46 +++++++++----- .../test/binary.test.js | 12 +++- .../test/fileset.test.js | 40 ++++++++++-- npm/testing-files-generator/test/helpers.js | 2 +- npm/testing-files-generator/test/run.test.js | 30 +++++++++ 13 files changed, 201 insertions(+), 49 deletions(-) create mode 100644 npm/testing-files-generator/test/run.test.js diff --git a/examples/playwright/README.md b/examples/playwright/README.md index 426d7b51..0945467b 100644 --- a/examples/playwright/README.md +++ b/examples/playwright/README.md @@ -16,7 +16,7 @@ From the root of the repository, build tfg first. The example finds it there. ```bash -go build -tags "$(cat .github/build-tags)" -o tfg.exe ./cmd/tfg +go build -tags "$(cat .github/build-tags)" -o "tfg$(go env GOEXE)" ./cmd/tfg ``` ```bash diff --git a/examples/playwright/check.mjs b/examples/playwright/check.mjs index e40f9830..14e189dc 100644 --- a/examples/playwright/check.mjs +++ b/examples/playwright/check.mjs @@ -6,7 +6,7 @@ // tests could come from the tests themselves rather than from the app. import { spawnSync } from 'node:child_process'; -import { readFileSync } from 'node:fs'; +import { existsSync, readFileSync, rmSync } from 'node:fs'; import { createRequire } from 'node:module'; // Playwright's own command line, run by this Node rather than through npx and @@ -32,12 +32,23 @@ const mistakes = { }, }; +const reportFile = 'results/check.json'; + +// run runs the tests once and reads their report. Red tests end the run with +// a code other than nought, which is what this is here to see, so the code says +// nothing. A report left by an earlier run is removed first, or a run that +// never got as far as writing one would be read as that earlier run. function run(fixed) { - spawnSync(process.execPath, [playwright, 'test', '--reporter=json'], { + rmSync(reportFile, { force: true }); + const done = spawnSync(process.execPath, [playwright, 'test', '--reporter=json'], { stdio: ['ignore', 'ignore', 'inherit'], - env: { ...process.env, FIXED: fixed ? '1' : '', PLAYWRIGHT_JSON_OUTPUT_NAME: 'results/check.json' }, + env: { ...process.env, FIXED: fixed ? '1' : '', PLAYWRIGHT_JSON_OUTPUT_NAME: reportFile }, }); - const report = JSON.parse(readFileSync('results/check.json', 'utf8')); + if (done.error || done.signal || !existsSync(reportFile)) { + const why = done.error?.message ?? (done.signal ? `it was stopped by ${done.signal}` : `it ended with code ${done.status} and wrote no ${reportFile}`); + throw new Error(`Playwright did not run the tests - ${why}. Run npm install and npx playwright install chromium, then try again.`); + } + const report = JSON.parse(readFileSync(reportFile, 'utf8')); const tests = []; const walk = (suite, titles) => { for (const spec of suite.specs ?? []) { diff --git a/internal/guard/manifestreadlimits_test.go b/internal/guard/manifestreadlimits_test.go index 0eb59d55..83790ae2 100644 --- a/internal/guard/manifestreadlimits_test.go +++ b/internal/guard/manifestreadlimits_test.go @@ -256,6 +256,15 @@ func TestAnotherRunsRecordListingOneFileOverAndOverHoldsItOnce(t *testing.T) { } t.Logf("reading the record naming the file allocated %d B, and the record naming other files %d B", same, apart) + // Not compared under the race detector, which allocates on its own + // account. Measured 2026-11-05 on the same machine, six runs each: without + // it the first record cost 0.68 to 0.82 MB less than the second, out of + // 20.5 MB, and with it about 85 MB and anything from 1.53 MB less to 0.94 + // MB more - one run in six over the ceiling, as on CI the night before. + // The jobs that run without the detector still hold the ceiling. + if raceEnabled { + return + } if grew := same - apart; grew > entries { t.Errorf("a record naming one file in each of its %d entries took %d B more to read than one naming none of the files, "+ "%.1f B an entry - the file was kept once for every entry rather than once", entries, grew, float64(grew)/entries) diff --git a/internal/guard/raceflag_test.go b/internal/guard/raceflag_test.go index 8b8db1b3..0d8a9bad 100644 --- a/internal/guard/raceflag_test.go +++ b/internal/guard/raceflag_test.go @@ -4,10 +4,12 @@ package guard // raceEnabled says whether this binary was built with the race detector. // -// It exists for exactly one guard: the object count in resources_test.go. The -// detector allocates on its own account, so a count taken under it is not the -// same measurement as a count taken without, and comparing either against one -// ceiling asks the number to mean two things. +// It exists for guards whose measurement the detector changes. Most are about +// memory - the object count in resources_test.go, the bytes compared in +// manifestreadlimits_test.go. The detector allocates on its own account, so a +// count taken under it is not the same measurement as a count taken without, +// and comparing either against one ceiling asks the number to mean two things. +// One is about time: jxlladder_test.go sweeps fewer seeds under it. // // Two files rather than a runtime lookup, because the build tag is the only // thing that knows. Go offers no way to ask at run time. diff --git a/npm/testing-files-generator/README.md b/npm/testing-files-generator/README.md index d3850bc7..d938db31 100644 --- a/npm/testing-files-generator/README.md +++ b/npm/testing-files-generator/README.md @@ -58,6 +58,8 @@ fileSet({ recipe: 'fixtures/uploads.yaml' }); fileSet({ options: { format: 'png', size: '5mb', count: 3, seed: 7, expected: 'accept' } }); ``` +A relative recipe path is taken from the directory the tests were started in. `path.join(__dirname, 'uploads.yaml')` names one beside the spec file, wherever the tests are started from. + Files made from flags expect nothing unless `expected` says what, so without it every one of them is skipped. | preset | the question it answers | @@ -81,8 +83,8 @@ await page.getByLabel('Files').setInputFiles(together); ## When the files are written 1. **While Playwright loads the spec file, nothing is written.** `fileSet` asks for the plan, the list of files and what each should meet, and Playwright declares the tests from it. Listing the tests with `--list` writes nothing either. -2. **The first test that asks for a path writes the whole set**, once, however many workers ask at the same moment. The default `upload-validation` set, 71 files and 116 MB, was written in under a second on the machine it was measured on. -3. **Every later run takes the same files.** The same options give the same bytes, so a set is written once and kept, in `node_modules/.cache/testing-files-generator` or the directory `TFG_CACHE_DIR` names. +2. **The first test that asks for a path writes the whole set**, once, however many workers ask at the same moment. The default `upload-validation` set, 71 files and 115 MB, was written in under a second on the machine it was measured on. +3. **Every later run takes the same files.** The same options give the same bytes, so a set is written once and kept, in `node_modules/.cache/testing-files-generator` or the directory `TFG_CACHE_DIR` names. Before a kept set is used, every byte of it is checked against its manifest, which took under a tenth of a second for the same set. A set a test wrote into, or a file someone changed, is refused with the names of the files that differ. A large set is better written before the tests start, so its time does not count against the first test: diff --git a/npm/testing-files-generator/index.d.ts b/npm/testing-files-generator/index.d.ts index fd257434..8cc5e5e1 100644 --- a/npm/testing-files-generator/index.d.ts +++ b/npm/testing-files-generator/index.d.ts @@ -60,7 +60,7 @@ export type OptionValue = string | number | boolean | ReadonlyArray; diff --git a/npm/testing-files-generator/lib/binary.js b/npm/testing-files-generator/lib/binary.js index 48cb522e..5b51346c 100644 --- a/npm/testing-files-generator/lib/binary.js +++ b/npm/testing-files-generator/lib/binary.js @@ -46,7 +46,14 @@ function binaryPath({ env = process.env, platform = process.platform, arch = pro const exe = platform === 'win32' ? 'tfg.exe' : 'tfg'; try { return resolve(`${pkg}/${exe}`); - } catch { + } catch (err) { + // Only a package that is not there is a matter of --omit. Anything else, + // such as a package that does not export the binary, is told as it is, + // because installing again would not change it. + if (err?.code !== 'MODULE_NOT_FOUND') { + throw new Error(`The binary for ${platform} ${arch} in the package ${pkg} could not be found - ${err?.message}. ` + + 'Set TFG_BINARY to the path of a tfg binary to go on.', { cause: err }); + } throw new Error(`The binary for ${platform} ${arch} comes in the package ${pkg}, and it is not installed. ` + 'npm leaves it out when it installs with --omit=optional or --no-optional. ' + 'Install again without that option, or set TFG_BINARY to the path of a tfg binary.'); diff --git a/npm/testing-files-generator/lib/fileset.js b/npm/testing-files-generator/lib/fileset.js index 095580db..e72d3b87 100644 --- a/npm/testing-files-generator/lib/fileset.js +++ b/npm/testing-files-generator/lib/fileset.js @@ -12,13 +12,17 @@ const os = require('node:os'); const path = require('node:path'); const { binaryPath } = require('./binary.js'); -const { run, runToFile } = require('./run.js'); +const { TfgError, run, runToFile } = require('./run.js'); const { TestFile, outcomes } = require('./testfile.js'); // The manifest major this package reads. Another major is refused rather than // read as if it were this one - docs/MANIFEST.md, the version of the contract. const manifestMajor = '1'; +// The code tfg verify ends with when a directory does not match its manifest, +// from the exit codes in docs/CLI.md. Any other code is verify failing to look. +const exitVerify = 7; + // Flags this package sets itself. An option of the same name would fight it. const reserved = { out: 'this package writes every set into a directory of its own', @@ -51,7 +55,13 @@ function fileSet(spec = {}) { if (typeof spec !== 'object' || spec === null) { throw new TypeError('fileSet takes an object, for example { preset: \'upload-validation\', options: { limit: \'2mb\' } }.'); } - const binary = spec.binary ?? binaryPath(); + const binary = spec.binary === undefined ? binaryPath() : named(spec.binary, 'binary', + 'Point it at a tfg binary, or leave it out to use the one this package installs.'); + if (spec.recipe) { + named(path.resolve(spec.recipe), 'recipe', + `A relative path is taken from the directory the tests started in, here ${process.cwd()}, ` + + 'so path.join(__dirname, ...) names a recipe beside the spec file.'); + } const args = argsOf(spec); const cacheDir = path.resolve(spec.cacheDir ?? defaultCacheDir()); const recipe = spec.recipe ? fs.readFileSync(path.resolve(spec.recipe), 'utf8') : ''; @@ -92,6 +102,15 @@ function argsOf(spec) { return args; } +// named gives back a file an option names, or says that there is none, before +// anything reads it and fails with only the system's word for it. +function named(file, option, wayOut) { + if (!fs.existsSync(file)) { + throw new Error(`The ${option} option names ${file}, and there is no file there. ${wayOut}`); + } + return file; +} + function defaultCacheDir() { if (process.env.TFG_CACHE_DIR) { return process.env.TFG_CACHE_DIR; @@ -169,10 +188,7 @@ class Source { async _write() { if (fs.existsSync(this.dir)) { - const problem = mismatch(this.dir, this.plan); - if (problem) { - throw stale(this.dir, problem); - } + await this._taken(); return this.dir; } const tmp = scratch(this.cacheDir); @@ -202,11 +218,8 @@ class Source { return this.dir; } catch (err) { if (fs.existsSync(this.dir)) { - const problem = mismatch(this.dir, this.plan); discard(this.binary, tmp); - if (problem) { - throw stale(this.dir, problem); - } + await this._taken(); events.emit('took', this.dir); return this.dir; } @@ -218,6 +231,25 @@ class Source { } } } + + // _taken checks a set this process did not write before it is used: against + // the plan, and then every byte against the manifest through tfg verify. A + // test that writes into a file it was given changes the set for every later + // run, and the size alone does not show it. + async _taken() { + const problem = mismatch(this.dir, this.plan); + if (problem) { + throw stale(this.dir, problem); + } + try { + await run(this.binary, ['verify', path.join(this.dir, 'manifest.json')]); + } catch (err) { + if (err instanceof TfgError && err.status === exitVerify) { + throw stale(this.dir, 'its files are not the ones it was written with', err.stderr.trim()); + } + throw err; + } + } } // readable refuses a manifest this package cannot read for what it says. @@ -236,8 +268,10 @@ function readable(plan) { } // mismatch says how a written set differs from its plan, or nothing when it -// does not. Sizes are compared and contents are not: a set is checked on every -// run, and hashing all of it each time would cost more than writing it. +// does not. It compares names and sizes, which is enough for a set tfg has just +// written. The bytes of a set written earlier are checked by _taken, through +// tfg verify - measured 2026-11-05, about 80 ms for the default upload set of +// 115 MB, against about 590 ms to write it. // // Every file has to be on the disk. A run that ends with code nought writes // every file it plans, and one that cannot ends with the partial code, which @@ -281,9 +315,10 @@ function mismatch(dir, plan) { return ''; } -function stale(dir, problem) { +function stale(dir, problem, details = '') { return new Error(`${dir} is where this set belongs, and it does not hold it - ${problem}. ` + - 'Nothing here removes that directory, because a person may have changed it on purpose. Delete it and run again.'); + 'Nothing here removes that directory, because a person may have changed it on purpose. Delete it and run again.' + + (details ? `\n${details}` : '')); } // discard removes a set this process wrote and will not use. The files go diff --git a/npm/testing-files-generator/lib/run.js b/npm/testing-files-generator/lib/run.js index 175b6b81..37ed53d0 100644 --- a/npm/testing-files-generator/lib/run.js +++ b/npm/testing-files-generator/lib/run.js @@ -11,12 +11,14 @@ const fs = require('node:fs'); // refusals are a few lines, and a run of many files can print a note for each. const keptError = 64 * 1024; -// TfgError is tfg ending with a code other than nought. The message carries -// what tfg said, because its refusals already name the value and the way out. +// TfgError is tfg ending with a code other than nought, or stopped by a signal, +// whose name is then the status. The message carries what tfg said, because +// its refusals already name the value and the way out. class TfgError extends Error { constructor(args, status, stderr) { const said = stderr.trim() || 'tfg said nothing about why.'; - super(`tfg ${args.join(' ')} ended with code ${status}.\n${said}`); + const ended = typeof status === 'string' ? `was stopped by ${status}` : `ended with code ${status}`; + super(`tfg ${args.join(' ')} ${ended}.\n${said}`); this.name = 'TfgError'; this.status = status; this.stderr = stderr; @@ -29,23 +31,39 @@ function notStarted(binary, err) { // runToFile runs tfg to the end, writing its standard output to a file. A file // and not memory, because the manifest of a large set outgrows the buffer Node -// gives a child process by default. +// gives a child process by default. The error stream goes to a file of its own +// for the same reason - a buffer that fills stops tfg part way - and only its +// end is kept. function runToFile(binary, args, outFile) { - const fd = fs.openSync(outFile, 'w'); - let done; + const errFile = `${outFile}.err`; + const out = fs.openSync(outFile, 'w'); + let err; try { - done = spawnSync(binary, args, { stdio: ['ignore', fd, 'pipe'], windowsHide: true, maxBuffer: keptError * 16 }); + err = fs.openSync(errFile, 'w+'); + const done = spawnSync(binary, args, { stdio: ['ignore', out, err], windowsHide: true }); + if (done.error) { + throw notStarted(binary, done.error); + } + if (done.status !== 0) { + throw new TfgError(args, done.status ?? done.signal, tail(err)); + } } finally { - fs.closeSync(fd); - } - if (done.error) { - throw notStarted(binary, done.error); - } - if (done.status !== 0) { - throw new TfgError(args, done.status, done.stderr.toString('utf8').slice(-keptError)); + fs.closeSync(out); + if (err !== undefined) { + fs.closeSync(err); + } + fs.rmSync(errFile, { force: true }); } } +// tail reads the last keptError bytes of an open file. +function tail(fd) { + const size = fs.fstatSync(fd).size; + const kept = Buffer.alloc(Math.min(size, keptError)); + fs.readSync(fd, kept, 0, kept.length, size - kept.length); + return kept.toString('utf8'); +} + // run runs tfg without holding the event loop, so a test runner keeps its own // timers and messages going while files are written. function run(binary, args) { diff --git a/npm/testing-files-generator/test/binary.test.js b/npm/testing-files-generator/test/binary.test.js index 12948b3d..c5cfffef 100644 --- a/npm/testing-files-generator/test/binary.test.js +++ b/npm/testing-files-generator/test/binary.test.js @@ -6,7 +6,8 @@ const { test } = require('node:test'); const { binaryPath, platforms } = require('../lib/binary.js'); -const nothing = () => { throw new Error('not installed'); }; +// What require.resolve throws for a package that is not installed. +const nothing = (id) => { throw Object.assign(new Error(`Cannot find module '${id}'`), { code: 'MODULE_NOT_FOUND' }); }; test('TFG_BINARY is taken as given when there is a file there', () => { assert.equal(binaryPath({ env: { TFG_BINARY: __filename }, resolve: nothing }), __filename); @@ -39,3 +40,12 @@ test('a platform package left out by --omit=optional is named, with the way back assert.throws(() => binaryPath({ env: {}, platform: 'linux', arch: 'x64', resolve: nothing }), (err) => err.message.includes('@testing-files-generator/linux-x64') && err.message.includes('--omit=optional')); }); + +test('a platform package that is there and still gives no binary is told as it is, not as left out', () => { + const hidden = () => { + throw Object.assign(new Error('Package subpath \'./tfg\' is not defined by "exports"'), { code: 'ERR_PACKAGE_PATH_NOT_EXPORTED' }); + }; + assert.throws(() => binaryPath({ env: {}, platform: 'linux', arch: 'x64', resolve: hidden }), + (err) => err.message.includes('is not defined by "exports"') && !err.message.includes('--omit=optional') && + err.cause?.code === 'ERR_PACKAGE_PATH_NOT_EXPORTED'); +}); diff --git a/npm/testing-files-generator/test/fileset.test.js b/npm/testing-files-generator/test/fileset.test.js index 93e3ce41..69b953c2 100644 --- a/npm/testing-files-generator/test/fileset.test.js +++ b/npm/testing-files-generator/test/fileset.test.js @@ -47,7 +47,9 @@ test('a set is written once, whole, and its files are where their paths say', as }); // racer plans a set, waits until every racer has planned, and only then -// writes, so that all of them find the set unwritten and write it at once. +// writes, so that all of them find the set unwritten and write it at once. A +// racer that fails before it reaches the barrier would hold the others there +// for ever, so the wait has an end. function racer(cacheDir, barrier, racers) { const lib = JSON.stringify(path.join(__dirname, '..', 'lib', 'fileset.js')); const at = JSON.stringify(barrier); @@ -57,19 +59,22 @@ const said = []; for (const e of ['wrote', 'moved', 'took']) events.on(e, () => said.push(e)); const set = fileSet({ ...${JSON.stringify(small)}, cacheDir: ${JSON.stringify(cacheDir)} }); fs.writeFileSync(require('node:path').join(${at}, String(process.pid)), ''); -const ready = () => fs.readdirSync(${at}).length >= ${racers} ? Promise.resolve() : new Promise((r) => setTimeout(r, 5)).then(ready); +const until = Date.now() + 30000; +const ready = () => fs.readdirSync(${at}).length >= ${racers} ? Promise.resolve() + : Date.now() > until ? Promise.reject(new Error('not every racer planned within 30 s')) + : new Promise((r) => setTimeout(r, 5)).then(ready); ready().then(() => set.write()).then((dir) => console.log(JSON.stringify({ dir, said })), (err) => { console.error(err.message); process.exit(3); });`; } -test('workers writing one set at once end with one copy and nothing left over', async (t) => { +test('workers writing one set at once end with one copy and nothing left over', { timeout: 120000 }, async (t) => { const cacheDir = tempDir(t); const barrier = tempDir(t); const racers = 4; const runs = await Promise.all(Array.from({ length: racers }, () => node(racer(cacheDir, barrier, racers)))); - for (const r of runs) { - assert.equal(r.status, 0, r.stderr); - } + // Every failed racer is shown, because the first to fail may only be one + // that waited for another. + assert.deepEqual(runs.filter((r) => r.status !== 0).map((r) => r.stderr), [], 'a racer failed'); const results = runs.map((r) => JSON.parse(r.stdout)); const said = results.flatMap((r) => r.said); const count = (e) => said.filter((s) => s === e).length; @@ -125,6 +130,29 @@ test('a file of the set cut short by hand is refused', async (t) => { assert.match(again.stderr, /1mb_at_limit\.jpg is 10 B and its manifest says 1048576 B/); }); +test('a file of the set written over by hand, its size the same, is refused', async (t) => { + const cacheDir = tempDir(t); + const dir = (await node(writer(cacheDir))).stdout.trim(); + const changed = path.join(dir, '1mb_at_limit.jpg'); + const fd = fs.openSync(changed, 'r+'); + fs.writeSync(fd, Buffer.from('changed'), 0, 7, 0); + fs.closeSync(fd); + const again = await node(writer(cacheDir)); + assert.equal(again.status, 3, again.stderr); + assert.match(again.stderr, /its files are not the ones it was written with\. Nothing here removes that directory/); + assert.match(again.stderr, /wrong-hash\s+1mb_at_limit\.jpg/); + assert.ok(fs.existsSync(changed), 'the changed set was removed'); +}); + +test('a binary or a recipe the options name and that is not there is named, with what to do', (t) => { + const cacheDir = tempDir(t); + const missing = path.join(cacheDir, 'no-such-tfg'); + assert.throws(() => fileSet({ ...small, binary: missing, cacheDir }), + (err) => err.message.includes(`The binary option names ${missing}, and there is no file there.`) && err.message.includes('leave it out')); + assert.throws(() => fileSet({ recipe: 'no-such-recipe.yaml', binary: tfg(), cacheDir }), + (err) => err.message.includes(path.resolve('no-such-recipe.yaml')) && err.message.includes('path.join(__dirname')); +}); + test('a set a stopped run left behind is named once outside the workers, and never used', async (t) => { const cacheDir = tempDir(t); const gone = await node(''); diff --git a/npm/testing-files-generator/test/helpers.js b/npm/testing-files-generator/test/helpers.js index 75689fe5..af64e127 100644 --- a/npm/testing-files-generator/test/helpers.js +++ b/npm/testing-files-generator/test/helpers.js @@ -15,7 +15,7 @@ function tfg() { const binary = process.env.TFG_BINARY || built; if (!fs.existsSync(binary)) { throw new Error(`No tfg binary at ${binary}. Build one from the repository root with ` + - 'go build -tags "$(cat .github/build-tags)" -o tfg.exe ./cmd/tfg, or set TFG_BINARY.'); + `go build -tags "$(cat .github/build-tags)" -o ${path.basename(built)} ./cmd/tfg, or set TFG_BINARY.`); } return binary; } diff --git a/npm/testing-files-generator/test/run.test.js b/npm/testing-files-generator/test/run.test.js new file mode 100644 index 00000000..ace16476 --- /dev/null +++ b/npm/testing-files-generator/test/run.test.js @@ -0,0 +1,30 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { test } = require('node:test'); + +const { TfgError, run, runToFile } = require('../lib/run.js'); +const { tempDir } = require('./helpers.js'); + +// Node stands in for tfg here, so a child can be made to end any way at all. +const child = (script) => [process.execPath, ['-e', script]]; + +test('a child stopped by a signal is named by the signal, from both ways of running it', { + skip: process.platform === 'win32' && 'Windows has no signals to stop a process with', +}, async (t) => { + const [binary, args] = child("process.kill(process.pid, 'SIGTERM')"); + const out = path.join(tempDir(t), 'out.json'); + assert.throws(() => runToFile(binary, args, out), + (err) => err instanceof TfgError && err.status === 'SIGTERM' && /was stopped by SIGTERM/.test(err.message)); + await assert.rejects(run(binary, args), + (err) => err instanceof TfgError && err.status === 'SIGTERM' && /was stopped by SIGTERM/.test(err.message)); +}); + +test('a child that says a great deal on its error stream runs to its own end, and the end is kept', (t) => { + const [binary, args] = child("process.stderr.write('x'.repeat(4 * 1024 * 1024) + 'the last words'); process.exitCode = 3"); + const out = path.join(tempDir(t), 'out.json'); + assert.throws(() => runToFile(binary, args, out), + (err) => err instanceof TfgError && err.status === 3 && err.stderr.endsWith('the last words') && + err.stderr.length <= 64 * 1024); +});