diff --git a/.github/scripts/dependency_gate.py b/.github/scripts/dependency_gate.py index 7b8c6d08..987f1381 100644 --- a/.github/scripts/dependency_gate.py +++ b/.github/scripts/dependency_gate.py @@ -89,6 +89,15 @@ # in third_party/go-gl-gl/PATCH.md. This names one directory, and a second # copy of something else needs a line of its own here. "./third_party/go-gl-gl": "a copy of github.com/go-gl/gl, MIT, held to the published module plus one patch by a guard", + # This repository's own npm package, linked into its Playwright example + # with file: since 2026-11-04 (#220). The graph reports it twice with no + # licence, once by its name and once by its path, measured on that pull + # request - a linked directory is not on any registry either. Its licence + # is MPL-2.0, in its package.json and its LICENSE, written by this project, + # and TestTheLinkedPackageIsUnderTheLicenceTheGateIsToldItIs holds both to + # that. The two names are one package. + "testing-files-generator": "this repository's npm package, linked into examples/playwright, MPL-2.0 held by a guard", + "../../npm/testing-files-generator": "the same package, as the graph names it by its path", } # GitHub reports license: null for every action, measured on this repository on diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47a5ee43..dae8548b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1482,6 +1482,66 @@ jobs: run: go test -tags "$(cat .github/build-tags)" ./internal/guard/ -count=1 -run 'ReferenceTool' -v shell: bash + npm: + name: npm package on ${{ matrix.os }} + # The npm package (npm/testing-files-generator) runs tfg and reads what it + # writes from JavaScript, so nothing in the Go suite runs it. Its own tests + # run here against the binary this commit builds, on the three systems a + # release has binaries for - Windows above all, because the rename that + # settles a race between test workers is refused there in ways it is not + # elsewhere. Node 22 is the oldest the package declares and Playwright + # supports, so it is the one tested. + # + # The Playwright example runs on Linux only. It is the package's acceptance + # test: an upload form with four mistakes, where the generated set has to + # find all four, the tests written with Buffer.alloc have to find none, and + # with the mistakes fixed nothing may fail - that last run is the control + # without which a red test could come from the tests. One browser on one + # system is enough to show that, and the package itself is tested on all + # three. docs/PLAYWRIGHT-2026-11-04.md. + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + env: + CGO_ENABLED: "0" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: false + cache: true + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: build tfg where the package's tests and the example look for it + run: go build -tags "$(cat .github/build-tags)" -o "tfg$(go env GOEXE)" ./cmd/tfg + shell: bash + + - name: the package's own tests + working-directory: npm/testing-files-generator + run: node --test "test/*.test.js" + shell: bash + + - name: the Playwright example finds the four mistakes, and nothing once they are fixed + if: runner.os == 'Linux' + working-directory: examples/playwright + run: | + set -euo pipefail + npm ci --no-audit --no-fund + npx playwright install --with-deps chromium + npx tsc --noEmit + node check.mjs + shell: bash + fuzz: name: fuzzing with a time budget # Weekly and by hand, never on a push. Fuzzing searches without end, so a diff --git a/examples/playwright/.gitignore b/examples/playwright/.gitignore new file mode 100644 index 00000000..3b28d5e2 --- /dev/null +++ b/examples/playwright/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +results/ +test-results/ diff --git a/examples/playwright/README.md b/examples/playwright/README.md new file mode 100644 index 00000000..0945467b --- /dev/null +++ b/examples/playwright/README.md @@ -0,0 +1,40 @@ +# An upload form, the tests it usually gets, and the tests a generated set gives it + +`app/server.mjs` is an upload form with four mistakes, each one a mistake real upload forms have: + +1. The server never checks the size. Only the page does. +2. The content and the extension are each checked against the allowed types, and never against each other, so a PDF named `.jpg` goes through. +3. An allowed extension anywhere in the name counts, so `invoice.jpg.exe` is taken for a picture. +4. The page refuses a file of exactly the limit, which is the largest size allowed. + +`tests/typical.spec.ts` is what such a form usually gets: a real picture that should go through, and files made in the test with `Buffer.alloc` that should not. Every one of those tests passes. The file over the limit is zeros rather than a PDF, so the server turns it away for what it holds, and the test meant to prove the size limit passes on a server that has none. + +`tests/upload.spec.ts` sends the `upload-validation` set from [testing-files-generator](../../npm/testing-files-generator) through the page and straight to the server. It finds all four mistakes, and each failure says what it means. + +## Run it + +From the root of the repository, build tfg first. The example finds it there. + +```bash +go build -tags "$(cat .github/build-tags)" -o "tfg$(go env GOEXE)" ./cmd/tfg +``` + +```bash +cd examples/playwright && npm install && npx playwright install chromium +``` + +```bash +npx playwright test +``` + +Eleven tests fail, five through the page and six through the API, and every failure names its file and the mistake it shows: + +``` +1mb_over_1b.jpg was taken, and it should have been turned away (size_limit). +A JPEG file 1 B over the limit of 1mb. Your system should turn it away - taking it means +the limit is not enforced, or enforced only in the browser. +``` + +Ten are skipped, because what a form does with an empty file or a name full of emoji is its own policy, and the set says so rather than pass them. + +`FIXED=1 npx playwright test` corrects all four mistakes, and nothing fails. `npm run check` runs both and says whether the example still shows what it is for. The repository's CI runs that. diff --git a/examples/playwright/app/index.html b/examples/playwright/app/index.html new file mode 100644 index 00000000..fe0390c4 --- /dev/null +++ b/examples/playwright/app/index.html @@ -0,0 +1,13 @@ + + +Upload + +
+ + + +
+

+ + + diff --git a/examples/playwright/app/page.js b/examples/playwright/app/page.js new file mode 100644 index 00000000..44dcacdf --- /dev/null +++ b/examples/playwright/app/page.js @@ -0,0 +1,22 @@ +// The form's own check, in the browser, before anything is sent. + +const limit = Number(document.body.dataset.limit); +const fixed = document.body.dataset.fixed === '1'; + +document.getElementById('upload').addEventListener('submit', async (event) => { + event.preventDefault(); + const status = document.getElementById('status'); + const file = document.getElementById('file').files[0]; + // Mistake 4: a file of exactly the limit is refused. The limit is the + // largest size allowed, so the comparison should be >. + const tooLarge = fixed ? file.size > limit : file.size >= limit; + if (tooLarge) { + status.textContent = 'Refused (size_limit)'; + return; + } + const body = new FormData(); + body.append('file', file); + const response = await fetch('/upload', { method: 'POST', body }); + const answer = await response.json(); + status.textContent = response.ok ? 'Uploaded' : `Refused (${answer.reason})`; +}); diff --git a/examples/playwright/app/server.mjs b/examples/playwright/app/server.mjs new file mode 100644 index 00000000..e83c0e6d --- /dev/null +++ b/examples/playwright/app/server.mjs @@ -0,0 +1,108 @@ +// An upload form with four mistakes made on purpose, each one a mistake real +// upload forms have. playwright.config.ts starts it. FIXED=1 corrects all four, +// which is how check.mjs shows that the red tests come from the mistakes and +// not from the tests. +// +// It uses nothing outside Node, so the example has no server dependency. + +import { readFileSync } from 'node:fs'; +import http from 'node:http'; +import { Readable } from 'node:stream'; + +const limit = Number(process.env.UPLOAD_LIMIT_BYTES); +const port = Number(process.env.PORT); +const fixed = process.env.FIXED === '1'; +const allowed = ['jpg', 'png', 'pdf']; + +if (!Number.isInteger(limit) || !Number.isInteger(port)) { + throw new Error('Set UPLOAD_LIMIT_BYTES and PORT. playwright.config.ts sets both.'); +} + +// What a file holds, read from its first bytes rather than from its name. +function sniff(bytes) { + if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) { + return 'jpg'; + } + if (bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) { + return 'png'; + } + if (bytes.subarray(0, 5).toString('latin1') === '%PDF-') { + return 'pdf'; + } + return ''; +} + +function extensionOf(name) { + const lower = name.toLowerCase(); + if (fixed) { + const dot = lower.lastIndexOf('.'); + return dot > 0 ? lower.slice(dot + 1) : ''; + } + // Mistake 3: an allowed extension anywhere in the name counts, so + // invoice.jpg.exe is taken for a picture. + return allowed.find((ext) => lower.includes(`.${ext}`)) ?? ''; +} + +function verdict(name, bytes) { + // Mistake 1: the server never looks at the size. The page does, so a test + // through the page sees a limit that a request straight to the server does not. + if (fixed && bytes.length > limit) { + return { status: 413, reason: 'size_limit' }; + } + const extension = extensionOf(name); + if (!allowed.includes(extension)) { + return { status: 422, reason: 'extension_rule' }; + } + const content = sniff(bytes); + if (!allowed.includes(content)) { + return { status: 415, reason: 'content_malformed' }; + } + // Mistake 2: the content and the extension are each checked against the + // allowed list and never against each other, so a PDF named .jpg goes through. + if (fixed && content !== extension) { + return { status: 415, reason: 'mime_mismatch' }; + } + return { status: 201 }; +} + +// The page carries the limit and the mode, and page.js reads them from it. +const page = readFileSync(new URL('index.html', import.meta.url), 'utf8') + .replace('{{limit}}', String(limit)) + .replace('{{fixed}}', fixed ? '1' : ''); +const script = readFileSync(new URL('page.js', import.meta.url), 'utf8'); + +async function upload(req, res) { + const request = new Request(`http://localhost${req.url}`, { + method: 'POST', headers: req.headers, body: Readable.toWeb(req), duplex: 'half', + }); + const file = (await request.formData()).get('file'); + if (!file || typeof file === 'string') { + return answer(res, 400, { reason: 'none' }); + } + const { status, reason } = verdict(file.name, Buffer.from(await file.arrayBuffer())); + return answer(res, status, reason ? { reason } : { stored: file.name }); +} + +function answer(res, status, body) { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(JSON.stringify(body)); +} + +http.createServer((req, res) => { + if (req.method === 'GET' && req.url === '/') { + res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }); + res.end(page); + return; + } + if (req.method === 'GET' && req.url === '/page.js') { + res.writeHead(200, { 'content-type': 'text/javascript; charset=utf-8' }); + res.end(script); + return; + } + if (req.method === 'POST' && req.url === '/upload') { + upload(req, res).catch((err) => answer(res, 500, { reason: 'none', error: String(err) })); + return; + } + res.writeHead(404); + res.end(); +}).listen(port, () => console.log(`upload form on http://localhost:${port}${fixed ? ', mistakes fixed' : ''}`)); diff --git a/examples/playwright/check.mjs b/examples/playwright/check.mjs new file mode 100644 index 00000000..14e189dc --- /dev/null +++ b/examples/playwright/check.mjs @@ -0,0 +1,111 @@ +// Runs the example twice and says whether it shows what it is for. +// +// With the four mistakes, the tests from the generated set fail on exactly the +// files each mistake lets through or turns away, and every typical test passes. +// With FIXED=1, nothing fails. The second run is the control: without it, red +// tests could come from the tests themselves rather than from the app. + +import { spawnSync } from 'node:child_process'; +import { existsSync, readFileSync, rmSync } from 'node:fs'; +import { createRequire } from 'node:module'; + +// Playwright's own command line, run by this Node rather than through npx and +// a shell, which Windows would need for npx. +const playwright = createRequire(import.meta.url).resolve('@playwright/test/cli'); + +// Each mistake in app/server.mjs, and the tests it turns red. +const mistakes = { + 'through the page': { + '1mb_at_limit.jpg': 4, + 'pdf_as_jpg.jpg': 2, + 'png_as_pdf.pdf': 2, + 'jpg_as_png.png': 2, + 'invoice.jpg.exe': 3, + }, + 'through the API': { + '1mb_over_1b.jpg': 1, + '1mb_far_over_2x.jpg': 1, + 'pdf_as_jpg.jpg': 2, + 'png_as_pdf.pdf': 2, + 'jpg_as_png.png': 2, + 'invoice.jpg.exe': 3, + }, +}; + +const reportFile = 'results/check.json'; + +// run runs the tests once and reads their report. Red tests end the run with +// a code other than nought, which is what this is here to see, so the code says +// nothing. A report left by an earlier run is removed first, or a run that +// never got as far as writing one would be read as that earlier run. +function run(fixed) { + rmSync(reportFile, { force: true }); + const done = spawnSync(process.execPath, [playwright, 'test', '--reporter=json'], { + stdio: ['ignore', 'ignore', 'inherit'], + env: { ...process.env, FIXED: fixed ? '1' : '', PLAYWRIGHT_JSON_OUTPUT_NAME: reportFile }, + }); + if (done.error || done.signal || !existsSync(reportFile)) { + const why = done.error?.message ?? (done.signal ? `it was stopped by ${done.signal}` : `it ended with code ${done.status} and wrote no ${reportFile}`); + throw new Error(`Playwright did not run the tests - ${why}. Run npm install and npx playwright install chromium, then try again.`); + } + const report = JSON.parse(readFileSync(reportFile, 'utf8')); + const tests = []; + const walk = (suite, titles) => { + for (const spec of suite.specs ?? []) { + for (const one of spec.tests) { + tests.push({ file: spec.file, describe: titles.at(-1) ?? '', title: spec.title, status: one.status }); + } + } + for (const child of suite.suites ?? []) { + walk(child, [...titles, child.title]); + } + }; + for (const suite of report.suites) { + walk(suite, []); + } + if (report.errors?.length) { + throw new Error(`The run did not get as far as its tests: ${report.errors.map((e) => e.message).join('\n')}`); + } + return tests; +} + +const problems = []; +const say = (problem) => problems.push(problem); + +for (const fixed of [false, true]) { + const mode = fixed ? 'with the mistakes fixed' : 'with the four mistakes'; + const tests = run(fixed); + const ran = tests.filter((t) => t.status !== 'skipped'); + const generated = tests.filter((t) => t.file.endsWith('upload.spec.ts')); + if (ran.length === 0 || generated.every((t) => t.status === 'skipped')) { + say(`${mode}: no test from the generated set ran`); + } + for (const t of tests) { + const name = t.title.split(' - expected ')[0]; + const mistake = mistakes[t.describe]?.[name]; + const shouldFail = !fixed && mistake !== undefined && t.file.endsWith('upload.spec.ts'); + const failed = t.status === 'unexpected'; + if (shouldFail && !failed) { + say(`${mode}: ${t.describe} - ${name} should be red for mistake ${mistake} and is ${t.status}`); + } + if (!shouldFail && failed) { + say(`${mode}: ${t.describe ? `${t.describe} - ` : ''}${t.title} failed, and nothing in the app should make it`); + } + } + const red = tests.filter((t) => t.status === 'unexpected').length; + const skipped = tests.filter((t) => t.status === 'skipped').length; + console.log(`${mode}: ${red} red, ${tests.length - red - skipped} green, ${skipped} skipped as unspecified`); +} + +const found = new Set(Object.values(mistakes).flatMap((files) => Object.values(files))); +for (const n of [1, 2, 3, 4]) { + if (!found.has(n)) { + say(`mistake ${n} has no test that finds it`); + } +} + +if (problems.length > 0) { + console.error(`The example does not show what it is for:\n- ${problems.join('\n- ')}`); + process.exit(1); +} +console.log('The generated set finds all four mistakes, the typical tests find none, and with the mistakes fixed nothing is red.'); diff --git a/examples/playwright/package-lock.json b/examples/playwright/package-lock.json new file mode 100644 index 00000000..d6820d11 --- /dev/null +++ b/examples/playwright/package-lock.json @@ -0,0 +1,469 @@ +{ + "name": "testing-files-generator-playwright-example", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "testing-files-generator-playwright-example", + "license": "MPL-2.0", + "devDependencies": { + "@playwright/test": "1.64.1", + "@types/node": "26.6.6", + "testing-files-generator": "file:../../npm/testing-files-generator", + "typescript": "7.0.2" + } + }, + "../../npm/testing-files-generator": { + "version": "0.0.0-development", + "dev": true, + "license": "MPL-2.0", + "bin": { + "tfg": "bin/tfg.js" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@playwright/test": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.64.1.tgz", + "integrity": "sha512-IaYGml/nSOiZWROfPtuIL7neuu9qDOZ3TNvNljUIrhDMxiGTgPjzj/g9p9RxpS2tSepPDRY+UwAaEeiKtUvIAg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.64.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@types/node": { + "version": "26.6.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.6.tgz", + "integrity": "sha512-MzQj+8Kx3XI40OmUAE6Qoh58ZMNstNNMwn2nfd3qWsfe7nDNSqJNzes4njhz6AHwrSHOffSLwXU6yb474Q4aog==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/@typescript/typescript-aix-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", + "integrity": "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.0.2.tgz", + "integrity": "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.0.2.tgz", + "integrity": "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-arm64/-/typescript-freebsd-arm64-7.0.2.tgz", + "integrity": "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-x64/-/typescript-freebsd-x64-7.0.2.tgz", + "integrity": "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.0.2.tgz", + "integrity": "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.0.2.tgz", + "integrity": "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-loong64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-loong64/-/typescript-linux-loong64-7.0.2.tgz", + "integrity": "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-mips64el": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-mips64el/-/typescript-linux-mips64el-7.0.2.tgz", + "integrity": "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-ppc64/-/typescript-linux-ppc64-7.0.2.tgz", + "integrity": "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-riscv64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-riscv64/-/typescript-linux-riscv64-7.0.2.tgz", + "integrity": "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-s390x": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-s390x/-/typescript-linux-s390x-7.0.2.tgz", + "integrity": "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.0.2.tgz", + "integrity": "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-arm64/-/typescript-netbsd-arm64-7.0.2.tgz", + "integrity": "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-x64/-/typescript-netbsd-x64-7.0.2.tgz", + "integrity": "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-arm64/-/typescript-openbsd-arm64-7.0.2.tgz", + "integrity": "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-x64/-/typescript-openbsd-x64-7.0.2.tgz", + "integrity": "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-sunos-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-sunos-x64/-/typescript-sunos-x64-7.0.2.tgz", + "integrity": "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.0.2.tgz", + "integrity": "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.0.2.tgz", + "integrity": "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/playwright": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.64.1.tgz", + "integrity": "sha512-HCrVhHr5QQuwYoswEUGTb7gGp522yflbv/ST8nF4sFWGuLjQCt17Q/EubpGGSNKYn1tsTCPZxqHhHCfVURu7UA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.64.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.64.1.tgz", + "integrity": "sha512-BC4s8y6SEVA6Q1rSxZTto8hI1KUI3xXlHGZgvGqP4IqPdI/a6FX9H29IDNz7vKo9l21b6LfYosScPL6M3xxyDQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/testing-files-generator": { + "resolved": "../../npm/testing-files-generator", + "link": true + }, + "node_modules/typescript": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", + "integrity": "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc" + }, + "engines": { + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-aix-ppc64": "7.0.2", + "@typescript/typescript-darwin-arm64": "7.0.2", + "@typescript/typescript-darwin-x64": "7.0.2", + "@typescript/typescript-freebsd-arm64": "7.0.2", + "@typescript/typescript-freebsd-x64": "7.0.2", + "@typescript/typescript-linux-arm": "7.0.2", + "@typescript/typescript-linux-arm64": "7.0.2", + "@typescript/typescript-linux-loong64": "7.0.2", + "@typescript/typescript-linux-mips64el": "7.0.2", + "@typescript/typescript-linux-ppc64": "7.0.2", + "@typescript/typescript-linux-riscv64": "7.0.2", + "@typescript/typescript-linux-s390x": "7.0.2", + "@typescript/typescript-linux-x64": "7.0.2", + "@typescript/typescript-netbsd-arm64": "7.0.2", + "@typescript/typescript-netbsd-x64": "7.0.2", + "@typescript/typescript-openbsd-arm64": "7.0.2", + "@typescript/typescript-openbsd-x64": "7.0.2", + "@typescript/typescript-sunos-x64": "7.0.2", + "@typescript/typescript-win32-arm64": "7.0.2", + "@typescript/typescript-win32-x64": "7.0.2" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/examples/playwright/package.json b/examples/playwright/package.json new file mode 100644 index 00000000..af04827c --- /dev/null +++ b/examples/playwright/package.json @@ -0,0 +1,17 @@ +{ + "name": "testing-files-generator-playwright-example", + "private": true, + "description": "An upload form with four mistakes, the tests it usually gets, which pass, and tests from a generated set, which find all four.", + "license": "MPL-2.0", + "scripts": { + "test": "playwright test", + "check": "node check.mjs", + "types": "tsc --noEmit" + }, + "devDependencies": { + "@playwright/test": "1.64.1", + "@types/node": "26.6.6", + "testing-files-generator": "file:../../npm/testing-files-generator", + "typescript": "7.0.2" + } +} diff --git a/examples/playwright/playwright.config.ts b/examples/playwright/playwright.config.ts new file mode 100644 index 00000000..c1388dbc --- /dev/null +++ b/examples/playwright/playwright.config.ts @@ -0,0 +1,26 @@ +import { defineConfig } from '@playwright/test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import { limitBytes, port } from './shared'; + +// Inside the repository the package is linked rather than installed, so its +// binary comes from the repository's own build. A copy of this example outside +// the repository installs testing-files-generator from npm and needs none of this. +const built = path.join(__dirname, '..', '..', process.platform === 'win32' ? 'tfg.exe' : 'tfg'); +if (!process.env.TFG_BINARY && fs.existsSync(built)) { + process.env.TFG_BINARY = built; +} + +export default defineConfig({ + testDir: 'tests', + fullyParallel: true, + reporter: [['list'], ['json', { outputFile: 'results/report.json' }], ['html', { open: 'never', outputFolder: 'results/html' }]], + use: { baseURL: `http://localhost:${port}` }, + webServer: { + command: 'node app/server.mjs', + url: `http://localhost:${port}/`, + reuseExistingServer: false, + env: { UPLOAD_LIMIT_BYTES: String(limitBytes), PORT: String(port), FIXED: process.env.FIXED ?? '' }, + }, +}); diff --git a/examples/playwright/shared.ts b/examples/playwright/shared.ts new file mode 100644 index 00000000..5be9e232 --- /dev/null +++ b/examples/playwright/shared.ts @@ -0,0 +1,5 @@ +// The one limit the form declares, read by the app, the tests made in code and +// the generated set alike. Units count in 1024s, as they do in tfg. +export const limit = '1mb'; +export const limitBytes = 1024 * 1024; +export const port = 3210; diff --git a/examples/playwright/tests/typical.spec.ts b/examples/playwright/tests/typical.spec.ts new file mode 100644 index 00000000..22221360 --- /dev/null +++ b/examples/playwright/tests/typical.spec.ts @@ -0,0 +1,41 @@ +import { test, expect } from '@playwright/test'; + +import { limitBytes } from '../shared'; + +// The tests an upload form usually gets: one real picture that should go +// through, and files made in the test that should not. Against this app every +// one of them passes, while the app has the four mistakes upload.spec.ts finds. +// +// The file over the limit is the reason. Buffer.alloc gives zeros, not a PDF, +// so the server turns it away for what it holds - and the test that was meant +// to prove a size limit passes on a server that has none. + +// A picture of one pixel, the kind of fixture kept as text in a test. +const photo = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==', 'base64'); +const tooLarge = { name: 'big.pdf', mimeType: 'application/pdf', buffer: Buffer.alloc(limitBytes + 1) }; + +test('a photo is uploaded', async ({ page }) => { + await page.goto('/'); + await page.getByLabel('File').setInputFiles({ name: 'photo.png', mimeType: 'image/png', buffer: photo }); + await page.getByRole('button', { name: 'Upload' }).click(); + await expect(page.getByRole('status')).toHaveText('Uploaded'); +}); + +test('a file over the limit is refused in the page', async ({ page }) => { + await page.goto('/'); + await page.getByLabel('File').setInputFiles(tooLarge); + await page.getByRole('button', { name: 'Upload' }).click(); + await expect(page.getByRole('status')).toHaveText(/^Refused/); +}); + +test('a file over the limit is refused by the server', async ({ request }) => { + const response = await request.post('/upload', { multipart: { file: tooLarge } }); + expect(response.ok()).toBe(false); +}); + +test('an executable is refused', async ({ request }) => { + const response = await request.post('/upload', { + multipart: { file: { name: 'setup.exe', mimeType: 'application/octet-stream', buffer: Buffer.from('MZ') } }, + }); + expect(response.ok()).toBe(false); +}); diff --git a/examples/playwright/tests/upload.spec.ts b/examples/playwright/tests/upload.spec.ts new file mode 100644 index 00000000..7ad4e8f1 --- /dev/null +++ b/examples/playwright/tests/upload.spec.ts @@ -0,0 +1,45 @@ +import { test, expect } from '@playwright/test'; +import { createReadStream } from 'node:fs'; +import { fileSet, type Observed } from 'testing-files-generator'; + +import { limit } from '../shared'; + +// The set the upload-validation preset builds around the form's limit: either +// side of it, types the form takes and turns away, content under the wrong +// name, names that are hard to store. The form takes one file at a time, so +// the mass upload is left out. +const uploads = fileSet({ preset: 'upload-validation', options: { limit, bulk: 0 } }); + +test.beforeAll(() => uploads.write()); + +test.describe('through the page', () => { + for (const file of uploads.files) { + test(file.title, file.details, async ({ page }) => { + test.skip(file.unspecified, file.whyUnspecified); + await page.goto('/'); + await page.getByLabel('File').setInputFiles(await file.path()); + await page.getByRole('button', { name: 'Upload' }).click(); + const status = page.getByRole('status'); + await expect(status).not.toBeEmpty(); + file.check(shown(await status.textContent())); + }); + } +}); + +// The same files sent to the server directly, the way anything other than this +// page can send them. +test.describe('through the API', () => { + for (const file of uploads.files) { + test(file.title, file.details, async ({ request }) => { + test.skip(file.unspecified, file.whyUnspecified); + const response = await request.post('/upload', { multipart: { file: createReadStream(await file.path()) } }); + const answer = await response.json(); + file.check(response.ok() ? 'accept' : { outcome: 'reject', reason: answer.reason }); + }); + } +}); + +function shown(text: string | null): Observed { + const refused = /^Refused \((\w+)\)$/.exec(text ?? ''); + return refused ? { outcome: 'reject', reason: refused[1] } : 'accept'; +} diff --git a/examples/playwright/tsconfig.json b/examples/playwright/tsconfig.json new file mode 100644 index 00000000..d82f15ef --- /dev/null +++ b/examples/playwright/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "skipLibCheck": false, + "types": ["node"] + }, + "include": ["playwright.config.ts", "shared.ts", "tests/**/*.ts"] +} diff --git a/internal/guard/ascii_test.go b/internal/guard/ascii_test.go index dd62a961..1de7d63d 100644 --- a/internal/guard/ascii_test.go +++ b/internal/guard/ascii_test.go @@ -65,6 +65,11 @@ var englishFiles = []string{ // the copy. Repository text somebody reads to learn what was changed in // a module they thought they knew - the same criterion again. "third_party/go-gl-gl/PATCH.md", + // The npm package's page and its example's, added 2026-11-04 with them. + // The first is what npm shows a person deciding whether to install it, + // the second what they read before running it - the same criterion again. + "npm/testing-files-generator/README.md", + "examples/playwright/README.md", } // symbolsAllowed names the files that may hold a symbol such as an emoji. diff --git a/internal/guard/manifestreadlimits_test.go b/internal/guard/manifestreadlimits_test.go index 0eb59d55..83790ae2 100644 --- a/internal/guard/manifestreadlimits_test.go +++ b/internal/guard/manifestreadlimits_test.go @@ -256,6 +256,15 @@ func TestAnotherRunsRecordListingOneFileOverAndOverHoldsItOnce(t *testing.T) { } t.Logf("reading the record naming the file allocated %d B, and the record naming other files %d B", same, apart) + // Not compared under the race detector, which allocates on its own + // account. Measured 2026-11-05 on the same machine, six runs each: without + // it the first record cost 0.68 to 0.82 MB less than the second, out of + // 20.5 MB, and with it about 85 MB and anything from 1.53 MB less to 0.94 + // MB more - one run in six over the ceiling, as on CI the night before. + // The jobs that run without the detector still hold the ceiling. + if raceEnabled { + return + } if grew := same - apart; grew > entries { t.Errorf("a record naming one file in each of its %d entries took %d B more to read than one naming none of the files, "+ "%.1f B an entry - the file was kept once for every entry rather than once", entries, grew, float64(grew)/entries) diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index 8147e36c..2fd25b5f 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -16844,4 +16844,62 @@ "\t\tif code != ExitOK {", "TestGenerateSendsItsManifestAPartAtATime", ), + # --- the npm package is held to the release and the manifest it reads + # (2026-11-04, docs/PLAYWRIGHT-2026-11-04.md) --- + ( + "the npm package forgets a platform the release builds a binary for", + "npm/testing-files-generator/lib/binary.js", + " 'linux arm64': '@testing-files-generator/linux-arm64',\n", + "", + "TestThePackageKnowsEveryPlatformTheReleaseBuilds", + ), + ( + "the npm package looks for a binary in a package the release does not publish", + "npm/testing-files-generator/lib/binary.js", + "'@testing-files-generator/win32-arm64'", + "'@testing-files-generator/windows-arm64'", + "TestThePackageKnowsEveryPlatformTheReleaseBuilds", + ), + ( + "the npm package's types lose a reason the program declares", + "npm/testing-files-generator/index.d.ts", + " | 'duplicate'\n", + "", + "TestThePackageHoldsTheManifestsClosedLists", + ), + ( + "the npm package refuses a manifest that expects sanitize", + "npm/testing-files-generator/lib/testfile.js", + "'sanitize', 'unspecified']", + "'unspecified']", + "TestThePackageHoldsTheManifestsClosedLists", + ), + ( + "the npm package's types promise a field the manifest does not write", + "npm/testing-files-generator/index.d.ts", + " target_id: string;\n", + " target: string;\n", + "TestThePackageTypesOnlyPromiseFieldsTheManifestHas", + ), + ( + "an error the npm package throws into a test picks up a semicolon", + "npm/testing-files-generator/lib/binary.js", + "or unset it to use", + "; unset it to use", + "TestTheTextOfThePackageIsAsciiWithPlainPunctuation", + ), + ( + "the linked npm package changes its licence while the dependency gate still lets it through as MPL-2.0", + "npm/testing-files-generator/package.json", + '"license": "MPL-2.0",', + '"license": "MIT",', + "TestTheLinkedPackageIsUnderTheLicenceTheGateIsToldItIs", + ), + ( + "the example's page says something with a semicolon", + "examples/playwright/app/page.js", + "'Uploaded'", + "'Uploaded; done'", + "TestTheTextOfThePackageIsAsciiWithPlainPunctuation", + ), ] diff --git a/internal/guard/npmpackage_test.go b/internal/guard/npmpackage_test.go new file mode 100644 index 00000000..82797f91 --- /dev/null +++ b/internal/guard/npmpackage_test.go @@ -0,0 +1,391 @@ +package guard + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "sort" + "strings" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/manifest" + "github.com/donislawdev/TestingFilesGenerator/internal/recipe" +) + +// The npm package (npm/testing-files-generator) runs this program and reads +// what it writes, from JavaScript. Nothing in Go calls it, so nothing in Go +// notices when the two drift apart - a platform the release builds and the +// package cannot find, an outcome the program declares and the package refuses, +// a manifest field the package's types promise and the manifest does not have. +// These guards read the package's files the way the release and the manifest +// are read here, and hold one to the other. docs/PLAYWRIGHT-2026-11-04.md. + +const npmPackage = "npm/testing-files-generator" + +func npmFile(t *testing.T, rel string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(repoRoot(t), filepath.FromSlash(rel))) + if err != nil { + t.Fatalf("reading %s: %v", rel, err) + } + return string(b) +} + +// releaseTargets is the list of GOOS/GOARCH pairs the release loop builds a +// command line for. +func releaseTargets(t *testing.T) []string { + t.Helper() + body := npmFile(t, ".github/workflows/release.yml") + loop := regexp.MustCompile(`(?s)for target in \\\n(.*?)\n\s*do\n`).FindStringSubmatch(body) + if loop == nil { + t.Fatal("the release workflow has no loop of command line targets - this guard no longer reads it") + } + targets := regexp.MustCompile(`[a-z0-9]+/[a-z0-9]+`).FindAllString(loop[1], -1) + if len(targets) == 0 { + t.Fatal("the release loop names no targets this guard can read") + } + sort.Strings(targets) + return targets +} + +func TestThePackageKnowsEveryPlatformTheReleaseBuilds(t *testing.T) { + node := map[string]string{"windows": "win32", "linux": "linux", "darwin": "darwin", "amd64": "x64", "arm64": "arm64"} + var want []string + for _, target := range releaseTargets(t) { + goos, goarch, _ := strings.Cut(target, "/") + if node[goos] == "" || node[goarch] == "" { + t.Fatalf("the release builds %s, and this guard does not know what Node calls it", target) + } + want = append(want, node[goos]+" "+node[goarch]) + } + + body := npmFile(t, npmPackage+"/lib/binary.js") + entries := regexp.MustCompile(`'([a-z0-9]+ [a-z0-9]+)': '(@testing-files-generator/[a-z0-9-]+)'`).FindAllStringSubmatch(body, -1) + var got []string + for _, e := range entries { + got = append(got, e[1]) + if name := "@testing-files-generator/" + strings.ReplaceAll(e[1], " ", "-"); e[2] != name { + t.Errorf("the binary for %s comes in %s, and the name the release publishes for it is %s", e[1], e[2], name) + } + } + sort.Strings(got) + sort.Strings(want) + if !reflect.DeepEqual(got, want) { + t.Errorf("the package looks for a binary on %v, and the release builds one for %v", got, want) + } +} + +// The dependency gate lets the package through by name, because the graph has +// no licence for a directory linked with file: (#220). That is only true while +// the package is what the gate's exception says it is. +func TestTheLinkedPackageIsUnderTheLicenceTheGateIsToldItIs(t *testing.T) { + var pkg struct { + Name string `json:"name"` + License string `json:"license"` + } + if err := json.Unmarshal([]byte(npmFile(t, npmPackage+"/package.json")), &pkg); err != nil { + t.Fatalf("reading the package's package.json: %v", err) + } + if pkg.Name != "testing-files-generator" || pkg.License != "MPL-2.0" { + t.Errorf("the package is %s under %s, and the dependency gate lets testing-files-generator through as MPL-2.0", pkg.Name, pkg.License) + } + if licence := npmFile(t, npmPackage+"/LICENSE"); !strings.HasPrefix(licence, "Mozilla Public License Version 2.0\n") { + t.Errorf("the package's LICENSE is not the text of the Mozilla Public License 2.0") + } + gate := npmFile(t, ".github/scripts/dependency_gate.py") + for _, name := range []string{`"testing-files-generator": "`, `"../../npm/testing-files-generator": "`} { + if !strings.Contains(gate, name) || !strings.Contains(gate, "MPL-2.0") { + t.Errorf("the dependency gate has no exception %s naming the licence", name) + } + } +} + +// unionOf reads the members of a TypeScript union of string literals. +func unionOf(t *testing.T, body, name string) []string { + t.Helper() + decl := regexp.MustCompile(`(?s)export type ` + name + ` =(.*?);`).FindStringSubmatch(body) + if decl == nil { + t.Fatalf("index.d.ts has no type %s", name) + } + var out []string + for _, m := range regexp.MustCompile(`'([a-z_]+)'`).FindAllStringSubmatch(decl[1], -1) { + out = append(out, m[1]) + } + sort.Strings(out) + return out +} + +func TestThePackageHoldsTheManifestsClosedLists(t *testing.T) { + types := npmFile(t, npmPackage+"/index.d.ts") + if got, want := unionOf(t, types, "Outcome"), recipe.Outcomes(); !reflect.DeepEqual(got, want) { + t.Errorf("index.d.ts says an outcome is one of %v, and the program declares %v", got, want) + } + if got, want := unionOf(t, types, "Reason"), recipe.Reasons(); !reflect.DeepEqual(got, want) { + t.Errorf("index.d.ts says a reason is one of %v, and the program declares %v", got, want) + } + + runtime := regexp.MustCompile(`const outcomes = \[([^\]]*)\]`).FindStringSubmatch(npmFile(t, npmPackage+"/lib/testfile.js")) + if runtime == nil { + t.Fatal("lib/testfile.js has no list of outcomes this guard can read") + } + var got []string + for _, m := range regexp.MustCompile(`'([a-z_]+)'`).FindAllStringSubmatch(runtime[1], -1) { + got = append(got, m[1]) + } + sort.Strings(got) + if want := recipe.Outcomes(); !reflect.DeepEqual(got, want) { + t.Errorf("the package accepts the outcomes %v, and the program declares %v", got, want) + } +} + +// jsonFields is the set of names a manifest type writes. +func jsonFields(typ reflect.Type) map[string]bool { + out := map[string]bool{} + for i := 0; i < typ.NumField(); i++ { + name, _, _ := strings.Cut(typ.Field(i).Tag.Get("json"), ",") + if name != "" && name != "-" { + out[name] = true + } + } + return out +} + +// typeScriptFields reads the field names of one block of index.d.ts - an +// interface, or an object written inside one - leaving out the index +// signature that lets a manifest carry more than the package reads. +func typeScriptFields(block string) []string { + var out []string + for _, m := range regexp.MustCompile(`(?m)(?:^\s*|[{;]\s*)([a-z_]+)\??:`).FindAllStringSubmatch(block, -1) { + out = append(out, m[1]) + } + return out +} + +func TestThePackageTypesOnlyPromiseFieldsTheManifestHas(t *testing.T) { + types := npmFile(t, npmPackage+"/index.d.ts") + iface := func(name string) string { + m := regexp.MustCompile(`(?s)export interface ` + name + ` \{\n(.*?)\n\}`).FindStringSubmatch(types) + if m == nil { + t.Fatalf("index.d.ts has no interface %s", name) + } + return m[1] + } + inline := func(block, name string) string { + m := regexp.MustCompile(`(?m)^\s*` + name + `: \{(.*)\};$`).FindStringSubmatch(block) + if m == nil { + t.Fatalf("the Manifest interface in index.d.ts has no %s block on one line", name) + } + return m[1] + } + whole := iface("Manifest") + top := regexp.MustCompile(`(?m)^\s*\w+: \{.*\};$`).ReplaceAllString(whole, "") + cases := []struct { + where string + block string + typ reflect.Type + }{ + {"ManifestFile", iface("ManifestFile"), reflect.TypeOf(manifest.File{})}, + {"Expected", iface("Expected"), reflect.TypeOf(manifest.Expected{})}, + {"Manifest", top + "\ntool: x;\nrun: x;\nsummary: x;", reflect.TypeOf(manifest.Manifest{})}, + {"Manifest.tool", inline(whole, "tool"), reflect.TypeOf(manifest.Tool{})}, + {"Manifest.run", inline(whole, "run"), reflect.TypeOf(manifest.Run{})}, + {"Manifest.summary", inline(whole, "summary"), reflect.TypeOf(manifest.Summary{})}, + } + for _, c := range cases { + have := jsonFields(c.typ) + fields := typeScriptFields(c.block) + if len(fields) == 0 { + t.Errorf("%s in index.d.ts declares no field this guard can read", c.where) + } + for _, f := range fields { + if !have[f] { + t.Errorf("%s in index.d.ts promises %s, and the manifest's %s writes no such field", c.where, f, c.typ.Name()) + } + } + } +} + +// The text of the package and of its example is what a person reads - an +// error thrown into their test, a comment their editor shows, a line the +// example prints. It is repository text, so it is English, ASCII and rule 13. +var npmTextRoots = []string{npmPackage + "/", "examples/playwright/"} + +// npmLiterals are the files whose strings are sentences somebody reads. In +// the rest a string is a test's data, a script for a child process or a +// header value, and judging it as prose would be crying wolf - the same line +// goFileFaults draws for Go. +var npmLiterals = map[string]bool{ + npmPackage + "/index.js": true, + npmPackage + "/bin/tfg.js": true, + npmPackage + "/lib/binary.js": true, + npmPackage + "/lib/fileset.js": true, + npmPackage + "/lib/run.js": true, + npmPackage + "/lib/testfile.js": true, + "examples/playwright/app/page.js": true, + "examples/playwright/check.mjs": true, +} + +func npmTextFiles(t *testing.T) []string { + t.Helper() + var out []string + for _, name := range strings.Fields(gitOutput(t, "ls-files", "--cached", "--others", "--exclude-standard")) { + for _, root := range npmTextRoots { + if strings.HasPrefix(name, root) { + out = append(out, name) + } + } + } + return out +} + +func TestTheTextOfThePackageIsAsciiWithPlainPunctuation(t *testing.T) { + files := npmTextFiles(t) + sources, literals := 0, 0 + for _, name := range files { + body := npmFile(t, name) + for n, line := range strings.Split(body, "\n") { + if i := strings.IndexFunc(line, func(r rune) bool { return r > 127 }); i >= 0 { + t.Errorf("%s:%d holds %q - text in the repository is English", name, n+1, []rune(line[i:])[0]) + } + } + switch filepath.Ext(name) { + case ".js", ".mjs", ".ts": + default: + continue + } + sources++ + prose, err := jsProse(body, npmLiterals[name]) + if err != nil { + t.Errorf("%s cannot be read for its prose: %v", name, err) + continue + } + if npmLiterals[name] { + literals++ + } + for _, p := range prose { + for _, fault := range proseFaults(p.text, false) { + t.Errorf("%s:%d holds %s", name, p.line, fault) + } + } + } + if sources == 0 || literals < 6 { + t.Fatalf("read %d sources and the strings of %d - the package is not where this guard looks", sources, literals) + } +} + +type jsText struct { + line int + text string +} + +// jsProse returns what a person reads in a JavaScript or TypeScript file: its +// comments, and with literals the words of its strings, the code inside a +// template's ${} left out. It reads only as much of the language as that +// takes, and says so rather than guess when a file ends inside a string. +func jsProse(src string, literals bool) ([]jsText, error) { + var out []jsText + line := 1 + add := func(at int, text string) { + if strings.TrimSpace(text) != "" { + out = append(out, jsText{at, text}) + } + } + // last is the last character of code that is not space, which tells a + // regular expression from a division. + last := byte('\n') + for i := 0; i < len(src); i++ { + c := src[i] + switch { + case c == '\n': + line++ + case c == '/' && i+1 < len(src) && src[i+1] == '/': + end := strings.IndexByte(src[i:], '\n') + if end < 0 { + end = len(src) - i + } + add(line, src[i+2:i+end]) + i += end - 1 + continue + case c == '/' && i+1 < len(src) && src[i+1] == '*': + end := strings.Index(src[i+2:], "*/") + if end < 0 { + return nil, errorAt(line, "a comment that never ends") + } + body := src[i+2 : i+2+end] + add(line, strings.NewReplacer("\n *", "\n", "\n*", "\n").Replace(body)) + line += strings.Count(body, "\n") + i += end + 3 + continue + case c == '/' && strings.IndexByte("(,=:[!&|?{};+\n", last) >= 0: + j := i + 1 + class := false + for ; j < len(src) && (src[j] != '/' || class); j++ { + switch src[j] { + case '\\': + j++ + case '[': + class = true + case ']': + class = false + case '\n': + return nil, errorAt(line, "a regular expression that never ends") + } + } + i = j + last = '/' + continue + case c == '\'' || c == '"' || c == '`': + start, j := line, i+1 + var text strings.Builder + for ; j < len(src) && src[j] != c; j++ { + switch { + case src[j] == '\\': + j++ + case c == '`' && src[j] == '$' && j+1 < len(src) && src[j+1] == '{': + // The code of an interpolation is skipped to its closing + // brace. A brace inside a string in there would end it + // early, and the package writes none. + j++ + for open := 1; open > 0 && j+1 < len(src); { + j++ + switch src[j] { + case '{': + open++ + case '}': + open-- + } + } + text.WriteByte(' ') + case src[j] == '\n': + if c != '`' { + return nil, errorAt(line, "a string that never ends on its line") + } + line++ + text.WriteByte('\n') + default: + text.WriteByte(src[j]) + } + } + if j >= len(src) { + return nil, errorAt(start, "a string that never ends") + } + if literals { + add(start, text.String()) + } + i = j + last = c + continue + } + if c != ' ' && c != '\t' && c != '\r' { + last = c + } + } + return out, nil +} + +func errorAt(line int, what string) error { + return fmt.Errorf("line %d starts %s", line, what) +} diff --git a/internal/guard/raceflag_test.go b/internal/guard/raceflag_test.go index 8b8db1b3..0d8a9bad 100644 --- a/internal/guard/raceflag_test.go +++ b/internal/guard/raceflag_test.go @@ -4,10 +4,12 @@ package guard // raceEnabled says whether this binary was built with the race detector. // -// It exists for exactly one guard: the object count in resources_test.go. The -// detector allocates on its own account, so a count taken under it is not the -// same measurement as a count taken without, and comparing either against one -// ceiling asks the number to mean two things. +// It exists for guards whose measurement the detector changes. Most are about +// memory - the object count in resources_test.go, the bytes compared in +// manifestreadlimits_test.go. The detector allocates on its own account, so a +// count taken under it is not the same measurement as a count taken without, +// and comparing either against one ceiling asks the number to mean two things. +// One is about time: jxlladder_test.go sweeps fewer seeds under it. // // Two files rather than a runtime lookup, because the build tag is the only // thing that knows. Go offers no way to ask at run time. diff --git a/npm/testing-files-generator/LICENSE b/npm/testing-files-generator/LICENSE new file mode 100644 index 00000000..d0a1fa14 --- /dev/null +++ b/npm/testing-files-generator/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at https://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/npm/testing-files-generator/README.md b/npm/testing-files-generator/README.md new file mode 100644 index 00000000..d938db31 --- /dev/null +++ b/npm/testing-files-generator/README.md @@ -0,0 +1,112 @@ +# testing-files-generator + +Real files of exact sizes and formats for your tests, and what the system under test should do with each one. + +A PDF of exactly 1 MiB and one byte is a real PDF, so a server that looks inside it can only turn it away for its size. A file made with `Buffer.alloc` is zeros, so the same server turns it away for what it holds, and a test meant to prove a size limit passes on a server that has none. [The example](../../examples/playwright) shows exactly that. + +## In a Playwright test + +```ts +import { test, expect } from '@playwright/test'; +import { fileSet } from 'testing-files-generator'; + +const uploads = fileSet({ preset: 'upload-validation', options: { limit: '2mb' } }); + +for (const file of uploads.except({ group: 'bulk-upload' }).files) { + test(file.title, file.details, async ({ page }) => { + test.skip(file.unspecified, file.whyUnspecified); + await page.goto('/upload'); + await page.getByLabel('File').setInputFiles(await file.path()); + await page.getByRole('button', { name: 'Upload' }).click(); + const status = page.getByRole('status'); + await expect(status).not.toBeEmpty(); + file.check((await status.textContent())?.includes('Uploaded') ? 'accept' : 'reject'); + }); +} +``` + +You write the two things only you know: how a file is uploaded in your app, and how it shows that a file was taken or turned away. The set brings the rest. + +- **One test for each file.** The title names the file and what it should meet, such as `1mb_over_1b.jpg - expected reject (size_limit)`. The report shows what the file is for, and tags such as `@reject` and `@size-limit` narrow a run with `--grep`. +- **A failure says what it means.** `check` throws with the sentence the set carries for that file: + + ``` + 1mb_over_1b.jpg was taken, and it should have been turned away (size_limit). + A JPEG file 1 B over the limit of 1mb. Your system should turn it away - taking it means + the limit is not enforced, or enforced only in the browser. + ``` + +- **Files whose outcome is your policy are skipped, never passed.** An empty file or a name with an emoji in it has no right answer until you decide one. `whyUnspecified` says so in the report, and `check` refuses to pass such a file. +- **The same files can go straight to your server**, past the page and its checks: + + ```ts + import { createReadStream } from 'node:fs'; + + const response = await request.post('/upload', { multipart: { file: createReadStream(await file.path()) } }); + file.check(response.ok() ? 'accept' : 'reject'); + ``` + + `check` also takes `{ outcome: 'reject', reason: 'size_limit' }`. A reason is compared only for a refusal, because it names the rule a file is about - a file a byte under a limit is expected to be accepted with `size_limit` as the rule in play. + +## The sets + +`fileSet` takes a preset, a recipe file or plain flags of `tfg generate`: + +```ts +fileSet({ preset: 'size-boundaries', options: { limit: '10mb', format: 'pdf' } }); +fileSet({ recipe: 'fixtures/uploads.yaml' }); +fileSet({ options: { format: 'png', size: '5mb', count: 3, seed: 7, expected: 'accept' } }); +``` + +A relative recipe path is taken from the directory the tests were started in. `path.join(__dirname, 'uploads.yaml')` names one beside the spec file, wherever the tests are started from. + +Files made from flags expect nothing unless `expected` says what, so without it every one of them is skipped. + +| preset | the question it answers | +|---|---| +| `upload-validation` | Does my upload form take what it should and turn the rest away? | +| `size-boundaries` | Is a size limit enforced exactly where it is declared? | +| `filename-handling` | Will my system store, show and give back a file name it did not expect? | +| `tabular-import` | Does my table import survive what real tools export? | +| `text-encoding` | Does my reader know which encoding a file is in, or is it guessing? | +| `empty-and-minimal` | Does a file that is valid and as small as the format allows get through? | + +`npx tfg preset show ` lists what a preset takes, and `npx tfg formats` lists the formats. + +`only` and `except` take part of a set by `group`, `target` or `outcome`, and `paths()` gives the files of a part together, for a form that takes several at once: + +```ts +const together = await uploads.only({ group: 'bulk-upload' }).paths(); +await page.getByLabel('Files').setInputFiles(together); +``` + +## When the files are written + +1. **While Playwright loads the spec file, nothing is written.** `fileSet` asks for the plan, the list of files and what each should meet, and Playwright declares the tests from it. Listing the tests with `--list` writes nothing either. +2. **The first test that asks for a path writes the whole set**, once, however many workers ask at the same moment. The default `upload-validation` set, 71 files and 115 MB, was written in under a second on the machine it was measured on. +3. **Every later run takes the same files.** The same options give the same bytes, so a set is written once and kept, in `node_modules/.cache/testing-files-generator` or the directory `TFG_CACHE_DIR` names. Before a kept set is used, every byte of it is checked against its manifest, which took under a tenth of a second for the same set. A set a test wrote into, or a file someone changed, is refused with the names of the files that differ. + +A large set is better written before the tests start, so its time does not count against the first test: + +```ts +test.beforeAll(async () => { + test.setTimeout(120_000); + await uploads.write(); +}); +``` + +Nothing is ever removed on its own. Delete the cache directory whenever you like. A run stopped part way leaves an unfinished set, which nothing uses, and the next run names it so it does not sit there unseen. + +## Where tfg comes from + +The package installs the `tfg` program for your system: Windows on x64 and Arm, Linux on x64 and Arm, macOS on Apple silicon. `npx tfg` runs it, with the same arguments, output and exit codes as tfg run directly. + +To use a tfg installed another way, set `TFG_BINARY` to its path, or pass `binary` to `fileSet`. On a system with no binary of its own, such as an Intel Mac, the container image `ghcr.io/donislawdev/tfg` runs the same program. + +## Not only Playwright + +`fileSet` gives plain data and plain promises, and imports nothing from Playwright. The same loop works in Vitest, Jest or a script, with `file.title` as a test name. + +## Licences + +This package is under the [Mozilla Public License 2.0](LICENSE). The `tfg` program it runs is under the GNU General Public License 3.0 and runs as a separate program. The files it generates are yours: they are output of the program and not derived works of it, so you can commit them and ship them inside a closed source product with no obligation of any kind. diff --git a/npm/testing-files-generator/bin/tfg.js b/npm/testing-files-generator/bin/tfg.js new file mode 100644 index 00000000..09847900 --- /dev/null +++ b/npm/testing-files-generator/bin/tfg.js @@ -0,0 +1,52 @@ +#!/usr/bin/env node +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +'use strict'; + +// npx tfg runs the binary this package installed, with the same arguments, the +// same streams and the same exit code, so a script cannot tell it from tfg run +// directly. + +const { spawn } = require('node:child_process'); +const { binaryPath } = require('../lib/binary.js'); + +// RUNTIME in docs/CLI.md section 3 - the tool could not do its work, and the +// arguments were never looked at. +const cannotRun = 1; + +let binary; +try { + binary = binaryPath(); +} catch (err) { + process.stderr.write(`tfg: ${err.message}\n`); + process.exit(cannotRun); +} + +const child = spawn(binary, process.argv.slice(2), { stdio: 'inherit', windowsHide: false }); + +// A Ctrl+C in a terminal reaches tfg as well as this process. This one waits, +// so tfg can stop the way it stops on its own and its exit code is the one a +// script sees. A signal sent to this process alone is passed on. +for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) { + process.on(signal, () => { + if (signal !== 'SIGINT') { + child.kill(signal); + } + }); +} + +child.on('error', (err) => { + process.stderr.write(`tfg: tfg could not be started from ${binary}: ${err.message}\n`); + process.exit(cannotRun); +}); + +child.on('close', (status, signal) => { + if (signal) { + process.removeAllListeners(signal); + process.kill(process.pid, signal); + return; + } + process.exit(status ?? cannotRun); +}); diff --git a/npm/testing-files-generator/index.d.ts b/npm/testing-files-generator/index.d.ts new file mode 100644 index 00000000..8cc5e5e1 --- /dev/null +++ b/npm/testing-files-generator/index.d.ts @@ -0,0 +1,129 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +/** What a system under test is expected to do with a file. */ +export type Outcome = 'accept' | 'reject' | 'sanitize' | 'unspecified'; + +/** The rule a file is about. It names the rule and not what the system did, so a file a byte under a limit is expected to be accepted with size_limit. */ +export type Reason = + | 'content_malformed' + | 'count_limit' + | 'dimensions_limit' + | 'duplicate' + | 'encoding_invalid' + | 'extension_rule' + | 'filename_invalid' + | 'filename_too_long' + | 'filename_traversal' + | 'malware_signature' + | 'mime_mismatch' + | 'nesting_depth' + | 'none' + | 'size_limit' + | 'size_zero'; + +export interface Expected { + outcome: Outcome; + reason?: Reason; + detail?: string; + confidence?: string; +} + +/** One entry of a manifest, as tfg writes it. */ +export interface ManifestFile { + id: string; + path: string; + name: string; + materialized: boolean; + target_id: string; + bytes: number; + format: string; + group?: string; + purpose?: string; + expected: Expected; + [field: string]: unknown; +} + +/** A manifest, as tfg writes it. For a plan nothing is written yet and every file has materialized false. */ +export interface Manifest { + manifest_version: string; + tool: { name: string; version: string; generators: Record; [field: string]: unknown }; + run: { id: string; seed: number; complete: boolean; recipe_hash?: string; [field: string]: unknown }; + summary: { file_count: number; total_bytes: number; materialized: number; [field: string]: unknown }; + files: ManifestFile[]; + [field: string]: unknown; +} + +export type OptionValue = string | number | boolean | ReadonlyArray; + +export interface FileSetSpec { + /** A preset, as tfg preset list names it. */ + preset?: string; + /** A recipe file. A relative path is taken from the directory the tests were started in. */ + recipe?: string; + /** Flags of tfg generate and of the preset, without the dashes: { limit: '2mb', seed: 7, clean: true }. */ + options?: Record; + /** Where sets are written. TFG_CACHE_DIR, or node_modules/.cache/testing-files-generator, or the system's temporary directory. */ + cacheDir?: string; + /** The tfg binary to run. TFG_BINARY, or the one this package installed. */ + binary?: string; +} + +export interface Filter { + group?: string | readonly string[]; + target?: string | readonly string[]; + outcome?: Outcome | readonly Outcome[]; +} + +export type Observed = + | Exclude + | { outcome: Exclude; reason?: string }; + +/** The shape test(title, details, body) takes in Playwright. */ +export interface TestDetails { + tag: string[]; + annotation: Array<{ type: string; description: string }>; +} + +export interface TestFile { + readonly entry: ManifestFile; + readonly relativePath: string; + readonly name: string; + readonly format: string; + readonly bytes: number; + readonly group: string; + readonly target: string; + readonly purpose: string; + readonly expected: Expected; + readonly unspecified: boolean; + readonly whyUnspecified: string; + /** The file's path and what it is expected to meet. Unique within a set. */ + readonly title: string; + readonly details: TestDetails; + /** Writes the set the first time any of its files is asked for, and gives this file's absolute path. */ + path(): Promise; + /** Throws when what the system did differs from what is expected, with the sentence that says what the difference means. */ + check(observed: Observed): void; +} + +export interface FileSet { + readonly files: readonly TestFile[]; + readonly plan: Manifest; + only(filter: Filter): FileSet; + except(filter: Filter): FileSet; + /** Writes the whole set once per process and gives its directory. */ + write(): Promise; + paths(): Promise; +} + +/** Plans a set of files, writing nothing. The files are written the first time one of them is asked for. */ +export function fileSet(spec: FileSetSpec): FileSet; + +/** The tfg binary this package runs. */ +export function binaryPath(): string; + +export class TfgError extends Error { + readonly status: number | string; + readonly stderr: string; +} diff --git a/npm/testing-files-generator/index.js b/npm/testing-files-generator/index.js new file mode 100644 index 00000000..76530289 --- /dev/null +++ b/npm/testing-files-generator/index.js @@ -0,0 +1,12 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +'use strict'; + +// Each export is assigned on its own line, the form Node reads the names of +// when an ES module imports this file, so `import { fileSet }` works as well as +// require does. +exports.fileSet = require('./lib/fileset.js').fileSet; +exports.binaryPath = require('./lib/binary.js').binaryPath; +exports.TfgError = require('./lib/run.js').TfgError; diff --git a/npm/testing-files-generator/lib/binary.js b/npm/testing-files-generator/lib/binary.js new file mode 100644 index 00000000..5b51346c --- /dev/null +++ b/npm/testing-files-generator/lib/binary.js @@ -0,0 +1,63 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +'use strict'; + +const fs = require('node:fs'); + +// The systems a release builds a command line for, as Node names them, and the +// package that carries each binary. A guard holds this to the list of targets +// in .github/workflows/release.yml, because a platform here with nothing behind +// it installs a package that cannot run, and a platform there missing here is a +// binary nobody can reach. Intel Macs are not on either list, by the owner's +// decision of 2026-08-20. +const platforms = { + 'win32 x64': '@testing-files-generator/win32-x64', + 'win32 arm64': '@testing-files-generator/win32-arm64', + 'linux x64': '@testing-files-generator/linux-x64', + 'linux arm64': '@testing-files-generator/linux-arm64', + 'darwin arm64': '@testing-files-generator/darwin-arm64', +}; + +// binaryPath finds the tfg binary this package runs. +// +// TFG_BINARY comes first and is taken as given - a tfg installed another way, +// or one built from the repository. PATH is never searched: a different tfg +// found there would answer with its own version and say nothing about it. +// Everything is passed in so the tests can ask about a system they are not on. +function binaryPath({ env = process.env, platform = process.platform, arch = process.arch, resolve = require.resolve } = {}) { + const given = env.TFG_BINARY; + if (given) { + if (!fs.existsSync(given)) { + throw new Error(`TFG_BINARY is set to ${given} and there is no file there. ` + + 'Point it at a tfg binary, or unset it to use the one this package installs.'); + } + return given; + } + const pkg = platforms[`${platform} ${arch}`]; + const known = Object.keys(platforms).join(', '); + if (!pkg) { + throw new Error(`testing-files-generator has no binary for ${platform} ${arch}. ` + + `It comes for ${known}. ` + + 'On another system, run tfg from its container image ghcr.io/donislawdev/tfg, ' + + 'or install it another way and set TFG_BINARY to its path.'); + } + const exe = platform === 'win32' ? 'tfg.exe' : 'tfg'; + try { + return resolve(`${pkg}/${exe}`); + } catch (err) { + // Only a package that is not there is a matter of --omit. Anything else, + // such as a package that does not export the binary, is told as it is, + // because installing again would not change it. + if (err?.code !== 'MODULE_NOT_FOUND') { + throw new Error(`The binary for ${platform} ${arch} in the package ${pkg} could not be found - ${err?.message}. ` + + 'Set TFG_BINARY to the path of a tfg binary to go on.', { cause: err }); + } + throw new Error(`The binary for ${platform} ${arch} comes in the package ${pkg}, and it is not installed. ` + + 'npm leaves it out when it installs with --omit=optional or --no-optional. ' + + 'Install again without that option, or set TFG_BINARY to the path of a tfg binary.'); + } +} + +module.exports = { binaryPath, platforms }; diff --git a/npm/testing-files-generator/lib/fileset.js b/npm/testing-files-generator/lib/fileset.js new file mode 100644 index 00000000..e72d3b87 --- /dev/null +++ b/npm/testing-files-generator/lib/fileset.js @@ -0,0 +1,414 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +'use strict'; + +const { spawnSync } = require('node:child_process'); +const crypto = require('node:crypto'); +const { EventEmitter } = require('node:events'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +const { binaryPath } = require('./binary.js'); +const { TfgError, run, runToFile } = require('./run.js'); +const { TestFile, outcomes } = require('./testfile.js'); + +// The manifest major this package reads. Another major is refused rather than +// read as if it were this one - docs/MANIFEST.md, the version of the contract. +const manifestMajor = '1'; + +// The code tfg verify ends with when a directory does not match its manifest, +// from the exit codes in docs/CLI.md. Any other code is verify failing to look. +const exitVerify = 7; + +// Flags this package sets itself. An option of the same name would fight it. +const reserved = { + out: 'this package writes every set into a directory of its own', + json: 'this package reads the manifest from the directory it writes', + 'dry-run': 'this package plans each set with a dry run and then writes it', + preset: 'a preset is named by the preset property beside the options', +}; + +// Everything this package leaves while it works starts with this, followed by +// the process id, so a later run can tell a set being written now from one a +// stopped run left behind. +const unfinished = '.tmp-'; + +// A rename the system refuses for a moment. On Windows a scanner holding a new +// file open is enough, so these are tried again before anything is said. +const busy = new Set(['EPERM', 'EACCES', 'EBUSY']); +const renameTries = 10; + +// sources keeps one plan per set and process. A spec file is loaded once in the +// runner and again in every worker, and two spec files can ask for one set. +const sources = new Map(); +let noted = false; + +// events says what a write did - wrote a copy, moved it to its name, or took +// one another process moved first. Not part of the package's interface: the +// tests listen, because a race that never happened passes as well as one won. +const events = new EventEmitter(); + +function fileSet(spec = {}) { + if (typeof spec !== 'object' || spec === null) { + throw new TypeError('fileSet takes an object, for example { preset: \'upload-validation\', options: { limit: \'2mb\' } }.'); + } + const binary = spec.binary === undefined ? binaryPath() : named(spec.binary, 'binary', + 'Point it at a tfg binary, or leave it out to use the one this package installs.'); + if (spec.recipe) { + named(path.resolve(spec.recipe), 'recipe', + `A relative path is taken from the directory the tests started in, here ${process.cwd()}, ` + + 'so path.join(__dirname, ...) names a recipe beside the spec file.'); + } + const args = argsOf(spec); + const cacheDir = path.resolve(spec.cacheDir ?? defaultCacheDir()); + const recipe = spec.recipe ? fs.readFileSync(path.resolve(spec.recipe), 'utf8') : ''; + const key = digest([identity(binary), args, recipe, cacheDir]); + let source = sources.get(key); + if (!source) { + source = new Source(binary, args, cacheDir, labelOf(spec)); + sources.set(key, source); + } + return new FileSet(source, source.files); +} + +function argsOf(spec) { + const args = ['generate']; + if (spec.recipe) { + args.push(path.resolve(spec.recipe)); + } + if (spec.preset) { + args.push(`--preset=${spec.preset}`); + } + for (const [key, value] of Object.entries(spec.options ?? {})) { + if (Object.hasOwn(reserved, key)) { + throw new Error(`The option ${key} cannot be set here - ${reserved[key]}.`); + } + if (!/^[a-z][a-z0-9-]*$/.test(key)) { + throw new Error(`${JSON.stringify(key)} is not a flag tfg has. Options are flag names without the dashes, such as limit or far-over.`); + } + for (const one of Array.isArray(value) ? value : [value]) { + if (one === true) { + args.push(`--${key}`); + } else if (typeof one === 'string' || typeof one === 'number') { + args.push(`--${key}=${one}`); + } else if (one !== false && one !== undefined && one !== null) { + throw new TypeError(`The option ${key} takes a string, a number, true or a list of them. It was given a ${typeof one}.`); + } + } + } + return args; +} + +// named gives back a file an option names, or says that there is none, before +// anything reads it and fails with only the system's word for it. +function named(file, option, wayOut) { + if (!fs.existsSync(file)) { + throw new Error(`The ${option} option names ${file}, and there is no file there. ${wayOut}`); + } + return file; +} + +function defaultCacheDir() { + if (process.env.TFG_CACHE_DIR) { + return process.env.TFG_CACHE_DIR; + } + const modules = path.join(process.cwd(), 'node_modules'); + if (fs.existsSync(modules)) { + return path.join(modules, '.cache', 'testing-files-generator'); + } + return path.join(os.tmpdir(), 'testing-files-generator'); +} + +function labelOf(spec) { + const name = spec.preset ?? (spec.recipe ? path.basename(spec.recipe).replace(/\.[^.]*$/, '') : 'flags'); + return name.replace(/[^A-Za-z0-9_-]+/g, '-').slice(0, 40); +} + +// identity is what tells two binaries apart without running them, so a new +// build or another version plans again rather than reusing an old plan. +function identity(binary) { + const stat = fs.statSync(binary); + return [fs.realpathSync(binary), stat.size, stat.mtimeMs]; +} + +function digest(value) { + return crypto.createHash('sha256').update(JSON.stringify(value)).digest('hex'); +} + +function scratch(cacheDir) { + return path.join(cacheDir, `${unfinished}${process.pid}-${crypto.randomBytes(4).toString('hex')}`); +} + +// Source is one set: its plan, where it is written, and the one write a process +// makes of it however many files ask. +class Source { + constructor(binary, args, cacheDir, label) { + this.binary = binary; + this.args = args; + this.cacheDir = cacheDir; + fs.mkdirSync(cacheDir, { recursive: true }); + noteUnfinished(cacheDir); + this.plan = this._planned(); + // The recipe hash leaves out the seed and --clean, measured 2026-11-04, so + // the arguments are part of the name and two seeds never share a set. + const run = this.plan.run; + this.dir = path.join(cacheDir, `${label}-${digest([args, run.recipe_hash, run.seed, this.plan.tool]).slice(0, 20)}`); + this.files = this.plan.files.map((entry) => new TestFile(this, entry)); + this._written = undefined; + } + + // _planned asks tfg what the set holds without writing any of it. The dry run + // names a directory that is never made, because a dry run into a directory + // holding a manifest is refused like a real one. + _planned() { + const out = `${scratch(this.cacheDir)}.json`; + let plan; + try { + runToFile(this.binary, [...this.args, `--out=${scratch(this.cacheDir)}`, '--dry-run', '--json'], out); + plan = JSON.parse(fs.readFileSync(out, 'utf8')); + } finally { + fs.rmSync(out, { force: true }); + } + readable(plan); + return plan; + } + + write() { + if (!this._written) { + this._written = this._write().catch((err) => { + this._written = undefined; + throw err; + }); + } + return this._written; + } + + async _write() { + if (fs.existsSync(this.dir)) { + await this._taken(); + return this.dir; + } + const tmp = scratch(this.cacheDir); + fs.mkdirSync(tmp); + try { + await run(this.binary, [...this.args, `--out=${tmp}`]); + const problem = mismatch(tmp, this.plan); + if (problem) { + throw new Error(`tfg wrote a set that does not match its own plan - ${problem}. This is a fault in tfg or in this package. Please report it.`); + } + } catch (err) { + discard(this.binary, tmp); + throw err; + } + events.emit('wrote', tmp); + return this._settle(tmp); + } + + // _settle moves a finished set to its name. Several workers can write the + // same set at once, and the first to move it wins - the others find the + // winner whole, take it, and remove their own copy through its manifest. + async _settle(tmp) { + for (let attempt = 1; ; attempt++) { + try { + fs.renameSync(tmp, this.dir); + events.emit('moved', this.dir); + return this.dir; + } catch (err) { + if (fs.existsSync(this.dir)) { + discard(this.binary, tmp); + await this._taken(); + events.emit('took', this.dir); + return this.dir; + } + if (!busy.has(err.code) || attempt === renameTries) { + discard(this.binary, tmp); + throw new Error(`The set written in ${tmp} could not be moved to ${this.dir} - ${err.message}`); + } + await new Promise((resolve) => setTimeout(resolve, attempt * 100)); + } + } + } + + // _taken checks a set this process did not write before it is used: against + // the plan, and then every byte against the manifest through tfg verify. A + // test that writes into a file it was given changes the set for every later + // run, and the size alone does not show it. + async _taken() { + const problem = mismatch(this.dir, this.plan); + if (problem) { + throw stale(this.dir, problem); + } + try { + await run(this.binary, ['verify', path.join(this.dir, 'manifest.json')]); + } catch (err) { + if (err instanceof TfgError && err.status === exitVerify) { + throw stale(this.dir, 'its files are not the ones it was written with', err.stderr.trim()); + } + throw err; + } + } +} + +// readable refuses a manifest this package cannot read for what it says. +function readable(plan) { + const version = String(plan?.manifest_version ?? ''); + if (version.split('.')[0] !== manifestMajor) { + throw new Error(`tfg wrote a manifest of version ${version || 'none'}, and this package reads version ${manifestMajor}. ` + + 'Install the version of testing-files-generator that comes with that tfg, or point TFG_BINARY at a tfg of the same version as this package.'); + } + for (const entry of plan.files) { + if (!outcomes.includes(entry.expected?.outcome)) { + throw new Error(`${entry.path} expects ${JSON.stringify(entry.expected?.outcome)}, which this package does not know. ` + + `It knows ${outcomes.join(', ')}. Use the version of this package that comes with the tfg it runs.`); + } + } +} + +// mismatch says how a written set differs from its plan, or nothing when it +// does not. It compares names and sizes, which is enough for a set tfg has just +// written. The bytes of a set written earlier are checked by _taken, through +// tfg verify - measured 2026-11-05, about 80 ms for the default upload set of +// 115 MB, against about 590 ms to write it. +// +// Every file has to be on the disk. A run that ends with code nought writes +// every file it plans, and one that cannot ends with the partial code, which +// run turns into an error. That holds until a run can describe a file without +// writing it on purpose - the archive and manifest-only output of D10. +function mismatch(dir, plan) { + let written; + try { + written = JSON.parse(fs.readFileSync(path.join(dir, 'manifest.json'), 'utf8')); + } catch (err) { + return `its manifest cannot be read (${err.message})`; + } + if (written.run?.recipe_hash !== plan.run.recipe_hash || written.run?.seed !== plan.run.seed) { + return 'it was written from another recipe or seed'; + } + if (written.run?.complete !== true) { + return 'the run that wrote it did not finish'; + } + if (!Array.isArray(written.files) || written.files.length !== plan.files.length) { + return `it holds ${written.files?.length ?? 'no'} files where the plan has ${plan.files.length}`; + } + for (let i = 0; i < plan.files.length; i++) { + const want = plan.files[i]; + const got = written.files[i]; + if (got.path !== want.path || got.bytes !== want.bytes) { + return `its file ${i + 1} is ${got.path} of ${got.bytes} B where the plan has ${want.path} of ${want.bytes} B`; + } + if (!got.materialized) { + return `${got.path} was not written`; + } + let stat; + try { + stat = fs.statSync(path.join(dir, got.path)); + } catch { + return `${got.path} is missing`; + } + if (stat.size !== got.bytes) { + return `${got.path} is ${stat.size} B and its manifest says ${got.bytes} B`; + } + } + return ''; +} + +function stale(dir, problem, details = '') { + return new Error(`${dir} is where this set belongs, and it does not hold it - ${problem}. ` + + 'Nothing here removes that directory, because a person may have changed it on purpose. Delete it and run again.' + + (details ? `\n${details}` : '')); +} + +// discard removes a set this process wrote and will not use. The files go +// through tfg cleanup, which removes only what the manifest lists, and then +// the manifest and the directories it leaves empty. Whatever is left stays, +// and a later run says so. +function discard(binary, dir) { + const manifest = path.join(dir, 'manifest.json'); + if (fs.existsSync(manifest)) { + const done = spawnSync(binary, ['cleanup', manifest, '--yes'], { stdio: 'ignore', windowsHide: true }); + if (done.status !== 0) { + return; + } + fs.rmSync(manifest, { force: true }); + fs.rmSync(path.join(dir, 'manifest.instructions.md'), { force: true }); + } + removeEmpty(dir); +} + +function removeEmpty(dir) { + let entries; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + if (entry.isDirectory()) { + removeEmpty(path.join(dir, entry.name)); + } + } + try { + fs.rmdirSync(dir); + } catch { + // Not empty: left where it is. + } +} + +// noteUnfinished says, once per process and only outside a test worker, which +// sets a stopped run left behind. Nothing reads them and nothing removes them +// on its own, so without this they would fill the disk in silence. +function noteUnfinished(cacheDir) { + if (noted || process.env.TEST_WORKER_INDEX !== undefined) { + return; + } + noted = true; + const left = fs.readdirSync(cacheDir).filter((name) => { + const pid = Number(name.slice(unfinished.length).split('-')[0]); + return name.startsWith(unfinished) && Number.isInteger(pid) && pid > 0 && !alive(pid); + }); + if (left.length > 0) { + console.warn(`testing-files-generator: ${left.length} unfinished ${left.length === 1 ? 'set was' : 'sets were'} left in ${cacheDir} ` + + `by a run that stopped part way - ${left.join(', ')}. Nothing uses them and they can be deleted.`); + } +} + +function alive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (err) { + return err.code === 'EPERM'; + } +} + +// FileSet is a set, or a part of one. Every part shares the set's one write. +class FileSet { + constructor(source, files) { + this._source = source; + this.files = files; + } + + get plan() { return this._source.plan; } + + only(filter) { return new FileSet(this._source, this.files.filter((file) => matches(file, filter))); } + + except(filter) { return new FileSet(this._source, this.files.filter((file) => !matches(file, filter))); } + + // write writes the whole set, once per process, and gives its directory. + write() { return this._source.write(); } + + async paths() { + const dir = await this._source.write(); + return this.files.map((file) => path.join(dir, file.relativePath)); + } +} + +function matches(file, filter = {}) { + const has = (wanted, value) => wanted === undefined || [].concat(wanted).includes(value); + return has(filter.group, file.group) && has(filter.target, file.target) && has(filter.outcome, file.expected.outcome); +} + +module.exports = { events, fileSet, readable }; diff --git a/npm/testing-files-generator/lib/run.js b/npm/testing-files-generator/lib/run.js new file mode 100644 index 00000000..37ed53d0 --- /dev/null +++ b/npm/testing-files-generator/lib/run.js @@ -0,0 +1,88 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +'use strict'; + +const { spawn, spawnSync } = require('node:child_process'); +const fs = require('node:fs'); + +// How much of what tfg says on its error stream is kept for a message. Its +// refusals are a few lines, and a run of many files can print a note for each. +const keptError = 64 * 1024; + +// TfgError is tfg ending with a code other than nought, or stopped by a signal, +// whose name is then the status. The message carries what tfg said, because +// its refusals already name the value and the way out. +class TfgError extends Error { + constructor(args, status, stderr) { + const said = stderr.trim() || 'tfg said nothing about why.'; + const ended = typeof status === 'string' ? `was stopped by ${status}` : `ended with code ${status}`; + super(`tfg ${args.join(' ')} ${ended}.\n${said}`); + this.name = 'TfgError'; + this.status = status; + this.stderr = stderr; + } +} + +function notStarted(binary, err) { + return new Error(`tfg could not be started from ${binary}: ${err.message}`); +} + +// runToFile runs tfg to the end, writing its standard output to a file. A file +// and not memory, because the manifest of a large set outgrows the buffer Node +// gives a child process by default. The error stream goes to a file of its own +// for the same reason - a buffer that fills stops tfg part way - and only its +// end is kept. +function runToFile(binary, args, outFile) { + const errFile = `${outFile}.err`; + const out = fs.openSync(outFile, 'w'); + let err; + try { + err = fs.openSync(errFile, 'w+'); + const done = spawnSync(binary, args, { stdio: ['ignore', out, err], windowsHide: true }); + if (done.error) { + throw notStarted(binary, done.error); + } + if (done.status !== 0) { + throw new TfgError(args, done.status ?? done.signal, tail(err)); + } + } finally { + fs.closeSync(out); + if (err !== undefined) { + fs.closeSync(err); + } + fs.rmSync(errFile, { force: true }); + } +} + +// tail reads the last keptError bytes of an open file. +function tail(fd) { + const size = fs.fstatSync(fd).size; + const kept = Buffer.alloc(Math.min(size, keptError)); + fs.readSync(fd, kept, 0, kept.length, size - kept.length); + return kept.toString('utf8'); +} + +// run runs tfg without holding the event loop, so a test runner keeps its own +// timers and messages going while files are written. +function run(binary, args) { + return new Promise((resolve, reject) => { + const child = spawn(binary, args, { stdio: ['ignore', 'ignore', 'pipe'], windowsHide: true }); + let stderr = ''; + child.stderr.setEncoding('utf8'); + child.stderr.on('data', (chunk) => { + stderr = (stderr + chunk).slice(-keptError); + }); + child.on('error', (err) => reject(notStarted(binary, err))); + child.on('close', (status, signal) => { + if (status === 0) { + resolve(); + return; + } + reject(new TfgError(args, status ?? signal, stderr)); + }); + }); +} + +module.exports = { TfgError, run, runToFile }; diff --git a/npm/testing-files-generator/lib/testfile.js b/npm/testing-files-generator/lib/testfile.js new file mode 100644 index 00000000..5ca45136 --- /dev/null +++ b/npm/testing-files-generator/lib/testfile.js @@ -0,0 +1,129 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +'use strict'; + +const path = require('node:path'); +const { inspect } = require('node:util'); + +// The closed list of outcomes a manifest declares, from docs/MANIFEST.md. A +// guard holds this copy, and the one in index.d.ts, to recipe.Outcomes in the +// program, because two copies of a closed list is how they drift. +const outcomes = ['accept', 'reject', 'sanitize', 'unspecified']; + +// How each outcome reads in a sentence about what a system did with a file. +const done = { + accept: 'taken', + reject: 'turned away', + sanitize: 'taken and cleaned', +}; + +// A tag is a word Playwright filters on, so anything else in a group's name +// becomes a hyphen rather than breaking the filter. +function tag(word) { + return `@${String(word).replace(/[^A-Za-z0-9_-]+/g, '-')}`; +} + +function expectedText(expected) { + return expected.reason ? `${expected.outcome} (${expected.reason})` : expected.outcome; +} + +// TestFile is one file of a set, with what the system under test should do +// with it. It is what a test declares itself from: a title, the details a test +// takes, the path once the set is written, and a check against the manifest. +class TestFile { + constructor(source, entry) { + this._source = source; + this.entry = entry; + } + + get relativePath() { return this.entry.path; } + get name() { return this.entry.name; } + get format() { return this.entry.format; } + get bytes() { return this.entry.bytes; } + get group() { return this.entry.group ?? ''; } + get target() { return this.entry.target_id; } + get purpose() { return this.entry.purpose ?? ''; } + get expected() { return this.entry.expected; } + get unspecified() { return this.entry.expected.outcome === 'unspecified'; } + + // The path, which is unique in a set and is what a test title has to be. It + // equals the name today, because a name cannot hold a separator, and the + // path stays the right one if a set ever has directories. + get title() { + return `${this.entry.path} - expected ${expectedText(this.entry.expected)}`; + } + + // details has the shape test(title, details, body) takes, so a report shows + // what each file is for and a run can be narrowed by its tags. + get details() { + const tags = [tag(this.entry.expected.outcome)]; + if (this.group) { + tags.push(tag(this.group)); + } + const annotation = [{ type: 'expected', description: expectedText(this.entry.expected) }]; + if (this.purpose) { + annotation.push({ type: 'purpose', description: this.purpose }); + } + return { tag: tags, annotation }; + } + + get whyUnspecified() { + const why = `${this.entry.path} has no expected outcome - what your system does with it is its own policy.`; + return this.purpose ? `${why} ${this.purpose}` : why; + } + + // path writes the set the first time any of its files is asked for, and + // gives the absolute path of this one. + async path() { + const dir = await this._source.write(); + return path.join(dir, this.entry.path); + } + + // check compares what the system did with what the manifest expects, and + // throws with the file's purpose when they differ - that sentence says what + // the difference means. A reason is compared only for a refusal: a reason + // names the rule a file is about, so a file a byte under a limit is accepted + // with size_limit and there is nothing to compare it with. + check(observed) { + try { + this._compare(observed); + } catch (err) { + // The stack starts at whoever called check, so a test runner shows the + // line of the test rather than a line of this package. + Error.captureStackTrace(err, TestFile.prototype.check); + throw err; + } + } + + _compare(observed) { + const want = this.entry.expected; + if (want.outcome === 'unspecified') { + throw new Error(`${this.entry.path} has no expected outcome, so there is nothing to check it against. ` + + 'Skip it with test.skip(file.unspecified, file.whyUnspecified), or decide what your system should do with it and check that.'); + } + const got = typeof observed === 'string' ? { outcome: observed } : observed; + if (!got || typeof got !== 'object' || !Object.hasOwn(done, got.outcome)) { + throw new TypeError('check takes accept, reject or sanitize, or an object with an outcome and a reason. ' + + `It was given ${inspect(observed)}.`); + } + if (got.outcome !== want.outcome) { + throw mismatch(this, `${this.entry.path} was ${done[got.outcome]}, and it should have been ${done[want.outcome]}` + + (want.reason ? ` (${want.reason}).` : '.'), got); + } + if (want.outcome === 'reject' && want.reason && got.reason && got.reason !== want.reason) { + throw mismatch(this, `${this.entry.path} was turned away for ${got.reason}, and the rule it is about is ${want.reason}.`, got); + } + } +} + +function mismatch(file, sentence, observed) { + const err = new Error(file.purpose ? `${sentence}\n${file.purpose}` : sentence); + err.name = 'ExpectationError'; + err.expected = file.entry.expected; + err.observed = observed; + return err; +} + +module.exports = { TestFile, outcomes }; diff --git a/npm/testing-files-generator/package.json b/npm/testing-files-generator/package.json new file mode 100644 index 00000000..acaea500 --- /dev/null +++ b/npm/testing-files-generator/package.json @@ -0,0 +1,51 @@ +{ + "name": "testing-files-generator", + "version": "0.0.0-development", + "description": "Real files of exact sizes and formats for tests, and what the system under test should do with each one. For Playwright and any other test runner.", + "license": "MPL-2.0", + "author": "DonislawDev", + "homepage": "https://github.com/donislawdev/TestingFilesGenerator/tree/main/npm/testing-files-generator#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/donislawdev/TestingFilesGenerator.git", + "directory": "npm/testing-files-generator" + }, + "bugs": "https://github.com/donislawdev/TestingFilesGenerator/issues", + "keywords": [ + "playwright", + "test files", + "fixtures", + "file upload", + "upload testing", + "test data", + "boundary testing", + "qa" + ], + "type": "commonjs", + "main": "index.js", + "types": "index.d.ts", + "exports": { + ".": { + "types": "./index.d.ts", + "default": "./index.js" + }, + "./package.json": "./package.json" + }, + "bin": { + "tfg": "bin/tfg.js" + }, + "files": [ + "index.js", + "index.d.ts", + "lib/", + "bin/", + "LICENSE", + "README.md" + ], + "engines": { + "node": ">=22" + }, + "scripts": { + "test": "node --test \"test/*.test.js\"" + } +} diff --git a/npm/testing-files-generator/test/binary.test.js b/npm/testing-files-generator/test/binary.test.js new file mode 100644 index 00000000..c5cfffef --- /dev/null +++ b/npm/testing-files-generator/test/binary.test.js @@ -0,0 +1,51 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { test } = require('node:test'); + +const { binaryPath, platforms } = require('../lib/binary.js'); + +// What require.resolve throws for a package that is not installed. +const nothing = (id) => { throw Object.assign(new Error(`Cannot find module '${id}'`), { code: 'MODULE_NOT_FOUND' }); }; + +test('TFG_BINARY is taken as given when there is a file there', () => { + assert.equal(binaryPath({ env: { TFG_BINARY: __filename }, resolve: nothing }), __filename); +}); + +test('TFG_BINARY naming nothing is refused with the path it named', () => { + const missing = path.join(__dirname, 'no-such-tfg'); + assert.throws(() => binaryPath({ env: { TFG_BINARY: missing }, resolve: nothing }), + (err) => err.message.includes(missing) && err.message.includes('unset it')); +}); + +test('each platform a release builds resolves to its own package', () => { + for (const [key, pkg] of Object.entries(platforms)) { + const [platform, arch] = key.split(' '); + const exe = platform === 'win32' ? 'tfg.exe' : 'tfg'; + const asked = []; + const found = binaryPath({ env: {}, platform, arch, resolve: (id) => { asked.push(id); return `/resolved/${id}`; } }); + assert.deepEqual(asked, [`${pkg}/${exe}`]); + assert.equal(found, `/resolved/${pkg}/${exe}`); + } +}); + +test('an Intel Mac is told what it can do instead', () => { + assert.throws(() => binaryPath({ env: {}, platform: 'darwin', arch: 'x64', resolve: nothing }), + (err) => err.message.includes('no binary for darwin x64') && err.message.includes('TFG_BINARY') && + err.message.includes('ghcr.io/donislawdev/tfg') && err.message.includes('darwin arm64')); +}); + +test('a platform package left out by --omit=optional is named, with the way back', () => { + assert.throws(() => binaryPath({ env: {}, platform: 'linux', arch: 'x64', resolve: nothing }), + (err) => err.message.includes('@testing-files-generator/linux-x64') && err.message.includes('--omit=optional')); +}); + +test('a platform package that is there and still gives no binary is told as it is, not as left out', () => { + const hidden = () => { + throw Object.assign(new Error('Package subpath \'./tfg\' is not defined by "exports"'), { code: 'ERR_PACKAGE_PATH_NOT_EXPORTED' }); + }; + assert.throws(() => binaryPath({ env: {}, platform: 'linux', arch: 'x64', resolve: hidden }), + (err) => err.message.includes('is not defined by "exports"') && !err.message.includes('--omit=optional') && + err.cause?.code === 'ERR_PACKAGE_PATH_NOT_EXPORTED'); +}); diff --git a/npm/testing-files-generator/test/fileset.test.js b/npm/testing-files-generator/test/fileset.test.js new file mode 100644 index 00000000..69b953c2 --- /dev/null +++ b/npm/testing-files-generator/test/fileset.test.js @@ -0,0 +1,206 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { test } = require('node:test'); + +const { fileSet, readable } = require('../lib/fileset.js'); +const { TfgError } = require('../lib/run.js'); +const { node, pkg, small, tempDir, tfg } = require('./helpers.js'); + +function plannedIn(cacheDir, spec = small) { + return fileSet({ ...spec, binary: tfg(), cacheDir }); +} + +// The script a fresh process runs to write a set, printing its directory. +function writer(cacheDir, spec = small) { + return `require(${pkg}).fileSet({ ...${JSON.stringify(spec)}, cacheDir: ${JSON.stringify(cacheDir)} })` + + '.write().then((dir) => console.log(dir), (err) => { console.error(err.message); process.exit(3); });'; +} + +function entries(cacheDir) { + return fs.readdirSync(cacheDir).sort(); +} + +test('a plan writes nothing, and says every file with what is expected of it', (t) => { + const cacheDir = tempDir(t); + const set = plannedIn(cacheDir); + assert.deepEqual(entries(cacheDir), [], 'a plan left something in the cache'); + assert.equal(set.files.length, set.plan.summary.file_count); + assert.ok(set.files.length >= 20, `the set this test asks for holds ${set.files.length} files`); + assert.equal(new Set(set.files.map((f) => f.title)).size, set.files.length, 'two files share a title'); + assert.equal(set.only({ outcome: 'unspecified' }).files.length, set.plan.summary.by_expected.unspecified); + assert.equal(set.only({ group: 'bulk-upload' }).files.length, 2); + assert.equal(set.except({ group: 'bulk-upload' }).files.length, set.files.length - 2); +}); + +test('a set is written once, whole, and its files are where their paths say', async (t) => { + const cacheDir = tempDir(t); + const set = plannedIn(cacheDir); + const dir = await set.write(); + assert.equal(await set.write(), dir); + assert.deepEqual(entries(cacheDir), [path.basename(dir)]); + for (const file of set.files) { + assert.equal(fs.statSync(await file.path()).size, file.bytes, file.relativePath); + } +}); + +// racer plans a set, waits until every racer has planned, and only then +// writes, so that all of them find the set unwritten and write it at once. A +// racer that fails before it reaches the barrier would hold the others there +// for ever, so the wait has an end. +function racer(cacheDir, barrier, racers) { + const lib = JSON.stringify(path.join(__dirname, '..', 'lib', 'fileset.js')); + const at = JSON.stringify(barrier); + return `const fs = require('node:fs'); +const { fileSet, events } = require(${lib}); +const said = []; +for (const e of ['wrote', 'moved', 'took']) events.on(e, () => said.push(e)); +const set = fileSet({ ...${JSON.stringify(small)}, cacheDir: ${JSON.stringify(cacheDir)} }); +fs.writeFileSync(require('node:path').join(${at}, String(process.pid)), ''); +const until = Date.now() + 30000; +const ready = () => fs.readdirSync(${at}).length >= ${racers} ? Promise.resolve() + : Date.now() > until ? Promise.reject(new Error('not every racer planned within 30 s')) + : new Promise((r) => setTimeout(r, 5)).then(ready); +ready().then(() => set.write()).then((dir) => console.log(JSON.stringify({ dir, said })), + (err) => { console.error(err.message); process.exit(3); });`; +} + +test('workers writing one set at once end with one copy and nothing left over', { timeout: 120000 }, async (t) => { + const cacheDir = tempDir(t); + const barrier = tempDir(t); + const racers = 4; + const runs = await Promise.all(Array.from({ length: racers }, () => node(racer(cacheDir, barrier, racers)))); + // Every failed racer is shown, because the first to fail may only be one + // that waited for another. + assert.deepEqual(runs.filter((r) => r.status !== 0).map((r) => r.stderr), [], 'a racer failed'); + const results = runs.map((r) => JSON.parse(r.stdout)); + const said = results.flatMap((r) => r.said); + const count = (e) => said.filter((s) => s === e).length; + assert.equal(count('wrote'), racers, `not every worker wrote, so there was no race: ${said.join(', ')}`); + assert.equal(count('moved'), 1, said.join(', ')); + assert.equal(count('took'), racers - 1, said.join(', ')); + const dirs = new Set(results.map((r) => r.dir)); + assert.equal(dirs.size, 1, `the workers were given ${[...dirs].join(', ')}`); + assert.deepEqual(entries(cacheDir), [path.basename([...dirs][0])]); + const set = plannedIn(cacheDir); + assert.equal(await set.write(), [...dirs][0]); +}); + +test('a later run takes the set an earlier one wrote', async (t) => { + const cacheDir = tempDir(t); + const first = await node(writer(cacheDir)); + assert.equal(first.status, 0, first.stderr); + const dir = first.stdout.trim(); + const before = fs.statSync(path.join(dir, 'manifest.json')).mtimeMs; + const second = await node(writer(cacheDir)); + assert.equal(second.stdout.trim(), dir); + assert.equal(fs.statSync(path.join(dir, 'manifest.json')).mtimeMs, before, 'the set was written again'); +}); + +test('another seed or --clean is another set, though the recipe hash is the same', async (t) => { + const cacheDir = tempDir(t); + const plain = plannedIn(cacheDir); + const seeded = plannedIn(cacheDir, { ...small, options: { ...small.options, seed: 7 } }); + const clean = plannedIn(cacheDir, { ...small, options: { ...small.options, clean: true } }); + assert.equal(seeded.plan.run.recipe_hash, plain.plan.run.recipe_hash, 'the hash now covers the seed - this test is no longer about anything'); + const dirs = new Set([await plain.write(), await seeded.write(), await clean.write()]); + assert.equal(dirs.size, 3); +}); + +test('a set changed by hand is refused, and left where it is', async (t) => { + const cacheDir = tempDir(t); + const first = await node(writer(cacheDir)); + const dir = first.stdout.trim(); + const gone = path.join(dir, '1mb_at_limit.jpg'); + fs.rmSync(gone); + const second = await node(writer(cacheDir)); + assert.equal(second.status, 3); + assert.match(second.stderr, /does not hold it - 1mb_at_limit\.jpg is missing\. Nothing here removes that directory/); + assert.ok(fs.existsSync(path.join(dir, 'manifest.json')), 'the changed set was removed'); +}); + +test('a file of the set cut short by hand is refused', async (t) => { + const cacheDir = tempDir(t); + const dir = (await node(writer(cacheDir))).stdout.trim(); + fs.truncateSync(path.join(dir, '1mb_at_limit.jpg'), 10); + const again = await node(writer(cacheDir)); + assert.equal(again.status, 3); + assert.match(again.stderr, /1mb_at_limit\.jpg is 10 B and its manifest says 1048576 B/); +}); + +test('a file of the set written over by hand, its size the same, is refused', async (t) => { + const cacheDir = tempDir(t); + const dir = (await node(writer(cacheDir))).stdout.trim(); + const changed = path.join(dir, '1mb_at_limit.jpg'); + const fd = fs.openSync(changed, 'r+'); + fs.writeSync(fd, Buffer.from('changed'), 0, 7, 0); + fs.closeSync(fd); + const again = await node(writer(cacheDir)); + assert.equal(again.status, 3, again.stderr); + assert.match(again.stderr, /its files are not the ones it was written with\. Nothing here removes that directory/); + assert.match(again.stderr, /wrong-hash\s+1mb_at_limit\.jpg/); + assert.ok(fs.existsSync(changed), 'the changed set was removed'); +}); + +test('a binary or a recipe the options name and that is not there is named, with what to do', (t) => { + const cacheDir = tempDir(t); + const missing = path.join(cacheDir, 'no-such-tfg'); + assert.throws(() => fileSet({ ...small, binary: missing, cacheDir }), + (err) => err.message.includes(`The binary option names ${missing}, and there is no file there.`) && err.message.includes('leave it out')); + assert.throws(() => fileSet({ recipe: 'no-such-recipe.yaml', binary: tfg(), cacheDir }), + (err) => err.message.includes(path.resolve('no-such-recipe.yaml')) && err.message.includes('path.join(__dirname')); +}); + +test('a set a stopped run left behind is named once outside the workers, and never used', async (t) => { + const cacheDir = tempDir(t); + const gone = await node(''); + const left = path.join(cacheDir, `.tmp-${gone.pid}-0bad0bad`); + fs.mkdirSync(left); + const plan = `require(${pkg}).fileSet({ ...${JSON.stringify(small)}, cacheDir: ${JSON.stringify(cacheDir)} })`; + const runner = await node(plan, { env: { TEST_WORKER_INDEX: undefined } }); + assert.equal(runner.status, 0, runner.stderr); + const worker = await node(plan, { env: { TEST_WORKER_INDEX: '0' } }); + assert.equal(worker.status, 0, worker.stderr); + assert.equal(worker.stderr, '', 'a worker spoke about the cache'); + assert.match(runner.stderr, new RegExp(`1 unfinished set was left in .*\\.tmp-${gone.pid}-0bad0bad\\. Nothing uses them`)); + assert.ok(fs.existsSync(left), 'the unfinished set was removed'); +}); + +test('what tfg refuses reaches the test with its own words and code', (t) => { + const cacheDir = tempDir(t); + assert.throws(() => plannedIn(cacheDir, { preset: 'upload-validation', options: { limit: '1kb' } }), + (err) => err instanceof TfgError && err.status === 4 && /smallest PDF/.test(err.message)); +}); + +test('the flags this package sets itself cannot be set again', (t) => { + const cacheDir = tempDir(t); + for (const flag of ['out', 'json', 'dry-run', 'preset']) { + assert.throws(() => plannedIn(cacheDir, { preset: 'upload-validation', options: { [flag]: 'x' } }), + new RegExp(`The option ${flag} cannot be set here`)); + } + assert.throws(() => plannedIn(cacheDir, { preset: 'upload-validation', options: { '--limit': '1mb' } }), /not a flag tfg has/); + assert.throws(() => plannedIn(cacheDir, { preset: 'upload-validation', options: { limit: { mb: 1 } } }), TypeError); +}); + +test('a manifest of another major, or an outcome this package does not know, is refused', () => { + const plan = { manifest_version: '1.0', files: [{ path: 'a', expected: { outcome: 'accept' } }] }; + readable(plan); + assert.throws(() => readable({ ...plan, manifest_version: '2.0' }), /manifest of version 2\.0, and this package reads version 1/); + assert.throws(() => readable({ ...plan, files: [{ path: 'a', expected: { outcome: 'quarantine' } }] }), /"quarantine", which this package does not know/); +}); + +test('npx tfg is tfg - the same output and the same exit code', async () => { + const bin = path.join(__dirname, '..', 'bin', 'tfg.js'); + const viaPackage = await node('', { file: bin, args: ['generate', '--format', 'nope', '--dry-run'] }); + const direct = await new Promise((resolve) => { + const child = require('node:child_process').spawn(tfg(), ['generate', '--format', 'nope', '--dry-run']); + let stderr = ''; + child.stderr.on('data', (c) => { stderr += c; }); + child.on('close', (status) => resolve({ status, stderr })); + }); + assert.notEqual(direct.status, 0); + assert.equal(viaPackage.status, direct.status); + assert.equal(viaPackage.stderr, direct.stderr); +}); diff --git a/npm/testing-files-generator/test/helpers.js b/npm/testing-files-generator/test/helpers.js new file mode 100644 index 00000000..af64e127 --- /dev/null +++ b/npm/testing-files-generator/test/helpers.js @@ -0,0 +1,58 @@ +'use strict'; + +const { spawn } = require('node:child_process'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +const root = path.resolve(__dirname, '..', '..', '..'); +const built = path.join(root, process.platform === 'win32' ? 'tfg.exe' : 'tfg'); + +// tfg is the binary these tests run: TFG_BINARY, or the one the repository's +// build command leaves at its root. Neither is a failure and not a skip - a +// suite that skips without a binary is green while testing nothing. +function tfg() { + const binary = process.env.TFG_BINARY || built; + if (!fs.existsSync(binary)) { + throw new Error(`No tfg binary at ${binary}. Build one from the repository root with ` + + `go build -tags "$(cat .github/build-tags)" -o ${path.basename(built)} ./cmd/tfg, or set TFG_BINARY.`); + } + return binary; +} + +function tempDir(t) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tfg-npm-test-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + return dir; +} + +// node runs a script in a fresh process, with this package and a binary, and +// gives its exit code and both streams. A variable given as undefined is taken +// out of the child's environment rather than set to an empty string. +function node(script, { env = {}, file = '', args = [] } = {}) { + const childEnv = { ...process.env, TFG_BINARY: tfg(), ...env }; + for (const [key, value] of Object.entries(env)) { + if (value === undefined) { + delete childEnv[key]; + } + } + return new Promise((resolve) => { + const child = spawn(process.execPath, file ? [file, ...args] : ['-e', script, ...args], { + env: childEnv, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { stdout += chunk; }); + child.stderr.on('data', (chunk) => { stderr += chunk; }); + child.on('close', (status) => resolve({ status, stdout, stderr, pid: child.pid })); + }); +} + +const pkg = JSON.stringify(path.resolve(__dirname, '..')); + +// A small set that still holds every kind of case: one either side of the +// limit, unspecified files and a group of several files. +const small = { preset: 'upload-validation', options: { limit: '1mb', bulk: 2, 'far-over': 'off' } }; + +module.exports = { node, pkg, small, tempDir, tfg }; diff --git a/npm/testing-files-generator/test/run.test.js b/npm/testing-files-generator/test/run.test.js new file mode 100644 index 00000000..ace16476 --- /dev/null +++ b/npm/testing-files-generator/test/run.test.js @@ -0,0 +1,30 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { test } = require('node:test'); + +const { TfgError, run, runToFile } = require('../lib/run.js'); +const { tempDir } = require('./helpers.js'); + +// Node stands in for tfg here, so a child can be made to end any way at all. +const child = (script) => [process.execPath, ['-e', script]]; + +test('a child stopped by a signal is named by the signal, from both ways of running it', { + skip: process.platform === 'win32' && 'Windows has no signals to stop a process with', +}, async (t) => { + const [binary, args] = child("process.kill(process.pid, 'SIGTERM')"); + const out = path.join(tempDir(t), 'out.json'); + assert.throws(() => runToFile(binary, args, out), + (err) => err instanceof TfgError && err.status === 'SIGTERM' && /was stopped by SIGTERM/.test(err.message)); + await assert.rejects(run(binary, args), + (err) => err instanceof TfgError && err.status === 'SIGTERM' && /was stopped by SIGTERM/.test(err.message)); +}); + +test('a child that says a great deal on its error stream runs to its own end, and the end is kept', (t) => { + const [binary, args] = child("process.stderr.write('x'.repeat(4 * 1024 * 1024) + 'the last words'); process.exitCode = 3"); + const out = path.join(tempDir(t), 'out.json'); + assert.throws(() => runToFile(binary, args, out), + (err) => err instanceof TfgError && err.status === 3 && err.stderr.endsWith('the last words') && + err.stderr.length <= 64 * 1024); +}); diff --git a/npm/testing-files-generator/test/testfile.test.js b/npm/testing-files-generator/test/testfile.test.js new file mode 100644 index 00000000..19fccd75 --- /dev/null +++ b/npm/testing-files-generator/test/testfile.test.js @@ -0,0 +1,59 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { TestFile } = require('../lib/testfile.js'); + +function file(expected, extra = {}) { + return new TestFile({ write: async () => '/set' }, { + path: 'big.jpg', name: 'big.jpg', target_id: 'over', bytes: 9, format: 'jpg', + group: 'size limit', purpose: 'Taking it means the limit is not enforced.', expected, ...extra, + }); +} + +test('a file says what it is expected to meet in its title, its tags and its annotations', () => { + const f = file({ outcome: 'reject', reason: 'size_limit' }); + assert.equal(f.title, 'big.jpg - expected reject (size_limit)'); + assert.deepEqual(f.details.tag, ['@reject', '@size-limit']); + assert.deepEqual(f.details.annotation, [ + { type: 'expected', description: 'reject (size_limit)' }, + { type: 'purpose', description: 'Taking it means the limit is not enforced.' }, + ]); +}); + +test('a difference is thrown with the sentence that says what it means', () => { + const f = file({ outcome: 'reject', reason: 'size_limit' }); + assert.throws(() => f.check('accept'), (err) => err.name === 'ExpectationError' && + err.message === 'big.jpg was taken, and it should have been turned away (size_limit).\nTaking it means the limit is not enforced.' && + err.observed.outcome === 'accept'); + f.check('reject'); + f.check({ outcome: 'reject', reason: 'size_limit' }); + f.check({ outcome: 'reject' }); +}); + +test('a refusal for another rule is a difference, a reason on an accepted file is not', () => { + assert.throws(() => file({ outcome: 'reject', reason: 'size_limit' }).check({ outcome: 'reject', reason: 'mime_mismatch' }), + /turned away for mime_mismatch, and the rule it is about is size_limit/); + file({ outcome: 'accept', reason: 'size_limit' }).check({ outcome: 'accept', reason: 'anything' }); +}); + +test('an unspecified file is never passed by check', () => { + const f = file({ outcome: 'unspecified', reason: 'size_zero' }); + assert.equal(f.unspecified, true); + assert.match(f.whyUnspecified, /^big\.jpg has no expected outcome/); + assert.throws(() => f.check('accept'), /nothing to check it against/); + assert.throws(() => f.check('reject'), /nothing to check it against/); +}); + +test('check refuses what is not an outcome, the names an object inherits included', () => { + const f = file({ outcome: 'accept' }); + for (const bad of ['toString', 'unspecified', 'ACCEPT', undefined, null, 3, {}]) { + assert.throws(() => f.check(bad), TypeError, String(bad)); + } +}); + +test('the path is the set directory joined with the file', async () => { + const path = require('node:path'); + assert.equal(await file({ outcome: 'accept' }).path(), path.join('/set', 'big.jpg')); +});