diff --git a/examples/playwright/README.md b/examples/playwright/README.md index 0945467b..0ff415db 100644 --- a/examples/playwright/README.md +++ b/examples/playwright/README.md @@ -13,11 +13,25 @@ ## Run it -From the root of the repository, build tfg first. The example finds it there. +You need Node 22 or later and the tfg program. Inside the repository the package is linked rather than installed from npm, so tfg does not come with it here. Get it one of two ways: -```bash -go build -tags "$(cat .github/build-tags)" -o "tfg$(go env GOEXE)" ./cmd/tfg -``` +- **Download it.** Take the archive for your system from [the latest release](https://github.com/donislawdev/TestingFilesGenerator/releases/latest), such as `tfg__windows_amd64.zip`, unpack it, and set `TFG_BINARY` to the `tfg` inside, `tfg.exe` on Windows: + + ```bash + export TFG_BINARY=/path/to/tfg + ``` + + ```powershell + $env:TFG_BINARY = 'C:/path/to/tfg.exe' + ``` + +- **Build it with Go**, from the root of the repository. The example finds it there: + + ```bash + go build -tags "$(cat .github/build-tags)" -o "tfg$(go env GOEXE)" ./cmd/tfg + ``` + +Then install the example and the browser it drives, and run it: ```bash cd examples/playwright && npm install && npx playwright install chromium @@ -37,4 +51,12 @@ the limit is not enforced, or enforced only in the browser. Ten are skipped, because what a form does with an empty file or a name full of emoji is its own policy, and the set says so rather than pass them. +`npx playwright show-report results/html` opens the report, where every test of the set carries what its file expects and what the file is for. + `FIXED=1 npx playwright test` corrects all four mistakes, and nothing fails. `npm run check` runs both and says whether the example still shows what it is for. The repository's CI runs that. + +## What to read, in this order + +1. `app/server.mjs` and `app/page.js` - the form, with each mistake marked where it is made. +2. `tests/typical.spec.ts` - the tests such a form usually gets, which all pass. +3. `tests/upload.spec.ts` - the generated set, through the page and through the API. How the package works is in [its README](../../npm/testing-files-generator#readme). diff --git a/examples/playwright/tests/upload.spec.ts b/examples/playwright/tests/upload.spec.ts index 7ad4e8f1..12f0dee2 100644 --- a/examples/playwright/tests/upload.spec.ts +++ b/examples/playwright/tests/upload.spec.ts @@ -20,7 +20,7 @@ test.describe('through the page', () => { await page.getByLabel('File').setInputFiles(await file.path()); await page.getByRole('button', { name: 'Upload' }).click(); const status = page.getByRole('status'); - await expect(status).not.toBeEmpty(); + await expect(status).toHaveText(/^(Uploaded|Refused \(\w+\))$/); file.check(shown(await status.textContent())); }); } @@ -33,6 +33,7 @@ test.describe('through the API', () => { test(file.title, file.details, async ({ request }) => { test.skip(file.unspecified, file.whyUnspecified); const response = await request.post('/upload', { multipart: { file: createReadStream(await file.path()) } }); + expect(response.status(), 'a server error is not a refusal').toBeLessThan(500); const answer = await response.json(); file.check(response.ok() ? 'accept' : { outcome: 'reject', reason: answer.reason }); }); diff --git a/npm/testing-files-generator/README.md b/npm/testing-files-generator/README.md index d938db31..d8623afc 100644 --- a/npm/testing-files-generator/README.md +++ b/npm/testing-files-generator/README.md @@ -2,65 +2,223 @@ Real files of exact sizes and formats for your tests, and what the system under test should do with each one. -A PDF of exactly 1 MiB and one byte is a real PDF, so a server that looks inside it can only turn it away for its size. A file made with `Buffer.alloc` is zeros, so the same server turns it away for what it holds, and a test meant to prove a size limit passes on a server that has none. [The example](../../examples/playwright) shows exactly that. +A PDF of exactly 1 MiB and one byte is a real PDF, so a server that looks inside it can only turn it away for its size. A file made with `Buffer.alloc` is zeros, so the same server turns it away for what it holds, and a test meant to prove a size limit passes on a server that has none. [The example](https://github.com/donislawdev/TestingFilesGenerator/tree/main/examples/playwright) shows exactly that. -## In a Playwright test +> **Not on npm yet.** It will be published with a release of tfg. Until then, [the example](https://github.com/donislawdev/TestingFilesGenerator/tree/main/examples/playwright) runs it from the repository. + +## Contents + +- [What it is](#what-it-is) +- [Install](#install) +- [Your first test](#your-first-test) +- [The words this package uses](#the-words-this-package-uses) +- [Telling the set what your system did](#telling-the-set-what-your-system-did) +- [Files with no right answer](#files-with-no-right-answer) +- [Choosing the files](#choosing-the-files) +- [When the files are written, and where they are kept](#when-the-files-are-written-and-where-they-are-kept) +- [In CI](#in-ci) +- [Other test runners](#other-test-runners) +- [Reference](#reference) +- [When something goes wrong](#when-something-goes-wrong) +- [The tfg program](#the-tfg-program) +- [Licences](#licences) + +## What it is + +tfg, the Testing Files Generator, is a free and open source program that writes real files for tests: a JPEG that opens in a viewer, a PDF a reader can read, a CSV a spreadsheet imports, each at exactly the size you ask for. Beside the files it writes a manifest, a list that says for every file what it is for and what a system should do with it. + +This package runs tfg from your tests: + +1. You choose a set of files. Usually that is a preset, a set tfg knows by name that answers one question, such as "does my upload form take what it should and turn the rest away?". +2. The package gives you one test for each file, with a title that says what the file should meet. +3. Each test hands its file to your system the way your system takes files, and ends with one line that tells the set what your system did. +4. When that is not what the file expects, the test fails with a sentence that says what the difference means. + +tfg comes with this package, so there is nothing else to install. It works offline and never connects to the network. + +## Install + +```bash +npm install --save-dev testing-files-generator +``` + +- **Node 22 or later.** Types for TypeScript come with the package, and both `import` and `require` work. +- **The tfg program comes in a second package** that npm picks for your system: Windows on x64 and Arm, Linux on x64 and Arm, macOS on Apple silicon. Nothing runs when the package is installed, and nothing is downloaded when the tests run. +- **Do not install with `--omit=optional`.** npm leaves that second package out with it, and the tests then stop with a message saying so. On a system with no package of its own, such as an Intel Mac, see [The tfg program](#the-tfg-program). + +## Your first test + +Say your app has an upload form at `/upload` that takes one file at a time, has a size limit of `2mb`, and says `Uploaded` when it takes a file. This spec gives it one test for each file of the `upload-validation` preset: ```ts +// tests/upload.spec.ts import { test, expect } from '@playwright/test'; import { fileSet } from 'testing-files-generator'; -const uploads = fileSet({ preset: 'upload-validation', options: { limit: '2mb' } }); +// 1. The set, planned around the limit your form declares. Nothing is written yet. +// bulk: 0 leaves out the mass upload, which a form taking one file at a time has no use for. +const uploads = fileSet({ preset: 'upload-validation', options: { limit: '2mb', bulk: 0 } }); -for (const file of uploads.except({ group: 'bulk-upload' }).files) { +// 2. One test for each file of the set. +for (const file of uploads.files) { test(file.title, file.details, async ({ page }) => { + // 3. A file whose right answer is your own policy is skipped, and the report says why. test.skip(file.unspecified, file.whyUnspecified); + + // 4. Upload the file the way a person does. The first call to file.path() writes the set. await page.goto('/upload'); await page.getByLabel('File').setInputFiles(await file.path()); await page.getByRole('button', { name: 'Upload' }).click(); + + // 5. Tell the set what your app did. check throws when the file expected something else. const status = page.getByRole('status'); - await expect(status).not.toBeEmpty(); + await expect(status).toHaveText(/Uploaded|Refused/); file.check((await status.textContent())?.includes('Uploaded') ? 'accept' : 'reject'); }); } ``` -You write the two things only you know: how a file is uploaded in your app, and how it shows that a file was taken or turned away. The set brings the rest. +Steps 4 and 5 are the only part about your app. Change them to however your app takes a file and shows what it did with it. Keep step 5 waiting for an answer your app gives on purpose, `Uploaded` or `Refused` here, so a page that shows an error fails the test instead of passing as a refusal. + +**See the tests before you run them.** Listing them writes none of the files: + +```bash +npx playwright test --list +``` + +Each test is named by its file and what the file expects: + +``` +2mb_under_1b.jpg - expected accept +2mb_at_limit.jpg - expected accept +2mb_over_1b.jpg - expected reject (size_limit) +2mb_far_over_2x.jpg - expected reject (size_limit) +empty.jpg - expected unspecified (size_zero) +allowed_jpg.jpg - expected accept +denied.exe - expected reject (extension_rule) +pdf_as_jpg.jpg - expected reject (mime_mismatch) +invoice.jpg.exe - expected reject (extension_rule) +PHOTO.JPG - expected unspecified (extension_rule) +my report (final) v2.jpg - expected accept +... +``` + +**Run them** with `npx playwright test`. The first test that asks for a file writes the whole set. A test fails when your app does something other than what its file expects, and the failure says what that means: + +``` +ExpectationError: 2mb_at_limit.jpg was turned away, and it should have been taken. +A JPEG file of exactly 2mb, the limit itself. Your system should take it. A refusal here usually +means the comparison is off by one, or your system counts a kilobyte as 1000 bytes where this +set counts 1024. +``` + +A file with no right answer is skipped, never passed, and the report gives the reason, such as `empty.jpg has no expected outcome - what your system does with it is its own policy`. [Files with no right answer](#files-with-no-right-answer) says what to do once you have decided. + +**Run part of the set.** Every test is tagged with its expected outcome and with its file's group: + +```bash +npx playwright test --grep @reject +npx playwright test --grep @size-limit +``` -- **One test for each file.** The title names the file and what it should meet, such as `1mb_over_1b.jpg - expected reject (size_limit)`. The report shows what the file is for, and tags such as `@reject` and `@size-limit` narrow a run with `--grep`. -- **A failure says what it means.** `check` throws with the sentence the set carries for that file: +## The words this package uses - ``` - 1mb_over_1b.jpg was taken, and it should have been turned away (size_limit). - A JPEG file 1 B over the limit of 1mb. Your system should turn it away - taking it means - the limit is not enforced, or enforced only in the browser. - ``` +| word | what it means | +|---|---| +| set | the files one `fileSet` call describes, each with what it should meet | +| preset | a set tfg knows by name, built around one question. `upload-validation` is one | +| recipe | a YAML file that describes a set file by file. A preset can be turned into one and edited | +| plan | the files of a set and what each expects, known before any of them is written | +| manifest | `manifest.json`, which tfg writes beside a set: every file with its size, its hash and what it expects | +| outcome | what your system should do with a file: `accept`, `reject`, `sanitize` or `unspecified` | +| reason | the rule a file is about, such as `size_limit` or `mime_mismatch`, from a closed list | +| group | the part of a set a file belongs to, such as `size-limit` or `hostile-names` | +| target | one entry of a recipe. A target can make several files | +| purpose | one sentence on what a file is for and what it means when a system gets it wrong. A failure shows it | -- **Files whose outcome is your policy are skipped, never passed.** An empty file or a name with an emoji in it has no right answer until you decide one. `whyUnspecified` says so in the report, and `check` refuses to pass such a file. -- **The same files can go straight to your server**, past the page and its checks: +The four outcomes: - ```ts - import { createReadStream } from 'node:fs'; +- `accept` - your system should take the file. +- `reject` - your system should turn it away. A file to be turned away usually names the rule as its reason. +- `sanitize` - your system should take the file and change it, for example store it under a safe name. No preset expects this today, and a recipe can. +- `unspecified` - there is no right answer until you decide one, as for an empty file. These files are skipped. - const response = await request.post('/upload', { multipart: { file: createReadStream(await file.path()) } }); - file.check(response.ok() ? 'accept' : 'reject'); - ``` +## Telling the set what your system did - `check` also takes `{ outcome: 'reject', reason: 'size_limit' }`. A reason is compared only for a refusal, because it names the rule a file is about - a file a byte under a limit is expected to be accepted with `size_limit` as the rule in play. +`file.check` compares what your system did with what the file expects, and throws when the two differ. It takes one of three words: -## The sets +```ts +file.check('accept'); // your system took the file +file.check('reject'); // it turned the file away +file.check('sanitize'); // it took the file and changed it +``` -`fileSet` takes a preset, a recipe file or plain flags of `tfg generate`: +When your system says why it turned a file away, pass the reason too, and a file turned away for the wrong rule fails as well. Here the same files go straight to the server, which answers with a JSON body holding a `reason`: ```ts -fileSet({ preset: 'size-boundaries', options: { limit: '10mb', format: 'pdf' } }); -fileSet({ recipe: 'fixtures/uploads.yaml' }); -fileSet({ options: { format: 'png', size: '5mb', count: 3, seed: 7, expected: 'accept' } }); +import { createReadStream } from 'node:fs'; + +for (const file of uploads.files) { + test(file.title, file.details, async ({ request }) => { + test.skip(file.unspecified, file.whyUnspecified); + const response = await request.post('/upload', { multipart: { file: createReadStream(await file.path()) } }); + expect(response.status(), 'a server error is not a refusal').toBeLessThan(500); + const answer = await response.json(); + file.check(response.ok() ? 'accept' : { outcome: 'reject', reason: answer.reason }); + }); +} ``` -A relative recipe path is taken from the directory the tests were started in. `path.join(__dirname, 'uploads.yaml')` names one beside the spec file, wherever the tests are started from. +- **A server error is not a refusal.** An answer of 500 or more fails the test before `check`, so a file that breaks your server is found rather than counted as turned away. +- **A reason is compared only for a file that should be turned away, and only when you give one.** A reason is a name from the list in [Reference](#reference), such as `size_limit`. If your system has codes of its own, turn them into those names, or pass the outcome alone. +- **`check` never passes a file with no right answer.** On an `unspecified` file it throws, so such a file cannot pass by accident. +- **What your system did can come from anywhere a test can read**: the text on the page, an HTTP status, the body of a response, a row in a database. +- **The same files can go through the page and straight to the server.** The checks of a page run in the browser and anything other than that page skips them, so a limit only the page enforces shows up only in the tests that go to the server. The example does both. + +## Files with no right answer + +Some files have no answer the set could know: whether an empty file should go through, whether `PHOTO.JPG` is the same file as `photo.jpg`, whether a name in Japanese is stored as it is. Their outcome is `unspecified`, and `test.skip(file.unspecified, file.whyUnspecified)` skips them with the reason in the report. In `filename-handling` nearly every file is like this, because what a system does with a strange name is its own rule. + +Once you have decided what your system should do with such a file, write that down, and the file is checked like any other: + +1. **Turn the preset into a recipe**, a file of your own to edit and commit: + + ```bash + npx tfg preset eject upload-validation --limit 2mb --bulk 0 -o tests/uploads.yaml + ``` + +2. **Find the file in the recipe** and change `unspecified` to what your system does. Change its `purpose` too, because that is the sentence a failure shows: + + ```yaml + - id: empty + format: txt + count: 1 + size: 0 + name: empty.jpg + group: degenerate + purpose: "An empty file. Our form turns it away, because an empty upload is always a mistake." + expected: + outcome: reject + reason: size_zero + ``` + +3. **Use the recipe instead of the preset:** + + ```ts + import * as path from 'node:path'; -Files made from flags expect nothing unless `expected` says what, so without it every one of them is skipped. + const uploads = fileSet({ recipe: path.join(__dirname, 'uploads.yaml') }); + ``` + + In an ES module, such as a `.mjs` file or a project whose `package.json` says `"type": "module"`, write `import.meta.dirname` in place of `__dirname`. + +`empty.jpg` is then a test like any other, titled `empty.jpg - expected reject (size_zero)`, and it fails when your form takes an empty file. + +## Choosing the files + +### Presets + +`fileSet({ preset })` takes one of these: | preset | the question it answers | |---|---| @@ -71,22 +229,87 @@ Files made from flags expect nothing unless `expected` says what, so without it | `text-encoding` | Does my reader know which encoding a file is in, or is it guessing? | | `empty-and-minimal` | Does a file that is valid and as small as the format allows get through? | -`npx tfg preset show ` lists what a preset takes, and `npx tfg formats` lists the formats. +`npx tfg preset show ` lists what a preset takes, what it would write and what it typically catches: + +``` +$ npx tfg preset show upload-validation +... +parameters: + --limit a size such as 2mb, or a plain byte count, default 10mb + --allow format ids separated by commas, default jpg,png,pdf + --deny extensions separated by commas, default svg,html,exe,sh + --far-over one of: 10x, 2x, off, default 2x + --bulk whole number of files from 0 to 10000, default 50 +... +``` + +### Options + +Every parameter there is an option of `fileSet`, written without the dashes: + +```ts +fileSet({ preset: 'upload-validation', options: { limit: '5mb', allow: 'jpg,png', 'far-over': 'off', bulk: 0 } }); +``` + +- **A list is one string with commas**, as `allow` above. +- **Sizes count in 1024s**, so `2mb` is 2 097 152 bytes. If your system means 2 000 000 bytes by 2 MB, give the number of bytes, `limit: 2000000`, and the files are named after it, such as `2000000_over_1b.jpg`. +- **The same options give the same files**, byte for byte, on every run and every machine, for as long as the major version of tfg stays the same. `seed: 7` gives other bytes, and those are again the same on every run. +- **Options also take the flags of `tfg generate` itself**, such as `seed` or `clean`. `true` stands for a flag with no value, `clean: true`, and an array gives a flag that repeats, `set: ['width=640', 'height=480']`. + +### Part of a set -`only` and `except` take part of a set by `group`, `target` or `outcome`, and `paths()` gives the files of a part together, for a form that takes several at once: +`only` and `except` take part of a set by `group`, `target` or `outcome`, one name or a list of them: ```ts -const together = await uploads.only({ group: 'bulk-upload' }).paths(); -await page.getByLabel('Files').setInputFiles(together); +const refusals = uploads.only({ outcome: 'reject' }); +const singleFiles = uploads.except({ group: ['bulk-upload', 'hostile-names'] }); ``` -## When the files are written +Every part shares the set's one write. `paths()` gives the files of a part together, for a form that takes several at once: -1. **While Playwright loads the spec file, nothing is written.** `fileSet` asks for the plan, the list of files and what each should meet, and Playwright declares the tests from it. Listing the tests with `--list` writes nothing either. -2. **The first test that asks for a path writes the whole set**, once, however many workers ask at the same moment. The default `upload-validation` set, 71 files and 115 MB, was written in under a second on the machine it was measured on. -3. **Every later run takes the same files.** The same options give the same bytes, so a set is written once and kept, in `node_modules/.cache/testing-files-generator` or the directory `TFG_CACHE_DIR` names. Before a kept set is used, every byte of it is checked against its manifest, which took under a tenth of a second for the same set. A set a test wrote into, or a file someone changed, is refused with the names of the files that differ. +```ts +const uploads = fileSet({ preset: 'upload-validation', options: { limit: '2mb' } }); + +test('the mass upload is taken whole', async ({ page }) => { + await page.goto('/upload'); + await page.getByLabel('Files').setInputFiles(await uploads.only({ group: 'bulk-upload' }).paths()); + // ... +}); +``` + +### Your own files + +A recipe describes a set of your own, file by file, and the [recipe section of the tfg README](https://github.com/donislawdev/TestingFilesGenerator#-recipes) says how to write one. A relative recipe path is taken from the directory the tests were started in, so `path.join(__dirname, 'uploads.yaml')` names one beside the spec file wherever the tests are started from. In an ES module, `import.meta.dirname` takes the place of `__dirname`. + +```ts +fileSet({ recipe: path.join(__dirname, 'fixtures.yaml') }); +``` -A large set is better written before the tests start, so its time does not count against the first test: +A few files of one kind need no recipe: + +```ts +fileSet({ options: { format: 'png', size: '5mb', count: 3, expected: 'accept' } }); +``` + +Files made this way expect nothing unless `expected` says what, so without it every one of them is skipped. `npx tfg formats` lists the formats. + +## When the files are written, and where they are kept + +``` +the spec file loads fileSet plans the set: tfg lists the files and writes none of them +the first file.path() tfg writes the whole set, and the first copy finished is the one kept +every later run the set is taken from where it was kept, once every byte of it is checked +``` + +1. **While Playwright loads the spec file, none of the set is written.** `fileSet` asks tfg for the plan, and Playwright declares the tests from it. That is why `--list` writes none of the files. The plan comes back through a short-lived file in the cache directory, so that directory has to be one the tests can write to, even for `--list`. +2. **The first test that asks for a path writes the whole set.** Workers that ask at the same moment each write a copy of their own. The first one finished is kept and the others are removed, so for a moment the disk holds a copy for each of those workers. A worker that asks later takes the kept set. The `upload-validation` set at its defaults, 71 files and 115 MB, was written in under a second on the machine it was measured on. +3. **Every later run takes the same files.** A set is kept in `node_modules/.cache/testing-files-generator`, or in the directory `TFG_CACHE_DIR` names. Before a kept set is used, every byte of it is checked against its manifest, which took under a tenth of a second for the same set. A set a test wrote into, or a file someone changed, is refused with the names of the files that differ. + +**Do not write into a file the set gave you.** Copy it first. A file changed in place is changed for every later run, and the next one refuses the set. + +**To look at the files yourself**, `await uploads.write()` gives the directory of the set. It holds the files, `manifest.json` and `manifest.instructions.md`, which says in plain words what each file is for. + +**A large set is better written before the tests start**, so the time it takes does not count against the first test: ```ts test.beforeAll(async () => { @@ -95,17 +318,145 @@ test.beforeAll(async () => { }); ``` -Nothing is ever removed on its own. Delete the cache directory whenever you like. A run stopped part way leaves an unfinished set, which nothing uses, and the next run names it so it does not sit there unseen. +**Nothing is ever removed on its own.** Delete the cache directory whenever you like, and the next run writes the sets again. A run stopped part way leaves an unfinished set, which nothing uses, and the next run names it so it does not sit there unseen. + +## In CI + +- **Nothing to set up.** `npm ci` installs tfg with the package, and the tests write their sets when they run. +- **`npm ci` removes `node_modules`, and the kept sets with it**, so every CI run writes its sets again. For the `upload-validation` set at its defaults that is under a second on the machine it was measured on. To keep sets from one run to the next, point `TFG_CACHE_DIR` at a directory your CI keeps. +- **Runners on Windows, Linux and macOS on Apple silicon all have a package of tfg.** An Intel macOS runner has none, see [The tfg program](#the-tfg-program). + +## Other test runners + +`fileSet` gives plain data and plain promises, and imports nothing from Playwright. Any runner that can declare tests in a loop works the same way. + +With the test runner built into Node, `node --test`: + +```js +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { openAsBlob } from 'node:fs'; +import { fileSet } from 'testing-files-generator'; + +const uploads = fileSet({ preset: 'upload-validation', options: { limit: '2mb', bulk: 0 } }); + +for (const file of uploads.files) { + test(file.title, { skip: file.unspecified && file.whyUnspecified }, async () => { + const body = new FormData(); + body.append('file', await openAsBlob(await file.path()), file.name); + const response = await fetch('http://localhost:3000/upload', { method: 'POST', body }); + assert.ok(response.status < 500, `the server failed with ${response.status}, which is not a refusal`); + file.check(response.ok ? 'accept' : 'reject'); + }); +} +``` + +With Vitest, the same loop skips with `test.skipIf`: + +```js +import { test, expect } from 'vitest'; +import { openAsBlob } from 'node:fs'; +import { fileSet } from 'testing-files-generator'; + +const uploads = fileSet({ preset: 'upload-validation', options: { limit: '2mb', bulk: 0 } }); + +for (const file of uploads.files) { + test.skipIf(file.unspecified)(file.title, async () => { + const body = new FormData(); + body.append('file', await openAsBlob(await file.path()), file.name); + const response = await fetch('http://localhost:3000/upload', { method: 'POST', body }); + expect(response.status, 'a server error is not a refusal').toBeLessThan(500); + file.check(response.ok ? 'accept' : 'reject'); + }); +} +``` + +## Reference + +### `fileSet(spec)` + +Plans a set and gives it back. Nothing is written until a test asks for a file. + +| spec | what it is | +|---|---| +| `preset` | a preset by name, as `npx tfg preset list` lists them | +| `recipe` | the path of a recipe file. A relative path is taken from the directory the tests were started in | +| `options` | the parameters of the preset and the flags of `tfg generate`, without the dashes | +| `cacheDir` | where sets are kept. By default `TFG_CACHE_DIR`, then `node_modules/.cache/testing-files-generator`, then the system's temporary directory | +| `binary` | the tfg to run. By default `TFG_BINARY`, then the one this package installed | -## Where tfg comes from +`fileSet` throws at once, while the spec file loads, when the binary or the recipe is not there or tfg refuses the plan. Two calls with the same spec in one process share one plan and one write. -The package installs the `tfg` program for your system: Windows on x64 and Arm, Linux on x64 and Arm, macOS on Apple silicon. `npx tfg` runs it, with the same arguments, output and exit codes as tfg run directly. +### A set + +| member | what it is | +|---|---| +| `files` | every file of the set, in order | +| `plan` | the manifest tfg gave for the plan: every file, nothing written yet | +| `only(filter)`, `except(filter)` | part of the set, by `group`, `target` or `outcome` | +| `write()` | writes the whole set, once for the process, and gives its directory | +| `paths()` | writes the set if it is not written yet, and gives the paths of this part's files | + +### A file + +| member | what it is | +|---|---| +| `title` | the file's path and what it expects, such as `2mb_over_1b.jpg - expected reject (size_limit)`. Unique within a set | +| `details` | tags and annotations for `test(title, details, body)`: the outcome and the group as tags, what the file expects and its purpose as annotations | +| `unspecified` | `true` for a file with no right answer | +| `whyUnspecified` | the sentence a skip shows | +| `path()` | writes the set the first time it is called, and gives this file's absolute path | +| `check(observed)` | `'accept'`, `'reject'`, `'sanitize'` or `{ outcome, reason }`. Throws when it is not what the file expects | +| `name`, `relativePath` | the file's name, and its path within the set | +| `format`, `bytes` | the format, such as `jpg`, and the exact size in bytes | +| `group`, `target`, `purpose` | the file's group, the recipe target that made it, and its purpose | +| `expected` | `{ outcome, reason, detail, confidence }` as the manifest has it | +| `entry` | the file's whole entry in the manifest | + +The reasons a file can name are `content_malformed`, `count_limit`, `dimensions_limit`, `duplicate`, `encoding_invalid`, `extension_rule`, `filename_invalid`, `filename_too_long`, `filename_traversal`, `malware_signature`, `mime_mismatch`, `nesting_depth`, `none`, `size_limit` and `size_zero`. + +### `TfgError` + +What `fileSet`, `write`, `path` and `paths` throw when tfg itself refuses or fails. `status` is the code tfg ended with, or the name of the signal that stopped it, and `stderr` is the end of everything tfg said. The codes are listed in [the tfg README](https://github.com/donislawdev/TestingFilesGenerator), under Using it in CI. + +### `binaryPath()` + +The path of the tfg binary this package runs. + +## When something goes wrong + +Every message names what it is about and what to do. The ones you are most likely to meet: + +| the message starts with | what it means and what to do | +|---|---| +| `... was taken, and it should have been turned away` | Your system did something other than what the file expects. That is the test doing its job, and the next line says what the difference means | +| `fileSet({ ... }) could not be planned` | tfg refused what the spec asks for. The line under it is tfg's own sentence, which names the value and the way out | +| `fileSet({ ... }) gives the option ...` | An option tfg does not have, often a typo. The message lists the flags tfg generate has | +| `fileSet was not told which files to write` | The spec has no `preset`, `recipe` or `options` | +| `The recipe option names ..., and there is no file there` | A relative recipe path is taken from the directory the tests were started in. Use `path.join(__dirname, ...)`, or `import.meta.dirname` in an ES module | +| `... is where this set belongs, and it does not hold it` | A kept set was changed, usually by a test writing into one of its files. Delete the directory the message names and run again | +| `... has no expected outcome, so there is nothing to check it against` | `check` was called on a file with no right answer. Skip it, or decide what your system should do and write it in a recipe | +| `Test timeout of 30000ms exceeded` in the first test | The set was being written inside the first test. Write it in `beforeAll` with a longer timeout | +| `The binary for ... comes in the package ..., and it is not installed` | npm installed with `--omit=optional`. Install again without it | +| `testing-files-generator has no binary for ...` | This system has no package of tfg. See [The tfg program](#the-tfg-program) | +| `tfg wrote a manifest of version ...` | `TFG_BINARY` points at a tfg whose manifest this package cannot read. Point it at a tfg of the same version as this package, or leave it unset | +| `testing-files-generator: 1 unfinished set was left in ...` | A warning. A run stopped part way, and what it left can be deleted | + +## The tfg program + +`npx tfg` runs the tfg this package installed, with the same arguments, output and exit codes as tfg run directly: + +```bash +npx tfg preset list +npx tfg formats +npx tfg generate --help +``` -To use a tfg installed another way, set `TFG_BINARY` to its path, or pass `binary` to `fileSet`. On a system with no binary of its own, such as an Intel Mac, the container image `ghcr.io/donislawdev/tfg` runs the same program. +To use a tfg installed another way, set `TFG_BINARY` to its path, or pass `binary` to `fileSet`. What decides is the manifest it writes. A tfg whose manifest is of another major version than this package reads is refused, because that manifest can mean something else, and a tfg of the same version as this package never is. -## Not only Playwright +On a system with no package of its own, such as an Intel Mac, run the tests in a Linux container, where npm installs the package for Linux. -`fileSet` gives plain data and plain promises, and imports nothing from Playwright. The same loop works in Vitest, Jest or a script, with `file.title` as a test name. +tfg also writes sets from the command line on its own, has a desktop window, and installs in other ways. All of that is in [the tfg README](https://github.com/donislawdev/TestingFilesGenerator). ## Licences diff --git a/npm/testing-files-generator/index.d.ts b/npm/testing-files-generator/index.d.ts index 8cc5e5e1..a994fe3d 100644 --- a/npm/testing-files-generator/index.d.ts +++ b/npm/testing-files-generator/index.d.ts @@ -2,10 +2,13 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -/** What a system under test is expected to do with a file. */ +/** + * What a system under test is expected to do with a file. accept is to take it, reject to turn it away, + * sanitize to take it and change it, and unspecified means there is no right answer until you decide one. + */ export type Outcome = 'accept' | 'reject' | 'sanitize' | 'unspecified'; -/** The rule a file is about. It names the rule and not what the system did, so a file a byte under a limit is expected to be accepted with size_limit. */ +/** The rule a file is about, from a closed list. It names the rule and not what a system did with the file. */ export type Reason = | 'content_malformed' | 'count_limit' @@ -23,107 +26,172 @@ export type Reason = | 'size_limit' | 'size_zero'; +/** What a file expects of the system under test, as the manifest has it. */ export interface Expected { + /** What the system should do with the file. */ outcome: Outcome; + /** The rule the file is about. A file to be turned away usually names one. */ reason?: Reason; + /** A sentence about the expectation, such as why a file expects nothing. */ detail?: string; + /** certain when the expectation was declared, policy_dependent when nobody declared one. */ confidence?: string; } -/** One entry of a manifest, as tfg writes it. */ +/** One entry of a manifest, as tfg writes it. It carries more fields than are named here. */ export interface ManifestFile { + /** The file's identity within the run. */ id: string; + /** The file's path within the set, with forward slashes. */ path: string; + /** The file's name. */ name: string; + /** Whether the file is on the disk. Every file of a plan has false. */ materialized: boolean; + /** The recipe target that made the file. */ target_id: string; + /** The file's exact size in bytes. */ bytes: number; + /** The file's format, such as jpg. */ format: string; + /** The part of the set the file belongs to, such as size-limit. */ group?: string; + /** One sentence on what the file is for and what it means when a system gets it wrong. */ purpose?: string; + /** What the system under test should do with the file. */ expected: Expected; [field: string]: unknown; } /** A manifest, as tfg writes it. For a plan nothing is written yet and every file has materialized false. */ export interface Manifest { + /** The version of the manifest's format. This package reads major version 1. */ manifest_version: string; + /** The program that wrote the manifest and its version. */ tool: { name: string; version: string; generators: Record; [field: string]: unknown }; + /** The run that made the set, and the seed every file's bytes come from. */ run: { id: string; seed: number; complete: boolean; recipe_hash?: string; [field: string]: unknown }; + /** How many files the set has and how many of them were written. */ summary: { file_count: number; total_bytes: number; materialized: number; [field: string]: unknown }; + /** Every file of the set, in order. */ files: ManifestFile[]; [field: string]: unknown; } +/** The value of one option. true stands for a flag with no value, and an array repeats the flag. */ export type OptionValue = string | number | boolean | ReadonlyArray; +/** What a set is made of, and where it is kept. Name a preset, a recipe or the options of a set. */ export interface FileSetSpec { - /** A preset, as tfg preset list names it. */ + /** A preset, as npx tfg preset list names them, such as upload-validation. */ preset?: string; - /** A recipe file. A relative path is taken from the directory the tests were started in. */ + /** + * A recipe file. A relative path is taken from the directory the tests were started in, + * so path.join(__dirname, 'uploads.yaml') names one beside the spec file. In an ES module, + * import.meta.dirname takes the place of __dirname. + */ recipe?: string; - /** Flags of tfg generate and of the preset, without the dashes: { limit: '2mb', seed: 7, clean: true }. */ + /** + * The parameters of the preset and the flags of tfg generate, without the dashes, such as + * { limit: '2mb', bulk: 0, seed: 7 }. npx tfg preset show lists what a preset takes. + */ options?: Record; - /** Where sets are written. TFG_CACHE_DIR, or node_modules/.cache/testing-files-generator, or the system's temporary directory. */ + /** Where sets are kept. TFG_CACHE_DIR, or node_modules/.cache/testing-files-generator, or the system's temporary directory. */ cacheDir?: string; /** The tfg binary to run. TFG_BINARY, or the one this package installed. */ binary?: string; } +/** Which files to take from a set. Each field takes one name or a list, and a file has to match every field given. */ export interface Filter { + /** The group, such as bulk-upload. */ group?: string | readonly string[]; + /** The recipe target. */ target?: string | readonly string[]; + /** The expected outcome. */ outcome?: Outcome | readonly Outcome[]; } +/** What the system under test did with a file: one word, or the outcome with the reason the system gave. */ export type Observed = | Exclude | { outcome: Exclude; reason?: string }; /** The shape test(title, details, body) takes in Playwright. */ export interface TestDetails { + /** The expected outcome and the group, such as @reject and @size-limit, for --grep. */ tag: string[]; + /** What the file expects and its purpose, which the report shows beside the test. */ annotation: Array<{ type: string; description: string }>; } +/** One file of a set, with what the system under test should do with it. */ export interface TestFile { + /** The file's whole entry in the manifest. */ readonly entry: ManifestFile; + /** The file's path within the set. */ readonly relativePath: string; + /** The file's name. */ readonly name: string; + /** The file's format, such as jpg. */ readonly format: string; + /** The file's exact size in bytes. */ readonly bytes: number; + /** The part of the set the file belongs to, or an empty string. */ readonly group: string; + /** The recipe target that made the file. */ readonly target: string; + /** One sentence on what the file is for and what it means when a system gets it wrong. A failure shows it. */ readonly purpose: string; + /** What the system under test should do with the file. */ readonly expected: Expected; + /** True for a file with no right answer. Skip it with test.skip(file.unspecified, file.whyUnspecified). */ readonly unspecified: boolean; + /** The sentence a skip shows for a file with no right answer. */ readonly whyUnspecified: string; - /** The file's path and what it is expected to meet. Unique within a set. */ + /** The file's path and what it expects, such as 2mb_over_1b.jpg - expected reject (size_limit). Unique within a set. */ readonly title: string; + /** Tags and annotations, as the second argument of test(title, details, body). */ readonly details: TestDetails; /** Writes the set the first time any of its files is asked for, and gives this file's absolute path. */ path(): Promise; - /** Throws when what the system did differs from what is expected, with the sentence that says what the difference means. */ + /** + * Compares what the system did with what the file expects, and throws when they differ, with the sentence + * that says what the difference means. A reason is compared only for a file to be turned away, and only when + * one is given. On a file with no right answer it always throws. + */ check(observed: Observed): void; } +/** A set of files, or a part of one. Every part shares the set's one write. */ export interface FileSet { + /** Every file of this set or part, in order. */ readonly files: readonly TestFile[]; + /** The manifest tfg gave for the plan of the whole set. Nothing in it is written yet. */ readonly plan: Manifest; + /** The files that match the filter. */ only(filter: Filter): FileSet; + /** The files that do not match the filter. */ except(filter: Filter): FileSet; - /** Writes the whole set once per process and gives its directory. */ + /** Writes the whole set once per process, or takes a set kept from an earlier run, and gives its directory. */ write(): Promise; + /** Writes the set if it is not written yet, and gives the absolute paths of this part's files. */ paths(): Promise; } -/** Plans a set of files, writing nothing. The files are written the first time one of them is asked for. */ +/** + * Plans a set of files, writing nothing, so tests can be declared from it while the spec file loads. + * The files are written the first time one of them is asked for. + */ export function fileSet(spec: FileSetSpec): FileSet; /** The tfg binary this package runs. */ export function binaryPath(): string; +/** tfg ending with a code other than nought, or stopped by a signal. */ export class TfgError extends Error { + /** The code tfg ended with, or the name of the signal that stopped it. */ readonly status: number | string; + /** The end of what tfg wrote to its error stream. */ readonly stderr: string; } diff --git a/npm/testing-files-generator/lib/binary.js b/npm/testing-files-generator/lib/binary.js index 5b51346c..d3248281 100644 --- a/npm/testing-files-generator/lib/binary.js +++ b/npm/testing-files-generator/lib/binary.js @@ -40,8 +40,8 @@ function binaryPath({ env = process.env, platform = process.platform, arch = pro if (!pkg) { throw new Error(`testing-files-generator has no binary for ${platform} ${arch}. ` + `It comes for ${known}. ` + - 'On another system, run tfg from its container image ghcr.io/donislawdev/tfg, ' + - 'or install it another way and set TFG_BINARY to its path.'); + 'On another system, run the tests in a Linux container, where npm installs the package for Linux, ' + + 'or install tfg another way and set TFG_BINARY to its path.'); } const exe = platform === 'win32' ? 'tfg.exe' : 'tfg'; try { diff --git a/npm/testing-files-generator/lib/fileset.js b/npm/testing-files-generator/lib/fileset.js index e72d3b87..663c106c 100644 --- a/npm/testing-files-generator/lib/fileset.js +++ b/npm/testing-files-generator/lib/fileset.js @@ -10,9 +10,10 @@ const { EventEmitter } = require('node:events'); const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); +const { inspect } = require('node:util'); const { binaryPath } = require('./binary.js'); -const { TfgError, run, runToFile } = require('./run.js'); +const { TfgError, ended, run, runToFile, said } = require('./run.js'); const { TestFile, outcomes } = require('./testfile.js'); // The manifest major this package reads. Another major is refused rather than @@ -55,20 +56,26 @@ function fileSet(spec = {}) { if (typeof spec !== 'object' || spec === null) { throw new TypeError('fileSet takes an object, for example { preset: \'upload-validation\', options: { limit: \'2mb\' } }.'); } + // Told from the flags and not from the spec: an option set to false or to an + // empty list adds no flag, and tfg asked for nothing answers in flags. + const args = argsOf(spec); + if (args.length === 1) { + throw new Error('fileSet was not told which files to write. Name a preset, a recipe or the options of a set, ' + + 'for example { preset: \'upload-validation\' }. npx tfg preset list lists the presets.'); + } const binary = spec.binary === undefined ? binaryPath() : named(spec.binary, 'binary', 'Point it at a tfg binary, or leave it out to use the one this package installs.'); if (spec.recipe) { named(path.resolve(spec.recipe), 'recipe', `A relative path is taken from the directory the tests started in, here ${process.cwd()}, ` + - 'so path.join(__dirname, ...) names a recipe beside the spec file.'); + 'so path.join(__dirname, ...) names a recipe beside the spec file, or path.join(import.meta.dirname, ...) in an ES module.'); } - const args = argsOf(spec); const cacheDir = path.resolve(spec.cacheDir ?? defaultCacheDir()); const recipe = spec.recipe ? fs.readFileSync(path.resolve(spec.recipe), 'utf8') : ''; const key = digest([identity(binary), args, recipe, cacheDir]); let source = sources.get(key); if (!source) { - source = new Source(binary, args, cacheDir, labelOf(spec)); + source = new Source(binary, args, cacheDir, spec); sources.set(key, source); } return new FileSet(source, source.files); @@ -127,6 +134,18 @@ function labelOf(spec) { return name.replace(/[^A-Za-z0-9_-]+/g, '-').slice(0, 40); } +// described is the call as the test wrote it, for a message - what makes the +// set, without where it is kept or which binary writes it. +function described(spec) { + const shown = {}; + for (const key of ['preset', 'recipe', 'options']) { + if (spec[key] !== undefined) { + shown[key] = spec[key]; + } + } + return `fileSet(${inspect(shown, { breakLength: Infinity, depth: Infinity })})`; +} + // identity is what tells two binaries apart without running them, so a new // build or another version plans again rather than reusing an old plan. function identity(binary) { @@ -145,17 +164,19 @@ function scratch(cacheDir) { // Source is one set: its plan, where it is written, and the one write a process // makes of it however many files ask. class Source { - constructor(binary, args, cacheDir, label) { + constructor(binary, args, cacheDir, spec) { this.binary = binary; this.args = args; this.cacheDir = cacheDir; + this.preset = spec.preset; + this.shown = described(spec); fs.mkdirSync(cacheDir, { recursive: true }); noteUnfinished(cacheDir); this.plan = this._planned(); // The recipe hash leaves out the seed and --clean, measured 2026-11-04, so // the arguments are part of the name and two seeds never share a set. const run = this.plan.run; - this.dir = path.join(cacheDir, `${label}-${digest([args, run.recipe_hash, run.seed, this.plan.tool]).slice(0, 20)}`); + this.dir = path.join(cacheDir, `${labelOf(spec)}-${digest([args, run.recipe_hash, run.seed, this.plan.tool]).slice(0, 20)}`); this.files = this.plan.files.map((entry) => new TestFile(this, entry)); this._written = undefined; } @@ -169,6 +190,8 @@ class Source { try { runToFile(this.binary, [...this.args, `--out=${scratch(this.cacheDir)}`, '--dry-run', '--json'], out); plan = JSON.parse(fs.readFileSync(out, 'utf8')); + } catch (err) { + throw err instanceof TfgError ? refusedPlan(this.shown, this.preset, err) : err; } finally { fs.rmSync(out, { force: true }); } @@ -252,6 +275,29 @@ class Source { } } +// refusedPlan tells a plan tfg refused in the words the test was written in. +// The command this package ran names a scratch directory and flags nobody +// wrote, and an option tfg does not know brings the whole help of tfg +// generate, every flag behind a dash, where one sentence names the mistake. +// Everything tfg said stays on the error, in stderr. The help lists every +// flag of generate, and a preset or a recipe refuses some of them, so the +// list is not offered as what this call takes. +function refusedPlan(shown, preset, err) { + const unknown = /^flag provided but not defined: -+(\S+)/m.exec(err.stderr); + const known = [...err.stderr.matchAll(/^\s+-([a-z][a-z0-9-]*)\b/gm)] + .map((m) => m[1]) + .filter((name) => !Object.hasOwn(reserved, name)); + if (unknown && known.length > 0) { + const help = preset + ? `npx tfg preset show ${preset} says what the preset's own options do, and npx tfg generate --help the rest.` + : 'npx tfg generate --help says what each one does.'; + return new TfgError([], err.status, err.stderr, + `${shown} gives the option ${unknown[1]}, and tfg generate has no flag of that name. ` + + `Its flags are ${known.join(', ')}. Not every one goes with every set, and tfg says why when one does not. ${help}`); + } + return new TfgError([], err.status, err.stderr, `${shown} could not be planned - tfg ${ended(err.status)}.\n${said(err.stderr)}`); +} + // readable refuses a manifest this package cannot read for what it says. function readable(plan) { const version = String(plan?.manifest_version ?? ''); diff --git a/npm/testing-files-generator/lib/run.js b/npm/testing-files-generator/lib/run.js index 37ed53d0..dbea3fdf 100644 --- a/npm/testing-files-generator/lib/run.js +++ b/npm/testing-files-generator/lib/run.js @@ -11,20 +11,29 @@ const fs = require('node:fs'); // refusals are a few lines, and a run of many files can print a note for each. const keptError = 64 * 1024; +// ended says how tfg stopped - with a code, or by a signal named as it is. +function ended(status) { + return typeof status === 'string' ? `was stopped by ${status}` : `ended with code ${status}`; +} + // TfgError is tfg ending with a code other than nought, or stopped by a signal, // whose name is then the status. The message carries what tfg said, because -// its refusals already name the value and the way out. +// its refusals already name the value and the way out. A caller that can say +// what went wrong in the words the test was written in passes its own message, +// and the status and everything tfg said stay on the error. class TfgError extends Error { - constructor(args, status, stderr) { - const said = stderr.trim() || 'tfg said nothing about why.'; - const ended = typeof status === 'string' ? `was stopped by ${status}` : `ended with code ${status}`; - super(`tfg ${args.join(' ')} ${ended}.\n${said}`); + constructor(args, status, stderr, message = `tfg ${args.join(' ')} ${ended(status)}.\n${said(stderr)}`) { + super(message); this.name = 'TfgError'; this.status = status; this.stderr = stderr; } } +function said(stderr) { + return stderr.trim() || 'tfg said nothing about why.'; +} + function notStarted(binary, err) { return new Error(`tfg could not be started from ${binary}: ${err.message}`); } @@ -85,4 +94,4 @@ function run(binary, args) { }); } -module.exports = { TfgError, run, runToFile }; +module.exports = { TfgError, ended, run, runToFile, said }; diff --git a/npm/testing-files-generator/test/binary.test.js b/npm/testing-files-generator/test/binary.test.js index c5cfffef..d0d8b993 100644 --- a/npm/testing-files-generator/test/binary.test.js +++ b/npm/testing-files-generator/test/binary.test.js @@ -33,7 +33,7 @@ test('each platform a release builds resolves to its own package', () => { test('an Intel Mac is told what it can do instead', () => { assert.throws(() => binaryPath({ env: {}, platform: 'darwin', arch: 'x64', resolve: nothing }), (err) => err.message.includes('no binary for darwin x64') && err.message.includes('TFG_BINARY') && - err.message.includes('ghcr.io/donislawdev/tfg') && err.message.includes('darwin arm64')); + err.message.includes('in a Linux container') && err.message.includes('darwin arm64')); }); test('a platform package left out by --omit=optional is named, with the way back', () => { diff --git a/npm/testing-files-generator/test/fileset.test.js b/npm/testing-files-generator/test/fileset.test.js index 69b953c2..70b88712 100644 --- a/npm/testing-files-generator/test/fileset.test.js +++ b/npm/testing-files-generator/test/fileset.test.js @@ -150,7 +150,8 @@ test('a binary or a recipe the options name and that is not there is named, with assert.throws(() => fileSet({ ...small, binary: missing, cacheDir }), (err) => err.message.includes(`The binary option names ${missing}, and there is no file there.`) && err.message.includes('leave it out')); assert.throws(() => fileSet({ recipe: 'no-such-recipe.yaml', binary: tfg(), cacheDir }), - (err) => err.message.includes(path.resolve('no-such-recipe.yaml')) && err.message.includes('path.join(__dirname')); + (err) => err.message.includes(path.resolve('no-such-recipe.yaml')) && err.message.includes('path.join(__dirname') && + err.message.includes('path.join(import.meta.dirname')); }); test('a set a stopped run left behind is named once outside the workers, and never used', async (t) => { @@ -174,6 +175,43 @@ test('what tfg refuses reaches the test with its own words and code', (t) => { (err) => err instanceof TfgError && err.status === 4 && /smallest PDF/.test(err.message)); }); +test('a refused plan is told in the words of the call, without the flags this package adds', (t) => { + const cacheDir = tempDir(t); + assert.throws(() => plannedIn(cacheDir, { preset: 'no-such-preset' }), + (err) => err instanceof TfgError && err.status === 2 && + err.message.startsWith("fileSet({ preset: 'no-such-preset' }) could not be planned - tfg ended with code 2.\n") && + /unknown preset "no-such-preset"/.test(err.message) && !/dry-run|--json|\.tmp-/.test(err.message)); +}); + +test('an option tfg does not know is named with the flags tfg has, and what tfg said stays on the error', (t) => { + const cacheDir = tempDir(t); + const spec = { ...small, options: { ...small.options, limt: '2mb' } }; + assert.throws(() => plannedIn(cacheDir, spec), (err) => { + assert.ok(err instanceof TfgError, err); + assert.equal(err.status, 2); + assert.ok(err.message.includes('gives the option limt, and tfg generate has no flag of that name.'), err.message); + assert.ok(!err.message.includes('flag provided'), 'the help of tfg is in the message'); + assert.ok(err.stderr.includes('flag provided but not defined: -limt'), err.stderr); + const taken = /Its flags are ([^.]+)\. Not every one goes with every set/.exec(err.message)?.[1].split(', ') ?? []; + for (const name of ['limit', 'far-over', 'bulk', 'seed']) { + assert.ok(taken.includes(name), `${name} is not among ${taken}`); + } + for (const name of ['out', 'json', 'dry-run', 'preset']) { + assert.ok(!taken.includes(name), `${name} is set by this package and is offered as an option`); + } + assert.ok(err.message.includes('npx tfg preset show upload-validation'), err.message); + return true; + }); +}); + +test('a call that names no files to write is told so before tfg runs', (t) => { + const cacheDir = tempDir(t); + for (const spec of [{}, { options: {} }, { options: { clean: false } }, { options: { set: [] } }]) { + assert.throws(() => fileSet({ ...spec, binary: tfg(), cacheDir }), /fileSet was not told which files to write/); + } + assert.deepEqual(entries(cacheDir), [], 'something ran'); +}); + test('the flags this package sets itself cannot be set again', (t) => { const cacheDir = tempDir(t); for (const flag of ['out', 'json', 'dry-run', 'preset']) {