From 7f12ae8970a14309a2686dc0562ed22b1cd13add Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Wed, 4 Nov 2026 10:26:04 +0100 Subject: [PATCH 1/2] The recipe reader counts the run's ceiling on files before it builds anything - A recipe whose targets together ask for more than a million files is refused while it is read, at the target that crosses the ceiling, in the words the engine used. The engine was the first to count the run, after every target's list of sizes had been built: tfg validate of fifty targets of a million took 2.5 GB to refuse it, and now takes 9 MiB. - A target the run has no room left for is still read for its other problems, but its list of sizes is not built. The ceiling is reported beside the recipe's other problems, and in a recipe built on a preset it names the file's own target rather than one counted past the preset's. - A target's list of sizes is reserved at its length rather than grown, so reading a million files allocates 8 B a file instead of 40. The recipe screen paid that on every key for every batch. - A recipe with a size under its format's minimum and a run over the ceiling now ends with code 3, the way every problem the reader finds does. - The sentence lives in core, so the reader and the engine say it once. The engine keeps its own count for targets handed to it directly, and a guard asks it there. Co-Authored-By: Claude Opus 5.5 --- .github/race-weights | 1 - CHANGELOG.md | 12 + internal/core/limits.go | 13 ++ internal/engine/engine.go | 18 +- internal/guard/mutation/entries.py | 87 ++++++- internal/guard/numberlimits_test.go | 32 ++- internal/guard/recipescreen_test.go | 24 +- internal/guard/refusaladdress_test.go | 33 +-- internal/guard/runceiling_test.go | 321 ++++++++++++++++++++++++++ internal/gui/text/locale/said/en.json | 34 +-- internal/gui/text/locale/said/pl.json | 6 +- internal/recipe/recipe.go | 34 ++- internal/recipe/target.go | 116 +++++++--- 13 files changed, 610 insertions(+), 121 deletions(-) create mode 100644 internal/guard/runceiling_test.go diff --git a/.github/race-weights b/.github/race-weights index 8bb69854..4fb51a5e 100644 --- a/.github/race-weights +++ b/.github/race-weights @@ -36,7 +36,6 @@ unlisted 0.58 35.5 TestJxlStillFitsWhenTheQualityIsNotTheOneTheCeilingsWereMeasuredAt 30.8 TestOurOwnGeneratorsHaveNotDrifted 29.9 TestTheSameSeedGivesTheSameBytes -27.7 TestTheCeilingCountsTheWholeRunNotOneTarget 26.8 TestEveryJxlRungStillFitsInsideItsDeclaredCeiling 26.6 TestTheMinimumStandsWhateverTheSeedIsUnderEveryDeclaredChoice 24.9 TestEveryEMLSettingSurvivesItsReferenceTool diff --git a/CHANGELOG.md b/CHANGELOG.md index 578b8169..c1363edf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,9 +49,21 @@ because it turns other people's test suites red. `--software-gl`, moves from 26.2.0 to 26.2.4, built with LLVM 23.1.2. Its licences are the same. `THIRD-PARTY-NOTICES.md` names the new files and their sums. +- **A recipe past the ceiling of a million files ends with code 3 even when a + target also asks for a size its format cannot make.** That size used to be + refused first, with code 4, and the ceiling came out only once it was + fixed. The ceiling is now found while the recipe is read, and like every + problem found there it comes first. ### Fixed +- **A recipe asking for more than a million files in all is refused before + it takes the memory.** Targets were held to a million each and the run only + once all their sizes were laid out, so `tfg validate` took 2.5 GB to refuse + fifty targets of a million, and the recipe screen spent 40 MB a key on each + such batch. The total is now counted as the recipe is read, beside its other + problems, and `validate --json` places it at the file's own target in a + recipe built on a preset. Reading takes a fifth of the memory. - **Cancel answers at once, and closing the window during a run no longer freezes it.** Both used to wait for the write in progress with the window standing still, which on a slow or network disk could last long enough for diff --git a/internal/core/limits.go b/internal/core/limits.go index ff98bcae..70fe1d7e 100644 --- a/internal/core/limits.go +++ b/internal/core/limits.go @@ -64,6 +64,19 @@ var ( "Or give the run more memory through the memory setting") ) +// TooManyFilesInRun is what a run past MaxFilesPerRun asks for: the files it +// came to at the target that crossed the ceiling, across all the targets it +// has. +// +// Both the recipe reader and the engine count the run, and a recipe refused by +// either reads the same - so the sentence is written here, once, rather than +// beside each of them (O302). +func TooManyFilesInRun(files, targets int) Said { + return Says("core.TooManyFilesInRun", "this run asks for %s across %s", + A("Files", SaysN("core.FilesInRun", "%d file", "%d files", A("Count", files))), + A("Targets", SaysN("core.TargetsInRun", "%d target", "%d targets", A("Count", targets)))) +} + // PlanTooLargeWhy is the same refusal for the quantity that actually runs out, // in the words of where its ceiling came from. // diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 4a5d49de..595b39c9 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -474,15 +474,19 @@ func PlanContext(ctx context.Context, targets []Target, opt Options) ([]PlannedF // so - but a refusal a window cannot place is a refusal at the // foot of a form with twenty batches above it and nothing marked. // - // Reachable only from here, and measured on 2026-08-25: the recipe - // reader bounds each target on its own, so three batches of four - // hundred thousand pass it and the total is refused here. Before - // this line "validate --json" carried no "at" for it. + // No surface reaches this any more. The recipe reader counts the + // same total since O302 and refuses first, in the same words, so + // a recipe never arrives here past the ceiling - and the flag + // path and the Generate screen are one target, bounded on its + // own. It stays because Plan is + // handed targets by whoever calls it, and planCapacity reserves + // nothing past the ceiling on the word of this refusal. Its + // position counts the merged list, which for a recipe built on a + // preset is not the file's (measured on 2026-11-04) - the + // reader's is. return nil, &RecipeError{ Setting: core.TargetAddress(i+1, SettingCount), - Detail: core.Says("engine.TooManyFiles", "this run asks for %s across %s", - core.A("Files", core.SaysN("engine.Files", "%d file", "%d files", core.A("Count", totalFiles))), - core.A("Targets", core.SaysN("engine.Targets", "%d target", "%d targets", core.A("Count", len(targets))))), + Detail: core.TooManyFilesInRun(totalFiles, len(targets)), Because: core.TooManyFilesWhy, Remedy: core.TooManyFilesFix} } diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index 5f91e058..3b59cccd 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -2542,7 +2542,9 @@ ), ( # The recipe reader took the worse of the two failures: it grew the list - # one entry at a time and reached a 13 GB allocation. + # one entry at a time and reached a 13 GB allocation. Since O302 the + # mutated count reaches no allocation - the run's ceiling turns the same + # recipe away and builds no list - so the guard asks which refusal it got. "the recipe reader stops asking whether the count can be planned", "internal/recipe/target.go", # Carries the line under it since 2026-08-20. The container entry gained @@ -2550,8 +2552,8 @@ # an ambiguous pattern is how a mutation quietly starts proving a # different guard than the one it names. The two are told apart by which # address they build: `where` for the target, `at` for the entry. - "case n > core.MaxFilesPerRun:\n\t\t\tp.add(where.of(\"count\")", - "case n > core.MaxFilesPerRun && false:\n\t\t\tp.add(where.of(\"count\")", + "case n > core.MaxFilesPerRun:\n\t\tp.add(where.of(\"count\")", + "case n > core.MaxFilesPerRun && false:\n\t\tp.add(where.of(\"count\")", "TestAnImpossibleFileCountInARecipeIsRefused", ), ( @@ -2568,12 +2570,84 @@ "TestAnImpossibleFileCountInsideAContainerIsRefused", ), ( + # Asked of the engine directly since O302. The recipe reader counts the + # run first, so the guard asking validate about pieces never reached + # this line again. "the ceiling is applied per target, so writing the number out in pieces gets past it", "internal/engine/engine.go", "totalFiles += len(t.Sizes)", "totalFiles = len(t.Sizes)", + "TestTheEngineCountsTheWholeRunOfTargetsHandedToItDirectly", + ), + ( + # The same break on the reader's side (O302): the targets before this + # one are forgotten, so twelve of 90 000 pass one at a time. + "the recipe reader counts each target against the run's ceiling alone", + "internal/recipe/recipe.go", + "planned = min(planned+files, core.MaxFilesPerRun+1)", + "planned = min(files, core.MaxFilesPerRun+1)", "TestTheCeilingCountsTheWholeRunNotOneTarget", ), + ( + # Asked beside another problem of the recipe, so the engine never runs + # and cannot stand in for the reader - which it does for every guard + # asking validate about the ceiling alone. + "the recipe reader stops refusing a run past the ceiling", + "internal/recipe/recipe.go", + "if room >= 0 && files > room {", + "if false && room >= 0 && files > room {", + "TestARunPastTheCeilingIsReportedBesideTheRecipesOtherProblems", + ), + ( + "the run's ceiling refuses a run of exactly a million", + "internal/recipe/recipe.go", + "if room >= 0 && files > room {", + "if room >= 0 && files >= room {", + "TestTheRunCeilingSitsExactlyAtAMillion", + ), + ( + # The engine's address: the merged list, preset's targets first. + "the run's ceiling names the target by the merged list rather than the file", + "internal/recipe/recipe.go", + "p.add(at(t.ID).of(KeyCount), core.TooManyFilesInRun(planned+files, len(raw.Targets)),", + "p.add(targetSpot(i, t.ID).of(KeyCount), core.TooManyFilesInRun(planned+files, len(raw.Targets)),", + "TestTheTargetThatCrossesTheRunCeilingIsPlacedInTheFile", + ), + ( + "the run's ceiling marks the size of the batch that crossed it rather than its count", + "internal/recipe/recipe.go", + "p.add(at(t.ID).of(KeyCount), core.TooManyFilesInRun(planned+files, len(raw.Targets)),", + "p.add(at(t.ID).of(KeySize), core.TooManyFilesInRun(planned+files, len(raw.Targets)),", + "TestARefusalAboutTheWholeRunMarksTheBatchThatCausedIt", + ), + ( + "a target past the run's ceiling has its list built before it is refused", + "internal/recipe/target.go", + "\tif count > room {\n\t\tbuilt = 0\n\t}", + "\tif false {\n\t\tbuilt = 0\n\t}", + "TestATargetPastTheRunCeilingIsNotBuilt", + ), + ( + "a target's list of sizes is grown one file at a time again", + "internal/recipe/target.go", + "\tsizes := make([]int64, count)\n\tfor i := range sizes {\n\t\tsizes[i] = each\n\t}", + "\tvar sizes []int64\n\tfor range count {\n\t\tsizes = append(sizes, each)\n\t}", + "TestATargetsListOfSizesIsReservedAtItsLength", + ), + ( + "a boundary set counts as the one file of the default count against the run's ceiling", + "internal/recipe/target.go", + "\tif rt.Boundary != nil {\n\t\tfiles = len(t.Sizes)\n\t}", + "\tif false {\n\t\tfiles = len(t.Sizes)\n\t}", + "TestTheRunCeilingSitsExactlyAtAMillion", + ), + ( + "a count refused on its own goes back to asking for the default", + "internal/recipe/target.go", + "\t\treturn int(n)\n\t}\n\treturn 0\n}", + "\t\treturn int(n)\n\t}\n\treturn DefaultCount\n}", + "TestTheRunCeilingSitsExactlyAtAMillion", + ), ( # The other direction. A ceiling low enough to refuse ordinary work # would satisfy every guard above and break the tool for everybody. @@ -5391,18 +5465,21 @@ "TestTheChangelogHasOneHeadingPerKindOfChange", ), ( + # Both asked of the engine directly since O302 - the window's batch + # screen reaches the reader's refusal now, and the guard over it moved + # with it (TestARefusalAboutTheWholeRunMarksTheBatchThatCausedIt). "the ceiling refusal stops naming the batch that caused it", "internal/engine/engine.go", "\t\t\t\tSetting: core.TargetAddress(i+1, SettingCount),", '\t\t\t\tSetting: "",', - "TestARefusalFromTheEngineAboutTheWholeRunMarksTheBatchThatCausedIt", + "TestTheEngineCountsTheWholeRunOfTargetsHandedToItDirectly", ), ( "the ceiling refusal names the batch before the one that caused it", "internal/engine/engine.go", "\t\t\t\tSetting: core.TargetAddress(i+1, SettingCount),", "\t\t\t\tSetting: core.TargetAddress(i, SettingCount),", - "TestARefusalFromTheEngineAboutTheWholeRunMarksTheBatchThatCausedIt", + "TestTheEngineCountsTheWholeRunOfTargetsHandedToItDirectly", ), ( "the report stops carrying the address of a refusal from below the reader", diff --git a/internal/guard/numberlimits_test.go b/internal/guard/numberlimits_test.go index 281033b6..9af4602f 100644 --- a/internal/guard/numberlimits_test.go +++ b/internal/guard/numberlimits_test.go @@ -1,13 +1,13 @@ package guard import ( - "path/filepath" "strconv" "strings" "testing" "github.com/donislawdev/TestingFilesGenerator/internal/cli" "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/recipe" ) // Two numbers a person can write that the tool could not honour, and used to @@ -90,6 +90,14 @@ targets: if strings.Contains(errOut, "fatal error") || strings.Contains(errOut, "out of memory") { t.Errorf("the allocator was reached instead of the check:\n%s", errOut) } + // Refused as the target's own count, not only as the run's. The run's + // ceiling turns this recipe away as well since O302, but it is the + // target's ceiling that keeps every term of the run's sum within a million + // - without it a count like this one, after the run is already past the + // ceiling, wraps the sum negative and every target after it is built again. + if !strings.Contains(errOut, "split the target across several recipes") { + t.Errorf("the count was not refused as the target's own:\n%s", errOut) + } if stdout != "" { t.Errorf("a failed run wrote to stdout:\n%s", stdout) } @@ -219,8 +227,13 @@ func TestABoundaryThatCannotHoldItsOwnSetIsRefused(t *testing.T) { // A run spread across many targets reaches the same ceiling. Checking only the // count of one target would leave the whole thing reachable by writing the // number out in pieces. +// +// Asked of the recipe reader since O302, which counts the run before it builds +// a target's list. Through validate this was asked of the reader and of the +// engine under it at once, and a refusal from either passed - so either one +// could lose its count and leave this green, and the engine planned 990 000 +// files to answer it (19 s, 28 s under the race detector). func TestTheCeilingCountsTheWholeRunNotOneTarget(t *testing.T) { - dir := t.TempDir() var b strings.Builder b.WriteString("version: 1\ntargets:\n") // Twelve targets of 90 000 files come to 1 080 000, which is over the @@ -230,16 +243,17 @@ func TestTheCeilingCountsTheWholeRunNotOneTarget(t *testing.T) { b.WriteString(string(rune('a' + i))) b.WriteString("\n format: txt\n size: 1b\n count: 90000\n") } - path := writeRecipe(t, dir, b.String()) - code, stdout, errOut := run(t, "validate", path) - if code == cli.ExitOK { - t.Fatalf("a run of over a million files spread across targets was accepted:\n%s", stdout) + _, err := recipe.Parse([]byte(b.String()), "pieces") + if err == nil { + t.Fatal("a run of over a million files spread across targets was accepted") } - if !strings.Contains(errOut, "1000000") { - t.Errorf("the refusal does not name the ceiling:\n%s", errOut) + said := err.Error() + for _, want := range []string{"1080000 files across 12 targets", "1000000"} { + if !strings.Contains(said, want) { + t.Errorf("the refusal does not say %q:\n%s", want, said) + } } - _ = filepath.Join } // The refusal names the ceiling the rule really uses. diff --git a/internal/guard/recipescreen_test.go b/internal/guard/recipescreen_test.go index e7f2d2dc..1167a628 100644 --- a/internal/guard/recipescreen_test.go +++ b/internal/guard/recipescreen_test.go @@ -518,20 +518,20 @@ func waitForNamedManifest(t *testing.T, settle func(), dir, name string) { manifestAfter(t, settle, dir, name) } -// A refusal from the ENGINE marks the batch it is about, the same as one from -// the recipe reader. +// A refusal about the whole run marks the batch that took it past the ceiling. // -// The sibling above covers the reader. This one covers the layer under it, and -// they are not the same question: the reader bounds each batch on its own, so -// two batches of six hundred thousand files each pass it and the total is -// refused by the engine. Measured on 2026-08-25, before the address existed: -// the refusal arrived with a sentence and nothing else, so a screen with twenty -// batches said the run could not go and marked nothing at all. +// The sibling above covers a refusal about one batch's own value. This one is +// about a fact of the run, which no single box holds: two batches each within +// the count ceiling come to more than a run may plan. Measured on 2026-08-25, +// before the address existed: the refusal arrived with a sentence and nothing +// else, so a screen with twenty batches said the run could not go and marked +// nothing at all. // -// The count is deliberately spread over two batches with the big one second. -// The engine tests the running total before planning a batch's files, so this -// reaches the refusal having planned one file rather than a million. -func TestARefusalFromTheEngineAboutTheWholeRunMarksTheBatchThatCausedIt(t *testing.T) { +// The engine refused this until O302, and the guard was named for it. The +// recipe reader counts the run now and refuses first, so this asks the reader - +// what the engine still says when it is handed targets directly is asked in +// runceiling_test.go. +func TestARefusalAboutTheWholeRunMarksTheBatchThatCausedIt(t *testing.T) { screen := window.NewRecipe(newFakeHost(t)) body := screen.Object() pressNamed(t, body, text.ButtonAddBatch()) diff --git a/internal/guard/refusaladdress_test.go b/internal/guard/refusaladdress_test.go index 23d6936f..72105974 100644 --- a/internal/guard/refusaladdress_test.go +++ b/internal/guard/refusaladdress_test.go @@ -479,11 +479,12 @@ func missingPart(why, fix string) string { // // Two things, and the second is what keeps the first honest. // -// A refusal the reader never sees is refused by the engine or by a format: the -// ceiling on files is checked on the total, so a recipe whose targets each pass -// the reader is refused underneath it. Until 2026-08-25 those arrived with a -// sentence and no address, so a script grouping a report by field had nothing -// to group by. +// A refusal the reader never sees is refused by the engine or by a format - a +// size the format cannot deliver, a name the host cannot store. Until +// 2026-08-25 those arrived with a sentence and no address, so a script grouping +// a report by field had nothing to group by. The ceiling on files was the first +// case here until O302 moved the run's count into the reader, so it is no +// longer refused from below - where it lands is asked in runceiling_test.go. // // And an address is only worth carrying if it says where. A refusal that names // "size" and not which of twenty targets asked points at all of them while @@ -522,27 +523,6 @@ func TestEveryAddressTheReportCarriesSaysWhereItIs(t *testing.T) { return parsed.Problems } - // The big target second, because the engine tests the running total before - // planning a target's files - so this refuses having planned one file - // rather than a million. - const ceiling = `version: 1 -targets: - - id: a - format: txt - count: 1 - size: 1kb - - id: b - format: txt - count: 1000000 - size: 1kb -` - if got := report(t, ceiling)[0].At; got != core.TargetAddress(2, recipe.KeyCount) { - t.Errorf("the ceiling was reported at %q and the target that crossed it is %q.\n"+ - "Reason: the ceiling belongs to the run, but the box somebody can change belongs to\n"+ - "a target - a report that cannot say which one leaves a script with the sentence and\n"+ - "nothing else.", got, core.TargetAddress(2, recipe.KeyCount)) - } - // One refusal per layer that can produce one below the reader: a format // refusing a size, the engine refusing a name, the engine refusing the name // of the manifest. The last of those is the one that says a document @@ -635,7 +615,6 @@ func TestARefusalFromBelowTheReaderArrivesInItsThreeParts(t *testing.T) { // so these parts come from one join rule rather than a sentence each, // and the join has to leave the sentence reading as it always did. {"a name the host cannot store", "version: 1\ntargets:\n - id: a\n format: txt\n size: 1kb\n name: \"a core.MaxFilesPerRun { + t.Errorf("reading a recipe whose second target takes the run past the ceiling allocated %d B, over "+ + "a byte a file of that target. Its list was built before the run was refused - which is what "+ + "took fifty such targets to 2.5 GB", notBuilt) + } +} + +// A list of sizes is reserved at its length, not grown to it. +// +// Grown one file at a time, a million files took 40 B a file from the +// collector for a list of eight - the window's batch screen paid that on every +// key, and the Generate screen, which reserves (engine.Uniform), paid eight. A +// list that was grown ends with room to spare, so one whose capacity is its +// length was reserved - the reading planreserve_test.go makes of the plan. +// Each of the three ways a count turns into a list is asked. +func TestATargetsListOfSizesIsReservedAtItsLength(t *testing.T) { + const count = 1000 + src := fmt.Sprintf(`version: 1 +targets: + - id: sized + format: txt + size: 1kb + count: %[1]d + - id: ranged + format: txt + size-range: 1kb-2kb + count: %[1]d + - id: holding + format: zip + count: %[1]d + contains: + - format: txt + count: 1 + size: 10 +`, count) + + rec, err := recipe.Parse([]byte(src), "reserved") + if err != nil { + t.Fatalf("the recipe was refused: %v", err) + } + for _, target := range rec.Targets { + if len(target.Sizes) != count { + t.Fatalf("target %q has %d sizes, expected %d - this is not the list it names", target.ID, len(target.Sizes), count) + } + if cap(target.Sizes) != len(target.Sizes) { + t.Errorf("target %q has a list of %d sizes with room for %d, so it was grown rather than reserved. "+ + "Grown, a million files cost 40 B a file to read instead of 8, on every key of the batch screen", + target.ID, len(target.Sizes), cap(target.Sizes)) + } + } +} + +// The run's ceiling sits exactly at a million, counted the way the engine counts. +// +// A million across two targets is legal, and one file more is not. A boundary +// set is three files whatever the count says, so it is asked at the edge as +// well: counted as the one file of the default count, it would let a run of a +// million and two files through to the engine. And a count +// refused on its own asks for nothing - counted as the default, it would take a +// million beside it past the ceiling and say one mistake twice. +func TestTheRunCeilingSitsExactlyAtAMillion(t *testing.T) { + boundary := func(before int) []byte { + return []byte(fmt.Sprintf("version: 1\ntargets:\n - id: t1\n format: txt\n size: 1b\n count: %d\n"+ + " - id: t2\n format: txt\n boundary: 1kb\n", before)) + } + for name, src := range map[string][]byte{ + "a million in two targets": targetsOf(core.MaxFilesPerRun-1, 1), + "a boundary set that takes the run to a million": boundary(core.MaxFilesPerRun - 3), + } { + if _, err := recipe.Parse(src, name); err != nil { + t.Errorf("%s was refused: %v", name, err) + } + } + for name, src := range map[string][]byte{ + "a million and one in two targets": targetsOf(core.MaxFilesPerRun, 1), + "a boundary set that takes the run one file past it": boundary(core.MaxFilesPerRun - 2), + } { + t.Run(name, func(t *testing.T) { ceilingAt(t, src, 2) }) + } + + _, err := recipe.Parse(targetsOf(0, core.MaxFilesPerRun), "refused count") + var invalid *recipe.ValidationError + if !errors.As(err, &invalid) || len(invalid.Problems) != 1 || invalid.Problems[0].At != core.TargetAddress(1, recipe.KeyCount) { + t.Errorf("a count of 0 beside a target of a million should be refused once, for its own count: %v", err) + } +} + +// A run past the ceiling is reported together with whatever else is wrong. +// +// Every problem of a recipe arrives at once (RC7). While the engine counted the +// run, a recipe with any other problem was refused for that one alone, and the +// ceiling came out only once it was fixed - one more round for somebody who +// could have been told both. +func TestARunPastTheCeilingIsReportedBesideTheRecipesOtherProblems(t *testing.T) { + path := writeRecipe(t, t.TempDir(), `version: 1 +targets: + - id: small + format: txt + size: 1kb + count: 1 + properties: + no-such-property: 1 + - id: big + format: txt + size: 1kb + count: 1000000 +`) + code, stdout, errOut := run(t, "validate", "--json", path) + if code != cli.ExitRecipe { + t.Errorf("exit %d, expected %d:\n%s", code, cli.ExitRecipe, errOut) + } + if stdout != "" { + t.Errorf("a refused recipe wrote to stdout:\n%s", stdout) + } + at := map[string]string{} + for _, p := range reportedProblems(t, errOut) { + at[p.At] = p.What + } + if _, ok := at["targets[1].properties.no-such-property"]; !ok { + t.Errorf("the property nobody declared is not in the report, so this is not the case it names:\n%s", errOut) + } + if what := at[core.TargetAddress(2, recipe.KeyCount)]; !strings.Contains(what, "1000001 files across 2 targets") { + t.Errorf("the run past the ceiling is not reported beside the other problem at %s.\n"+ + "Reason: a person fixes the property, runs again and only then hears about the ceiling.\n%s", + core.TargetAddress(2, recipe.KeyCount), errOut) + } +} + +// The target that crosses the ceiling is named where the file has it, also in a +// recipe built on a preset. +// +// The engine counts the merged list, the preset's targets first, and knows +// nothing of where the file's own begin - so a recipe of one target on top of +// size-boundaries was refused at targets[8].count, a box neither the file nor +// the screen has. Measured on 2026-11-04. The reader knows the offset +// (spotOfTarget), and the plain recipe is asked beside it so the two halves +// cannot both be wrong in the same way. +func TestTheTargetThatCrossesTheRunCeilingIsPlacedInTheFile(t *testing.T) { + cases := []struct{ name, src string }{ + {"a recipe of its own", `version: 1 +targets: + - id: small + format: txt + size: 1kb + count: 1 + - id: big + format: txt + size: 1kb + count: 1000000 +`}, + {"a recipe built on a preset", `version: 1 +extends: preset:size-boundaries +with: + limit: 4mb + format: txt +targets: + - id: small + format: txt + size: 1kb + count: 1 + - id: big + format: txt + size: 1kb + count: 1000000 +`}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + path := writeRecipe(t, t.TempDir(), c.src) + code, _, errOut := run(t, "validate", "--json", path) + if code != cli.ExitRecipe { + t.Fatalf("exit %d, expected %d:\n%s", code, cli.ExitRecipe, errOut) + } + problems := reportedProblems(t, errOut) + if len(problems) != 1 || !strings.Contains(problems[0].What, "files across") { + t.Fatalf("expected the ceiling and nothing else:\n%s", errOut) + } + if want := core.TargetAddress(2, recipe.KeyCount); problems[0].At != want { + t.Errorf("the ceiling was reported at %q and the target of the file that crossed it is %q.\n"+ + "Reason: an address is the box a screen marks and the line a script points at, and one\n"+ + "counted through the preset's targets names neither.", problems[0].At, want) + } + }) + } +} + +// The engine counts the run of targets it is handed directly. +// +// Every surface reads a recipe first, and the reader counts the run since O302 +// - so no surface reaches this refusal any more, and the guards that asked it +// through validate and through the window ask the reader now. Plan is handed +// targets by whoever calls it, though, and planCapacity reserves nothing past +// the ceiling on the word of this refusal. The memory setting is there so a +// build that lost the refusal plans until the plan ceiling stops it, a few +// megabytes in, rather than a million files. +func TestTheEngineCountsTheWholeRunOfTargetsHandedToItDirectly(t *testing.T) { + targets := []engine.Target{ + {ID: "small", Format: "txt", Sizes: engine.Uniform(1, 1024)}, + {ID: "big", Format: "txt", Sizes: engine.Uniform(core.MaxFilesPerRun, 1024)}, + } + _, err := engine.Plan(targets, engine.Options{OutDir: t.TempDir(), ManifestName: "manifest.json", Memory: core.MemoryOf(32 << 20)}) + if err == nil { + t.Fatal("a run of one file more than the ceiling was planned") + } + if !strings.Contains(err.Error(), "1000001 files across 2 targets") { + t.Fatalf("the run was refused for something other than its count of files: %v", err) + } + var about interface{ AboutSetting() string } + if !errors.As(err, &about) || about.AboutSetting() != core.TargetAddress(2, engine.SettingCount) { + t.Errorf("the refusal is not addressed to the target that crossed the ceiling, %s: %v", + core.TargetAddress(2, engine.SettingCount), err) + } +} + +// targetsOf is a recipe of txt targets of 1 B, one for each count given. +func targetsOf(counts ...int) []byte { + var b strings.Builder + b.WriteString("version: 1\ntargets:\n") + for i, n := range counts { + fmt.Fprintf(&b, " - id: t%d\n format: txt\n size: 1b\n count: %d\n", i+1, n) + } + return []byte(b.String()) +} + +// readingCost is what reading src allocates, warmed once first so the figure +// is about the recipe and not about what the first reading sets up. +func readingCost(t *testing.T, src []byte) uint64 { + t.Helper() + _, _ = recipe.Parse(src, "warm") + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + _, _ = recipe.Parse(src, "measured") + runtime.ReadMemStats(&after) + return after.TotalAlloc - before.TotalAlloc +} + +// ceilingAt asks that the reader refuses src for the run's count of files, at +// the target in the given position. +func ceilingAt(t *testing.T, src []byte, position int) { + t.Helper() + _, err := recipe.Parse(src, "over") + var invalid *recipe.ValidationError + if !errors.As(err, &invalid) { + t.Fatalf("the reader did not refuse a run past the ceiling: %v", err) + } + want := core.TargetAddress(position, recipe.KeyCount) + for _, p := range invalid.Problems { + if p.At == want && strings.Contains(p.What.String(), "files across") { + return + } + } + t.Fatalf("the reader did not refuse the run's count of files at %s: %v", want, err) +} + +// reportedProblem is one problem of validate --json. +type reportedProblem struct { + What string `json:"what"` + At string `json:"at"` +} + +// reportedProblems reads the problems validate --json put out. +func reportedProblems(t *testing.T, report string) []reportedProblem { + t.Helper() + var parsed struct { + Problems []reportedProblem `json:"problems"` + } + if err := json.Unmarshal([]byte(report), &parsed); err != nil { + t.Fatalf("the report is not readable as JSON: %v\n%s", err, report) + } + return parsed.Problems +} diff --git a/internal/gui/text/locale/said/en.json b/internal/gui/text/locale/said/en.json index 9247585c..f236195f 100644 --- a/internal/gui/text/locale/said/en.json +++ b/internal/gui/text/locale/said/en.json @@ -625,6 +625,12 @@ "hash": "sha256-6dc38cd1cb85", "other": "{{.Error}}" }, + "core.FilesInRun": { + "description": "Said by internal/core - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Count}}, written the way the program writes them - keep each spelled exactly that way.", + "hash": "sha256-83589d643302", + "one": "{{.Count}} file", + "other": "{{.Count}} files" + }, "core.MachineMemoryNone": { "description": "Said by internal/core - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Bytes}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-76987096904d", @@ -730,6 +736,12 @@ "hash": "sha256-58d85993b3db", "other": "the sizes in this run add up to more than a number of bytes can hold, so the total cannot be measured and the free space check cannot be trusted. Ask for fewer files or smaller ones" }, + "core.TargetsInRun": { + "description": "Said by internal/core - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Count}}, written the way the program writes them - keep each spelled exactly that way.", + "hash": "sha256-7f9636c1f771", + "one": "{{.Count}} target", + "other": "{{.Count}} targets" + }, "core.TooManyFiles": { "description": "Said by internal/core - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Why}}, {{.Fix}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-a0fd66f30b3f", @@ -740,6 +752,11 @@ "hash": "sha256-f03079434408", "other": "Ask for fewer files, or split the work into several runs" }, + "core.TooManyFilesInRun": { + "description": "Said by internal/core - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Files}}, {{.Targets}}, written the way the program writes them - keep each spelled exactly that way.", + "hash": "sha256-a18427eed894", + "other": "this run asks for {{.Files}} across {{.Targets}}" + }, "core.TooManyFilesWhy": { "description": "Said by internal/core - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Most}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-84bf666cf3a3", @@ -961,12 +978,6 @@ "hash": "sha256-1e90995301d1", "other": "writing {{.File}}" }, - "engine.Files": { - "description": "Said by internal/engine - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Count}}, written the way the program writes them - keep each spelled exactly that way.", - "hash": "sha256-83589d643302", - "one": "{{.Count}} file", - "other": "{{.Count}} files" - }, "engine.FilesSoFar": { "description": "Said by internal/engine - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Count}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-83589d643302", @@ -1258,12 +1269,6 @@ "hash": "sha256-f4062d7dbd6c", "other": "target {{.Target}}" }, - "engine.Targets": { - "description": "Said by internal/engine - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Count}}, written the way the program writes them - keep each spelled exactly that way.", - "hash": "sha256-7f9636c1f771", - "one": "{{.Count}} target", - "other": "{{.Count}} targets" - }, "engine.TheInstructions": { "description": "Said by internal/engine - a refusal under the box it is about or at the foot of the form, or a note after a run.", "hash": "sha256-074853502278", @@ -1274,11 +1279,6 @@ "hash": "sha256-b802ab9f53ca", "other": "the manifest" }, - "engine.TooManyFiles": { - "description": "Said by internal/engine - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Files}}, {{.Targets}}, written the way the program writes them - keep each spelled exactly that way.", - "hash": "sha256-a18427eed894", - "other": "this run asks for {{.Files}} across {{.Targets}}" - }, "format.AAlreadyNeedsThatMuchBefore": { "description": "Said by internal/format/pdf - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Opts}}, {{.Carrying}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-c48d23d9a134", diff --git a/internal/gui/text/locale/said/pl.json b/internal/gui/text/locale/said/pl.json index 663c1ae0..46a60665 100644 --- a/internal/gui/text/locale/said/pl.json +++ b/internal/gui/text/locale/said/pl.json @@ -124,6 +124,7 @@ "core.DurationTooLong": { "hash": "sha256-06c88d98ae65", "other": "{{.Value}} jest za długi, żeby policzyć go w milisekundach" }, "core.DurationUnknownUnit": { "hash": "sha256-96f50400bd65", "other": "{{.Value}} ma nieznaną jednostkę {{.Unit}}: użyj h, m, s albo ms" }, "core.Error": { "hash": "sha256-6dc38cd1cb85", "other": "{{.Error}}" }, + "core.FilesInRun": { "hash": "sha256-83589d643302", "one": "{{.Count}} plik", "few": "{{.Count}} pliki", "many": "{{.Count}} plików", "other": "{{.Count}} pliku" }, "core.MachineMemoryNone": { "hash": "sha256-76987096904d", "other": "system podał, że ta maszyna ma {{.Bytes}} B pamięci" }, "core.MemoryEmpty": { "hash": "sha256-4a0a99260fc6", "other": "pole {{.Setting}} jest puste: wpisz auto, off albo rozmiar, na przykład 8gb" }, "core.MemoryNegative": { "hash": "sha256-87bca52970de", "other": "{{.Setting}} {{.Value}} jest ujemna: wpisz auto, off albo rozmiar, na przykład 8gb" }, @@ -145,8 +146,10 @@ "core.SizeTooLarge": { "hash": "sha256-dd73ed43bac3", "other": "{{.Setting}} {{.Value}} jest za duży, żeby wyrazić go w bajtach" }, "core.SizeUnknownUnit": { "hash": "sha256-3cd2ef1da79b", "other": "{{.Setting}} {{.Value}} ma nieznaną jednostkę {{.Unit}}: użyj b, kb, mb, gb albo tb, wszystkie liczone po 1024, a kib, mib, gib albo tib to to samo zapisane w pełni" }, "core.SizesOverflow": { "hash": "sha256-58d85993b3db", "other": "rozmiary w tym przebiegu sumują się do więcej, niż mieści liczba bajtów, więc sumy nie da się zmierzyć, a sprawdzeniu wolnego miejsca nie można ufać. Poproś o mniej plików albo mniejsze" }, + "core.TargetsInRun": { "hash": "sha256-7f9636c1f771", "one": "{{.Count}} partii", "few": "{{.Count}} partiach", "many": "{{.Count}} partiach", "other": "{{.Count}} partiach" }, "core.TooManyFiles": { "hash": "sha256-a0fd66f30b3f", "other": "{{.Why}}. {{.Fix}}" }, "core.TooManyFilesFix": { "hash": "sha256-f03079434408", "other": "Poproś o mniej plików albo podziel pracę na kilka przebiegów" }, + "core.TooManyFilesInRun": { "hash": "sha256-a18427eed894", "other": "ten przebieg prosi o {{.Files}} w {{.Targets}}" }, "core.TooManyFilesWhy": { "hash": "sha256-84bf666cf3a3", "other": "ta wersja planuje najwyżej {{.Most}} plików w jednym przebiegu, bo cały plan powstaje w pamięci, zanim cokolwiek zostanie zapisane - dzięki temu przebieg, który nie może się udać, jest odrzucany przed pierwszym bajtem" }, "csv.HasToBeOneOf": { "hash": "sha256-150fed02f771", "other": "musi to być {{.Values}}" }, "csv.MinimumHeader": { "hash": "sha256-002d78cd1305", "other": "tabela zawiera nagłówek i całe wiersze, a po jednym z nich potrzeba tyle" }, @@ -191,7 +194,6 @@ "engine.CrashPlanning": { "hash": "sha256-90f7d5da615f", "other": "planowania pliku" }, "engine.CrashWhile": { "hash": "sha256-071bf3d7823e", "other": "{{.While}}" }, "engine.CrashWriting": { "hash": "sha256-1e90995301d1", "other": "zapisu {{.File}}" }, - "engine.Files": { "hash": "sha256-83589d643302", "one": "{{.Count}} plik", "few": "{{.Count}} pliki", "many": "{{.Count}} plików", "other": "{{.Count}} pliku" }, "engine.FilesSoFar": { "hash": "sha256-83589d643302", "one": "{{.Count}} pliku", "few": "{{.Count}} plikach", "many": "{{.Count}} plikach", "other": "{{.Count}} plikach" }, "engine.GeneratorCrash": { "hash": "sha256-82716d977373", "other": "generator {{.Format}} zatrzymał się na błędzie wewnętrznym w trakcie {{.While}}: {{.Value}}. To wada tego narzędzia, a nie coś, o co przepis może poprosić. Zgłoś ją razem z ustawieniami, które do niej doprowadziły" }, "engine.IDTwice": { "hash": "sha256-724959681477", "other": "nazwa partii {{.ID}} występuje dwa razy" }, @@ -250,10 +252,8 @@ "engine.RunTooLarge": { "hash": "sha256-aff817760566", "other": "partia {{.Target}} doprowadza przebieg do rozmiaru zbyt dużego, żeby go zmierzyć" }, "engine.SizeMoved": { "hash": "sha256-e0c05c6f958b", "other": "Rozmiar wylosowany z zakresu nie jest taki, jaki ten format potrafi zapisać, więc użyto najbliższego, który potrafi. Plik dalej mieści się w zakresie, o który proszono." }, "engine.Target": { "hash": "sha256-f4062d7dbd6c", "other": "partia {{.Target}}" }, - "engine.Targets": { "hash": "sha256-7f9636c1f771", "one": "{{.Count}} partii", "few": "{{.Count}} partiach", "many": "{{.Count}} partiach", "other": "{{.Count}} partiach" }, "engine.TheInstructions": { "hash": "sha256-074853502278", "other": "instrukcje obok manifestu" }, "engine.TheManifest": { "hash": "sha256-b802ab9f53ca", "other": "manifest" }, - "engine.TooManyFiles": { "hash": "sha256-a18427eed894", "other": "ten przebieg prosi o {{.Files}} w {{.Targets}}" }, "format.AAlreadyNeedsThatMuchBefore": { "hash": "sha256-c48d23d9a134", "other": "{{.Opts}}{{.Carrying}} już potrzebuje tyle, zanim dojdzie jakiekolwiek dopełnienie" }, "format.ABMPStatesItsOwnSize": { "hash": "sha256-dbd63989b4da", "other": "BMP zapisuje swój rozmiar w polu czterobajtowym, więc ten format nie opisze tak dużego pliku" }, "format.AByteOrderMarkIsB": { "hash": "sha256-4efd8df69acd", "other": "znacznik kolejności bajtów ma {{.Mark}} B, a plik musi go pomieścić" }, diff --git a/internal/recipe/recipe.go b/internal/recipe/recipe.go index 4a90df0a..8b97077c 100644 --- a/internal/recipe/recipe.go +++ b/internal/recipe/recipe.go @@ -425,14 +425,42 @@ func (raw rawRecipe) validate(p *problems, fromPreset int) *Recipe { return rec } + rec.Targets = raw.readTargets(p, rec.Defaults, fromPreset) + return rec +} + +// readTargets reads every target in order, with what only the whole list can +// tell: an id used twice, and a run past the ceiling on files. +func (raw rawRecipe) readTargets(p *problems, def Defaults, fromPreset int) []Target { + var targets []Target + // Which target first used an id, by position in the merged list. The // position is what tells a clash with the preset's target apart from a // clash with another target of the file - and the two get different // words, because only one of them has a line the reader can change. seen := map[string]int{} + // The files of the targets read so far. The run's ceiling is counted here, + // across every target, rather than left to the engine: the engine is handed + // each target with its list of sizes already built, so fifty targets at the + // count ceiling took 2.5 GB to validate before it refused them (O302). Held + // one past the ceiling once crossed, so the room left stays negative from + // there on. It cannot wrap however many targets follow because no target + // adds more than the ceiling: a count past it is refused as the target's + // own and asks for nothing. + planned := 0 for i, rt := range raw.Targets { at := spotOfTarget(i, fromPreset) - t := rt.validate(p, at, rec.Defaults) + room := core.MaxFilesPerRun - planned + t, files := rt.validate(p, at, def, room) + // Refused at the target that takes the run past the ceiling, which is + // the one somebody can shrink, and in the engine's words. Once, because + // every target after it is past the ceiling as well, and saying so of + // each would bury the one that matters. + if room >= 0 && files > room { + p.add(at(t.ID).of(KeyCount), core.TooManyFilesInRun(planned+files, len(raw.Targets)), + core.TooManyFilesWhy, core.TooManyFilesFix) + } + planned = min(planned+files, core.MaxFilesPerRun+1) if t.ID != "" { if first, dup := seen[t.ID]; dup { usedTwice(p, at(t.ID), t.ID, first < fromPreset && i >= fromPreset) @@ -440,9 +468,9 @@ func (raw rawRecipe) validate(p *problems, fromPreset int) *Recipe { seen[t.ID] = i } } - rec.Targets = append(rec.Targets, t) + targets = append(targets, t) } - return rec + return targets } // usedTwice refuses the second target carrying an id, in one of two wordings: diff --git a/internal/recipe/target.go b/internal/recipe/target.go index a7ad38fe..16c69377 100644 --- a/internal/recipe/target.go +++ b/internal/recipe/target.go @@ -64,14 +64,19 @@ type rawTarget struct { // that put one answer in tfg formats and another in the generator once already. const DefaultCount = 1 -// validate reads one target and reports everything wrong with it. +// validate reads one target and reports everything wrong with it, and says how +// many files it asks for - the number the run's ceiling counts. // // at is where this target is, as a function of its id - see spotOfTarget. It // is a function rather than a position because the prose names a target by // its id as soon as one is read, and the id is read here. -func (rt rawTarget) validate(p *problems, at func(id string) spot, def Defaults) Target { +// +// room is how many files the run has left before its ceiling. A target asking +// for more is read for everything else, but its list of sizes is not built: +// the run is refused at it whatever its sizes say, and building the list first +// is the allocation the ceiling is there to prevent. +func (rt rawTarget) validate(p *problems, at func(id string) spot, def Defaults, room int) (Target, int) { t := Target{Label: def.Label} - count := DefaultCount where := at("") if id, ok := oneValue(p, where.of("id"), where.String()+" {setting}", "id: invoices", rt.ID); ok && id != "" { @@ -92,34 +97,16 @@ func (rt rawTarget) validate(p *problems, at func(id string) spot, def Defaults) t.Format = format } - if rt.Count != nil { - n, ok := rt.Count.number() - switch { - case !ok: - p.add(where.of("count"), core.Says("recipe.HasSettingOfWhichIsNot", "%s has {a} {setting} of %q, which is not a whole number", core.A("Where", where), core.A("Text", rt.Count.text)), - core.Says("recipe.SettingIsHowManyFilesThis", "{a} {setting} is how many files this target produces, so it is read exactly as written"), - core.Says("recipe.WriteADecimalNumberSuchAs2", "write a decimal number such as count: 10")) - case n <= 0: - p.add(where.of("count"), core.Says("recipe.AsksForFiles", "%s asks for %d files", core.A("At", where), core.A("Value", n)), - core.Says("recipe.ATargetThatProducesNothingIs", "a target that produces nothing is almost always a mistake rather than an intention"), - core.Says("recipe.AskForAtLeastOneOr", "ask for at least one, or delete the target")) - // Judged before the list is built. The reader used to grow it one entry - // at a time and reached a 13 GB allocation on a count of 2^63 - so this - // has to refuse the number rather than the result of using it. - case n > core.MaxFilesPerRun: - p.add(where.of("count"), core.Says("recipe.AsksForFiles", "%s asks for %d files", core.A("At", where), core.A("Value", n)), - core.SaidOf(core.ErrTooManyFiles), - core.Says("recipe.UseSettingOfOrLessOr2", "use {a} {setting} of %d or less, or split the target across several recipes", core.A("MaxFilesPerRun", core.MaxFilesPerRun))) - default: - count = int(n) - } - } - + count := rt.readCount(p, where) rt.refuseSections(p, where) if rt.Contains != nil { t.Contains = contentGroups(p, where, rt.Contains) } - rt.resolveSize(p, where, count, &t) + built := count + if count > room { + built = 0 + } + rt.resolveSize(p, where, built, &t) if name, ok := oneValue(p, where.of("name"), where.String()+" {setting}", "name: invoice_{index:04}.pdf", rt.Name); ok { t.Name = name @@ -133,7 +120,50 @@ func (rt rawTarget) validate(p *problems, at func(id string) spot, def Defaults) refuseImpossibleExpectation(p, where, t) rt.describe(p, where, &t) t.Properties = properties(p, where, t.Format, rt.Properties) - return t + + // The count, whether or not its list was built and whether or not its size + // was refused, so a run past the ceiling is said beside the target's own + // problems (RC7). A boundary set is the sizes it has - three, built whatever + // room is left, because three is not an allocation. For a recipe this + // reader passes, it is the length of the list, which is what the engine + // counts. + files := count + if rt.Boundary != nil { + files = len(t.Sizes) + } + return t, files +} + +// readCount reads how many files this target asks for. +// +// A count refused here asks for nothing. Left at the default, the one file it +// stands for would take a target of a million beside it past the run's +// ceiling, and one mistake would be refused twice. +func (rt rawTarget) readCount(p *problems, where spot) int { + if rt.Count == nil { + return DefaultCount + } + n, ok := rt.Count.number() + switch { + case !ok: + p.add(where.of("count"), core.Says("recipe.HasSettingOfWhichIsNot", "%s has {a} {setting} of %q, which is not a whole number", core.A("Where", where), core.A("Text", rt.Count.text)), + core.Says("recipe.SettingIsHowManyFilesThis", "{a} {setting} is how many files this target produces, so it is read exactly as written"), + core.Says("recipe.WriteADecimalNumberSuchAs2", "write a decimal number such as count: 10")) + case n <= 0: + p.add(where.of("count"), core.Says("recipe.AsksForFiles", "%s asks for %d files", core.A("At", where), core.A("Value", n)), + core.Says("recipe.ATargetThatProducesNothingIs", "a target that produces nothing is almost always a mistake rather than an intention"), + core.Says("recipe.AskForAtLeastOneOr", "ask for at least one, or delete the target")) + // Judged before the list is built. The reader used to grow it one entry at + // a time and reached a 13 GB allocation on a count of 2^63 - so this has to + // refuse the number rather than the result of using it. + case n > core.MaxFilesPerRun: + p.add(where.of("count"), core.Says("recipe.AsksForFiles", "%s asks for %d files", core.A("At", where), core.A("Value", n)), + core.SaidOf(core.ErrTooManyFiles), + core.Says("recipe.UseSettingOfOrLessOr2", "use {a} {setting} of %d or less, or split the target across several recipes", core.A("MaxFilesPerRun", core.MaxFilesPerRun))) + default: + return int(n) + } + return 0 } // describe reads what a target says ABOUT its files rather than how to make @@ -215,9 +245,7 @@ func (rt rawTarget) resolveSize(p *problems, where spot, count int, t *Target) { core.Says("recipe.UseSettingSuchAs2mb512kb", "use {a} {setting} such as 2mb, 512kb or a plain byte count")) break } - for i := 0; i < count; i++ { - t.Sizes = append(t.Sizes, n) - } + t.Sizes = sizesOf(count, n) case rt.SizeRange != nil: // Placed above contains for the same reason size is: a stated size @@ -240,9 +268,7 @@ func (rt rawTarget) resolveSize(p *problems, where spot, count int, t *Target) { // The values stay zero here and the engine replaces them. What this // list carries at this point is the number of files, exactly as it // does for contains. - for i := 0; i < count; i++ { - t.Sizes = append(t.Sizes, 0) - } + t.Sizes = sizesOf(count, 0) case rt.Contains != nil: // The fourth way of declaring a size, and not an exception to the rule. @@ -250,9 +276,7 @@ func (rt rawTarget) resolveSize(p *problems, where spot, count int, t *Target) { // and a dry run still reports a number without generating anything. // See ARCHITECTURE.md section 9. t.SizeFromContents = true - for i := 0; i < count; i++ { - t.Sizes = append(t.Sizes, 0) - } + t.Sizes = sizesOf(count, 0) default: p.add(where.of("size"), core.Says("recipe.HasNoSetting", "%s has no {setting}", core.A("At", where)), @@ -376,6 +400,24 @@ func boundaryText(p *problems, where spot, v *scalar) string { return s } +// sizesOf is count files of one size, in a list reserved at its length. +// +// Grown one append at a time, the list for a million files cost 40 B a file to +// build for 8 B a file kept - every growth leaves the last array to the +// collector - and the batch screen builds it on every key (O302, measured +// 2026-11-04: 41.7 MB for one target of a million). engine.Uniform is the same +// list on the side of the engine, which this package cannot import. +func sizesOf(count int, each int64) []int64 { + if count <= 0 { + return nil + } + sizes := make([]int64, count) + for i := range sizes { + sizes[i] = each + } + return sizes +} + // boundarySizes turns a limit into the three sizes a boundary set means. // // This is the case the whole tool is pointed at: an application says it From 5997dccc4d6dfd5c6ccb8c135c15a12bb6cda736 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Wed, 4 Nov 2026 10:59:42 +0100 Subject: [PATCH 2/2] The ceiling refusal says the whole run's files, and the list of sizes is core's - A run past the ceiling on files is told every file its targets ask for. The reader and the engine both refuse where the run crosses the ceiling and said the running total there, so ten targets of a million were told they asked for two million. The reader now adds its one refusal once every target is counted, still at the target that crossed, and the engine counts every target's files when it refuses. The number is an int64, since a recipe of a megabyte can hold more targets of a million than an int of 32 bits counts. - The list of files of one size is a single function in core, which engine.Uniform and the recipe reader both use, rather than a copy in each. Co-Authored-By: Claude Opus 5.5 --- internal/core/limits.go | 13 +++++++----- internal/core/size.go | 20 +++++++++++++++++++ internal/engine/engine.go | 27 ++++++++++++++----------- internal/guard/mutation/entries.py | 32 +++++++++++++++++++++++------- internal/guard/runceiling_test.go | 29 ++++++++++++++++++++++++--- internal/recipe/recipe.go | 20 +++++++++++++------ internal/recipe/target.go | 24 +++------------------- 7 files changed, 111 insertions(+), 54 deletions(-) diff --git a/internal/core/limits.go b/internal/core/limits.go index 70fe1d7e..c4c6a9d2 100644 --- a/internal/core/limits.go +++ b/internal/core/limits.go @@ -64,14 +64,17 @@ var ( "Or give the run more memory through the memory setting") ) -// TooManyFilesInRun is what a run past MaxFilesPerRun asks for: the files it -// came to at the target that crossed the ceiling, across all the targets it -// has. +// TooManyFilesInRun is what a run past MaxFilesPerRun asks for: every file of +// every target, across all of them. // // Both the recipe reader and the engine count the run, and a recipe refused by // either reads the same - so the sentence is written here, once, rather than -// beside each of them (O302). -func TooManyFilesInRun(files, targets int) Said { +// beside each of them (O302). The number is the whole run's, not the running +// total at the target that crossed the ceiling where both of them refuse: ten +// targets of a million were told they asked for two million (review of #219). +// An int64 because a recipe of a megabyte holds tens of thousands of targets +// of a million, which is past what an int of 32 bits counts. +func TooManyFilesInRun(files int64, targets int) Said { return Says("core.TooManyFilesInRun", "this run asks for %s across %s", A("Files", SaysN("core.FilesInRun", "%d file", "%d files", A("Count", files))), A("Targets", SaysN("core.TargetsInRun", "%d target", "%d targets", A("Count", targets)))) diff --git a/internal/core/size.go b/internal/core/size.go index 45c4cfd7..4f4b78ff 100644 --- a/internal/core/size.go +++ b/internal/core/size.go @@ -332,3 +332,23 @@ func BoundarySizes(limit int64) ([]int64, error) { } return []int64{limit - 1, limit, limit + 1}, nil } + +// Uniform is n files of the same size, in a list reserved at its length. +// +// Here because the recipe reader builds the same list as the engine and cannot +// import it, and two copies of one list were what a review of #219 named. +// Reserved rather than grown: one append at a time, the list for a million +// files cost 40 B a file to build for 8 B a file kept - every growth leaves the +// last array to the collector - and the batch screen builds it on every key +// (O302, measured on 2026-11-04: 41.7 MB for a target of a million, 8.0 MB +// reserved). +func Uniform(n int, bytes int64) []int64 { + if n <= 0 { + return nil + } + out := make([]int64, n) + for i := range out { + out[i] = bytes + } + return out +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 595b39c9..c55af66b 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -100,17 +100,9 @@ type SizeRange struct { Min, Max int64 } -// Uniform is n files of the same size, which is what most targets ask for. -func Uniform(n int, bytes int64) []int64 { - if n <= 0 { - return nil - } - out := make([]int64, n) - for i := range out { - out[i] = bytes - } - return out -} +// Uniform is n files of the same size, which is what most targets ask for. It +// is core's, under the name the engine's callers reach for. +func Uniform(n int, bytes int64) []int64 { return core.Uniform(n, bytes) } // Options are the settings of one run. type Options struct { @@ -486,7 +478,7 @@ func PlanContext(ctx context.Context, targets []Target, opt Options) ([]PlannedF // reader's is. return nil, &RecipeError{ Setting: core.TargetAddress(i+1, SettingCount), - Detail: core.TooManyFilesInRun(totalFiles, len(targets)), + Detail: core.TooManyFilesInRun(filesIn(targets), len(targets)), Because: core.TooManyFilesWhy, Remedy: core.TooManyFilesFix} } @@ -498,6 +490,17 @@ func PlanContext(ctx context.Context, targets []Target, opt Options) ([]PlannedF return pl.out, nil } +// filesIn is every file the targets ask for, which is what a refusal of the +// run's ceiling says - rather than the running total at the target that +// crossed it, which is where the refusal is made. +func filesIn(targets []Target) int64 { + var n int64 + for i := range targets { + n += int64(len(targets[i].Sizes)) + } + return n +} + // Run writes a planned set of files. // // Each file is written under a temporary name and only then renamed, so the diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index 3b59cccd..64074d6c 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -2609,17 +2609,33 @@ # The engine's address: the merged list, preset's targets first. "the run's ceiling names the target by the merged list rather than the file", "internal/recipe/recipe.go", - "p.add(at(t.ID).of(KeyCount), core.TooManyFilesInRun(planned+files, len(raw.Targets)),", - "p.add(targetSpot(i, t.ID).of(KeyCount), core.TooManyFilesInRun(planned+files, len(raw.Targets)),", + "crossedAt = at(t.ID).of(KeyCount)", + "crossedAt = targetSpot(i, t.ID).of(KeyCount)", "TestTheTargetThatCrossesTheRunCeilingIsPlacedInTheFile", ), ( "the run's ceiling marks the size of the batch that crossed it rather than its count", "internal/recipe/recipe.go", - "p.add(at(t.ID).of(KeyCount), core.TooManyFilesInRun(planned+files, len(raw.Targets)),", - "p.add(at(t.ID).of(KeySize), core.TooManyFilesInRun(planned+files, len(raw.Targets)),", + "crossedAt = at(t.ID).of(KeyCount)", + "crossedAt = at(t.ID).of(KeySize)", "TestARefusalAboutTheWholeRunMarksTheBatchThatCausedIt", ), + ( + # Review of #219: the running total where the run crossed is what both + # layers said, and ten targets of a million were told two million. + "the run's ceiling says the files up to the target that crossed it rather than the whole run's", + "internal/recipe/recipe.go", + "p.add(crossedAt, core.TooManyFilesInRun(asked, len(raw.Targets)),", + "p.add(crossedAt, core.TooManyFilesInRun(int64(planned), len(raw.Targets)),", + "TestTheRunCeilingSaysWhatTheWholeRunAsksFor", + ), + ( + "the engine's ceiling says the files up to the target that crossed it rather than the whole run's", + "internal/engine/engine.go", + "Detail: core.TooManyFilesInRun(filesIn(targets), len(targets)),", + "Detail: core.TooManyFilesInRun(int64(totalFiles), len(targets)),", + "TestTheEngineCountsTheWholeRunOfTargetsHandedToItDirectly", + ), ( "a target past the run's ceiling has its list built before it is refused", "internal/recipe/target.go", @@ -2628,10 +2644,12 @@ "TestATargetPastTheRunCeilingIsNotBuilt", ), ( + # In core since the review of #219, which named the reader's own copy of + # engine.Uniform. Both now take this one. "a target's list of sizes is grown one file at a time again", - "internal/recipe/target.go", - "\tsizes := make([]int64, count)\n\tfor i := range sizes {\n\t\tsizes[i] = each\n\t}", - "\tvar sizes []int64\n\tfor range count {\n\t\tsizes = append(sizes, each)\n\t}", + "internal/core/size.go", + "\tout := make([]int64, n)\n\tfor i := range out {\n\t\tout[i] = bytes\n\t}", + "\tvar out []int64\n\tfor range n {\n\t\tout = append(out, bytes)\n\t}", "TestATargetsListOfSizesIsReservedAtItsLength", ), ( diff --git a/internal/guard/runceiling_test.go b/internal/guard/runceiling_test.go index f8f3bbb6..85c3d350 100644 --- a/internal/guard/runceiling_test.go +++ b/internal/guard/runceiling_test.go @@ -134,6 +134,23 @@ func TestTheRunCeilingSitsExactlyAtAMillion(t *testing.T) { } } +// The refusal says the files the whole run asks for, at the target that +// crossed the ceiling. +// +// Both are refused where the run crosses, so the running total there was the +// number they said - ten targets of a million were told they asked for two +// million across ten targets (review of #219, measured with the binary before +// O302 as well). The target after the one that crosses is what tells the two +// numbers apart. +func TestTheRunCeilingSaysWhatTheWholeRunAsksFor(t *testing.T) { + src := targetsOf(1, core.MaxFilesPerRun, core.MaxFilesPerRun) + ceilingAt(t, src, 2) + _, err := recipe.Parse(src, "whole run") + if !strings.Contains(err.Error(), "2000001 files across 3 targets") { + t.Errorf("the refusal does not say the files the whole run asks for, 2000001:\n%v", err) + } +} + // A run past the ceiling is reported together with whatever else is wrong. // // Every problem of a recipe arrives at once (RC7). While the engine counted the @@ -241,19 +258,25 @@ targets: // targets by whoever calls it, though, and planCapacity reserves nothing past // the ceiling on the word of this refusal. The memory setting is there so a // build that lost the refusal plans until the plan ceiling stops it, a few -// megabytes in, rather than a million files. +// megabytes in, rather than a million files. A third target follows the one +// that crosses, so the number the refusal says is asked to be the whole run's +// rather than the running total where it stopped. func TestTheEngineCountsTheWholeRunOfTargetsHandedToItDirectly(t *testing.T) { targets := []engine.Target{ {ID: "small", Format: "txt", Sizes: engine.Uniform(1, 1024)}, {ID: "big", Format: "txt", Sizes: engine.Uniform(core.MaxFilesPerRun, 1024)}, + {ID: "after", Format: "txt", Sizes: engine.Uniform(core.MaxFilesPerRun, 1024)}, } _, err := engine.Plan(targets, engine.Options{OutDir: t.TempDir(), ManifestName: "manifest.json", Memory: core.MemoryOf(32 << 20)}) if err == nil { - t.Fatal("a run of one file more than the ceiling was planned") + t.Fatal("a run of a million files more than the ceiling was planned") } - if !strings.Contains(err.Error(), "1000001 files across 2 targets") { + if !strings.Contains(err.Error(), "files across 3 targets") { t.Fatalf("the run was refused for something other than its count of files: %v", err) } + if !strings.Contains(err.Error(), "2000001 files across 3 targets") { + t.Errorf("the refusal does not say the files the whole run asks for, 2000001: %v", err) + } var about interface{ AboutSetting() string } if !errors.As(err, &about) || about.AboutSetting() != core.TargetAddress(2, engine.SettingCount) { t.Errorf("the refusal is not addressed to the target that crossed the ceiling, %s: %v", diff --git a/internal/recipe/recipe.go b/internal/recipe/recipe.go index 8b97077c..f85ea9d8 100644 --- a/internal/recipe/recipe.go +++ b/internal/recipe/recipe.go @@ -448,17 +448,21 @@ func (raw rawRecipe) readTargets(p *problems, def Defaults, fromPreset int) []Ta // adds more than the ceiling: a count past it is refused as the target's // own and asks for nothing. planned := 0 + // What every target asks for, which is the number the refusal says. The + // running total stops at the target that crossed the ceiling, and ten + // targets of a million were told they asked for two (review of #219). + var asked int64 + // The target that takes the run past the ceiling, which is the one + // somebody can shrink. Only one is: every target after it is past the + // ceiling as well, and saying so of each would bury the one that matters. + crossedAt := "" for i, rt := range raw.Targets { at := spotOfTarget(i, fromPreset) room := core.MaxFilesPerRun - planned t, files := rt.validate(p, at, def, room) - // Refused at the target that takes the run past the ceiling, which is - // the one somebody can shrink, and in the engine's words. Once, because - // every target after it is past the ceiling as well, and saying so of - // each would bury the one that matters. + asked += int64(files) if room >= 0 && files > room { - p.add(at(t.ID).of(KeyCount), core.TooManyFilesInRun(planned+files, len(raw.Targets)), - core.TooManyFilesWhy, core.TooManyFilesFix) + crossedAt = at(t.ID).of(KeyCount) } planned = min(planned+files, core.MaxFilesPerRun+1) if t.ID != "" { @@ -470,6 +474,10 @@ func (raw rawRecipe) readTargets(p *problems, def Defaults, fromPreset int) []Ta } targets = append(targets, t) } + // Said once every target is counted, in the engine's words. + if crossedAt != "" { + p.add(crossedAt, core.TooManyFilesInRun(asked, len(raw.Targets)), core.TooManyFilesWhy, core.TooManyFilesFix) + } return targets } diff --git a/internal/recipe/target.go b/internal/recipe/target.go index 16c69377..888e843d 100644 --- a/internal/recipe/target.go +++ b/internal/recipe/target.go @@ -245,7 +245,7 @@ func (rt rawTarget) resolveSize(p *problems, where spot, count int, t *Target) { core.Says("recipe.UseSettingSuchAs2mb512kb", "use {a} {setting} such as 2mb, 512kb or a plain byte count")) break } - t.Sizes = sizesOf(count, n) + t.Sizes = core.Uniform(count, n) case rt.SizeRange != nil: // Placed above contains for the same reason size is: a stated size @@ -268,7 +268,7 @@ func (rt rawTarget) resolveSize(p *problems, where spot, count int, t *Target) { // The values stay zero here and the engine replaces them. What this // list carries at this point is the number of files, exactly as it // does for contains. - t.Sizes = sizesOf(count, 0) + t.Sizes = core.Uniform(count, 0) case rt.Contains != nil: // The fourth way of declaring a size, and not an exception to the rule. @@ -276,7 +276,7 @@ func (rt rawTarget) resolveSize(p *problems, where spot, count int, t *Target) { // and a dry run still reports a number without generating anything. // See ARCHITECTURE.md section 9. t.SizeFromContents = true - t.Sizes = sizesOf(count, 0) + t.Sizes = core.Uniform(count, 0) default: p.add(where.of("size"), core.Says("recipe.HasNoSetting", "%s has no {setting}", core.A("At", where)), @@ -400,24 +400,6 @@ func boundaryText(p *problems, where spot, v *scalar) string { return s } -// sizesOf is count files of one size, in a list reserved at its length. -// -// Grown one append at a time, the list for a million files cost 40 B a file to -// build for 8 B a file kept - every growth leaves the last array to the -// collector - and the batch screen builds it on every key (O302, measured -// 2026-11-04: 41.7 MB for one target of a million). engine.Uniform is the same -// list on the side of the engine, which this package cannot import. -func sizesOf(count int, each int64) []int64 { - if count <= 0 { - return nil - } - sizes := make([]int64, count) - for i := range sizes { - sizes[i] = each - } - return sizes -} - // boundarySizes turns a limit into the three sizes a boundary set means. // // This is the case the whole tool is pointed at: an application says it