diff --git a/.github/race-weights b/.github/race-weights index 8bb69854..4fb51a5e 100644 --- a/.github/race-weights +++ b/.github/race-weights @@ -36,7 +36,6 @@ unlisted 0.58 35.5 TestJxlStillFitsWhenTheQualityIsNotTheOneTheCeilingsWereMeasuredAt 30.8 TestOurOwnGeneratorsHaveNotDrifted 29.9 TestTheSameSeedGivesTheSameBytes -27.7 TestTheCeilingCountsTheWholeRunNotOneTarget 26.8 TestEveryJxlRungStillFitsInsideItsDeclaredCeiling 26.6 TestTheMinimumStandsWhateverTheSeedIsUnderEveryDeclaredChoice 24.9 TestEveryEMLSettingSurvivesItsReferenceTool diff --git a/CHANGELOG.md b/CHANGELOG.md index 578b8169..c1363edf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,9 +49,21 @@ because it turns other people's test suites red. `--software-gl`, moves from 26.2.0 to 26.2.4, built with LLVM 23.1.2. Its licences are the same. `THIRD-PARTY-NOTICES.md` names the new files and their sums. +- **A recipe past the ceiling of a million files ends with code 3 even when a + target also asks for a size its format cannot make.** That size used to be + refused first, with code 4, and the ceiling came out only once it was + fixed. The ceiling is now found while the recipe is read, and like every + problem found there it comes first. ### Fixed +- **A recipe asking for more than a million files in all is refused before + it takes the memory.** Targets were held to a million each and the run only + once all their sizes were laid out, so `tfg validate` took 2.5 GB to refuse + fifty targets of a million, and the recipe screen spent 40 MB a key on each + such batch. The total is now counted as the recipe is read, beside its other + problems, and `validate --json` places it at the file's own target in a + recipe built on a preset. Reading takes a fifth of the memory. - **Cancel answers at once, and closing the window during a run no longer freezes it.** Both used to wait for the write in progress with the window standing still, which on a slow or network disk could last long enough for diff --git a/internal/core/limits.go b/internal/core/limits.go index ff98bcae..c4c6a9d2 100644 --- a/internal/core/limits.go +++ b/internal/core/limits.go @@ -64,6 +64,22 @@ var ( "Or give the run more memory through the memory setting") ) +// TooManyFilesInRun is what a run past MaxFilesPerRun asks for: every file of +// every target, across all of them. +// +// Both the recipe reader and the engine count the run, and a recipe refused by +// either reads the same - so the sentence is written here, once, rather than +// beside each of them (O302). The number is the whole run's, not the running +// total at the target that crossed the ceiling where both of them refuse: ten +// targets of a million were told they asked for two million (review of #219). +// An int64 because a recipe of a megabyte holds tens of thousands of targets +// of a million, which is past what an int of 32 bits counts. +func TooManyFilesInRun(files int64, targets int) Said { + return Says("core.TooManyFilesInRun", "this run asks for %s across %s", + A("Files", SaysN("core.FilesInRun", "%d file", "%d files", A("Count", files))), + A("Targets", SaysN("core.TargetsInRun", "%d target", "%d targets", A("Count", targets)))) +} + // PlanTooLargeWhy is the same refusal for the quantity that actually runs out, // in the words of where its ceiling came from. // diff --git a/internal/core/size.go b/internal/core/size.go index 45c4cfd7..4f4b78ff 100644 --- a/internal/core/size.go +++ b/internal/core/size.go @@ -332,3 +332,23 @@ func BoundarySizes(limit int64) ([]int64, error) { } return []int64{limit - 1, limit, limit + 1}, nil } + +// Uniform is n files of the same size, in a list reserved at its length. +// +// Here because the recipe reader builds the same list as the engine and cannot +// import it, and two copies of one list were what a review of #219 named. +// Reserved rather than grown: one append at a time, the list for a million +// files cost 40 B a file to build for 8 B a file kept - every growth leaves the +// last array to the collector - and the batch screen builds it on every key +// (O302, measured on 2026-11-04: 41.7 MB for a target of a million, 8.0 MB +// reserved). +func Uniform(n int, bytes int64) []int64 { + if n <= 0 { + return nil + } + out := make([]int64, n) + for i := range out { + out[i] = bytes + } + return out +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 4a5d49de..c55af66b 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -100,17 +100,9 @@ type SizeRange struct { Min, Max int64 } -// Uniform is n files of the same size, which is what most targets ask for. -func Uniform(n int, bytes int64) []int64 { - if n <= 0 { - return nil - } - out := make([]int64, n) - for i := range out { - out[i] = bytes - } - return out -} +// Uniform is n files of the same size, which is what most targets ask for. It +// is core's, under the name the engine's callers reach for. +func Uniform(n int, bytes int64) []int64 { return core.Uniform(n, bytes) } // Options are the settings of one run. type Options struct { @@ -474,15 +466,19 @@ func PlanContext(ctx context.Context, targets []Target, opt Options) ([]PlannedF // so - but a refusal a window cannot place is a refusal at the // foot of a form with twenty batches above it and nothing marked. // - // Reachable only from here, and measured on 2026-08-25: the recipe - // reader bounds each target on its own, so three batches of four - // hundred thousand pass it and the total is refused here. Before - // this line "validate --json" carried no "at" for it. + // No surface reaches this any more. The recipe reader counts the + // same total since O302 and refuses first, in the same words, so + // a recipe never arrives here past the ceiling - and the flag + // path and the Generate screen are one target, bounded on its + // own. It stays because Plan is + // handed targets by whoever calls it, and planCapacity reserves + // nothing past the ceiling on the word of this refusal. Its + // position counts the merged list, which for a recipe built on a + // preset is not the file's (measured on 2026-11-04) - the + // reader's is. return nil, &RecipeError{ Setting: core.TargetAddress(i+1, SettingCount), - Detail: core.Says("engine.TooManyFiles", "this run asks for %s across %s", - core.A("Files", core.SaysN("engine.Files", "%d file", "%d files", core.A("Count", totalFiles))), - core.A("Targets", core.SaysN("engine.Targets", "%d target", "%d targets", core.A("Count", len(targets))))), + Detail: core.TooManyFilesInRun(filesIn(targets), len(targets)), Because: core.TooManyFilesWhy, Remedy: core.TooManyFilesFix} } @@ -494,6 +490,17 @@ func PlanContext(ctx context.Context, targets []Target, opt Options) ([]PlannedF return pl.out, nil } +// filesIn is every file the targets ask for, which is what a refusal of the +// run's ceiling says - rather than the running total at the target that +// crossed it, which is where the refusal is made. +func filesIn(targets []Target) int64 { + var n int64 + for i := range targets { + n += int64(len(targets[i].Sizes)) + } + return n +} + // Run writes a planned set of files. // // Each file is written under a temporary name and only then renamed, so the diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index 5f91e058..64074d6c 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -2542,7 +2542,9 @@ ), ( # The recipe reader took the worse of the two failures: it grew the list - # one entry at a time and reached a 13 GB allocation. + # one entry at a time and reached a 13 GB allocation. Since O302 the + # mutated count reaches no allocation - the run's ceiling turns the same + # recipe away and builds no list - so the guard asks which refusal it got. "the recipe reader stops asking whether the count can be planned", "internal/recipe/target.go", # Carries the line under it since 2026-08-20. The container entry gained @@ -2550,8 +2552,8 @@ # an ambiguous pattern is how a mutation quietly starts proving a # different guard than the one it names. The two are told apart by which # address they build: `where` for the target, `at` for the entry. - "case n > core.MaxFilesPerRun:\n\t\t\tp.add(where.of(\"count\")", - "case n > core.MaxFilesPerRun && false:\n\t\t\tp.add(where.of(\"count\")", + "case n > core.MaxFilesPerRun:\n\t\tp.add(where.of(\"count\")", + "case n > core.MaxFilesPerRun && false:\n\t\tp.add(where.of(\"count\")", "TestAnImpossibleFileCountInARecipeIsRefused", ), ( @@ -2568,12 +2570,102 @@ "TestAnImpossibleFileCountInsideAContainerIsRefused", ), ( + # Asked of the engine directly since O302. The recipe reader counts the + # run first, so the guard asking validate about pieces never reached + # this line again. "the ceiling is applied per target, so writing the number out in pieces gets past it", "internal/engine/engine.go", "totalFiles += len(t.Sizes)", "totalFiles = len(t.Sizes)", + "TestTheEngineCountsTheWholeRunOfTargetsHandedToItDirectly", + ), + ( + # The same break on the reader's side (O302): the targets before this + # one are forgotten, so twelve of 90 000 pass one at a time. + "the recipe reader counts each target against the run's ceiling alone", + "internal/recipe/recipe.go", + "planned = min(planned+files, core.MaxFilesPerRun+1)", + "planned = min(files, core.MaxFilesPerRun+1)", "TestTheCeilingCountsTheWholeRunNotOneTarget", ), + ( + # Asked beside another problem of the recipe, so the engine never runs + # and cannot stand in for the reader - which it does for every guard + # asking validate about the ceiling alone. + "the recipe reader stops refusing a run past the ceiling", + "internal/recipe/recipe.go", + "if room >= 0 && files > room {", + "if false && room >= 0 && files > room {", + "TestARunPastTheCeilingIsReportedBesideTheRecipesOtherProblems", + ), + ( + "the run's ceiling refuses a run of exactly a million", + "internal/recipe/recipe.go", + "if room >= 0 && files > room {", + "if room >= 0 && files >= room {", + "TestTheRunCeilingSitsExactlyAtAMillion", + ), + ( + # The engine's address: the merged list, preset's targets first. + "the run's ceiling names the target by the merged list rather than the file", + "internal/recipe/recipe.go", + "crossedAt = at(t.ID).of(KeyCount)", + "crossedAt = targetSpot(i, t.ID).of(KeyCount)", + "TestTheTargetThatCrossesTheRunCeilingIsPlacedInTheFile", + ), + ( + "the run's ceiling marks the size of the batch that crossed it rather than its count", + "internal/recipe/recipe.go", + "crossedAt = at(t.ID).of(KeyCount)", + "crossedAt = at(t.ID).of(KeySize)", + "TestARefusalAboutTheWholeRunMarksTheBatchThatCausedIt", + ), + ( + # Review of #219: the running total where the run crossed is what both + # layers said, and ten targets of a million were told two million. + "the run's ceiling says the files up to the target that crossed it rather than the whole run's", + "internal/recipe/recipe.go", + "p.add(crossedAt, core.TooManyFilesInRun(asked, len(raw.Targets)),", + "p.add(crossedAt, core.TooManyFilesInRun(int64(planned), len(raw.Targets)),", + "TestTheRunCeilingSaysWhatTheWholeRunAsksFor", + ), + ( + "the engine's ceiling says the files up to the target that crossed it rather than the whole run's", + "internal/engine/engine.go", + "Detail: core.TooManyFilesInRun(filesIn(targets), len(targets)),", + "Detail: core.TooManyFilesInRun(int64(totalFiles), len(targets)),", + "TestTheEngineCountsTheWholeRunOfTargetsHandedToItDirectly", + ), + ( + "a target past the run's ceiling has its list built before it is refused", + "internal/recipe/target.go", + "\tif count > room {\n\t\tbuilt = 0\n\t}", + "\tif false {\n\t\tbuilt = 0\n\t}", + "TestATargetPastTheRunCeilingIsNotBuilt", + ), + ( + # In core since the review of #219, which named the reader's own copy of + # engine.Uniform. Both now take this one. + "a target's list of sizes is grown one file at a time again", + "internal/core/size.go", + "\tout := make([]int64, n)\n\tfor i := range out {\n\t\tout[i] = bytes\n\t}", + "\tvar out []int64\n\tfor range n {\n\t\tout = append(out, bytes)\n\t}", + "TestATargetsListOfSizesIsReservedAtItsLength", + ), + ( + "a boundary set counts as the one file of the default count against the run's ceiling", + "internal/recipe/target.go", + "\tif rt.Boundary != nil {\n\t\tfiles = len(t.Sizes)\n\t}", + "\tif false {\n\t\tfiles = len(t.Sizes)\n\t}", + "TestTheRunCeilingSitsExactlyAtAMillion", + ), + ( + "a count refused on its own goes back to asking for the default", + "internal/recipe/target.go", + "\t\treturn int(n)\n\t}\n\treturn 0\n}", + "\t\treturn int(n)\n\t}\n\treturn DefaultCount\n}", + "TestTheRunCeilingSitsExactlyAtAMillion", + ), ( # The other direction. A ceiling low enough to refuse ordinary work # would satisfy every guard above and break the tool for everybody. @@ -5391,18 +5483,21 @@ "TestTheChangelogHasOneHeadingPerKindOfChange", ), ( + # Both asked of the engine directly since O302 - the window's batch + # screen reaches the reader's refusal now, and the guard over it moved + # with it (TestARefusalAboutTheWholeRunMarksTheBatchThatCausedIt). "the ceiling refusal stops naming the batch that caused it", "internal/engine/engine.go", "\t\t\t\tSetting: core.TargetAddress(i+1, SettingCount),", '\t\t\t\tSetting: "",', - "TestARefusalFromTheEngineAboutTheWholeRunMarksTheBatchThatCausedIt", + "TestTheEngineCountsTheWholeRunOfTargetsHandedToItDirectly", ), ( "the ceiling refusal names the batch before the one that caused it", "internal/engine/engine.go", "\t\t\t\tSetting: core.TargetAddress(i+1, SettingCount),", "\t\t\t\tSetting: core.TargetAddress(i, SettingCount),", - "TestARefusalFromTheEngineAboutTheWholeRunMarksTheBatchThatCausedIt", + "TestTheEngineCountsTheWholeRunOfTargetsHandedToItDirectly", ), ( "the report stops carrying the address of a refusal from below the reader", diff --git a/internal/guard/numberlimits_test.go b/internal/guard/numberlimits_test.go index 281033b6..9af4602f 100644 --- a/internal/guard/numberlimits_test.go +++ b/internal/guard/numberlimits_test.go @@ -1,13 +1,13 @@ package guard import ( - "path/filepath" "strconv" "strings" "testing" "github.com/donislawdev/TestingFilesGenerator/internal/cli" "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/recipe" ) // Two numbers a person can write that the tool could not honour, and used to @@ -90,6 +90,14 @@ targets: if strings.Contains(errOut, "fatal error") || strings.Contains(errOut, "out of memory") { t.Errorf("the allocator was reached instead of the check:\n%s", errOut) } + // Refused as the target's own count, not only as the run's. The run's + // ceiling turns this recipe away as well since O302, but it is the + // target's ceiling that keeps every term of the run's sum within a million + // - without it a count like this one, after the run is already past the + // ceiling, wraps the sum negative and every target after it is built again. + if !strings.Contains(errOut, "split the target across several recipes") { + t.Errorf("the count was not refused as the target's own:\n%s", errOut) + } if stdout != "" { t.Errorf("a failed run wrote to stdout:\n%s", stdout) } @@ -219,8 +227,13 @@ func TestABoundaryThatCannotHoldItsOwnSetIsRefused(t *testing.T) { // A run spread across many targets reaches the same ceiling. Checking only the // count of one target would leave the whole thing reachable by writing the // number out in pieces. +// +// Asked of the recipe reader since O302, which counts the run before it builds +// a target's list. Through validate this was asked of the reader and of the +// engine under it at once, and a refusal from either passed - so either one +// could lose its count and leave this green, and the engine planned 990 000 +// files to answer it (19 s, 28 s under the race detector). func TestTheCeilingCountsTheWholeRunNotOneTarget(t *testing.T) { - dir := t.TempDir() var b strings.Builder b.WriteString("version: 1\ntargets:\n") // Twelve targets of 90 000 files come to 1 080 000, which is over the @@ -230,16 +243,17 @@ func TestTheCeilingCountsTheWholeRunNotOneTarget(t *testing.T) { b.WriteString(string(rune('a' + i))) b.WriteString("\n format: txt\n size: 1b\n count: 90000\n") } - path := writeRecipe(t, dir, b.String()) - code, stdout, errOut := run(t, "validate", path) - if code == cli.ExitOK { - t.Fatalf("a run of over a million files spread across targets was accepted:\n%s", stdout) + _, err := recipe.Parse([]byte(b.String()), "pieces") + if err == nil { + t.Fatal("a run of over a million files spread across targets was accepted") } - if !strings.Contains(errOut, "1000000") { - t.Errorf("the refusal does not name the ceiling:\n%s", errOut) + said := err.Error() + for _, want := range []string{"1080000 files across 12 targets", "1000000"} { + if !strings.Contains(said, want) { + t.Errorf("the refusal does not say %q:\n%s", want, said) + } } - _ = filepath.Join } // The refusal names the ceiling the rule really uses. diff --git a/internal/guard/recipescreen_test.go b/internal/guard/recipescreen_test.go index e7f2d2dc..1167a628 100644 --- a/internal/guard/recipescreen_test.go +++ b/internal/guard/recipescreen_test.go @@ -518,20 +518,20 @@ func waitForNamedManifest(t *testing.T, settle func(), dir, name string) { manifestAfter(t, settle, dir, name) } -// A refusal from the ENGINE marks the batch it is about, the same as one from -// the recipe reader. +// A refusal about the whole run marks the batch that took it past the ceiling. // -// The sibling above covers the reader. This one covers the layer under it, and -// they are not the same question: the reader bounds each batch on its own, so -// two batches of six hundred thousand files each pass it and the total is -// refused by the engine. Measured on 2026-08-25, before the address existed: -// the refusal arrived with a sentence and nothing else, so a screen with twenty -// batches said the run could not go and marked nothing at all. +// The sibling above covers a refusal about one batch's own value. This one is +// about a fact of the run, which no single box holds: two batches each within +// the count ceiling come to more than a run may plan. Measured on 2026-08-25, +// before the address existed: the refusal arrived with a sentence and nothing +// else, so a screen with twenty batches said the run could not go and marked +// nothing at all. // -// The count is deliberately spread over two batches with the big one second. -// The engine tests the running total before planning a batch's files, so this -// reaches the refusal having planned one file rather than a million. -func TestARefusalFromTheEngineAboutTheWholeRunMarksTheBatchThatCausedIt(t *testing.T) { +// The engine refused this until O302, and the guard was named for it. The +// recipe reader counts the run now and refuses first, so this asks the reader - +// what the engine still says when it is handed targets directly is asked in +// runceiling_test.go. +func TestARefusalAboutTheWholeRunMarksTheBatchThatCausedIt(t *testing.T) { screen := window.NewRecipe(newFakeHost(t)) body := screen.Object() pressNamed(t, body, text.ButtonAddBatch()) diff --git a/internal/guard/refusaladdress_test.go b/internal/guard/refusaladdress_test.go index 23d6936f..72105974 100644 --- a/internal/guard/refusaladdress_test.go +++ b/internal/guard/refusaladdress_test.go @@ -479,11 +479,12 @@ func missingPart(why, fix string) string { // // Two things, and the second is what keeps the first honest. // -// A refusal the reader never sees is refused by the engine or by a format: the -// ceiling on files is checked on the total, so a recipe whose targets each pass -// the reader is refused underneath it. Until 2026-08-25 those arrived with a -// sentence and no address, so a script grouping a report by field had nothing -// to group by. +// A refusal the reader never sees is refused by the engine or by a format - a +// size the format cannot deliver, a name the host cannot store. Until +// 2026-08-25 those arrived with a sentence and no address, so a script grouping +// a report by field had nothing to group by. The ceiling on files was the first +// case here until O302 moved the run's count into the reader, so it is no +// longer refused from below - where it lands is asked in runceiling_test.go. // // And an address is only worth carrying if it says where. A refusal that names // "size" and not which of twenty targets asked points at all of them while @@ -522,27 +523,6 @@ func TestEveryAddressTheReportCarriesSaysWhereItIs(t *testing.T) { return parsed.Problems } - // The big target second, because the engine tests the running total before - // planning a target's files - so this refuses having planned one file - // rather than a million. - const ceiling = `version: 1 -targets: - - id: a - format: txt - count: 1 - size: 1kb - - id: b - format: txt - count: 1000000 - size: 1kb -` - if got := report(t, ceiling)[0].At; got != core.TargetAddress(2, recipe.KeyCount) { - t.Errorf("the ceiling was reported at %q and the target that crossed it is %q.\n"+ - "Reason: the ceiling belongs to the run, but the box somebody can change belongs to\n"+ - "a target - a report that cannot say which one leaves a script with the sentence and\n"+ - "nothing else.", got, core.TargetAddress(2, recipe.KeyCount)) - } - // One refusal per layer that can produce one below the reader: a format // refusing a size, the engine refusing a name, the engine refusing the name // of the manifest. The last of those is the one that says a document @@ -635,7 +615,6 @@ func TestARefusalFromBelowTheReaderArrivesInItsThreeParts(t *testing.T) { // so these parts come from one join rule rather than a sentence each, // and the join has to leave the sentence reading as it always did. {"a name the host cannot store", "version: 1\ntargets:\n - id: a\n format: txt\n size: 1kb\n name: \"a core.MaxFilesPerRun { + t.Errorf("reading a recipe whose second target takes the run past the ceiling allocated %d B, over "+ + "a byte a file of that target. Its list was built before the run was refused - which is what "+ + "took fifty such targets to 2.5 GB", notBuilt) + } +} + +// A list of sizes is reserved at its length, not grown to it. +// +// Grown one file at a time, a million files took 40 B a file from the +// collector for a list of eight - the window's batch screen paid that on every +// key, and the Generate screen, which reserves (engine.Uniform), paid eight. A +// list that was grown ends with room to spare, so one whose capacity is its +// length was reserved - the reading planreserve_test.go makes of the plan. +// Each of the three ways a count turns into a list is asked. +func TestATargetsListOfSizesIsReservedAtItsLength(t *testing.T) { + const count = 1000 + src := fmt.Sprintf(`version: 1 +targets: + - id: sized + format: txt + size: 1kb + count: %[1]d + - id: ranged + format: txt + size-range: 1kb-2kb + count: %[1]d + - id: holding + format: zip + count: %[1]d + contains: + - format: txt + count: 1 + size: 10 +`, count) + + rec, err := recipe.Parse([]byte(src), "reserved") + if err != nil { + t.Fatalf("the recipe was refused: %v", err) + } + for _, target := range rec.Targets { + if len(target.Sizes) != count { + t.Fatalf("target %q has %d sizes, expected %d - this is not the list it names", target.ID, len(target.Sizes), count) + } + if cap(target.Sizes) != len(target.Sizes) { + t.Errorf("target %q has a list of %d sizes with room for %d, so it was grown rather than reserved. "+ + "Grown, a million files cost 40 B a file to read instead of 8, on every key of the batch screen", + target.ID, len(target.Sizes), cap(target.Sizes)) + } + } +} + +// The run's ceiling sits exactly at a million, counted the way the engine counts. +// +// A million across two targets is legal, and one file more is not. A boundary +// set is three files whatever the count says, so it is asked at the edge as +// well: counted as the one file of the default count, it would let a run of a +// million and two files through to the engine. And a count +// refused on its own asks for nothing - counted as the default, it would take a +// million beside it past the ceiling and say one mistake twice. +func TestTheRunCeilingSitsExactlyAtAMillion(t *testing.T) { + boundary := func(before int) []byte { + return []byte(fmt.Sprintf("version: 1\ntargets:\n - id: t1\n format: txt\n size: 1b\n count: %d\n"+ + " - id: t2\n format: txt\n boundary: 1kb\n", before)) + } + for name, src := range map[string][]byte{ + "a million in two targets": targetsOf(core.MaxFilesPerRun-1, 1), + "a boundary set that takes the run to a million": boundary(core.MaxFilesPerRun - 3), + } { + if _, err := recipe.Parse(src, name); err != nil { + t.Errorf("%s was refused: %v", name, err) + } + } + for name, src := range map[string][]byte{ + "a million and one in two targets": targetsOf(core.MaxFilesPerRun, 1), + "a boundary set that takes the run one file past it": boundary(core.MaxFilesPerRun - 2), + } { + t.Run(name, func(t *testing.T) { ceilingAt(t, src, 2) }) + } + + _, err := recipe.Parse(targetsOf(0, core.MaxFilesPerRun), "refused count") + var invalid *recipe.ValidationError + if !errors.As(err, &invalid) || len(invalid.Problems) != 1 || invalid.Problems[0].At != core.TargetAddress(1, recipe.KeyCount) { + t.Errorf("a count of 0 beside a target of a million should be refused once, for its own count: %v", err) + } +} + +// The refusal says the files the whole run asks for, at the target that +// crossed the ceiling. +// +// Both are refused where the run crosses, so the running total there was the +// number they said - ten targets of a million were told they asked for two +// million across ten targets (review of #219, measured with the binary before +// O302 as well). The target after the one that crosses is what tells the two +// numbers apart. +func TestTheRunCeilingSaysWhatTheWholeRunAsksFor(t *testing.T) { + src := targetsOf(1, core.MaxFilesPerRun, core.MaxFilesPerRun) + ceilingAt(t, src, 2) + _, err := recipe.Parse(src, "whole run") + if !strings.Contains(err.Error(), "2000001 files across 3 targets") { + t.Errorf("the refusal does not say the files the whole run asks for, 2000001:\n%v", err) + } +} + +// A run past the ceiling is reported together with whatever else is wrong. +// +// Every problem of a recipe arrives at once (RC7). While the engine counted the +// run, a recipe with any other problem was refused for that one alone, and the +// ceiling came out only once it was fixed - one more round for somebody who +// could have been told both. +func TestARunPastTheCeilingIsReportedBesideTheRecipesOtherProblems(t *testing.T) { + path := writeRecipe(t, t.TempDir(), `version: 1 +targets: + - id: small + format: txt + size: 1kb + count: 1 + properties: + no-such-property: 1 + - id: big + format: txt + size: 1kb + count: 1000000 +`) + code, stdout, errOut := run(t, "validate", "--json", path) + if code != cli.ExitRecipe { + t.Errorf("exit %d, expected %d:\n%s", code, cli.ExitRecipe, errOut) + } + if stdout != "" { + t.Errorf("a refused recipe wrote to stdout:\n%s", stdout) + } + at := map[string]string{} + for _, p := range reportedProblems(t, errOut) { + at[p.At] = p.What + } + if _, ok := at["targets[1].properties.no-such-property"]; !ok { + t.Errorf("the property nobody declared is not in the report, so this is not the case it names:\n%s", errOut) + } + if what := at[core.TargetAddress(2, recipe.KeyCount)]; !strings.Contains(what, "1000001 files across 2 targets") { + t.Errorf("the run past the ceiling is not reported beside the other problem at %s.\n"+ + "Reason: a person fixes the property, runs again and only then hears about the ceiling.\n%s", + core.TargetAddress(2, recipe.KeyCount), errOut) + } +} + +// The target that crosses the ceiling is named where the file has it, also in a +// recipe built on a preset. +// +// The engine counts the merged list, the preset's targets first, and knows +// nothing of where the file's own begin - so a recipe of one target on top of +// size-boundaries was refused at targets[8].count, a box neither the file nor +// the screen has. Measured on 2026-11-04. The reader knows the offset +// (spotOfTarget), and the plain recipe is asked beside it so the two halves +// cannot both be wrong in the same way. +func TestTheTargetThatCrossesTheRunCeilingIsPlacedInTheFile(t *testing.T) { + cases := []struct{ name, src string }{ + {"a recipe of its own", `version: 1 +targets: + - id: small + format: txt + size: 1kb + count: 1 + - id: big + format: txt + size: 1kb + count: 1000000 +`}, + {"a recipe built on a preset", `version: 1 +extends: preset:size-boundaries +with: + limit: 4mb + format: txt +targets: + - id: small + format: txt + size: 1kb + count: 1 + - id: big + format: txt + size: 1kb + count: 1000000 +`}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + path := writeRecipe(t, t.TempDir(), c.src) + code, _, errOut := run(t, "validate", "--json", path) + if code != cli.ExitRecipe { + t.Fatalf("exit %d, expected %d:\n%s", code, cli.ExitRecipe, errOut) + } + problems := reportedProblems(t, errOut) + if len(problems) != 1 || !strings.Contains(problems[0].What, "files across") { + t.Fatalf("expected the ceiling and nothing else:\n%s", errOut) + } + if want := core.TargetAddress(2, recipe.KeyCount); problems[0].At != want { + t.Errorf("the ceiling was reported at %q and the target of the file that crossed it is %q.\n"+ + "Reason: an address is the box a screen marks and the line a script points at, and one\n"+ + "counted through the preset's targets names neither.", problems[0].At, want) + } + }) + } +} + +// The engine counts the run of targets it is handed directly. +// +// Every surface reads a recipe first, and the reader counts the run since O302 +// - so no surface reaches this refusal any more, and the guards that asked it +// through validate and through the window ask the reader now. Plan is handed +// targets by whoever calls it, though, and planCapacity reserves nothing past +// the ceiling on the word of this refusal. The memory setting is there so a +// build that lost the refusal plans until the plan ceiling stops it, a few +// megabytes in, rather than a million files. A third target follows the one +// that crosses, so the number the refusal says is asked to be the whole run's +// rather than the running total where it stopped. +func TestTheEngineCountsTheWholeRunOfTargetsHandedToItDirectly(t *testing.T) { + targets := []engine.Target{ + {ID: "small", Format: "txt", Sizes: engine.Uniform(1, 1024)}, + {ID: "big", Format: "txt", Sizes: engine.Uniform(core.MaxFilesPerRun, 1024)}, + {ID: "after", Format: "txt", Sizes: engine.Uniform(core.MaxFilesPerRun, 1024)}, + } + _, err := engine.Plan(targets, engine.Options{OutDir: t.TempDir(), ManifestName: "manifest.json", Memory: core.MemoryOf(32 << 20)}) + if err == nil { + t.Fatal("a run of a million files more than the ceiling was planned") + } + if !strings.Contains(err.Error(), "files across 3 targets") { + t.Fatalf("the run was refused for something other than its count of files: %v", err) + } + if !strings.Contains(err.Error(), "2000001 files across 3 targets") { + t.Errorf("the refusal does not say the files the whole run asks for, 2000001: %v", err) + } + var about interface{ AboutSetting() string } + if !errors.As(err, &about) || about.AboutSetting() != core.TargetAddress(2, engine.SettingCount) { + t.Errorf("the refusal is not addressed to the target that crossed the ceiling, %s: %v", + core.TargetAddress(2, engine.SettingCount), err) + } +} + +// targetsOf is a recipe of txt targets of 1 B, one for each count given. +func targetsOf(counts ...int) []byte { + var b strings.Builder + b.WriteString("version: 1\ntargets:\n") + for i, n := range counts { + fmt.Fprintf(&b, " - id: t%d\n format: txt\n size: 1b\n count: %d\n", i+1, n) + } + return []byte(b.String()) +} + +// readingCost is what reading src allocates, warmed once first so the figure +// is about the recipe and not about what the first reading sets up. +func readingCost(t *testing.T, src []byte) uint64 { + t.Helper() + _, _ = recipe.Parse(src, "warm") + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + _, _ = recipe.Parse(src, "measured") + runtime.ReadMemStats(&after) + return after.TotalAlloc - before.TotalAlloc +} + +// ceilingAt asks that the reader refuses src for the run's count of files, at +// the target in the given position. +func ceilingAt(t *testing.T, src []byte, position int) { + t.Helper() + _, err := recipe.Parse(src, "over") + var invalid *recipe.ValidationError + if !errors.As(err, &invalid) { + t.Fatalf("the reader did not refuse a run past the ceiling: %v", err) + } + want := core.TargetAddress(position, recipe.KeyCount) + for _, p := range invalid.Problems { + if p.At == want && strings.Contains(p.What.String(), "files across") { + return + } + } + t.Fatalf("the reader did not refuse the run's count of files at %s: %v", want, err) +} + +// reportedProblem is one problem of validate --json. +type reportedProblem struct { + What string `json:"what"` + At string `json:"at"` +} + +// reportedProblems reads the problems validate --json put out. +func reportedProblems(t *testing.T, report string) []reportedProblem { + t.Helper() + var parsed struct { + Problems []reportedProblem `json:"problems"` + } + if err := json.Unmarshal([]byte(report), &parsed); err != nil { + t.Fatalf("the report is not readable as JSON: %v\n%s", err, report) + } + return parsed.Problems +} diff --git a/internal/gui/text/locale/said/en.json b/internal/gui/text/locale/said/en.json index 9247585c..f236195f 100644 --- a/internal/gui/text/locale/said/en.json +++ b/internal/gui/text/locale/said/en.json @@ -625,6 +625,12 @@ "hash": "sha256-6dc38cd1cb85", "other": "{{.Error}}" }, + "core.FilesInRun": { + "description": "Said by internal/core - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Count}}, written the way the program writes them - keep each spelled exactly that way.", + "hash": "sha256-83589d643302", + "one": "{{.Count}} file", + "other": "{{.Count}} files" + }, "core.MachineMemoryNone": { "description": "Said by internal/core - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Bytes}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-76987096904d", @@ -730,6 +736,12 @@ "hash": "sha256-58d85993b3db", "other": "the sizes in this run add up to more than a number of bytes can hold, so the total cannot be measured and the free space check cannot be trusted. Ask for fewer files or smaller ones" }, + "core.TargetsInRun": { + "description": "Said by internal/core - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Count}}, written the way the program writes them - keep each spelled exactly that way.", + "hash": "sha256-7f9636c1f771", + "one": "{{.Count}} target", + "other": "{{.Count}} targets" + }, "core.TooManyFiles": { "description": "Said by internal/core - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Why}}, {{.Fix}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-a0fd66f30b3f", @@ -740,6 +752,11 @@ "hash": "sha256-f03079434408", "other": "Ask for fewer files, or split the work into several runs" }, + "core.TooManyFilesInRun": { + "description": "Said by internal/core - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Files}}, {{.Targets}}, written the way the program writes them - keep each spelled exactly that way.", + "hash": "sha256-a18427eed894", + "other": "this run asks for {{.Files}} across {{.Targets}}" + }, "core.TooManyFilesWhy": { "description": "Said by internal/core - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Most}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-84bf666cf3a3", @@ -961,12 +978,6 @@ "hash": "sha256-1e90995301d1", "other": "writing {{.File}}" }, - "engine.Files": { - "description": "Said by internal/engine - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Count}}, written the way the program writes them - keep each spelled exactly that way.", - "hash": "sha256-83589d643302", - "one": "{{.Count}} file", - "other": "{{.Count}} files" - }, "engine.FilesSoFar": { "description": "Said by internal/engine - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Count}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-83589d643302", @@ -1258,12 +1269,6 @@ "hash": "sha256-f4062d7dbd6c", "other": "target {{.Target}}" }, - "engine.Targets": { - "description": "Said by internal/engine - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Count}}, written the way the program writes them - keep each spelled exactly that way.", - "hash": "sha256-7f9636c1f771", - "one": "{{.Count}} target", - "other": "{{.Count}} targets" - }, "engine.TheInstructions": { "description": "Said by internal/engine - a refusal under the box it is about or at the foot of the form, or a note after a run.", "hash": "sha256-074853502278", @@ -1274,11 +1279,6 @@ "hash": "sha256-b802ab9f53ca", "other": "the manifest" }, - "engine.TooManyFiles": { - "description": "Said by internal/engine - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Files}}, {{.Targets}}, written the way the program writes them - keep each spelled exactly that way.", - "hash": "sha256-a18427eed894", - "other": "this run asks for {{.Files}} across {{.Targets}}" - }, "format.AAlreadyNeedsThatMuchBefore": { "description": "Said by internal/format/pdf - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Opts}}, {{.Carrying}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-c48d23d9a134", diff --git a/internal/gui/text/locale/said/pl.json b/internal/gui/text/locale/said/pl.json index 663c1ae0..46a60665 100644 --- a/internal/gui/text/locale/said/pl.json +++ b/internal/gui/text/locale/said/pl.json @@ -124,6 +124,7 @@ "core.DurationTooLong": { "hash": "sha256-06c88d98ae65", "other": "{{.Value}} jest za długi, żeby policzyć go w milisekundach" }, "core.DurationUnknownUnit": { "hash": "sha256-96f50400bd65", "other": "{{.Value}} ma nieznaną jednostkę {{.Unit}}: użyj h, m, s albo ms" }, "core.Error": { "hash": "sha256-6dc38cd1cb85", "other": "{{.Error}}" }, + "core.FilesInRun": { "hash": "sha256-83589d643302", "one": "{{.Count}} plik", "few": "{{.Count}} pliki", "many": "{{.Count}} plików", "other": "{{.Count}} pliku" }, "core.MachineMemoryNone": { "hash": "sha256-76987096904d", "other": "system podał, że ta maszyna ma {{.Bytes}} B pamięci" }, "core.MemoryEmpty": { "hash": "sha256-4a0a99260fc6", "other": "pole {{.Setting}} jest puste: wpisz auto, off albo rozmiar, na przykład 8gb" }, "core.MemoryNegative": { "hash": "sha256-87bca52970de", "other": "{{.Setting}} {{.Value}} jest ujemna: wpisz auto, off albo rozmiar, na przykład 8gb" }, @@ -145,8 +146,10 @@ "core.SizeTooLarge": { "hash": "sha256-dd73ed43bac3", "other": "{{.Setting}} {{.Value}} jest za duży, żeby wyrazić go w bajtach" }, "core.SizeUnknownUnit": { "hash": "sha256-3cd2ef1da79b", "other": "{{.Setting}} {{.Value}} ma nieznaną jednostkę {{.Unit}}: użyj b, kb, mb, gb albo tb, wszystkie liczone po 1024, a kib, mib, gib albo tib to to samo zapisane w pełni" }, "core.SizesOverflow": { "hash": "sha256-58d85993b3db", "other": "rozmiary w tym przebiegu sumują się do więcej, niż mieści liczba bajtów, więc sumy nie da się zmierzyć, a sprawdzeniu wolnego miejsca nie można ufać. Poproś o mniej plików albo mniejsze" }, + "core.TargetsInRun": { "hash": "sha256-7f9636c1f771", "one": "{{.Count}} partii", "few": "{{.Count}} partiach", "many": "{{.Count}} partiach", "other": "{{.Count}} partiach" }, "core.TooManyFiles": { "hash": "sha256-a0fd66f30b3f", "other": "{{.Why}}. {{.Fix}}" }, "core.TooManyFilesFix": { "hash": "sha256-f03079434408", "other": "Poproś o mniej plików albo podziel pracę na kilka przebiegów" }, + "core.TooManyFilesInRun": { "hash": "sha256-a18427eed894", "other": "ten przebieg prosi o {{.Files}} w {{.Targets}}" }, "core.TooManyFilesWhy": { "hash": "sha256-84bf666cf3a3", "other": "ta wersja planuje najwyżej {{.Most}} plików w jednym przebiegu, bo cały plan powstaje w pamięci, zanim cokolwiek zostanie zapisane - dzięki temu przebieg, który nie może się udać, jest odrzucany przed pierwszym bajtem" }, "csv.HasToBeOneOf": { "hash": "sha256-150fed02f771", "other": "musi to być {{.Values}}" }, "csv.MinimumHeader": { "hash": "sha256-002d78cd1305", "other": "tabela zawiera nagłówek i całe wiersze, a po jednym z nich potrzeba tyle" }, @@ -191,7 +194,6 @@ "engine.CrashPlanning": { "hash": "sha256-90f7d5da615f", "other": "planowania pliku" }, "engine.CrashWhile": { "hash": "sha256-071bf3d7823e", "other": "{{.While}}" }, "engine.CrashWriting": { "hash": "sha256-1e90995301d1", "other": "zapisu {{.File}}" }, - "engine.Files": { "hash": "sha256-83589d643302", "one": "{{.Count}} plik", "few": "{{.Count}} pliki", "many": "{{.Count}} plików", "other": "{{.Count}} pliku" }, "engine.FilesSoFar": { "hash": "sha256-83589d643302", "one": "{{.Count}} pliku", "few": "{{.Count}} plikach", "many": "{{.Count}} plikach", "other": "{{.Count}} plikach" }, "engine.GeneratorCrash": { "hash": "sha256-82716d977373", "other": "generator {{.Format}} zatrzymał się na błędzie wewnętrznym w trakcie {{.While}}: {{.Value}}. To wada tego narzędzia, a nie coś, o co przepis może poprosić. Zgłoś ją razem z ustawieniami, które do niej doprowadziły" }, "engine.IDTwice": { "hash": "sha256-724959681477", "other": "nazwa partii {{.ID}} występuje dwa razy" }, @@ -250,10 +252,8 @@ "engine.RunTooLarge": { "hash": "sha256-aff817760566", "other": "partia {{.Target}} doprowadza przebieg do rozmiaru zbyt dużego, żeby go zmierzyć" }, "engine.SizeMoved": { "hash": "sha256-e0c05c6f958b", "other": "Rozmiar wylosowany z zakresu nie jest taki, jaki ten format potrafi zapisać, więc użyto najbliższego, który potrafi. Plik dalej mieści się w zakresie, o który proszono." }, "engine.Target": { "hash": "sha256-f4062d7dbd6c", "other": "partia {{.Target}}" }, - "engine.Targets": { "hash": "sha256-7f9636c1f771", "one": "{{.Count}} partii", "few": "{{.Count}} partiach", "many": "{{.Count}} partiach", "other": "{{.Count}} partiach" }, "engine.TheInstructions": { "hash": "sha256-074853502278", "other": "instrukcje obok manifestu" }, "engine.TheManifest": { "hash": "sha256-b802ab9f53ca", "other": "manifest" }, - "engine.TooManyFiles": { "hash": "sha256-a18427eed894", "other": "ten przebieg prosi o {{.Files}} w {{.Targets}}" }, "format.AAlreadyNeedsThatMuchBefore": { "hash": "sha256-c48d23d9a134", "other": "{{.Opts}}{{.Carrying}} już potrzebuje tyle, zanim dojdzie jakiekolwiek dopełnienie" }, "format.ABMPStatesItsOwnSize": { "hash": "sha256-dbd63989b4da", "other": "BMP zapisuje swój rozmiar w polu czterobajtowym, więc ten format nie opisze tak dużego pliku" }, "format.AByteOrderMarkIsB": { "hash": "sha256-4efd8df69acd", "other": "znacznik kolejności bajtów ma {{.Mark}} B, a plik musi go pomieścić" }, diff --git a/internal/recipe/recipe.go b/internal/recipe/recipe.go index 4a90df0a..f85ea9d8 100644 --- a/internal/recipe/recipe.go +++ b/internal/recipe/recipe.go @@ -425,14 +425,46 @@ func (raw rawRecipe) validate(p *problems, fromPreset int) *Recipe { return rec } + rec.Targets = raw.readTargets(p, rec.Defaults, fromPreset) + return rec +} + +// readTargets reads every target in order, with what only the whole list can +// tell: an id used twice, and a run past the ceiling on files. +func (raw rawRecipe) readTargets(p *problems, def Defaults, fromPreset int) []Target { + var targets []Target + // Which target first used an id, by position in the merged list. The // position is what tells a clash with the preset's target apart from a // clash with another target of the file - and the two get different // words, because only one of them has a line the reader can change. seen := map[string]int{} + // The files of the targets read so far. The run's ceiling is counted here, + // across every target, rather than left to the engine: the engine is handed + // each target with its list of sizes already built, so fifty targets at the + // count ceiling took 2.5 GB to validate before it refused them (O302). Held + // one past the ceiling once crossed, so the room left stays negative from + // there on. It cannot wrap however many targets follow because no target + // adds more than the ceiling: a count past it is refused as the target's + // own and asks for nothing. + planned := 0 + // What every target asks for, which is the number the refusal says. The + // running total stops at the target that crossed the ceiling, and ten + // targets of a million were told they asked for two (review of #219). + var asked int64 + // The target that takes the run past the ceiling, which is the one + // somebody can shrink. Only one is: every target after it is past the + // ceiling as well, and saying so of each would bury the one that matters. + crossedAt := "" for i, rt := range raw.Targets { at := spotOfTarget(i, fromPreset) - t := rt.validate(p, at, rec.Defaults) + room := core.MaxFilesPerRun - planned + t, files := rt.validate(p, at, def, room) + asked += int64(files) + if room >= 0 && files > room { + crossedAt = at(t.ID).of(KeyCount) + } + planned = min(planned+files, core.MaxFilesPerRun+1) if t.ID != "" { if first, dup := seen[t.ID]; dup { usedTwice(p, at(t.ID), t.ID, first < fromPreset && i >= fromPreset) @@ -440,9 +472,13 @@ func (raw rawRecipe) validate(p *problems, fromPreset int) *Recipe { seen[t.ID] = i } } - rec.Targets = append(rec.Targets, t) + targets = append(targets, t) } - return rec + // Said once every target is counted, in the engine's words. + if crossedAt != "" { + p.add(crossedAt, core.TooManyFilesInRun(asked, len(raw.Targets)), core.TooManyFilesWhy, core.TooManyFilesFix) + } + return targets } // usedTwice refuses the second target carrying an id, in one of two wordings: diff --git a/internal/recipe/target.go b/internal/recipe/target.go index a7ad38fe..888e843d 100644 --- a/internal/recipe/target.go +++ b/internal/recipe/target.go @@ -64,14 +64,19 @@ type rawTarget struct { // that put one answer in tfg formats and another in the generator once already. const DefaultCount = 1 -// validate reads one target and reports everything wrong with it. +// validate reads one target and reports everything wrong with it, and says how +// many files it asks for - the number the run's ceiling counts. // // at is where this target is, as a function of its id - see spotOfTarget. It // is a function rather than a position because the prose names a target by // its id as soon as one is read, and the id is read here. -func (rt rawTarget) validate(p *problems, at func(id string) spot, def Defaults) Target { +// +// room is how many files the run has left before its ceiling. A target asking +// for more is read for everything else, but its list of sizes is not built: +// the run is refused at it whatever its sizes say, and building the list first +// is the allocation the ceiling is there to prevent. +func (rt rawTarget) validate(p *problems, at func(id string) spot, def Defaults, room int) (Target, int) { t := Target{Label: def.Label} - count := DefaultCount where := at("") if id, ok := oneValue(p, where.of("id"), where.String()+" {setting}", "id: invoices", rt.ID); ok && id != "" { @@ -92,34 +97,16 @@ func (rt rawTarget) validate(p *problems, at func(id string) spot, def Defaults) t.Format = format } - if rt.Count != nil { - n, ok := rt.Count.number() - switch { - case !ok: - p.add(where.of("count"), core.Says("recipe.HasSettingOfWhichIsNot", "%s has {a} {setting} of %q, which is not a whole number", core.A("Where", where), core.A("Text", rt.Count.text)), - core.Says("recipe.SettingIsHowManyFilesThis", "{a} {setting} is how many files this target produces, so it is read exactly as written"), - core.Says("recipe.WriteADecimalNumberSuchAs2", "write a decimal number such as count: 10")) - case n <= 0: - p.add(where.of("count"), core.Says("recipe.AsksForFiles", "%s asks for %d files", core.A("At", where), core.A("Value", n)), - core.Says("recipe.ATargetThatProducesNothingIs", "a target that produces nothing is almost always a mistake rather than an intention"), - core.Says("recipe.AskForAtLeastOneOr", "ask for at least one, or delete the target")) - // Judged before the list is built. The reader used to grow it one entry - // at a time and reached a 13 GB allocation on a count of 2^63 - so this - // has to refuse the number rather than the result of using it. - case n > core.MaxFilesPerRun: - p.add(where.of("count"), core.Says("recipe.AsksForFiles", "%s asks for %d files", core.A("At", where), core.A("Value", n)), - core.SaidOf(core.ErrTooManyFiles), - core.Says("recipe.UseSettingOfOrLessOr2", "use {a} {setting} of %d or less, or split the target across several recipes", core.A("MaxFilesPerRun", core.MaxFilesPerRun))) - default: - count = int(n) - } - } - + count := rt.readCount(p, where) rt.refuseSections(p, where) if rt.Contains != nil { t.Contains = contentGroups(p, where, rt.Contains) } - rt.resolveSize(p, where, count, &t) + built := count + if count > room { + built = 0 + } + rt.resolveSize(p, where, built, &t) if name, ok := oneValue(p, where.of("name"), where.String()+" {setting}", "name: invoice_{index:04}.pdf", rt.Name); ok { t.Name = name @@ -133,7 +120,50 @@ func (rt rawTarget) validate(p *problems, at func(id string) spot, def Defaults) refuseImpossibleExpectation(p, where, t) rt.describe(p, where, &t) t.Properties = properties(p, where, t.Format, rt.Properties) - return t + + // The count, whether or not its list was built and whether or not its size + // was refused, so a run past the ceiling is said beside the target's own + // problems (RC7). A boundary set is the sizes it has - three, built whatever + // room is left, because three is not an allocation. For a recipe this + // reader passes, it is the length of the list, which is what the engine + // counts. + files := count + if rt.Boundary != nil { + files = len(t.Sizes) + } + return t, files +} + +// readCount reads how many files this target asks for. +// +// A count refused here asks for nothing. Left at the default, the one file it +// stands for would take a target of a million beside it past the run's +// ceiling, and one mistake would be refused twice. +func (rt rawTarget) readCount(p *problems, where spot) int { + if rt.Count == nil { + return DefaultCount + } + n, ok := rt.Count.number() + switch { + case !ok: + p.add(where.of("count"), core.Says("recipe.HasSettingOfWhichIsNot", "%s has {a} {setting} of %q, which is not a whole number", core.A("Where", where), core.A("Text", rt.Count.text)), + core.Says("recipe.SettingIsHowManyFilesThis", "{a} {setting} is how many files this target produces, so it is read exactly as written"), + core.Says("recipe.WriteADecimalNumberSuchAs2", "write a decimal number such as count: 10")) + case n <= 0: + p.add(where.of("count"), core.Says("recipe.AsksForFiles", "%s asks for %d files", core.A("At", where), core.A("Value", n)), + core.Says("recipe.ATargetThatProducesNothingIs", "a target that produces nothing is almost always a mistake rather than an intention"), + core.Says("recipe.AskForAtLeastOneOr", "ask for at least one, or delete the target")) + // Judged before the list is built. The reader used to grow it one entry at + // a time and reached a 13 GB allocation on a count of 2^63 - so this has to + // refuse the number rather than the result of using it. + case n > core.MaxFilesPerRun: + p.add(where.of("count"), core.Says("recipe.AsksForFiles", "%s asks for %d files", core.A("At", where), core.A("Value", n)), + core.SaidOf(core.ErrTooManyFiles), + core.Says("recipe.UseSettingOfOrLessOr2", "use {a} {setting} of %d or less, or split the target across several recipes", core.A("MaxFilesPerRun", core.MaxFilesPerRun))) + default: + return int(n) + } + return 0 } // describe reads what a target says ABOUT its files rather than how to make @@ -215,9 +245,7 @@ func (rt rawTarget) resolveSize(p *problems, where spot, count int, t *Target) { core.Says("recipe.UseSettingSuchAs2mb512kb", "use {a} {setting} such as 2mb, 512kb or a plain byte count")) break } - for i := 0; i < count; i++ { - t.Sizes = append(t.Sizes, n) - } + t.Sizes = core.Uniform(count, n) case rt.SizeRange != nil: // Placed above contains for the same reason size is: a stated size @@ -240,9 +268,7 @@ func (rt rawTarget) resolveSize(p *problems, where spot, count int, t *Target) { // The values stay zero here and the engine replaces them. What this // list carries at this point is the number of files, exactly as it // does for contains. - for i := 0; i < count; i++ { - t.Sizes = append(t.Sizes, 0) - } + t.Sizes = core.Uniform(count, 0) case rt.Contains != nil: // The fourth way of declaring a size, and not an exception to the rule. @@ -250,9 +276,7 @@ func (rt rawTarget) resolveSize(p *problems, where spot, count int, t *Target) { // and a dry run still reports a number without generating anything. // See ARCHITECTURE.md section 9. t.SizeFromContents = true - for i := 0; i < count; i++ { - t.Sizes = append(t.Sizes, 0) - } + t.Sizes = core.Uniform(count, 0) default: p.add(where.of("size"), core.Says("recipe.HasNoSetting", "%s has no {setting}", core.A("At", where)),