diff --git a/.github/workflows/reusable.yml b/.github/workflows/reusable.yml index 7db00d8..ebd8fe1 100644 --- a/.github/workflows/reusable.yml +++ b/.github/workflows/reusable.yml @@ -107,6 +107,22 @@ on: description: "`runs-on` for every job: one label, or a JSON array of them." type: string default: ubuntu-latest + min_positive: + description: >- + Evaluations a skill needs with `skill_should_trigger: true`. Blank + uses skillscope's own default. Zero for both this and min_negative + means a skill need not ship a dataset at all, which is how a + catalogue adopts these checks before it has written one. It lives + here, in the workflow, because the bar a skill is measured against is + not the skill author's to set. + type: string + default: "" + min_negative: + description: >- + Evaluations a skill needs with `skill_should_trigger: false`. Blank + uses skillscope's own default. See min_positive. + type: string + default: "" min_accuracy: description: >- Fail a routing leg below this accuracy (0-1). The default fails on any @@ -285,6 +301,13 @@ jobs: uses: ./.skillscope-action with: command: structural + args: >- + # `!= ''` rather than the bare truthiness the inputs above use: + # `0` is a meaningful value for these two, and the condition + # should say "the caller filled it in" rather than rely on a + # cast that happens to agree. + ${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }} + ${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }} skills: ${{ needs.discover.outputs.globs }} routing: @@ -337,6 +360,8 @@ jobs: args: >- --routing-room '${{ matrix.skill }}' --min-accuracy '${{ inputs.min_accuracy }}' + ${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }} + ${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }} --output routing-report.json --keep-logs routing-logs skills: ${{ needs.discover.outputs.globs }} @@ -394,6 +419,8 @@ jobs: args: >- --skill '${{ matrix.skill }}' --output behavioral-report.json + ${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }} + ${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }} skills: ${{ needs.discover.outputs.globs }} install-claude: "true" diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml index 78c2a5f..7e1415d 100644 --- a/.github/workflows/skill-evals.yml +++ b/.github/workflows/skill-evals.yml @@ -111,6 +111,22 @@ on: that answer a runner's IP with a 403. type: string default: "" + min_positive: + description: >- + Evaluations a skill needs with `skill_should_trigger: true`. Blank + uses skillscope's own default. Zero for both this and min_negative + means a skill need not ship a dataset at all, which is how a + catalogue adopts these checks before it has written one. It lives + here, in the workflow, because the bar a skill is measured against is + not the skill author's to set. + type: string + default: "" + min_negative: + description: >- + Evaluations a skill needs with `skill_should_trigger: false`. Blank + uses skillscope's own default. See min_positive. + type: string + default: "" skill_files: description: >- Files every skill must ship beside its SKILL.md, comma-separated and @@ -353,6 +369,12 @@ jobs: ${{ inputs.doc_globs && format('--docs ''{0}''', inputs.doc_globs) || '' }} ${{ inputs.skill_files && format('--skill-files ''{0}''', inputs.skill_files) || '' }} ${{ inputs.skill_sections && format('--skill-sections ''{0}''', inputs.skill_sections) || '' }} + # `!= ''` rather than the bare truthiness the inputs above use: + # `0` is a meaningful value for these two, and the condition + # should say "the caller filled it in" rather than rely on a + # cast that happens to agree. + ${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }} + ${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }} skills: ${{ inputs.skill_globs }} python-version: ${{ inputs.python_version }} @@ -504,6 +526,8 @@ jobs: --external ${{ inputs.doc_globs && format('--docs ''{0}''', inputs.doc_globs) || '' }} ${{ inputs.excluded_urls && format('--exclude-url ''{0}''', inputs.excluded_urls) || '' }} + ${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }} + ${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }} skills: ${{ inputs.skill_globs }} python-version: ${{ inputs.python_version }} @@ -565,6 +589,8 @@ jobs: ${{ inputs.only && format('--only ''{0}''', inputs.only) || '' }} --output routing-report.json --keep-logs routing-logs + ${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }} + ${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }} skills: ${{ inputs.skill_globs }} python-version: ${{ inputs.python_version }} setup-python: "false" @@ -627,7 +653,10 @@ jobs: uses: ./.skillscope-action with: command: behavioral - args: --skill '${{ matrix.skill }}' ${{ needs.discover.outputs.extended }} + args: >- + --skill '${{ matrix.skill }}' ${{ needs.discover.outputs.extended }} + ${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }} + ${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }} skills: ${{ inputs.skill_globs }} python-version: ${{ inputs.python_version }} setup-python: "false" @@ -684,7 +713,10 @@ jobs: uses: ./.skillscope-action with: command: behavioral - args: --skill '${{ matrix.skill }}' ${{ needs.discover.outputs.extended }} + args: >- + --skill '${{ matrix.skill }}' ${{ needs.discover.outputs.extended }} + ${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }} + ${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }} skills: ${{ inputs.skill_globs }} python-version: ${{ inputs.python_version }} setup-python: "false" diff --git a/docs/usage.md b/docs/usage.md index ee4c527..7a3468f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -29,6 +29,27 @@ Every skill ships one dataset at `/evals/evals.json`. A prompt alone grades routing; add expectations and it also runs end to end. How to write one is in [authoring-evals.md](authoring-evals.md). +By default a skill needs three evaluations that expect it to fire and two that +expect it not to, and a skill with no dataset at all fails the structural +check. That is the bar a new catalogue should hold, and it is the wrong bar for +an existing one: a repo adopting skillscope over fifty skills written before it +cannot write two hundred and fifty prompts before its first green run, and the +alternative it reaches for is to not run the checks. `--min-positive` and +`--min-negative` lower the bar so the folder, frontmatter and reference checks +can start working immediately, and get raised as the datasets land. Zero for +both turns the dataset requirement off; a skill that ships a dataset anyway is +still read against whatever the bar is. + +**Where that bar may be set is the point.** It is a flag, so under CI it is a +line in the workflow, which is a path a repo can put behind branch protection +and code owners. There is no file in the repo under test, none inside a skill, +and no environment variable the CLI will read. The bar a skill is measured +against is not the skill author's to set: were it a file beside the skill, the +same pull request that adds a thin dataset could lower the number it is judged +by, and a reviewer would have to notice the second change to understand the +first. The applied bar is printed at the top of every structural run, passing +or failing, so a report always says what it was holding skills to. + ## Configuring the repo under test There is no config file. Everything about your repo is a CLI flag, or the same @@ -46,6 +67,8 @@ places to disagree. | `--exclude-url` | `excluded_urls` | none | Regexes matching URLs the external reference check leaves alone. For hosts that are auth-gated or that answer a runner's IP with a 403. | | `--skill-files` | `skill_files` | none | Files every skill must ship beside its `SKILL.md`, such as a governance card. | | `--skill-sections` | `skill_sections` | none | `##` headings each of those markdown files must have something under, such as `Description,Owner,License`. | +| `--min-positive` | `min_positive` | `3` | Evaluations a skill needs with `skill_should_trigger: true`. | +| `--min-negative` | `min_negative` | `2` | Evaluations it needs with `skill_should_trigger: false`. Zero for both means a skill need not ship a dataset at all. | | `--behavior-runner` | `behavior_runner` | `["ubuntu-latest"]` | `runs-on` labels for a behavioral leg. | | `--behavior-os` | `behavior_os` | `Linux` | Platforms a skill runs on when its `machine.yml` does not narrow them. | | `--scoped-runner` | `scoped_runner` | reuses `behavior_runner` | Base labels for a leg whose skill asks for hardware labels. | @@ -410,6 +433,8 @@ Naming several, and holding them to different bars: | `behavioral` | `required` | `required`, `optional`, or `off`. | | `runner` | `ubuntu-latest` | `runs-on` for every job: one label, or a JSON array of them. | | `min_accuracy` | `1` | The routing bar. `0` reports the score without gating on it. | +| `min_positive` | (skillscope's own) | Evaluations a skill needs with `skill_should_trigger: true`. | +| `min_negative` | (skillscope's own) | Evaluations it needs with `skill_should_trigger: false`. Zero for both means a skill need not ship a dataset. | | `api_key` | (none) | The model API key, mapped from the caller's vault. One secret, not the whole set. | | `api_key_secret` | `ANTHROPIC_API_KEY` | Name to look up under `secrets: inherit`, if you would rather pass the vault than map one key. | | `api_base_url` | (none) | Base URL of the gateway that issued the key. Blank talks to the provider directly. | diff --git a/skillscope/cli.py b/skillscope/cli.py index bc9c406..74cb18a 100644 --- a/skillscope/cli.py +++ b/skillscope/cli.py @@ -299,12 +299,32 @@ def routing_gate(totals: dict, min_accuracy: float) -> str | None: def cmd_structural(args: argparse.Namespace) -> int: """Structural checks over every skill folder, dataset, and reference.""" + # Before the gate, not after it: a run that fails still has to say which + # bar it was holding skills to. Printing this only on success would hide + # exactly the case worth seeing -- a repo whose datasets are not required, + # failing on something else -- behind a passing run. + cfg = config.active() + # Flushed, because the failures below go to stderr: unflushed stdout is + # block-buffered down a pipe, and the line would land after the report it + # is meant to frame in every CI log that captures both. + if cfg.min_positive or cfg.min_negative: + print( + f"[evals] dataset bar: {cfg.min_positive} positive, " + f"{cfg.min_negative} negative per skill.", + flush=True, + ) + else: + print( + "[evals] dataset bar: off. Skills are not required to ship " + "`evals/evals.json`; the ones that do are still read.", + flush=True, + ) + found = _structural_or_exit() skills = datasets.skills_with_datasets() # Extended datasets are checked regardless of --extended, so count them # here too rather than reporting fewer cases than were checked. cases = datasets.load_all_cases(extended=True) - cfg = config.active() print(f"[evals] OK: {len(cfg.skills)} skill folder(s) in shape.") print( f"[evals] OK: {len(cases)} case(s) across {len(skills)} skill(s) " @@ -882,6 +902,37 @@ def _add_skills_argument(parser: argparse.ArgumentParser) -> None: ) +def _add_dataset_bar_arguments(parser: argparse.ArgumentParser) -> None: + """How much of a dataset this repo asks of a skill. + + A flag and nothing else. The bar a skill is measured against is the repo's + policy, not the skill author's, so it has to live somewhere a skill cannot + reach: see `min_positive` on `config.Config`. + """ + parser.add_argument( + "--min-positive", + default="", + metavar="N", + help=( + "Evaluations a skill needs with `skill_should_trigger: true`. " + f"Default: {config.DEFAULT_MIN_POSITIVE}. Lower it to adopt these " + "checks over a catalogue that predates them, then raise it as the " + "datasets land." + ), + ) + parser.add_argument( + "--min-negative", + default="", + metavar="N", + help=( + "Evaluations a skill needs with `skill_should_trigger: false`. " + f"Default: {config.DEFAULT_MIN_NEGATIVE}. Zero for both this and " + "--min-positive means a skill need not ship a dataset at all; the " + "folder, frontmatter and reference checks still run." + ), + ) + + def _add_docs_argument(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--docs", @@ -924,6 +975,10 @@ def _add_routing_room_argument( def _add_graded_arguments(parser: argparse.ArgumentParser) -> None: """Flags shared by the routing and behavioral commands.""" _add_skills_argument(parser) + # A graded run gates on the same dataset bar before it spends anything, so + # it takes the same flags. Without them a workflow that lowered the bar for + # `structural` would still be refused by `routing` on the next line. + _add_dataset_bar_arguments(parser) parser.add_argument( "--skill", default="", @@ -1114,6 +1169,7 @@ def build_parser() -> argparse.ArgumentParser: ), ) _add_skills_argument(structural_parser) + _add_dataset_bar_arguments(structural_parser) _add_docs_argument(structural_parser) structural_parser.add_argument( "--skill-files", @@ -1341,6 +1397,8 @@ def _configure(args: argparse.Namespace) -> None: "behavior_runner", "behavior_os", "scoped_runner", + "min_positive", + "min_negative", ) } settings["scoped_gate"] = getattr(args, "scoped_gate", "") or "" diff --git a/skillscope/config.py b/skillscope/config.py index bc7d98a..62e7aa9 100644 --- a/skillscope/config.py +++ b/skillscope/config.py @@ -61,6 +61,15 @@ DEFAULT_BEHAVIOR_RUNNER = ("ubuntu-latest",) DEFAULT_BEHAVIOR_OS = ("Linux",) +# Tier 0, the bar every skill clears before it can ship: five prompts, no +# hardware, no assertions, enforced structurally so a thin dataset fails +# without spending a token. These are the defaults, not the rule -- a repo +# adopting this harness over an existing catalogue lowers them, and raises them +# again as the datasets land. Where that decision may be made is the point of +# `min_positive` on Config below. +DEFAULT_MIN_POSITIVE = 3 +DEFAULT_MIN_NEGATIVE = 2 + # The two answers `--routing-room` takes instead of a list of names: every # skill in the repo that ships a dataset, and no routing run at all. Both are # words rather than shapes of an empty flag, because saying nothing means "work @@ -123,6 +132,24 @@ class Config: scoped_gate: str = "" scoped_environment: str = "" + # How many of a skill's evaluations must expect it to fire, and how many + # must expect it not to. Zero for both means a skill need not ship a + # dataset at all, which is how a catalogue adopts the structural checks + # before it has written one. + # + # This is reachable from the command line and from nowhere else, and that + # is the design rather than an omission. The bar a skill is held to is not + # the skill author's to set: a file inside a skill, or at the repo root, + # would let the same pull request that adds a thin dataset also lower the + # number it is measured against, and the reviewer would have to notice the + # second change to understand the first. On the command line it lives in + # the workflow, which is a path a repo can put behind branch protection and + # code owners. There is deliberately no environment variable either: a + # self-hosted runner's environment is set by whoever administers the + # machine, which is not the same set of people who review a workflow diff. + min_positive: int = DEFAULT_MIN_POSITIVE + min_negative: int = DEFAULT_MIN_NEGATIVE + @property def skills(self) -> dict[str, Path]: """Every skill this repo declares, as ``{name: folder}``. @@ -286,6 +313,31 @@ def wants_no_skills(value: object) -> bool: return _is_sentinel(value, NO_SKILLS) +def _bar(value: object, flag: str, default: int) -> int: + """A non-negative count from a flag, a workflow input, or a test fixture. + + Blank means the default, so a workflow input nobody filled in is the same + as no flag at all. Anything else has to be a whole number that is not + negative: a bar of -1 would read as "off" to whoever wrote it and as "every + count clears it" to the code, and those are the same outcome reached by + accident. Zero is the way to say off, and it says it once. + """ + if value is None or (isinstance(value, str) and not value.strip()): + return default + try: + count = int(str(value).strip()) + except ValueError: + raise SystemExit( + f"error: {flag} takes a whole number, not {value!r}." + ) from None + if count < 0: + raise SystemExit( + f"error: {flag} cannot be negative (got {count}). Zero is how a " + "repo says this bar does not apply yet." + ) + return count + + def build( root: Path | None = None, *, @@ -301,6 +353,8 @@ def build( scoped_runner: object = None, scoped_gate: str = "", scoped_environment: str = "", + min_positive: object = None, + min_negative: object = None, dataset_skills: list[str] | None = None, ) -> Config: """A Config from loose values: what the CLI hands over after parsing. @@ -347,6 +401,8 @@ def build( scoped_runner=_items(scoped_runner, "--scoped-runner"), scoped_gate=(scoped_gate or "").strip(), scoped_environment=(scoped_environment or "").strip(), + min_positive=_bar(min_positive, "--min-positive", DEFAULT_MIN_POSITIVE), + min_negative=_bar(min_negative, "--min-negative", DEFAULT_MIN_NEGATIVE), ) diff --git a/skillscope/datasets.py b/skillscope/datasets.py index aea25f9..82acf65 100644 --- a/skillscope/datasets.py +++ b/skillscope/datasets.py @@ -91,11 +91,13 @@ # What a new skill copies to start. TEMPLATE = PACKAGE_DIR / "data" / "TEMPLATE.json" -# Tier 0, the bar every skill clears before it can ship. Cheap to meet (five -# prompts, no hardware, no assertions) and enforced structurally so a thin -# dataset fails the structural checks without spending a single token. -MIN_POSITIVE_CASES = 3 -MIN_NEGATIVE_CASES = 2 +# Tier 0, the bar every skill clears before it can ship. Cheap to meet by +# default (five prompts, no hardware, no assertions) and enforced structurally +# so a thin dataset fails without spending a single token. The numbers live on +# the config rather than here, because a repo adopting this harness over a +# catalogue that predates it needs to lower them and ratchet them back up; see +# `min_positive` on `config.Config` for why the command line is the only place +# that decision can be made. # A dataset is one array of evaluations, and every evaluation answers the # routing question outright rather than leaving it to be inferred. @@ -504,33 +506,44 @@ def structural_errors(skills: list[str] | None = None) -> list[str]: def tier0_errors(skill: str, cases: list[Case]) -> list[str]: - """Whether `skill` meets the mandatory coverage bar. + """Whether `skill` meets this repo's coverage bar. - Counted over the required dataset alone (evals.json). + Counted over the required dataset alone (evals.json). The bar is + `min_positive` and `min_negative` on the active config, which only the + command line sets; both at zero is a repo saying datasets are not required + of it yet, and then a skill without one is not a finding. """ + cfg = config.active() + required = cfg.min_positive or cfg.min_negative + if not dataset_path(skill).is_file(): + if not required: + return [] return [ f"{skill}: no eval dataset. Every skill needs " f"`{DATASET_RELPATH.as_posix()}` in its folder with at least " - f"{MIN_POSITIVE_CASES} evaluations where `{TRIGGER_KEY}` is true " - f"and {MIN_NEGATIVE_CASES} where it is false. " + f"{cfg.min_positive} evaluations where `{TRIGGER_KEY}` is true " + f"and {cfg.min_negative} where it is false. " "Copy the template from `skillscope template` to start." ] + # A dataset that exists is still read against the bar, whatever the bar is. + # A repo that has turned the requirement off has said it does not yet + # expect every skill to ship one, not that the ones that do may be empty. errors: list[str] = [] positive = sum(1 for c in cases if c.skill_should_trigger) negative = len(cases) - positive - if positive < MIN_POSITIVE_CASES: + if positive < cfg.min_positive: errors.append( f"{skill}: {positive} evaluation(s) with `{TRIGGER_KEY}: true`; " - f"Tier 0 needs at least {MIN_POSITIVE_CASES}. Add prompts a real " - "user would type." + f"this repo asks for at least {cfg.min_positive}. Add prompts a " + "real user would type." ) - if negative < MIN_NEGATIVE_CASES: + if negative < cfg.min_negative: errors.append( f"{skill}: {negative} evaluation(s) with `{TRIGGER_KEY}: false`; " - f"Tier 0 needs at least {MIN_NEGATIVE_CASES}. Add prompts close to " - "this skill's domain that should NOT trigger it." + f"this repo asks for at least {cfg.min_negative}. Add prompts " + "close to this skill's domain that should NOT trigger it." ) return errors diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 8fb988d..e48ec42 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -104,11 +104,11 @@ def tier0_dataset(slug: str, **extra) -> dict: """The smallest dataset that clears the mandatory coverage bar.""" evaluations = [ {"id": f"{slug}-yes-{i}", TRIGGER_KEY: True, "prompt": f"please do {slug} work {i}"} - for i in range(datasets.MIN_POSITIVE_CASES) + for i in range(config.DEFAULT_MIN_POSITIVE) ] evaluations += [ {"id": f"{slug}-no-{i}", TRIGGER_KEY: False, "prompt": f"nothing to do with {slug} {i}"} - for i in range(datasets.MIN_NEGATIVE_CASES) + for i in range(config.DEFAULT_MIN_NEGATIVE) ] return {EVALUATIONS_KEY: evaluations, **extra} @@ -1393,6 +1393,113 @@ def test_a_skill_with_no_dataset_is_reported(self) -> None: self.assertIn("no eval dataset", errors[0]) +class TestTheDatasetBarIsTheRepos(unittest.TestCase): + """The coverage bar is configuration, and only the command line sets it. + + A repo adopting these checks over a catalogue written before them has to be + able to lower the bar, or it runs none of the checks at all. What that must + never become is a number the skill under review can move: the point of the + bar is that something outside the skill decides it. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + + def test_a_lowered_bar_accepts_what_the_default_rejects(self) -> None: + self.repo.skill("demo-skill", dataset=tier0_dataset("demo")) + self.repo.activate() + thin, _ = parse( + { + EVALUATIONS_KEY: [ + {"id": "yes", TRIGGER_KEY: True, "prompt": "do demo work"}, + {"id": "no", TRIGGER_KEY: False, "prompt": "unrelated"}, + ] + }, + skill="demo-skill", + ) + self.assertNotEqual(datasets.tier0_errors("demo-skill", thin), []) + self.repo.reactivate(min_positive=1, min_negative=1) + self.assertEqual(datasets.tier0_errors("demo-skill", thin), []) + + def test_a_raised_bar_rejects_what_the_default_accepts(self) -> None: + self.repo.skill("demo-skill", dataset=tier0_dataset("demo")) + self.repo.activate(min_positive=99) + cases, _ = parse(tier0_dataset("demo"), skill="demo-skill") + errors = datasets.tier0_errors("demo-skill", cases) + self.assertTrue(any("at least 99" in e for e in errors), errors) + + def test_zero_for_both_stops_requiring_a_dataset(self) -> None: + self.repo.skill("bare-skill") + self.repo.activate() + self.assertNotEqual(datasets.tier0_errors("bare-skill", []), []) + self.repo.reactivate(min_positive=0, min_negative=0) + self.assertEqual(datasets.tier0_errors("bare-skill", []), []) + + def test_one_bar_still_standing_still_requires_a_dataset(self) -> None: + """Only *both* at zero means "not required"; one of them does not.""" + self.repo.skill("bare-skill") + self.repo.activate(min_positive=0, min_negative=1) + errors = datasets.tier0_errors("bare-skill", []) + self.assertEqual(len(errors), 1) + self.assertIn("no eval dataset", errors[0]) + + def test_a_dataset_that_exists_is_read_even_with_the_bar_off(self) -> None: + """Off means "not required of every skill", not "unread where present".""" + self.repo.skill("demo-skill", dataset=tier0_dataset("demo")) + self.repo.activate(min_positive=0, min_negative=0) + self.assertIn("demo-skill", datasets.skills_with_datasets()) + cases = datasets.routing_cases(["demo-skill"]) + self.assertTrue(cases) + + def test_the_bar_is_not_settable_from_inside_a_skill(self) -> None: + """The governance property, asserted rather than assumed. + + A file a skill author can edit must not move the number their own + skill is judged by, whatever it is called or wherever in the skill it + sits. Nothing reads these; the test exists so that stays true. + """ + folder = self.repo.skill("bare-skill") + for name in ("skillscope.json", "config.yml", "evals/skillscope.yml"): + path = folder / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + '{"min_positive": 0, "min_negative": 0}', encoding="utf-8" + ) + self.repo.activate() + errors = datasets.tier0_errors("bare-skill", []) + self.assertEqual(len(errors), 1) + self.assertIn("no eval dataset", errors[0]) + + def test_the_bar_is_not_settable_from_the_environment(self) -> None: + """No environment fallback, unlike SKILLSCOPE_SKILLS. + + A self-hosted runner's environment is set by whoever administers the + machine, who is not the person reviewing the workflow diff. Adding a + fallback here would quietly widen who can lower the bar. + """ + self.repo.skill("bare-skill") + for key in ("SKILLSCOPE_MIN_POSITIVE", "SKILLSCOPE_MIN_NEGATIVE"): + os.environ[key] = "0" + self.addCleanup(os.environ.pop, key, None) + self.repo.activate() + errors = datasets.tier0_errors("bare-skill", []) + self.assertEqual(len(errors), 1) + self.assertIn("no eval dataset", errors[0]) + + def test_a_bar_that_is_not_a_count_is_refused(self) -> None: + for value in ("-1", "two", "1.5"): + with self.subTest(value=value): + with self.assertRaises(SystemExit) as caught: + config.build(self.repo.root, min_positive=value) + self.assertIn("--min-positive", str(caught.exception)) + + def test_a_blank_bar_is_the_default(self) -> None: + """A workflow input nobody filled in is the same as no flag at all.""" + cfg = config.build(self.repo.root, min_positive="", min_negative=None) + self.assertEqual(cfg.min_positive, config.DEFAULT_MIN_POSITIVE) + self.assertEqual(cfg.min_negative, config.DEFAULT_MIN_NEGATIVE) + + class TestExtendedDataset(unittest.TestCase): """The optional second dataset: same format, no bar, opt-in run."""