Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions .github/workflows/reusable.yml
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,22 @@ on:
description: "`runs-on` for every job: one label, or a JSON array of them."
type: string
default: ubuntu-latest
min_positive:
description: >-
Evaluations a skill needs with `skill_should_trigger: true`. Blank
uses skillscope's own default. Zero for both this and min_negative
means a skill need not ship a dataset at all, which is how a
catalogue adopts these checks before it has written one. It lives
here, in the workflow, because the bar a skill is measured against is
not the skill author's to set.
type: string
default: ""
min_negative:
description: >-
Evaluations a skill needs with `skill_should_trigger: false`. Blank
uses skillscope's own default. See min_positive.
type: string
default: ""
min_accuracy:
description: >-
Fail a routing leg below this accuracy (0-1). The default fails on any
Expand Down Expand Up @@ -285,6 +301,13 @@ jobs:
uses: ./.skillscope-action
with:
command: structural
args: >-
# `!= ''` rather than the bare truthiness the inputs above use:
# `0` is a meaningful value for these two, and the condition
# should say "the caller filled it in" rather than rely on a
# cast that happens to agree.
${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }}
${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }}
skills: ${{ needs.discover.outputs.globs }}

routing:
Expand Down Expand Up @@ -337,6 +360,8 @@ jobs:
args: >-
--routing-room '${{ matrix.skill }}'
--min-accuracy '${{ inputs.min_accuracy }}'
${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }}
${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }}
--output routing-report.json
--keep-logs routing-logs
skills: ${{ needs.discover.outputs.globs }}
Expand Down Expand Up @@ -394,6 +419,8 @@ jobs:
args: >-
--skill '${{ matrix.skill }}'
--output behavioral-report.json
${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }}
${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }}
skills: ${{ needs.discover.outputs.globs }}
install-claude: "true"

Expand Down
36 changes: 34 additions & 2 deletions .github/workflows/skill-evals.yml
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,22 @@ on:
that answer a runner's IP with a 403.
type: string
default: ""
min_positive:
description: >-
Evaluations a skill needs with `skill_should_trigger: true`. Blank
uses skillscope's own default. Zero for both this and min_negative
means a skill need not ship a dataset at all, which is how a
catalogue adopts these checks before it has written one. It lives
here, in the workflow, because the bar a skill is measured against is
not the skill author's to set.
type: string
default: ""
min_negative:
description: >-
Evaluations a skill needs with `skill_should_trigger: false`. Blank
uses skillscope's own default. See min_positive.
type: string
default: ""
skill_files:
description: >-
Files every skill must ship beside its SKILL.md, comma-separated and
Expand Down Expand Up @@ -353,6 +369,12 @@ jobs:
${{ inputs.doc_globs && format('--docs ''{0}''', inputs.doc_globs) || '' }}
${{ inputs.skill_files && format('--skill-files ''{0}''', inputs.skill_files) || '' }}
${{ inputs.skill_sections && format('--skill-sections ''{0}''', inputs.skill_sections) || '' }}
# `!= ''` rather than the bare truthiness the inputs above use:
# `0` is a meaningful value for these two, and the condition
# should say "the caller filled it in" rather than rely on a
# cast that happens to agree.
${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }}
${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }}
skills: ${{ inputs.skill_globs }}
python-version: ${{ inputs.python_version }}

Expand Down Expand Up @@ -504,6 +526,8 @@ jobs:
--external
${{ inputs.doc_globs && format('--docs ''{0}''', inputs.doc_globs) || '' }}
${{ inputs.excluded_urls && format('--exclude-url ''{0}''', inputs.excluded_urls) || '' }}
${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }}
${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }}
skills: ${{ inputs.skill_globs }}
python-version: ${{ inputs.python_version }}

Expand Down Expand Up @@ -565,6 +589,8 @@ jobs:
${{ inputs.only && format('--only ''{0}''', inputs.only) || '' }}
--output routing-report.json
--keep-logs routing-logs
${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }}
${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }}
skills: ${{ inputs.skill_globs }}
python-version: ${{ inputs.python_version }}
setup-python: "false"
Expand Down Expand Up @@ -627,7 +653,10 @@ jobs:
uses: ./.skillscope-action
with:
command: behavioral
args: --skill '${{ matrix.skill }}' ${{ needs.discover.outputs.extended }}
args: >-
--skill '${{ matrix.skill }}' ${{ needs.discover.outputs.extended }}
${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }}
${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }}
skills: ${{ inputs.skill_globs }}
python-version: ${{ inputs.python_version }}
setup-python: "false"
Expand Down Expand Up @@ -684,7 +713,10 @@ jobs:
uses: ./.skillscope-action
with:
command: behavioral
args: --skill '${{ matrix.skill }}' ${{ needs.discover.outputs.extended }}
args: >-
--skill '${{ matrix.skill }}' ${{ needs.discover.outputs.extended }}
${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }}
${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }}
skills: ${{ inputs.skill_globs }}
python-version: ${{ inputs.python_version }}
setup-python: "false"
Expand Down
25 changes: 25 additions & 0 deletions docs/usage.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,27 @@ Every skill ships one dataset at `<skill>/evals/evals.json`. A prompt alone
grades routing; add expectations and it also runs end to end. How to write one
is in [authoring-evals.md](authoring-evals.md).

By default a skill needs three evaluations that expect it to fire and two that
expect it not to, and a skill with no dataset at all fails the structural
check. That is the bar a new catalogue should hold, and it is the wrong bar for
an existing one: a repo adopting skillscope over fifty skills written before it
cannot write two hundred and fifty prompts before its first green run, and the
alternative it reaches for is to not run the checks. `--min-positive` and
`--min-negative` lower the bar so the folder, frontmatter and reference checks
can start working immediately, and get raised as the datasets land. Zero for
both turns the dataset requirement off; a skill that ships a dataset anyway is
still read against whatever the bar is.

**Where that bar may be set is the point.** It is a flag, so under CI it is a
line in the workflow, which is a path a repo can put behind branch protection
and code owners. There is no file in the repo under test, none inside a skill,
and no environment variable the CLI will read. The bar a skill is measured
against is not the skill author's to set: were it a file beside the skill, the
same pull request that adds a thin dataset could lower the number it is judged
by, and a reviewer would have to notice the second change to understand the
first. The applied bar is printed at the top of every structural run, passing
or failing, so a report always says what it was holding skills to.

## Configuring the repo under test

There is no config file. Everything about your repo is a CLI flag, or the same
Expand All @@ -46,6 +67,8 @@ places to disagree.
| `--exclude-url` | `excluded_urls` | none | Regexes matching URLs the external reference check leaves alone. For hosts that are auth-gated or that answer a runner's IP with a 403. |
| `--skill-files` | `skill_files` | none | Files every skill must ship beside its `SKILL.md`, such as a governance card. |
| `--skill-sections` | `skill_sections` | none | `##` headings each of those markdown files must have something under, such as `Description,Owner,License`. |
| `--min-positive` | `min_positive` | `3` | Evaluations a skill needs with `skill_should_trigger: true`. |
| `--min-negative` | `min_negative` | `2` | Evaluations it needs with `skill_should_trigger: false`. Zero for both means a skill need not ship a dataset at all. |
| `--behavior-runner` | `behavior_runner` | `["ubuntu-latest"]` | `runs-on` labels for a behavioral leg. |
| `--behavior-os` | `behavior_os` | `Linux` | Platforms a skill runs on when its `machine.yml` does not narrow them. |
| `--scoped-runner` | `scoped_runner` | reuses `behavior_runner` | Base labels for a leg whose skill asks for hardware labels. |
Expand Down Expand Up @@ -410,6 +433,8 @@ Naming several, and holding them to different bars:
| `behavioral` | `required` | `required`, `optional`, or `off`. |
| `runner` | `ubuntu-latest` | `runs-on` for every job: one label, or a JSON array of them. |
| `min_accuracy` | `1` | The routing bar. `0` reports the score without gating on it. |
| `min_positive` | (skillscope's own) | Evaluations a skill needs with `skill_should_trigger: true`. |
| `min_negative` | (skillscope's own) | Evaluations it needs with `skill_should_trigger: false`. Zero for both means a skill need not ship a dataset. |
| `api_key` | (none) | The model API key, mapped from the caller's vault. One secret, not the whole set. |
| `api_key_secret` | `ANTHROPIC_API_KEY` | Name to look up under `secrets: inherit`, if you would rather pass the vault than map one key. |
| `api_base_url` | (none) | Base URL of the gateway that issued the key. Blank talks to the provider directly. |
Expand Down
60 changes: 59 additions & 1 deletion skillscope/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -299,12 +299,32 @@ def routing_gate(totals: dict, min_accuracy: float) -> str | None:

def cmd_structural(args: argparse.Namespace) -> int:
"""Structural checks over every skill folder, dataset, and reference."""
# Before the gate, not after it: a run that fails still has to say which
# bar it was holding skills to. Printing this only on success would hide
# exactly the case worth seeing -- a repo whose datasets are not required,
# failing on something else -- behind a passing run.
cfg = config.active()
# Flushed, because the failures below go to stderr: unflushed stdout is
# block-buffered down a pipe, and the line would land after the report it
# is meant to frame in every CI log that captures both.
if cfg.min_positive or cfg.min_negative:
print(
f"[evals] dataset bar: {cfg.min_positive} positive, "
f"{cfg.min_negative} negative per skill.",
flush=True,
)
else:
print(
"[evals] dataset bar: off. Skills are not required to ship "
"`evals/evals.json`; the ones that do are still read.",
flush=True,
)

found = _structural_or_exit()
skills = datasets.skills_with_datasets()
# Extended datasets are checked regardless of --extended, so count them
# here too rather than reporting fewer cases than were checked.
cases = datasets.load_all_cases(extended=True)
cfg = config.active()
print(f"[evals] OK: {len(cfg.skills)} skill folder(s) in shape.")
print(
f"[evals] OK: {len(cases)} case(s) across {len(skills)} skill(s) "
Expand Down Expand Up @@ -882,6 +902,37 @@ def _add_skills_argument(parser: argparse.ArgumentParser) -> None:
)


def _add_dataset_bar_arguments(parser: argparse.ArgumentParser) -> None:
"""How much of a dataset this repo asks of a skill.

A flag and nothing else. The bar a skill is measured against is the repo's
policy, not the skill author's, so it has to live somewhere a skill cannot
reach: see `min_positive` on `config.Config`.
"""
parser.add_argument(
"--min-positive",
default="",
metavar="N",
help=(
"Evaluations a skill needs with `skill_should_trigger: true`. "
f"Default: {config.DEFAULT_MIN_POSITIVE}. Lower it to adopt these "
"checks over a catalogue that predates them, then raise it as the "
"datasets land."
),
)
parser.add_argument(
"--min-negative",
default="",
metavar="N",
help=(
"Evaluations a skill needs with `skill_should_trigger: false`. "
f"Default: {config.DEFAULT_MIN_NEGATIVE}. Zero for both this and "
"--min-positive means a skill need not ship a dataset at all; the "
"folder, frontmatter and reference checks still run."
),
)


def _add_docs_argument(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
"--docs",
Expand Down Expand Up @@ -924,6 +975,10 @@ def _add_routing_room_argument(
def _add_graded_arguments(parser: argparse.ArgumentParser) -> None:
"""Flags shared by the routing and behavioral commands."""
_add_skills_argument(parser)
# A graded run gates on the same dataset bar before it spends anything, so
# it takes the same flags. Without them a workflow that lowered the bar for
# `structural` would still be refused by `routing` on the next line.
_add_dataset_bar_arguments(parser)
parser.add_argument(
"--skill",
default="",
Expand Down Expand Up @@ -1114,6 +1169,7 @@ def build_parser() -> argparse.ArgumentParser:
),
)
_add_skills_argument(structural_parser)
_add_dataset_bar_arguments(structural_parser)
_add_docs_argument(structural_parser)
structural_parser.add_argument(
"--skill-files",
Expand Down Expand Up @@ -1341,6 +1397,8 @@ def _configure(args: argparse.Namespace) -> None:
"behavior_runner",
"behavior_os",
"scoped_runner",
"min_positive",
"min_negative",
)
}
settings["scoped_gate"] = getattr(args, "scoped_gate", "") or ""
Expand Down
56 changes: 56 additions & 0 deletions skillscope/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,15 @@
DEFAULT_BEHAVIOR_RUNNER = ("ubuntu-latest",)
DEFAULT_BEHAVIOR_OS = ("Linux",)

# Tier 0, the bar every skill clears before it can ship: five prompts, no
# hardware, no assertions, enforced structurally so a thin dataset fails
# without spending a token. These are the defaults, not the rule -- a repo
# adopting this harness over an existing catalogue lowers them, and raises them
# again as the datasets land. Where that decision may be made is the point of
# `min_positive` on Config below.
DEFAULT_MIN_POSITIVE = 3
DEFAULT_MIN_NEGATIVE = 2

# The two answers `--routing-room` takes instead of a list of names: every
# skill in the repo that ships a dataset, and no routing run at all. Both are
# words rather than shapes of an empty flag, because saying nothing means "work
Expand Down Expand Up @@ -123,6 +132,24 @@ class Config:
scoped_gate: str = ""
scoped_environment: str = ""

# How many of a skill's evaluations must expect it to fire, and how many
# must expect it not to. Zero for both means a skill need not ship a
# dataset at all, which is how a catalogue adopts the structural checks
# before it has written one.
#
# This is reachable from the command line and from nowhere else, and that
# is the design rather than an omission. The bar a skill is held to is not
# the skill author's to set: a file inside a skill, or at the repo root,
# would let the same pull request that adds a thin dataset also lower the
# number it is measured against, and the reviewer would have to notice the
# second change to understand the first. On the command line it lives in
# the workflow, which is a path a repo can put behind branch protection and
# code owners. There is deliberately no environment variable either: a
# self-hosted runner's environment is set by whoever administers the
# machine, which is not the same set of people who review a workflow diff.
min_positive: int = DEFAULT_MIN_POSITIVE
min_negative: int = DEFAULT_MIN_NEGATIVE

@property
def skills(self) -> dict[str, Path]:
"""Every skill this repo declares, as ``{name: folder}``.
Expand Down Expand Up @@ -286,6 +313,31 @@ def wants_no_skills(value: object) -> bool:
return _is_sentinel(value, NO_SKILLS)


def _bar(value: object, flag: str, default: int) -> int:
"""A non-negative count from a flag, a workflow input, or a test fixture.

Blank means the default, so a workflow input nobody filled in is the same
as no flag at all. Anything else has to be a whole number that is not
negative: a bar of -1 would read as "off" to whoever wrote it and as "every
count clears it" to the code, and those are the same outcome reached by
accident. Zero is the way to say off, and it says it once.
"""
if value is None or (isinstance(value, str) and not value.strip()):
return default
try:
count = int(str(value).strip())
except ValueError:
raise SystemExit(
f"error: {flag} takes a whole number, not {value!r}."
) from None
if count < 0:
raise SystemExit(
f"error: {flag} cannot be negative (got {count}). Zero is how a "
"repo says this bar does not apply yet."
)
return count


def build(
root: Path | None = None,
*,
Expand All @@ -301,6 +353,8 @@ def build(
scoped_runner: object = None,
scoped_gate: str = "",
scoped_environment: str = "",
min_positive: object = None,
min_negative: object = None,
dataset_skills: list[str] | None = None,
) -> Config:
"""A Config from loose values: what the CLI hands over after parsing.
Expand Down Expand Up @@ -347,6 +401,8 @@ def build(
scoped_runner=_items(scoped_runner, "--scoped-runner"),
scoped_gate=(scoped_gate or "").strip(),
scoped_environment=(scoped_environment or "").strip(),
min_positive=_bar(min_positive, "--min-positive", DEFAULT_MIN_POSITIVE),
min_negative=_bar(min_negative, "--min-negative", DEFAULT_MIN_NEGATIVE),
)


Expand Down
Loading
Loading