Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions .agents/skills/docs_consistency_audit/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,24 @@ rationale, authority reason, and suggested resolution class. A candidate without
matching valid adjudication remains unverified and cannot enter findings or notifications.
The script rejects unsupported quotes and schema-incomplete adjudications.

When consulting saved finding metadata, use the candidate's `candidate_id` from the
prepass output, not its array index or category:

```bash
python3 .agents/skills/docs_consistency_audit/scripts/audit_consistency.py prior-finding \
--candidates /tmp/docs-consistency-candidates.json \
--candidate-id CANDIDATE_ID \
--state-dir STATE_WORKTREE/.agents/state/docs_consistency_audit
```

Replace `CANDIDATE_ID` with the selected candidate's ID. This read-only command prints
JSON with `finding` set to the saved finding for the same stable claim pair, or `null`
when none exists. Categories describe findings, not identities; a `cli` topic can have
a `permissions` category. A missing match is normal: write a fresh adjudication without
saved metadata. Multiple matches or an unknown candidate block comparison. Treat saved
metadata as history only; recheck current public scope, verdict, severity, and confidence
before submitting an adjudication. Do not copy a prior decision as current evidence.

Two narrower claims with disjoint qualifiers do not conflict. A universal claim can
conflict with a narrower counterexample unless the universal claim contains matching
exception language.
Expand Down
41 changes: 41 additions & 0 deletions .agents/skills/docs_consistency_audit/scripts/audit_consistency.py
Original file line number Diff line number Diff line change
Expand Up @@ -756,6 +756,38 @@ def generate_candidates(
return candidates


def prior_finding_for_candidate(
candidate: dict[str, Any],
previous_state: dict[str, Any],
) -> dict[str, Any] | None:
matches = [
finding for finding in previous_state["findings"].values()
if len(finding["claims"]) == 2
and candidate_identity(*finding["claims"]) == candidate["candidate_id"]
]
if len(matches) > 1:
raise AuditError("Multiple saved findings match the candidate's stable claim pair")
return copy.deepcopy(matches[0]) if matches else None


def prior_finding_command(args: argparse.Namespace) -> int:
candidates = load_json(Path(args.candidates))["candidates"]
matches = [
candidate for candidate in candidates
if candidate["candidate_id"] == args.candidate_id
]
if len(matches) != 1:
raise AuditError("Prior-finding lookup requires exactly one candidate with that ID")
previous = load_json(Path(args.state_dir) / "findings.json")
result = {
"candidate_id": args.candidate_id,
"finding": prior_finding_for_candidate(matches[0], previous),
}
ensure_no_private_reference(result)
print(canonical_json(result))
return 0


def finding_identity_payload(finding: dict[str, Any]) -> dict[str, Any]:
return {
"category": finding["category"],
Expand Down Expand Up @@ -2586,6 +2618,15 @@ def build_parser() -> argparse.ArgumentParser:
adapters.add_argument("--output", required=True)
adapters.set_defaults(func=adapters_command)

prior_finding = subparsers.add_parser(
"prior-finding",
help="Read historical context for a candidate's stable claim pair",
)
prior_finding.add_argument("--candidates", required=True)
prior_finding.add_argument("--candidate-id", required=True)
prior_finding.add_argument("--state-dir", required=True)
prior_finding.set_defaults(func=prior_finding_command)

run = subparsers.add_parser("run", help="Compare validated claims and update generated state")
run.add_argument("--claims", required=True)
run.add_argument("--inventory")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@

import copy
import concurrent.futures
import contextlib
import datetime as dt
import importlib.util
import io
import json
import shutil
import tempfile
Expand Down Expand Up @@ -606,6 +608,90 @@ def test_incomplete_run_never_resolves_prior_finding(self):
self.assertTrue(carried["carried_forward_due_to_incomplete_run"])


class PriorFindingTests(unittest.TestCase):
def setUp(self):
self.claims = [
claim(path="overview.mdx", topic="cli", value=5000),
claim(path="reference.mdx", topic="cli", value=3000),
]
self.candidate = audit.generate_candidates(self.claims, RULES)[0]
self.saved = audit.normalize_finding({
**finding(*reversed(self.claims)),
"category": "permissions",
"topic": "cli",
})
self.previous = {"findings": {self.saved["finding_id"]: self.saved}}

def test_matches_stable_claim_pair_not_category_or_claim_order(self):
result = audit.prior_finding_for_candidate(self.candidate, self.previous)
self.assertEqual(result, self.saved)
self.assertEqual(result["category"], "permissions")
self.assertEqual(self.candidate["left"]["topic"], "cli")
result["title"] = "Changed locally"
self.assertEqual(self.saved["title"], "Conflicting limit")

def test_no_match_returns_none_even_with_same_topic_and_category(self):
unrelated = audit.normalize_finding({
**finding(self.claims[0], claim(path="other.mdx", topic="cli")),
"category": "cli",
"topic": "cli",
})
previous = {"findings": {unrelated["finding_id"]: unrelated}}
self.assertIsNone(audit.prior_finding_for_candidate(self.candidate, previous))
self.assertIsNone(audit.prior_finding_for_candidate(self.candidate, {"findings": {}}))

def test_ambiguous_saved_claim_pairs_are_rejected(self):
duplicate = copy.deepcopy(self.saved)
duplicate["category"] = "cli"
duplicate = audit.normalize_finding(duplicate)
previous = copy.deepcopy(self.previous)
previous["findings"][duplicate["finding_id"]] = duplicate
with self.assertRaisesRegex(audit.AuditError, "Multiple saved findings"):
audit.prior_finding_for_candidate(self.candidate, previous)

def test_command_reads_saved_state_without_mutating_it(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
state = root / "state"
candidates = root / "candidates.json"
audit.write_json(candidates, {"candidates": [self.candidate]})
audit.write_json(state / "findings.json", self.previous)
before = {path: path.read_bytes() for path in state.rglob("*") if path.is_file()}
output = io.StringIO()
with contextlib.redirect_stdout(output):
status = audit.main([
"prior-finding", "--candidates", str(candidates),
"--candidate-id", self.candidate["candidate_id"],
"--state-dir", str(state),
])
self.assertEqual(status, 0)
self.assertEqual(json.loads(output.getvalue()), {
"candidate_id": self.candidate["candidate_id"],
"finding": self.saved,
})
self.assertEqual(before, {
path: path.read_bytes() for path in state.rglob("*") if path.is_file()
})

def test_command_rejects_unknown_or_duplicate_candidate_ids(self):
with tempfile.TemporaryDirectory() as directory:
candidates = Path(directory) / "candidates.json"
for items in ([], [self.candidate, self.candidate]):
with self.subTest(candidates=len(items)):
audit.write_json(candidates, {"candidates": items})
output = io.StringIO()
errors = io.StringIO()
with contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors):
status = audit.main([
"prior-finding", "--candidates", str(candidates),
"--candidate-id", self.candidate["candidate_id"],
"--state-dir", directory,
])
self.assertEqual(status, 2)
self.assertEqual(output.getvalue(), "")
self.assertIn("exactly one candidate", errors.getvalue())


class PipelinePrepassTests(unittest.TestCase):
def test_prepass_writes_semantic_candidates_without_state(self):
with tempfile.TemporaryDirectory() as directory:
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,9 @@ jobs:
python3 .agents/skills/missing_docs/scripts/test_audit_docs.py
python3 .agents/skills/missing_docs/scripts/test_check_new_release.py

- name: Test docs consistency audit
run: python3 .agents/skills/docs_consistency_audit/scripts/test_audit_consistency.py

# Stdlib-only tests for the PR body integrity checker, including the
# feature-summary lead section that drafting PRs must open with.
- name: Test create_pr body checker
Expand Down
Loading