diff --git a/.agents/skills/docs_consistency_audit/SKILL.md b/.agents/skills/docs_consistency_audit/SKILL.md index 9c13001c5..55f8c7698 100644 --- a/.agents/skills/docs_consistency_audit/SKILL.md +++ b/.agents/skills/docs_consistency_audit/SKILL.md @@ -221,6 +221,24 @@ rationale, authority reason, and suggested resolution class. A candidate without matching valid adjudication remains unverified and cannot enter findings or notifications. The script rejects unsupported quotes and schema-incomplete adjudications. +When consulting saved finding metadata, use the candidate's `candidate_id` from the +prepass output, not its array index or category: + +```bash +python3 .agents/skills/docs_consistency_audit/scripts/audit_consistency.py prior-finding \ + --candidates /tmp/docs-consistency-candidates.json \ + --candidate-id CANDIDATE_ID \ + --state-dir STATE_WORKTREE/.agents/state/docs_consistency_audit +``` + +Replace `CANDIDATE_ID` with the selected candidate's ID. This read-only command prints +JSON with `finding` set to the saved finding for the same stable claim pair, or `null` +when none exists. Categories describe findings, not identities; a `cli` topic can have +a `permissions` category. A missing match is normal: write a fresh adjudication without +saved metadata. Multiple matches or an unknown candidate block comparison. Treat saved +metadata as history only; recheck current public scope, verdict, severity, and confidence +before submitting an adjudication. Do not copy a prior decision as current evidence. + Two narrower claims with disjoint qualifiers do not conflict. A universal claim can conflict with a narrower counterexample unless the universal claim contains matching exception language. diff --git a/.agents/skills/docs_consistency_audit/scripts/audit_consistency.py b/.agents/skills/docs_consistency_audit/scripts/audit_consistency.py index e0d8bb5fd..682e440d0 100644 --- a/.agents/skills/docs_consistency_audit/scripts/audit_consistency.py +++ b/.agents/skills/docs_consistency_audit/scripts/audit_consistency.py @@ -756,6 +756,38 @@ def generate_candidates( return candidates +def prior_finding_for_candidate( + candidate: dict[str, Any], + previous_state: dict[str, Any], +) -> dict[str, Any] | None: + matches = [ + finding for finding in previous_state["findings"].values() + if len(finding["claims"]) == 2 + and candidate_identity(*finding["claims"]) == candidate["candidate_id"] + ] + if len(matches) > 1: + raise AuditError("Multiple saved findings match the candidate's stable claim pair") + return copy.deepcopy(matches[0]) if matches else None + + +def prior_finding_command(args: argparse.Namespace) -> int: + candidates = load_json(Path(args.candidates))["candidates"] + matches = [ + candidate for candidate in candidates + if candidate["candidate_id"] == args.candidate_id + ] + if len(matches) != 1: + raise AuditError("Prior-finding lookup requires exactly one candidate with that ID") + previous = load_json(Path(args.state_dir) / "findings.json") + result = { + "candidate_id": args.candidate_id, + "finding": prior_finding_for_candidate(matches[0], previous), + } + ensure_no_private_reference(result) + print(canonical_json(result)) + return 0 + + def finding_identity_payload(finding: dict[str, Any]) -> dict[str, Any]: return { "category": finding["category"], @@ -2586,6 +2618,15 @@ def build_parser() -> argparse.ArgumentParser: adapters.add_argument("--output", required=True) adapters.set_defaults(func=adapters_command) + prior_finding = subparsers.add_parser( + "prior-finding", + help="Read historical context for a candidate's stable claim pair", + ) + prior_finding.add_argument("--candidates", required=True) + prior_finding.add_argument("--candidate-id", required=True) + prior_finding.add_argument("--state-dir", required=True) + prior_finding.set_defaults(func=prior_finding_command) + run = subparsers.add_parser("run", help="Compare validated claims and update generated state") run.add_argument("--claims", required=True) run.add_argument("--inventory") diff --git a/.agents/skills/docs_consistency_audit/scripts/test_audit_consistency.py b/.agents/skills/docs_consistency_audit/scripts/test_audit_consistency.py index 0fbbdac10..5620d42c4 100644 --- a/.agents/skills/docs_consistency_audit/scripts/test_audit_consistency.py +++ b/.agents/skills/docs_consistency_audit/scripts/test_audit_consistency.py @@ -5,8 +5,10 @@ import copy import concurrent.futures +import contextlib import datetime as dt import importlib.util +import io import json import shutil import tempfile @@ -606,6 +608,90 @@ def test_incomplete_run_never_resolves_prior_finding(self): self.assertTrue(carried["carried_forward_due_to_incomplete_run"]) +class PriorFindingTests(unittest.TestCase): + def setUp(self): + self.claims = [ + claim(path="overview.mdx", topic="cli", value=5000), + claim(path="reference.mdx", topic="cli", value=3000), + ] + self.candidate = audit.generate_candidates(self.claims, RULES)[0] + self.saved = audit.normalize_finding({ + **finding(*reversed(self.claims)), + "category": "permissions", + "topic": "cli", + }) + self.previous = {"findings": {self.saved["finding_id"]: self.saved}} + + def test_matches_stable_claim_pair_not_category_or_claim_order(self): + result = audit.prior_finding_for_candidate(self.candidate, self.previous) + self.assertEqual(result, self.saved) + self.assertEqual(result["category"], "permissions") + self.assertEqual(self.candidate["left"]["topic"], "cli") + result["title"] = "Changed locally" + self.assertEqual(self.saved["title"], "Conflicting limit") + + def test_no_match_returns_none_even_with_same_topic_and_category(self): + unrelated = audit.normalize_finding({ + **finding(self.claims[0], claim(path="other.mdx", topic="cli")), + "category": "cli", + "topic": "cli", + }) + previous = {"findings": {unrelated["finding_id"]: unrelated}} + self.assertIsNone(audit.prior_finding_for_candidate(self.candidate, previous)) + self.assertIsNone(audit.prior_finding_for_candidate(self.candidate, {"findings": {}})) + + def test_ambiguous_saved_claim_pairs_are_rejected(self): + duplicate = copy.deepcopy(self.saved) + duplicate["category"] = "cli" + duplicate = audit.normalize_finding(duplicate) + previous = copy.deepcopy(self.previous) + previous["findings"][duplicate["finding_id"]] = duplicate + with self.assertRaisesRegex(audit.AuditError, "Multiple saved findings"): + audit.prior_finding_for_candidate(self.candidate, previous) + + def test_command_reads_saved_state_without_mutating_it(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + state = root / "state" + candidates = root / "candidates.json" + audit.write_json(candidates, {"candidates": [self.candidate]}) + audit.write_json(state / "findings.json", self.previous) + before = {path: path.read_bytes() for path in state.rglob("*") if path.is_file()} + output = io.StringIO() + with contextlib.redirect_stdout(output): + status = audit.main([ + "prior-finding", "--candidates", str(candidates), + "--candidate-id", self.candidate["candidate_id"], + "--state-dir", str(state), + ]) + self.assertEqual(status, 0) + self.assertEqual(json.loads(output.getvalue()), { + "candidate_id": self.candidate["candidate_id"], + "finding": self.saved, + }) + self.assertEqual(before, { + path: path.read_bytes() for path in state.rglob("*") if path.is_file() + }) + + def test_command_rejects_unknown_or_duplicate_candidate_ids(self): + with tempfile.TemporaryDirectory() as directory: + candidates = Path(directory) / "candidates.json" + for items in ([], [self.candidate, self.candidate]): + with self.subTest(candidates=len(items)): + audit.write_json(candidates, {"candidates": items}) + output = io.StringIO() + errors = io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors): + status = audit.main([ + "prior-finding", "--candidates", str(candidates), + "--candidate-id", self.candidate["candidate_id"], + "--state-dir", directory, + ]) + self.assertEqual(status, 2) + self.assertEqual(output.getvalue(), "") + self.assertIn("exactly one candidate", errors.getvalue()) + + class PipelinePrepassTests(unittest.TestCase): def test_prepass_writes_semantic_candidates_without_state(self): with tempfile.TemporaryDirectory() as directory: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddca77de7..8d5c1624e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,6 +170,9 @@ jobs: python3 .agents/skills/missing_docs/scripts/test_audit_docs.py python3 .agents/skills/missing_docs/scripts/test_check_new_release.py + - name: Test docs consistency audit + run: python3 .agents/skills/docs_consistency_audit/scripts/test_audit_consistency.py + # Stdlib-only tests for the PR body integrity checker, including the # feature-summary lead section that drafting PRs must open with. - name: Test create_pr body checker