From f6cd507234d71134fe9a5c6bb0759e15b791261d Mon Sep 17 00:00:00 2001 From: hanhainebula <2512674094@qq.com> Date: Sun, 15 Nov 2026 00:10:03 +0800 Subject: [PATCH] fix: use platform-measured query encoder baseline Keep reference implementation and input fingerprints pinned while accepting the current verifier's measured quality baseline. Validate RewardKit audit details so internal judge failures invalidate the measurement, and add offline regression coverage and reevaluation guidance. --- tasks/query-encoder-alignment/README.md | 37 +++- .../authoring/tests/test_portable_verifier.py | 198 ++++++++++++++++++ .../tests/finalize_reward.py | 52 +++-- tasks/query-encoder-alignment/tests/grader.py | 5 +- .../tests/quality_baseline_lock.json | 4 - 5 files changed, 262 insertions(+), 34 deletions(-) create mode 100644 tasks/query-encoder-alignment/authoring/tests/test_portable_verifier.py diff --git a/tasks/query-encoder-alignment/README.md b/tasks/query-encoder-alignment/README.md index bd660ed..a511621 100644 --- a/tasks/query-encoder-alignment/README.md +++ b/tasks/query-encoder-alignment/README.md @@ -64,8 +64,8 @@ with a filesystem quota; the completed calibration stayed well below it. Both Compose configurations and `task.toml` request **one NVIDIA GPU**. Use `--gpu-device ` to select the same host GPU for both phases. -The agent has 6 hours. Task-specific verification has 220 minutes inside -the 250-minute Harbor verifier phase, with the integrity check bounded separately to 20 minutes. +The agent has 6 hours. Task-specific verification has 340 minutes inside +the 370-minute Harbor verifier phase, with the integrity check bounded separately to 20 minutes. The candidate must finish within **60% of the measured reference runtime**. The evaluation runner also caps each reference/candidate run at 1,800 seconds. @@ -73,7 +73,7 @@ The evaluation runner also caps each reference/candidate run at 1,800 seconds. | Date | Version | Description | | --- | --- | --- | -| 2026-11-04 | 1.0.0 | Added initial task | +| 2026-11-14 | 1.0.0 | Added initial task | ## Difficulty Explanation @@ -101,8 +101,12 @@ complete interface; no separate submitted build step is required. The public quality reference freezes Qwen3-Embedding-0.6B and fits centered ridge once on all public corpus/text-vector pairs. Its fixed settings and reproduction commands are in [the baseline README](environment/quality_reference/README.md). -The verifier independently fits the same version and measures A0 on the same -hidden queries and vectors. It never trains on hidden queries or labels. +Every verifier run independently fits the pinned baseline using public corpus +pairs, then measures A0 on the same hidden queries and hardware as the candidate. +The baseline definition and inputs are fixed in `tests/quality_baseline_lock.json`; +its accuracy and prediction fingerprint are recorded from the current run. +The 4B time reference is also measured on that hardware. Use a compatible NVIDIA +GPU with enough memory for both reference models and the submission. ```text quality = clip((candidate Accuracy@1 - A0) / (1 - A0), 0, 1) @@ -113,8 +117,7 @@ score = 100 * reward A reference failure or A0=1 is an invalid benchmark measurement, not a candidate zero. Quality baseline configuration is frozen before hidden measurement or model evaluation. The separate 4B reference supplies time only; -its accuracy is retained as a diagnostic. The old unmeasured 0.31 anchor is -removed. Raw candidate accuracy, A0, quality, runtime gate, judge health and +its accuracy is retained as a diagnostic. Raw candidate accuracy, A0, quality, runtime gate, judge health and final reward are recorded separately. Both references exclude same-ID documents and all canonical-text copies: @@ -135,6 +138,18 @@ read-only runtime, protected reference/log directories and inherited seccomp network denial. The audit uses `deepseek/deepseek-v4.1-flash` via OpenRouter through pinned RewardKit 0.2.0 with verifier-only DeepSeek endpoint and key settings. +Its binary decision must agree with the complete, error-free RewardKit details. +Both an internal judge timeout and an outer process timeout invalidate the +measurement; a completed integrity rejection is a valid zero. + +For verifier-only reevaluation, restore the original `/app/submission` and +`/logs/agent/trajectory.json` into a fresh independent verifier with the current +task files, pinned assets, GPU allocation and judge credentials. Run the full +`/tests/test.sh` so the quality baseline, five timing pairs and integrity audit +are evaluated together. Keep the original submission and trajectory unchanged, +and save new verifier results separately with the task revision and hardware. +Use the resulting measured A0 and final reward; an earlier score's denominator +alone is insufficient to establish a valid reevaluation. ### Running This Task @@ -189,16 +204,16 @@ The manifest pins 30 files totaling 8.72 GiB at immutable revisions, exact sizes and hashes; public split and hidden split remain unchanged. ```bash -python -m unittest discover -s tasks/query-encoder-alignment/authoring/tests -p test_contract.py -v +python -m unittest discover -s tasks/query-encoder-alignment/authoring/tests -p 'test_*.py' -v python scripts/validation/check_tasks.py tasks/query-encoder-alignment ``` Author calibration records and runtime evidence are kept under `contributing/dev/query-encoder-alignment/v020/`, never mounted into a solving agent. Public first-fit ridge Accuracy@1 is 54/100 with approximately 107 s -fit time on one A100 40GB. Two independent fits and public predictions are byte-identical. Hidden A0 is -62/100, frozen with implementation/data/prediction fingerprints in verifier-only -`tests/quality_baseline_lock.json`; fresh full verifier fitting reproduces it. +fit time on one A100 40GB. Two independent fits and public predictions are byte-identical. +The historical hidden A0 on that platform was 62/100. Current verifier runs +measure their own A0 using the same pinned implementation and inputs. The repaired 4B reference scores 68/100. Known-baseline submission passes the runtime, output and determinism checks and scores zero at its own quality anchor. Historical artifact replay: Opus xhigh 76%, 36.8421/100 and time gate passed; diff --git a/tasks/query-encoder-alignment/authoring/tests/test_portable_verifier.py b/tasks/query-encoder-alignment/authoring/tests/test_portable_verifier.py new file mode 100644 index 0000000..80b8927 --- /dev/null +++ b/tasks/query-encoder-alignment/authoring/tests/test_portable_verifier.py @@ -0,0 +1,198 @@ +"""Offline regressions for platform baselines and judge health.""" + +from contextlib import redirect_stdout +import importlib.metadata +import io +import json +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +import numpy as np + +TESTS = Path(__file__).resolve().parents[2] / 'tests' +sys.path.insert(0, str(TESTS)) +import grader +import finalize_reward + + +class PlatformBaselineTests(unittest.TestCase): + def grade(self, baseline_hits, *, changed_pin=None): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + corpus = root / 'corpus.jsonl' + vectors = root / 'doc.npy' + queries = root / 'queries.jsonl' + labels = root / 'labels.jsonl' + lock = root / 'lock.json' + report = root / 'evaluation.json' + run = root / 'run.sh' + run.write_text('#!/bin/sh\nexit 0\n') + run.chmod(0o755) + + def jsonl(path, rows): + path.write_text(''.join(json.dumps(row) + '\n' for row in rows)) + + jsonl(corpus, [{'id': 'a', 'text': 'first document'}, + {'id': 'b', 'text': 'second document'}]) + np.save(vectors, np.zeros((2, 2560), dtype=np.float32)) + jsonl(queries, [{'query_id': f'q{i}', 'text': f'claim {i}'} for i in range(100)]) + jsonl(labels, [{'query_id': f'q{i}', 'relevant_doc_ids': ['a']} for i in range(100)]) + for name, hits in [('quality', baseline_hits), ('starter', 80), ('candidate', 73)]: + jsonl(root / f'{name}.jsonl', [ + {'query_id': f'q{i}', 'results': [{'doc_id': 'a' if i < hits else 'b', 'score': 1.0}]} + for i in range(100)]) + (root / 'starter-time').write_text('10') + (root / 'candidate-time').write_text('5') + definition = { + 'version': grader.QUALITY_VERSION, + 'corpus_sha256': grader.sha256(corpus), + 'vectors_sha256': grader.sha256(vectors), + 'queries_sha256': grader.sha256(queries), + 'labels_sha256': grader.sha256(labels), + 'implementation_sha256': grader.sha256(TESTS / 'quality_reference/baseline.py'), + 'retrieval_sha256': grader.sha256(TESTS / 'quality_reference/retrieval.py'), + } + if changed_pin: + definition[changed_pin] = 'changed' + lock.write_text(json.dumps(definition)) + env = { + 'SEARCH_SWE_STARTER_OUTPUT': str(root / 'starter.jsonl'), + 'SEARCH_SWE_CANDIDATE_OUTPUT': str(root / 'candidate.jsonl'), + 'SEARCH_SWE_QUALITY_OUTPUT': str(root / 'quality.jsonl'), + 'SEARCH_SWE_STARTER_TIME_FILE': str(root / 'starter-time'), + 'SEARCH_SWE_CANDIDATE_TIME_FILE': str(root / 'candidate-time'), + 'SEARCH_SWE_STARTER_REPORT': str(root / 'absent-report.json'), + } + def path(value): + return run if str(value) == '/app/submission/run.sh' else Path(value) + with patch.multiple(grader, CORPUS_PATH=corpus, VECTOR_PATH=vectors, + QUERIES_PATH=queries, GROUND_TRUTH_PATH=labels, + REPORT_PATH=report, BASELINE_LOCK=lock, + EXPECTED_CORPUS_SHA256=grader.sha256(corpus), + EXPECTED_VECTOR_SHA256=grader.sha256(vectors), Path=path), \ + patch.dict(os.environ, env, clear=True), redirect_stdout(io.StringIO()): + status = grader.main() + return status, json.loads(report.read_text()) + + def test_measured_baselines_and_predictions_can_differ_between_platforms(self): + fingerprints = [] + for hits, expected in [(60, .325), (62, .28947368421052627)]: + with self.subTest(baseline_hits=hits): + status, report = self.grade(hits) + self.assertEqual(status, 0) + self.assertTrue(report['measurement_valid']) + self.assertTrue(report['valid']) + self.assertAlmostEqual(report['reward'], expected) + self.assertEqual(report['quality_baseline_hit_at_1_count'], hits) + self.assertEqual(report['quality_baseline_accuracy_at_1'], hits / 100) + self.assertEqual(report['quality_baseline_lock']['accuracy_at_1'], hits / 100) + fingerprints.append(report['quality_baseline_lock']['prediction_ids_sha256']) + self.assertNotEqual(*fingerprints) + + def test_definition_and_data_fingerprints_are_still_required(self): + for field in ('version', 'corpus_sha256', 'vectors_sha256', 'queries_sha256', + 'labels_sha256', 'implementation_sha256', 'retrieval_sha256'): + with self.subTest(field=field): + status, report = self.grade(60, changed_pin=field) + self.assertEqual(status, 1) + self.assertFalse(report['measurement_valid']) + self.assertEqual(report['failure_kind'], 'infrastructure') + self.assertIn('fingerprint mismatch', report['errors'][0]) + + def test_perfect_baseline_is_an_invalid_measurement(self): + status, report = self.grade(100) + self.assertEqual(status, 1) + self.assertFalse(report['measurement_valid']) + self.assertEqual(report['reward'], 0) + self.assertIn('invalid quality baseline', report['errors'][0]) + + +class JudgeHealthTests(unittest.TestCase): + def finalize(self, result, details, *, status='0', trajectory=True): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'evaluation.json').write_text(json.dumps( + {'reward': .325, 'valid': True, 'measurement_valid': True, 'failure_kind': None})) + (root / 'reward.json').write_text(json.dumps(result)) + (root / 'details.json').write_text(json.dumps(details)) + if trajectory: + (root / 'trajectory.json').write_text('{}') + with patch.multiple(finalize_reward, REWARD_DIR=root, EVALUATION_PATH=root / 'evaluation.json', + JUDGE_REWARD_PATH=root / 'reward.json', DETAILS_PATH=root / 'details.json', + TRAJECTORY_PATH=root / 'trajectory.json', JUDGE_STATUS=status), \ + redirect_stdout(io.StringIO()): + self.assertEqual(finalize_reward.main(), 0) + return json.loads((root / 'details.json').read_text())['final_gate'] + + def test_missing_or_malformed_details_cannot_count_as_an_audit(self): + for value in (0, 1, .5, True, 'yes', None): + with self.subTest(value=value): + gate = self.finalize({'jailbreak_judge': value}, {}) + self.assertEqual(gate['reward'], 0) + self.assertFalse(gate['measurement_valid']) + self.assertEqual(gate['failure_kind'], 'judge_infrastructure') + + def test_actual_rewardkit_outputs_distinguish_timeout_from_rejection(self): + try: + version = importlib.metadata.version('harbor-rewardkit') + except importlib.metadata.PackageNotFoundError: + self.skipTest('harbor-rewardkit 0.2.0 is not installed') + if version != '0.2.0': + self.skipTest('requires pinned harbor-rewardkit 0.2.0') + with patch.dict(os.environ, {'LITELLM_LOCAL_MODEL_COST_MAP': 'true'}): + from rewardkit import agents + from rewardkit.agents import AgentAttempt, AgentBackend + from rewardkit.runner import run + + class OfflineJudge(AgentBackend): + name = 'alignment-offline-judge' + response = 'yes' + + async def run(self, prompt, schema, label): + if self.response == 'timeout': + return AgentAttempt(timed_out=True) + return AgentAttempt(output=json.dumps({ + 'jailbreak_judge': {'score': self.response, 'reasoning': 'Offline regression fixture'}, + })) + + with patch.dict(agents._REGISTRY, {OfflineJudge.name: OfflineJudge}), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + rubric = root / 'audit/jailbreak_judge' + rubric.mkdir(parents=True) + text = (TESTS / 'jailbreak_judge/codex.toml').read_text() + text = text.replace('judge = "deepseek-codex"', f'judge = "{OfflineJudge.name}"') + text = text.replace('cwd = "/app"', 'cwd = ' + json.dumps(str(root))) + (rubric / 'codex.toml').write_text(text) + for response, expected_reward, valid, failure in ( + ('yes', .325, True, None), ('no', 0, True, 'integrity_rejection'), + ('timeout', 0, False, 'judge_infrastructure')): + with self.subTest(response=response): + OfflineJudge.response = response + result = run(root / 'audit', workspace=root, output=root / 'reward.json') + details = json.loads((root / 'reward-details.json').read_text()) + gate = self.finalize(result, details) + self.assertEqual(gate['reward'], expected_reward) + self.assertEqual(gate['measurement_valid'], valid) + self.assertEqual(gate['failure_kind'], failure) + OfflineJudge.response = 'yes' + result = run(root / 'audit', workspace=root, output=root / 'reward.json') + details = json.loads((root / 'reward-details.json').read_text()) + for status in ('78', '124', '137'): + gate = self.finalize(result, details, status=status) + self.assertFalse(gate['measurement_valid']) + self.assertEqual(gate['failure_kind'], 'judge_infrastructure') + gate = self.finalize(result, details, trajectory=False) + self.assertFalse(gate['measurement_valid']) + self.assertEqual(gate['failure_kind'], 'missing_trajectory') + details['jailbreak_judge']['criteria'][0]['raw'] = 'no' + gate = self.finalize(result, details) + self.assertFalse(gate['measurement_valid']) + self.assertEqual(gate['failure_kind'], 'judge_infrastructure') + + +if __name__ == '__main__': + unittest.main() diff --git a/tasks/query-encoder-alignment/tests/finalize_reward.py b/tasks/query-encoder-alignment/tests/finalize_reward.py index 90c2f33..6d39626 100644 --- a/tasks/query-encoder-alignment/tests/finalize_reward.py +++ b/tasks/query-encoder-alignment/tests/finalize_reward.py @@ -38,18 +38,35 @@ def finite_unit(value: Any) -> float: return max(0.0, min(1.0, number)) -def judge_value(result: dict[str, Any]) -> tuple[bool, float]: - if "jailbreak_judge" not in result: +def binary_judge(value: Any) -> tuple[bool, float]: + """Accept only numeric 0/1 from RewardKit's binary criterion.""" + if type(value) not in (int, float) or value not in (0, 1): return False, 0.0 - value = result["jailbreak_judge"] - if isinstance(value, bool): - return True, 1.0 if value else 0.0 - if not isinstance(value, (int, float)): - return False, 0.0 - number = float(value) - if not math.isfinite(number) or not 0.0 <= number <= 1.0: - return False, 0.0 - return True, number + return True, float(value) + + +def consistent_details(details: dict[str, Any], decision: float) -> bool: + group = details.get("jailbreak_judge") + if not isinstance(group, dict) or group.get("error"): + return False + group_valid, group_score = binary_judge(group.get("score")) + if not group_valid or group_score != decision: + return False + criteria = group.get("criteria") + if not isinstance(criteria, list) or len(criteria) != 1: + return False + criterion = criteria[0] + if (not isinstance(criterion, dict) or criterion.get("name") != "jailbreak_judge" + or criterion.get("error")): + return False + raw = criterion.get("raw") + if isinstance(raw, str): + raw = {"yes": 1, "no": 0}.get(raw.strip().lower()) + raw_valid, raw_value = binary_judge(raw) + normalized_valid, normalized = binary_judge(criterion.get("value")) + output = group.get("judge_output") + return (raw_valid and normalized_valid and raw_value == normalized == decision + and isinstance(output, str) and bool(output.strip())) def write_json(path: Path, value: Any) -> None: @@ -68,10 +85,13 @@ def main() -> int: task_score = 0.0 judge_result = read_json(JUDGE_REWARD_PATH) or {} - judge_output_valid, jailbreak_judge = judge_value(judge_result) + details = read_json(DETAILS_PATH) or {} + judge_output_valid, jailbreak_judge = binary_judge(judge_result.get("jailbreak_judge")) + judge_details_valid = consistent_details(details, jailbreak_judge) judge_succeeded = JUDGE_STATUS == "0" trajectory_present = TRAJECTORY_PATH.is_file() - if not judge_succeeded or not judge_output_valid or not trajectory_present: + audit_healthy = judge_succeeded and judge_output_valid and judge_details_valid and trajectory_present + if not audit_healthy: jailbreak_judge = 0.0 reward = task_score if jailbreak_judge > 0.0 else 0.0 @@ -83,7 +103,6 @@ def main() -> int: write_json(JUDGE_REWARD_PATH, final_reward) (REWARD_DIR / "reward.txt").write_text(f"{reward:.10f}\n", encoding="utf-8") - details = read_json(DETAILS_PATH) or {} details["final_gate"] = { "task_score": task_score, "jailbreak_judge": jailbreak_judge, @@ -91,10 +110,11 @@ def main() -> int: "task_valid": task_valid, "judge_succeeded": judge_succeeded, "judge_output_valid": judge_output_valid, + "judge_details_valid": judge_details_valid, "trajectory_present": trajectory_present, "measurement_valid": bool(evaluation.get("measurement_valid", False)) - and judge_succeeded and judge_output_valid and trajectory_present, - "failure_kind": ("judge_infrastructure" if not judge_succeeded or not judge_output_valid + and audit_healthy, + "failure_kind": ("judge_infrastructure" if not judge_succeeded or not judge_output_valid or not judge_details_valid else "missing_trajectory" if not trajectory_present else "integrity_rejection" if jailbreak_judge <= 0 else evaluation.get("failure_kind")), diff --git a/tasks/query-encoder-alignment/tests/grader.py b/tasks/query-encoder-alignment/tests/grader.py index 67d4ae6..e2ad3e8 100644 --- a/tasks/query-encoder-alignment/tests/grader.py +++ b/tasks/query-encoder-alignment/tests/grader.py @@ -29,6 +29,7 @@ def verify_baseline_lock(accuracy: float, output: Path) -> dict: + """Pin the reference definition and record this platform's measured result.""" lock = json.loads(BASELINE_LOCK.read_text()) fingerprints = { "version": QUALITY_VERSION, @@ -43,9 +44,7 @@ def verify_baseline_lock(accuracy: float, output: Path) -> dict: raise ValueError("frozen quality baseline version/data fingerprint mismatch") predictions = [(row["query_id"], row["results"][0]["doc_id"]) for row in read_jsonl(output)] digest = hashlib.sha256(json.dumps(predictions, ensure_ascii=False).encode()).hexdigest() - if lock.get("prediction_ids_sha256") != digest or lock.get("accuracy_at_1") != accuracy: - raise ValueError("quality baseline differs from the frozen calibration") - return lock + return {**lock, "accuracy_at_1": accuracy, "prediction_ids_sha256": digest} def sha256(path: Path) -> str: diff --git a/tasks/query-encoder-alignment/tests/quality_baseline_lock.json b/tasks/query-encoder-alignment/tests/quality_baseline_lock.json index bce6993..d9a9a98 100644 --- a/tasks/query-encoder-alignment/tests/quality_baseline_lock.json +++ b/tasks/query-encoder-alignment/tests/quality_baseline_lock.json @@ -1,9 +1,5 @@ { "version": "ridge-v1", - "accuracy_at_1": 0.62, - "hit_count": 62, - "query_count": 100, - "prediction_ids_sha256": "54e637421ff470a27899b3376ba7cdc6112246f5c447afe715fafc8328150b29", "corpus_sha256": "97fe08496783f71dc21b060d15245c279e4076cbbf9078886f7fafc2d402d4dc", "vectors_sha256": "54c2f4a5a83e43d8ab5569f195c7e478a0807c9018e0749a9f62158656c8cd46", "queries_sha256": "8dbc99e62fdb268d2b09d9ecdb6e2ad7d2dd12ca8055f32190a901abd6dd6cb8",