From 9eee46b849c5df1f7b20766aa0780cb43e295a0d Mon Sep 17 00:00:00 2001 From: duongthimy Date: Fri, 20 Nov 2026 08:35:16 +0100 Subject: [PATCH] feat(gooddata-eval): evaluate AI Knowledge answers in chat - New agentic kind `agentic_knowledge`: fixed user turns in one conversation, last answer graded - Retrieval checks from knowledge tool results (search results and the get_knowledge_chunks document): searched, expected documents found, forbidden (disabled/other-workspace) documents absent, source label (one of gen-ai's labels) - Judge criteria must_include / must_not_include / rubric on the last answer; judge input holds the earlier turns - Each code check is its own Langfuse score: a fail shows retrieval vs reasoning; quality_score counts code checks and judge verdicts once - A run the judge could not grade is an ERROR, not a fail - `knowledge_input` (turns, workspace role) parsed from Langfuse item input/metadata; the CLI refuses a workspace role other than parent - Calibration fixture + `make calibrate-knowledge` jira: QA-30178 risk: low --- packages/gooddata-eval/AGENTS.md | 4 +- packages/gooddata-eval/Makefile | 4 + packages/gooddata-eval/README.md | 60 ++- .../scripts/calibrate_knowledge_judge.py | 62 +++ .../src/gooddata_eval/cli/agentic_runner.py | 25 + .../gooddata_eval/core/agentic/knowledge.py | 443 ++++++++++++++++++ .../core/dataset/langfuse_source.py | 34 +- .../core/evaluators/knowledge.py | 252 ++++++++++ .../src/gooddata_eval/core/interaction.py | 1 + .../src/gooddata_eval/core/models.py | 30 ++ .../tests/fixtures/knowledge_calibration.json | 98 ++++ .../tests/test_agentic_knowledge.py | 295 ++++++++++++ .../tests/test_agentic_runner.py | 55 ++- packages/gooddata-eval/tests/test_cli.py | 43 ++ .../tests/test_knowledge_evaluator.py | 386 +++++++++++++++ .../tests/test_langfuse_source.py | 62 +++ packages/gooddata-eval/tests/test_models.py | 16 + .../gooddata-eval/tests/test_trace_linker.py | 1 + 18 files changed, 1862 insertions(+), 9 deletions(-) create mode 100644 packages/gooddata-eval/scripts/calibrate_knowledge_judge.py create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/agentic/knowledge.py create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/evaluators/knowledge.py create mode 100644 packages/gooddata-eval/tests/fixtures/knowledge_calibration.json create mode 100644 packages/gooddata-eval/tests/test_agentic_knowledge.py create mode 100644 packages/gooddata-eval/tests/test_knowledge_evaluator.py diff --git a/packages/gooddata-eval/AGENTS.md b/packages/gooddata-eval/AGENTS.md index 3923fd891..fd37d776e 100644 --- a/packages/gooddata-eval/AGENTS.md +++ b/packages/gooddata-eval/AGENTS.md @@ -63,7 +63,9 @@ implementation, sometimes under different `test_kind` strings (`search_tool` vs its own shape. `test_kind` on the item is what labels the result, not the evaluator class, which is why `knowledge_question` can reuse `GeneralQuestionEvaluator` verbatim. `dashboard_summary` items additionally need `summary_input`, `document_takeaway` items -`takeaway_input`, `catalog_metadata` items `catalog_input`. +`takeaway_input`, `catalog_metadata` items `catalog_input`. `agentic_knowledge` items may +carry `knowledge_input` (fixed turns, workspace role); their evaluator lives in +`core/evaluators/knowledge.py` but is agentic-only, so it is not in the single-shot registry. ## Running the pipeline diff --git a/packages/gooddata-eval/Makefile b/packages/gooddata-eval/Makefile index 4bf22f465..cbbea6d79 100644 --- a/packages/gooddata-eval/Makefile +++ b/packages/gooddata-eval/Makefile @@ -8,3 +8,7 @@ calibrate-takeaway: .PHONY: calibrate-catalog-metadata calibrate-catalog-metadata: uv run --package gooddata-eval --extra llm-judge python scripts/calibrate_catalog_metadata_judge.py + +.PHONY: calibrate-knowledge +calibrate-knowledge: + uv run --package gooddata-eval --extra llm-judge python scripts/calibrate_knowledge_judge.py diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index 9749259d1..1469ce59c 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -132,7 +132,7 @@ gd-eval run \ | `--runs K` | `2` | Independent runs per item. | | `--gate` | `any` | Which verdict decides an item: `any` = pass@K (a run passing is enough), `power` = pass^K (every run must pass, so the verdict measures stability). Identical at `--runs 1`. Kinds that repeat K runs only — `power` is refused when the dataset also has non-agentic items or `agentic_conversation`, which are always decided on pass@K. | | `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests — see *Concurrency and workspace safety* below. | -| `--judge-model MODEL` | `gpt-4o` | Model used for LLM-as-judge scoring — `agentic_general_question`, `agentic_guardrail`, `general_question`, `guardrail`, `dashboard_summary`, `document_takeaway` and `catalog_metadata`. Also settable via `GD_EVAL_JUDGE_MODEL`. Two things to weigh before changing it: the gpt-5 family rejects `temperature=0`, so verdicts stop being reproducible (the run warns when this happens); and choosing the same model the agent runs means the judge grades its own family's output. | +| `--judge-model MODEL` | `gpt-4o` | Model used for LLM-as-judge scoring — `agentic_general_question`, `agentic_guardrail`, `general_question`, `guardrail`, `dashboard_summary`, `document_takeaway`, `catalog_metadata` and `agentic_knowledge`. Also settable via `GD_EVAL_JUDGE_MODEL`. Two things to weigh before changing it: the gpt-5 family rejects `temperature=0`, so verdicts stop being reproducible (the run warns when this happens); and choosing the same model the agent runs means the judge grades its own family's output. | | `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary`, `document_takeaway` and `catalog_metadata` items go through their own endpoints, which have no such option. | **Concurrency and workspace safety.** Agentic kinds that create workspace objects @@ -323,8 +323,8 @@ their eval kind, and states latency for each: `agentic_general_question`, `agentic_guardrail` and `agentic_search`. - **Multi-turn** — kinds that drive a simulated user or fixture turns: `agentic_visualization` (and `vis_agentic`), `agentic_metric_skill`, `agentic_alert_skill`, `agentic_dashboard_skill`, - `agentic_document_skill`, `agentic_kda_skill`, `agentic_anomaly_detection`, `agentic_what_if` and - `agentic_conversation`. + `agentic_document_skill`, `agentic_kda_skill`, `agentic_anomaly_detection`, `agentic_what_if`, + `agentic_knowledge` and `agentic_conversation`. ```json "summary": { @@ -912,6 +912,59 @@ A 200 answer with no usable text (a null value without a note, or the placeholde `OPENAI_API_KEY`); re-run it after changing them. Request errors are handled as for `document_takeaway`; the CI entry point is `evaluate_catalog_metadata`. +### `agentic_knowledge` items + +Knowledge items ask the AI chat questions it answers from knowledge documents (tools +`search_knowledge` and `get_knowledge_chunks`). Each run is a fresh conversation that sends +fixed user turns in order (no simulated user) and grades the last answer. `question` is the +first turn; `knowledge_input.turns` lists every turn when there are several (`turns[0]` must +equal `question`). `knowledge_input.workspace` is an optional caller-defined workspace role +(e.g. `parent`, `child`), carried for the caller to map to a workspace id. `gd-eval run` +cannot map it: it runs an item with no role or `parent` in `--workspace` and refuses any +other role. In a Langfuse dataset `knowledge_input` goes in the item `metadata` (or the `input` +object); an item carrying it is an `agentic_knowledge` item unless it declares another +`test_kind`. Unknown keys are rejected. + +```json +{ + "id": "kn-001", + "dataset_name": "agent_knowledge", + "test_kind": "agentic_knowledge", + "question": "Who must approve a 30 % discount?", + "knowledge_input": {"turns": ["Who must approve a 30 % discount?", "And for 45 %?"], "workspace": "child"}, + "expected_output": { + "expected_documents": ["Discount Policy"], + "forbidden_documents": ["Discount Policy 2023"], + "expected_source": "the parent workspace", + "must_include": ["Says that discounts above 40 % are not allowed."], + "must_not_include": ["Says that the CFO must approve the discount."] + } +} +``` + +Every `expected_output` key is optional; unknown keys and wrong types are rejected before +any request. Code checks over the knowledge tool results of all turns gate the verdict, so a +failed item shows whether retrieval (document never found) or reasoning (found, answered +wrong) failed: + +- `expect_search` (default `true`) — at least one `search_knowledge` call (`knowledge_searched`). +- `expected_documents` — each title is in some knowledge tool result (`expected_documents_found`). +- `forbidden_documents` — no title is in any knowledge tool result (`forbidden_documents_absent`). +- `expected_source` — an expected document's result carries this `source` label + (`expected_source_found`); needs `expected_documents`. One of gen-ai's labels: `this + workspace`, `the parent workspace`, `an ancestor workspace`, `AI Hub`. +- `must_include`, `must_not_include`, `rubric` — judge criteria on the last answer, graded + like `dashboard_summary`; the judge reads the earlier turns and answers as INPUT. + +Titles match case-insensitively on a result's `title` or `filename`. A last turn without +text fails (`answered`). Diagnostics that never gate: `documents_retrieved`, +`sources_cited` (titles in the answer's trailing "Sources" list) and `turn_count`. Langfuse +gets `knowledge_pass` plus each code check as its own score per run; `quality_score` counts the +code checks and judge verdicts. A run the judge could not grade is an ERROR (`eval_error`), not +a fail. The judge steps are calibrated with +`make calibrate-knowledge` (needs `OPENAI_API_KEY`); the CI entry point is +`evaluate_agentic_knowledge`. + ### Items asked in a dashboard context (`user_context`) An item can be asked the way a user asks from an open dashboard or an attached widget. @@ -1013,6 +1066,7 @@ from that turn on until another turn changes it, as the attached context does in | `dashboard_summary` | Dashboard summary (via `/summary` endpoint) scored against a rubric by LLM | `[llm-judge]` | | `document_takeaway` | One short takeaway for a page (via `/takeaway` endpoint): deterministic format checks plus a rubric scored by LLM | `[llm-judge]` | | `catalog_metadata` | Analytics Catalog title or description for one object (via the `analyticsCatalog/generate*` endpoints): format checks plus a rubric scored by LLM | `[llm-judge]` | +| `agentic_knowledge` | Answer from knowledge documents over fixed chat turns: retrieval checks on the knowledge tool results plus a rubric on the last answer scored by LLM | `[llm-judge]` | ## Optional extras diff --git a/packages/gooddata-eval/scripts/calibrate_knowledge_judge.py b/packages/gooddata-eval/scripts/calibrate_knowledge_judge.py new file mode 100644 index 000000000..c6104f400 --- /dev/null +++ b/packages/gooddata-eval/scripts/calibrate_knowledge_judge.py @@ -0,0 +1,62 @@ +# (C) 2026 GoodData Corporation +"""Does the real judge grade agentic_knowledge criteria on the right side? + +tests/test_knowledge_evaluator.py pins the wiring, not the verdict. This pins the verdict, +against tests/fixtures/knowledge_calibration.json: each case is one question, one criterion +and one real answer, and the criterion must be satisfied ("pass") or not ("fail") on every run. + +Run it after changing the judge steps in core/evaluators/knowledge.py: + + OPENAI_API_KEY=... make calibrate-knowledge + +A script, not a test: CI has no OPENAI_API_KEY. Exits 1 on any mismatch. +""" + +import json +import os +import sys +from pathlib import Path + +from gooddata_eval.core.evaluators.knowledge import AgenticKnowledgeEvaluator +from gooddata_eval.core.models import ChatResult, DatasetItem + +RUNS = 3 +FIXTURE = Path(__file__).resolve().parent.parent / "tests" / "fixtures" / "knowledge_calibration.json" +_FIELDS = {"include": "must_include", "exclude": "must_not_include"} + + +def main() -> int: + if not os.environ.get("OPENAI_API_KEY"): + raise SystemExit("OPENAI_API_KEY required") + cases = json.loads(FIXTURE.read_text(encoding="utf-8")) + evaluator = AgenticKnowledgeEvaluator() + print(f"judge model: {evaluator._positive_judge.model}, runs: {RUNS}") + mismatches = 0 + for case in cases: + item = DatasetItem( + id=case["label"], + dataset_name="knowledge_calibration", + test_kind="agentic_knowledge", + question=case["question"], + # No tool calls here: only the judge is calibrated. + expected_output={_FIELDS[case["kind"]]: [case["criterion"]], "expect_search": False}, + ) + chat = ChatResult(text_response=case["text"]) + key = f"{case['kind']}_0" + verdicts = [evaluator.evaluate(item, [chat]).detail.get(key) for _ in range(RUNS)] + ungraded = sum(not isinstance(v, bool) for v in verdicts) + satisfied = sum(v is True for v in verdicts) + want = RUNS if case["want"] == "pass" else 0 + # An ungraded run proves nothing, on either side. + ok = satisfied == want and not ungraded + mismatches += not ok + print( + f"{case['label'][:44]:44} {case['kind']:7} want {case['want']:4} satisfied {satisfied}/{RUNS}" + f"{f' ungraded {ungraded}' if ungraded else ''} {'OK' if ok else 'MISMATCH'}" + ) + print(f"\n{'all calibration answers on the right side' if not mismatches else f'{mismatches} mismatch(es)'}") + return 1 if mismatches else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index fb2984dce..39c1c9823 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -23,6 +23,7 @@ from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail from gooddata_eval.core.agentic.kda_skill import evaluate_agentic_kda_skill +from gooddata_eval.core.agentic.knowledge import evaluate_agentic_knowledge from gooddata_eval.core.agentic.metric_skill import evaluate_agentic_metric_skill from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization @@ -63,6 +64,7 @@ class _LfKw(TypedDict, total=False): "agentic_dashboard_summary", "agentic_anomaly_detection", "agentic_what_if", + "agentic_knowledge", } ) @@ -385,6 +387,29 @@ def _dispatch_agentic( user_context=item.user_context, **lf_kw, ) + elif kind == "agentic_knowledge": + knowledge_input = item.knowledge_input + role = knowledge_input.workspace if knowledge_input is not None else None + # Only "parent" is the --workspace itself; any other role names a workspace the CLI cannot resolve. + if role is not None and role.strip().lower() != "parent": + raise ValueError( + f"agentic_knowledge item '{item.id}' sets knowledge_input.workspace={role!r}: the workspace role " + "needs a caller that maps it to a workspace id (e.g. the gdc-nas shim); the CLI would run it in " + "--workspace instead." + ) + return evaluate_agentic_knowledge( + host=host, + token=token, + workspace_id=workspace_id, + question=item.question, + expected_output=eo, + turns=knowledge_input.turns if knowledge_input is not None else None, + k=k, + gate=gate, + agent_id=agent_id, + user_context=item.user_context, + **lf_kw, + ) elif kind == "agentic_conversation": fixture_data = eo.get("fixture") or eo if isinstance(eo, dict) else {} return evaluate_agentic_conversation( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/knowledge.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/knowledge.py new file mode 100644 index 000000000..883b9596f --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/knowledge.py @@ -0,0 +1,443 @@ +# (C) 2026 GoodData Corporation +"""Agentic knowledge evaluation runner. + +Each run is a fresh conversation that sends the item's fixed user turns in order (no +simulated user) and grades the last answer with `AgenticKnowledgeEvaluator`. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any + +from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) +from gooddata_eval.core.agentic._outcome import ( + agentic_success, + attach_labels, + chat_failure_exit, + failure_message_head, + label_runs, + mark_unscored_run, + raise_agentic_failure, +) +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + conversations_to_link, + open_item_trace, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) +from gooddata_eval.core.chat.sse_client import ChatClient, ChatError +from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError +from gooddata_eval.core.evaluators.base import ItemEvaluation +from gooddata_eval.core.evaluators.knowledge import AgenticKnowledgeEvaluator, resolve_turns +from gooddata_eval.core.langfuse.item_scope import item_scope +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ChatResult, + DatasetItem, + KnowledgeInput, + LoopExit, + ReasoningStepEvent, + ToolCallEvent, + build_latency_breakdown, + shift_and_index_events, +) +from gooddata_eval.core.timing import PhaseTimings, log_timer, run_latency_s, sum_timings + +_DEFAULT_K = 1 +# Code checks that gate the verdict; each is its own Langfuse score, to split retrieval from reasoning. +CODE_CHECKS = ( + "knowledge_searched", + "expected_documents_found", + "forbidden_documents_absent", + "expected_source_found", + "answered", +) + + +@dataclass +class KnowledgeRunResult: + """Outcome of one K-run conversation.""" + + conversation_id: str + evaluation: ItemEvaluation + turn_results: list[ChatResult] = field(default_factory=list) + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None + timings: PhaseTimings = field(default_factory=PhaseTimings) + run_latency_s: float = 0.0 + # All turns, rebased onto one timeline. + tool_call_events: list[ToolCallEvent] = field(default_factory=list) + reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) + # Set when a turn or the conversation creation failed; such a run is not graded. + chat_error: str | None = None + exit_reason: LoopExit | None = None + + @property + def judge_error(self) -> str | None: + """Why the judge left a run without a verdict; None when it graded or was not asked. + + `grade_rubric` sets `evaluation.error` only when every code check held and no gating + criterion got a readable verdict, so the judge's verdict was the one still open. + """ + return self.evaluation.error if self.chat_error is None else None + + @property + def passed(self) -> bool: + return self.chat_error is None and self.evaluation.passed + + +@dataclass +class AgenticKnowledgeSummary: + """Aggregated outcome of K runs for one knowledge item.""" + + run_results: list[KnowledgeRunResult] + pass_at_k: bool + pass_power_k: bool + best: KnowledgeRunResult + + +def _failed_run( + conversation_id: str, message: str, turn_results: list[ChatResult], agent_s: float, exit_reason: LoopExit +) -> KnowledgeRunResult: + """A run that never reached its last answer, kept so completed runs survive.""" + return KnowledgeRunResult( + exit_reason=exit_reason, + conversation_id=conversation_id, + evaluation=ItemEvaluation(passed=False, rank_key=(-1, 0.0), detail={}, error=message), + turn_results=turn_results, + timings=PhaseTimings(agent_s=agent_s), + run_latency_s=agent_s, + chat_error=message, + ) + + +def _run_single_knowledge( + client: ChatClient, + evaluator: AgenticKnowledgeEvaluator, + conversation_id: str, + item: DatasetItem, + turns: list[str], +) -> KnowledgeRunResult: + agent_started = time.monotonic() + turn_results: list[ChatResult] = [] + for turn in turns: + try: + turn_results.append(client.send_message(conversation_id, turn)) + except ChatError as exc: # noqa: PERF203 -- one failed turn ends this run only + message = f"chat failed on turn {len(turn_results) + 1}/{len(turns)}: {exc}" + elapsed = time.monotonic() - agent_started + return _failed_run(conversation_id, message, turn_results, elapsed, chat_failure_exit(exc)) + agent_elapsed = time.monotonic() - agent_started + + judge_started = time.monotonic() + evaluation = evaluator.evaluate(item, turn_results) + judge_elapsed = time.monotonic() - judge_started + log_timer(f"[timer] knowledge {conversation_id} agent {agent_elapsed:.2f}s, judge {judge_elapsed:.2f}s") + + tool_events: list[ToolCallEvent] = [] + reasoning_events: list[ReasoningStepEvent] = [] + reasoning_steps: list[str] = [] + offsets = (0.0, 0, 0) + for result in turn_results: + offsets = shift_and_index_events( + result, turn_offset=offsets[0], tool_index_offset=offsets[1], reasoning_index_offset=offsets[2] + ) + tool_events += result.tool_call_events or [] + reasoning_events += result.reasoning_step_events or [] + reasoning_steps += result.reasoning_steps or [] + + return KnowledgeRunResult( + conversation_id=conversation_id, + evaluation=evaluation, + turn_results=turn_results, + reasoning_steps=reasoning_steps, + response_id=turn_results[-1].response_id, + timings=PhaseTimings(agent_s=agent_elapsed, judge_s=judge_elapsed), + run_latency_s=agent_elapsed + judge_elapsed, + tool_call_events=tool_events, + reasoning_step_events=reasoning_events, + ) + + +def run_agentic_knowledge( + host: str, + token: str, + workspace_id: str, + question: str, + expected_output: Any, + turns: list[str] | None = None, + k: int = _DEFAULT_K, + initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, + user_context: dict[str, Any] | None = None, + dataset_item_id: str = "", + dataset_name: str = "agent_knowledge", +) -> AgenticKnowledgeSummary: + """Run the knowledge item K times, each on a fresh conversation, and return a summary. + + Raises: + ValueError: The item is invalid; raised before any request. + """ + item = DatasetItem( + id=dataset_item_id or "agentic_knowledge", + dataset_name=dataset_name, + test_kind="agentic_knowledge", + question=question, + expected_output=expected_output, + knowledge_input=KnowledgeInput(turns=turns) if turns else None, + ) + AgenticKnowledgeEvaluator.validate(item) + all_turns = resolve_turns(question, turns) + client = ChatClient( + host=host, + token=token, + workspace_id=workspace_id, + reasoning_effort=reasoning_effort, + agent_id=agent_id, + user_context=user_context, + ) + run_results: list[KnowledgeRunResult] = [] + try: + evaluator = AgenticKnowledgeEvaluator() + for run_idx in range(k): + owned = not (run_idx == 0 and initial_conversation_id is not None) + if owned: + try: + conv_id = client.create_conversation() + except Exception as exc: # noqa: BLE001 -- a lost conversation must not discard completed runs + message = f"conversation creation failed: {exc}" + run_results.append(_failed_run("", message, [], 0.0, LoopExit.CHAT_ERROR)) + continue + else: + conv_id = str(initial_conversation_id) + try: + run_results.append(_run_single_knowledge(client, evaluator, conv_id, item, all_turns)) + finally: + if owned: + client.delete_conversation(conv_id) + finally: + client.close() + + # Chat failures and judge failures both leave a run without a verdict. + scored = [r for r in run_results if r.chat_error is None and r.judge_error is None] + pass_at_k = any(r.passed for r in scored) + pass_power_k = len(scored) == len(run_results) and bool(scored) and all(r.passed for r in scored) + best = max(scored or run_results, key=lambda r: r.evaluation.rank_key) + return AgenticKnowledgeSummary(run_results=run_results, pass_at_k=pass_at_k, pass_power_k=pass_power_k, best=best) + + +def _run_detail(run: KnowledgeRunResult) -> dict: + """Diagnostics for one run: evaluation detail, latency breakdown, chat error.""" + detail = dict(run.evaluation.detail) + if run.chat_error is not None: + detail["chat_error"] = run.chat_error + detail["latency_breakdown"] = build_latency_breakdown(run.tool_call_events, run.reasoning_step_events) + return detail + + +def _check_scores(run: KnowledgeRunResult) -> dict[str, bool]: + """Per-run Langfuse scores: `knowledge_pass` plus each code check the run recorded.""" + detail = run.evaluation.detail + checks = {name: detail[name] for name in CODE_CHECKS if isinstance(detail.get(name), bool)} + return {"knowledge_pass": run.evaluation.passed, **checks} + + +def _quality_checks(run: KnowledgeRunResult) -> dict[str, bool]: + """Every bool verdict in the detail: code checks and judge criteria, no `knowledge_pass`. + + Same set as `rank_key` quality and the report's `quality_score`, so nothing counts twice. + """ + return {name: value for name, value in run.evaluation.detail.items() if isinstance(value, bool)} + + +def _failure_reason(run: KnowledgeRunResult) -> str: + """Failed gating checks as `name=False`, plus the judge or chat error.""" + detail = run.evaluation.detail + failed = [ + f"{name}=False" + for name, value in detail.items() + if value is False and (name in CODE_CHECKS or name.startswith(("include_", "exclude_"))) + ] + parts = [", ".join(failed)] if failed else [] + if run.evaluation.error: + parts.append(run.evaluation.error) + return "; ".join(parts) or "criteria not satisfied" + + +class KnowledgeAssertionError(AgenticAssertionError): + """Raised when an agentic knowledge evaluation fails.""" + + +def evaluate_agentic_knowledge( + host: str, + token: str, + workspace_id: str, + question: str, + expected_output: Any, + turns: list[str] | None = None, + k: int = _DEFAULT_K, + initial_conversation_id: str | None = None, + gate: EvalGate = DEFAULT_GATE, + agent_id: str | None = None, + user_context: dict[str, Any] | None = None, + langfuse: object | None = None, + dataset_item_id: str = "", + dataset_name: str = "agent_knowledge", + run_timestamp: str | None = None, + model_version_override: str | None = None, + run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, +) -> AgenticEvalOutcome: + """Run the evaluation, log to Langfuse, and raise KnowledgeAssertionError on failure.""" + identity = RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ) + langfuse, window_start, scope = open_item_trace(langfuse, identity, dataset_item_id, suffix_runs=k > 1) + with item_scope(scope): + summary = run_agentic_knowledge( + host=host, + token=token, + workspace_id=workspace_id, + question=question, + expected_output=expected_output, + turns=turns, + k=k, + initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, + agent_id=agent_id, + user_context=user_context, + dataset_item_id=dataset_item_id, + dataset_name=dataset_name, + ) + + # Before the Langfuse scores, so a run the judge could not grade gets its ERROR marker there. + run_labels = label_runs(summary.run_results, passed=lambda r: r.passed, judge_error=lambda r: r.judge_error) + + if langfuse is not None and dataset_item_id: + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate) + for run_idx, run in enumerate(summary.run_results): + if run.chat_error is not None: + continue + if run.judge_error is not None: + # No verdict: a knowledge_pass=0 would publish a failure nobody assessed. + mark_unscored_run(ctx, run_idx, run.conversation_id, run_labels.error_label(run_idx)) + continue + pt = ctx.trace(run.conversation_id) + scores = _check_scores(run) + with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=scores) as tid: + for name, value in scores.items(): + ctx.score(tid, name=name, value=float(value), data_type="BOOLEAN") + ctx.quality( + tid, + strict_checks=_quality_checks(run), + latency_sec=pt.latency if pt else None, + cost_usd=pt.total_cost if pt else None, + ) + log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) + + all_turns = resolve_turns(question, turns) + submit_trace_scoring( + submit_trace_link, + identity, + langfuse=langfuse, + dataset_item_id=dataset_item_id, + conversation_ids=conversations_to_link( + [r.conversation_id for r in summary.run_results if r.chat_error is None and r.judge_error is None], + [r.conversation_id for r in summary.run_results if r.judge_error is not None], + ), + window_start=window_start, + window_end=window_end, + suffix_runs=k > 1, + write_scores=_write_scores, + scope=scope, + item_input=question if len(all_turns) == 1 else all_turns, + ) + + best = summary.best + unscored = [r.judge_error for r in summary.run_results if r.judge_error is not None] + detail = _run_detail(best) + if unscored: + detail["unscored_runs"] = len(unscored) + detail["judge_errors"] = unscored + timings = sum_timings([r.timings for r in summary.run_results]) + runs_passed = sum(1 for r in summary.run_results if r.passed) + runs_effective = len(summary.run_results) + run_latencies_s = [r.run_latency_s for r in summary.run_results] + failed_runs = build_failed_runs(summary.run_results, labels=run_labels, detail=_run_detail) + + if unscored and not any(r.chat_error is None and r.judge_error is None for r in summary.run_results): + # No run got a verdict: the item is a harness error, not K failures. + error = JudgeResponseError( + f"judge returned no readable verdict for any of the {len(unscored)} graded run(s): " + " | ".join(unscored) + ) + error.reasoning_steps = best.reasoning_steps + error.conversation_id = best.conversation_id + error.response_id = best.response_id + error.detail = detail + error.runs_passed = runs_passed + error.runs_effective = runs_effective + error.failed_runs = failed_runs + error.run_latencies_s = run_latencies_s + error.best_run_latency_s = run_latency_s(best.timings) + error.timings = timings + attach_labels(error, run_labels, gate) + raise error + + if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + head = failure_message_head(gate, run_labels, runs_passed, failed_runs, len(unscored)) + raise_agentic_failure( + KnowledgeAssertionError, + f"Knowledge answer failed for '{dataset_item_id or question}'. {head} {_failure_reason(best)}", + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + failed_runs=failed_runs, + run_latencies_s=run_latencies_s, + best_run_latency_s=run_latency_s(best.timings), + timings=timings, + ) + return agentic_success( + shadow=run_labels.shadow(gate), + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + failed_runs=failed_runs, + run_latencies_s=run_latencies_s, + best_run_latency_s=run_latency_s(best.timings), + timings=timings, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py b/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py index 878bac8e7..c7e668235 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py @@ -18,7 +18,7 @@ from pydantic import ValidationError from gooddata_eval.core.langfuse._env import make_http_client -from gooddata_eval.core.models import CatalogInput, DatasetItem, SummaryInput, TakeawayInput +from gooddata_eval.core.models import CatalogInput, DatasetItem, KnowledgeInput, SummaryInput, TakeawayInput _PAGE_SIZE = 100 @@ -81,6 +81,29 @@ def _catalog_input_from_raw(raw: dict, expected_output: Any) -> CatalogInput | N return None +def _knowledge_input_from_raw(raw: dict) -> KnowledgeInput | None: + """Locate an agentic_knowledge item's `knowledge_input`, in `input` or `metadata`. + + Not read from expected_output: the knowledge evaluator rejects unknown keys there. + + Raises: + ValueError: The value found is not a valid `knowledge_input` object. + """ + for source in (raw.get("input"), raw.get("metadata")): + if not isinstance(source, dict) or source.get("knowledge_input") is None: + continue + found = source["knowledge_input"] + if not isinstance(found, dict): + raise ValueError( + f"Langfuse item {raw.get('id')!r}: knowledge_input must be a JSON object, got {type(found).__name__}" + ) + try: + return KnowledgeInput.model_validate(found) + except ValidationError as e: + raise ValueError(f"Langfuse item {raw.get('id')!r}: invalid knowledge_input: {e}") from e + return None + + def _takeaway_input_from_raw(raw: dict, expected_output: Any) -> TakeawayInput | None: """Locate a document_takeaway item's `takeaway_input`. @@ -160,9 +183,13 @@ def _item_from_raw(raw: dict, *, dataset_name: str, test_kind: str) -> DatasetIt expected_output = raw.get("expectedOutput") or raw.get("expected_output") takeaway_input = _takeaway_input_from_raw(raw, expected_output) catalog_input = _catalog_input_from_raw(raw, expected_output) - # An item carrying `takeaway_input` or `catalog_input` can only be of that kind, so it stands - # in for the CLI default; an explicit test_kind or a structural match still wins. + knowledge_input = _knowledge_input_from_raw(raw) + # An item carrying `takeaway_input`, `catalog_input` or `knowledge_input` can only be of that + # kind, so it stands in for the CLI default; an explicit test_kind or a structural match still + # wins. When several are set: takeaway, then catalog, then knowledge. default_kind = test_kind + if knowledge_input is not None: + default_kind = "agentic_knowledge" if catalog_input is not None: default_kind = "catalog_metadata" if takeaway_input is not None: @@ -177,6 +204,7 @@ def _item_from_raw(raw: dict, *, dataset_name: str, test_kind: str) -> DatasetIt summary_input=_summary_input_from_raw(raw, expected_output), takeaway_input=takeaway_input, catalog_input=catalog_input, + knowledge_input=knowledge_input, user_context=_user_context_from_raw(raw), ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/knowledge.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/knowledge.py new file mode 100644 index 000000000..98de23e42 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/knowledge.py @@ -0,0 +1,252 @@ +# (C) 2026 GoodData Corporation +"""Evaluator for agentic_knowledge: retrieval checks plus an LLM-judge rubric on the last answer. + +The AI chat answers from knowledge documents through the `search_knowledge` and +`get_knowledge_chunks` tools. An item sends fixed user turns in one conversation and grades +the last answer. `expected_output` is: + + { + "expect_search": true, + "expected_documents": ["...document titles some knowledge tool must return..."], + "forbidden_documents": ["...document titles no knowledge tool may return..."], + "expected_source": "the parent workspace", + "must_include": ["...criteria the last answer must satisfy..."], + "must_not_include": ["...characteristics the last answer must not show..."], + "rubric": ["...soft quality dimensions..."] + } + +Every key is optional. Retrieval checks gate the verdict before the judge, so a failed item +shows whether the document was never found (retrieval) or found and misused (reasoning). +""" + +import re +from collections.abc import Sequence +from typing import Any + +from gooddata_eval.core.chat.render import render_answer_text +from gooddata_eval.core.evaluators._llm_judge import LLMJudge +from gooddata_eval.core.evaluators._rubric import Criterion, grade_rubric +from gooddata_eval.core.evaluators.base import ItemEvaluation +from gooddata_eval.core.models import ChatResult, DatasetItem, ToolCallEvent + +SEARCH_TOOL = "search_knowledge" +CHUNKS_TOOL = "get_knowledge_chunks" + +_POSITIVE_STEPS = [ + "Read the INPUT (the conversation so far, ending with the user's last question) and the " + "EXPECTED OUTPUT (one criterion the assistant's last answer must satisfy).", + "Read the ACTUAL OUTPUT (the assistant's last answer).", + "Score 1 if the actual output clearly satisfies the criterion.", + "Score 0 if the criterion is missing, contradicted, or only partially addressed.", +] + +# Presence question, inverted in code. +_VIOLATION_STEPS = [ + "Read the CHARACTERISTIC described in EXPECTED OUTPUT, including any exceptions it states.", + "Read the ACTUAL OUTPUT (the assistant's last answer).", + "Decide only whether the actual output exhibits the characteristic, not whether it is correct or wanted.", + "Score 1 if the actual output clearly exhibits the described characteristic.", + "Score 0 if it does not exhibit it.", +] + +_BOOL_KEYS = ("expect_search",) +_LIST_KEYS = ("expected_documents", "forbidden_documents", "must_include", "must_not_include", "rubric") +_STR_KEYS = ("expected_source",) +_KNOWN_KEYS = frozenset(_BOOL_KEYS + _LIST_KEYS + _STR_KEYS) +# The `source` labels gen-ai puts on knowledge tool results. +SOURCE_LABELS = ("this workspace", "the parent workspace", "an ancestor workspace", "AI Hub") + +# Earlier user turns and answers shown to the judge are cut to this many characters each. +_EARLIER_TURN_CHARS = 1500 + +# "**Sources**", "### Sources" or "Sources:" on a line of its own. +_SOURCES_HEADING = re.compile(r"^\s*(?:\*\*Sources:?\*\*|#{1,6}\s*Sources:?|Sources:)\s*$", re.IGNORECASE) +_BULLET = re.compile(r"^\s*(?:[-*•]|\d+[.)])\s+(.+?)\s*$") +_EMPHASIS = re.compile(r"^(\*\*|\*|_)(.+?)\1") + + +def _norm(text: Any) -> str: + return str(text or "").strip().casefold() + + +def resolve_turns(question: str, turns: Sequence[str] | None) -> list[str]: + """All user turns of an item: `turns` when given, else the question alone.""" + return list(turns) if turns else [question] + + +def item_turns(item: DatasetItem) -> list[str]: + turns = item.knowledge_input.turns if item.knowledge_input is not None else None + return resolve_turns(item.question, turns) + + +def knowledge_results(events: Sequence[ToolCallEvent]) -> list[dict[str, Any]]: + """One row per document any knowledge tool returned, in call order. + + - `search_knowledge`: `{"results": [{"title", "filename", "source", ...}]}`, one row each + - `get_knowledge_chunks`: `{"title", "filename", "source", "chunks": [...]}`, one row; + a document it did not find (no title, no filename) adds none + """ + rows: list[dict[str, Any]] = [] + for event in events: + parsed = event.parsed_result() if event.result else None + if not isinstance(parsed, dict): + continue + if event.function_name == SEARCH_TOOL and isinstance(parsed.get("results"), list): + rows.extend(row for row in parsed["results"] if isinstance(row, dict)) + elif event.function_name == CHUNKS_TOOL and _row_names(parsed): + rows.append({key: parsed.get(key) for key in ("title", "filename", "source")}) + return rows + + +def _row_names(row: dict[str, Any]) -> set[str]: + """Normalized title and filename of a result row; either can match a document.""" + return {name for name in (_norm(row.get("title")), _norm(row.get("filename"))) if name} + + +def _row_label(row: dict[str, Any]) -> str: + return str(row.get("title") or "").strip() or str(row.get("filename") or "").strip() + + +def parse_sources_cited(answer: str) -> list[str]: + """Titles in the last "Sources" section of an answer; [] when there is none. + + - heading line: `**Sources**`, `### Sources` or `Sources:` + - each following bullet: `- *Title* (filename), pp. 1-2 — from *source*`, or `- Title (...)` + - stops at the first non-blank line that is not a bullet + """ + lines = answer.splitlines() + start = next((i for i in range(len(lines) - 1, -1, -1) if _SOURCES_HEADING.match(lines[i])), None) + if start is None: + return [] + titles: list[str] = [] + for line in lines[start + 1 :]: + if not line.strip(): + continue + bullet = _BULLET.match(line) + if bullet is None: + break + text = bullet.group(1) + emphasis = _EMPHASIS.match(text) + title = emphasis.group(2) if emphasis else text.split(" (", 1)[0] + if title.strip(): + titles.append(title.strip()) + return titles + + +def _clip(text: str) -> str: + return text if len(text) <= _EARLIER_TURN_CHARS else text[:_EARLIER_TURN_CHARS] + "… (truncated)" + + +def judge_input(turns: Sequence[str], answers: Sequence[str]) -> str: + """What the judge reads as INPUT: the question, or the conversation up to the last question.""" + if len(turns) == 1: + return turns[0] + parts: list[str] = [] + for turn, answer in zip(turns[:-1], answers, strict=False): + parts.append(f"User: {_clip(turn)}\nAssistant: {_clip(answer)}") + parts.append(f"User (last question): {turns[-1]}") + return "\n\n".join(parts) + + +class AgenticKnowledgeEvaluator: + test_kind = "agentic_knowledge" + + def __init__(self): + self._positive_judge = LLMJudge(evaluation_steps=_POSITIVE_STEPS) + self._violation_judge = LLMJudge(evaluation_steps=_VIOLATION_STEPS) + + @staticmethod + def validate(item: DatasetItem) -> None: + """Reject an item this evaluator cannot read, before any request. + + Raises: + ValueError: Blank question, turns not starting with the question, `expected_output` + not an object, an unknown key, or a value of the wrong type. + """ + prefix = f"agentic_knowledge item '{item.id}'" + if not item.question.strip(): + raise ValueError(f"{prefix}: question must be a non-empty string") + turns = item.knowledge_input.turns if item.knowledge_input is not None else None + if turns and turns[0].strip() != item.question.strip(): + raise ValueError(f"{prefix}: knowledge_input.turns[0] must equal the question") + expected = item.expected_output if item.expected_output is not None else {} + if not isinstance(expected, dict): + raise ValueError(f"{prefix}: expected_output must be a JSON object") + # `test_kind` routes the item (langfuse_source) and is not an expectation. + unknown = sorted(set(expected) - _KNOWN_KEYS - {"test_kind"}) + if unknown: + raise ValueError(f"{prefix}: unknown expected_output key(s) {unknown}; known: {sorted(_KNOWN_KEYS)}") + for key in _BOOL_KEYS: + if key in expected and not isinstance(expected[key], bool): + raise ValueError(f"{prefix}: {key} must be true or false") + for key in _LIST_KEYS: + value = expected.get(key, []) + if not isinstance(value, list) or not all(isinstance(v, str) and v.strip() for v in value): + raise ValueError(f"{prefix}: {key} must be a JSON array of non-empty strings") + for key in _STR_KEYS: + if key in expected and not (isinstance(expected[key], str) and expected[key].strip()): + raise ValueError(f"{prefix}: {key} must be a non-empty string") + source = expected.get("expected_source") + if isinstance(source, str) and _norm(source) not in {_norm(label) for label in SOURCE_LABELS}: + raise ValueError(f"{prefix}: expected_source must be one of {list(SOURCE_LABELS)}") + if "expected_source" in expected and not expected.get("expected_documents"): + raise ValueError(f"{prefix}: expected_source needs expected_documents to say which document") + + def evaluate(self, item: DatasetItem, turn_results: Sequence[ChatResult]) -> ItemEvaluation: + """Grade one run: every turn's result, in order; the last one is the graded answer.""" + self.validate(item) + expected: dict[str, Any] = item.expected_output if isinstance(item.expected_output, dict) else {} + turns = item_turns(item) + answers = [render_answer_text(result) for result in turn_results] + last = answers[-1] if answers else "" + detail: dict[str, Any] = {"actual_output": last, **self.retrieval_checks(expected, turn_results)} + detail["answered"] = bool(last) + detail["sources_cited"] = parse_sources_cited(last) + detail["turn_count"] = len(turn_results) + + passed = all(v for v in detail.values() if isinstance(v, bool)) + if not last: + return ItemEvaluation(passed=False, rank_key=(0, 0.0), detail=detail) + + def _criteria(key: str, prefix: str) -> list[Criterion]: + return [(f"{prefix}_{i}", str(c)) for i, c in enumerate(expected.get(key, []))] + + return grade_rubric( + self._positive_judge, + self._violation_judge, + question=judge_input(turns, answers), + actual=last, + must_include=_criteria("must_include", "include"), + must_not_include=_criteria("must_not_include", "exclude"), + rubric=_criteria("rubric", "rubric"), + detail=detail, + passed=passed, + ) + + @staticmethod + def retrieval_checks(expected: dict[str, Any], turn_results: Sequence[ChatResult]) -> dict[str, Any]: + """Code checks over every turn's knowledge tool calls. + + - bools gate the verdict; each is present only when its key is set + - `documents_retrieved` is a list, so it never gates or counts towards quality + """ + events = [event for result in turn_results for event in result.tool_call_events or []] + rows = knowledge_results(events) + found = set().union(*(_row_names(row) for row in rows)) if rows else set() + checks: dict[str, Any] = {} + if expected.get("expect_search", True): + checks["knowledge_searched"] = any(e.function_name == SEARCH_TOOL for e in events) + wanted = expected.get("expected_documents") or [] + if wanted: + checks["expected_documents_found"] = all(_norm(title) in found for title in wanted) + forbidden = expected.get("forbidden_documents") or [] + if forbidden: + checks["forbidden_documents_absent"] = not any(_norm(title) in found for title in forbidden) + source = expected.get("expected_source") + if source: + wanted_names = {_norm(title) for title in wanted} + checks["expected_source_found"] = any( + _row_names(row) & wanted_names and _norm(row.get("source")) == _norm(source) for row in rows + ) + checks["documents_retrieved"] = sorted({label for row in rows if (label := _row_label(row))}) + return checks diff --git a/packages/gooddata-eval/src/gooddata_eval/core/interaction.py b/packages/gooddata-eval/src/gooddata_eval/core/interaction.py index 645625697..1fffe2215 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/interaction.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/interaction.py @@ -46,6 +46,7 @@ "agentic_anomaly_detection", "agentic_what_if", "agentic_forecasting", + "agentic_knowledge", } ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index eceda9f54..1094b7561 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -579,6 +579,34 @@ class CatalogInput(BaseModel): output: Literal["title", "description"] +class KnowledgeInput(BaseModel): + """Structured input for the `agentic_knowledge` test kind. + + - `turns`: every user turn, in order, sent in one conversation; None means the item's + question is the only turn. + - `workspace`: a caller-defined workspace role (e.g. "parent"); carried, not used here. + """ + + model_config = ConfigDict(extra="forbid") + + turns: list[str] | None = Field(default=None, min_length=1) + workspace: str | None = None + + @field_validator("turns") + @classmethod + def _turns_not_blank(cls, value: list[str] | None) -> list[str] | None: + if value is not None and any(not turn.strip() for turn in value): + raise ValueError("every turn must be a non-empty string") + return value + + @field_validator("workspace") + @classmethod + def _workspace_not_blank(cls, value: str | None) -> str | None: + if value is not None and not value.strip(): + raise ValueError("workspace must be a non-empty string") + return value + + class DatasetItem(BaseModel): """Common dataset envelope. `expected_output` stays raw; each evaluator parses its own shape.""" @@ -595,6 +623,8 @@ class DatasetItem(BaseModel): takeaway_input: TakeawayInput | None = None # Only used by the `catalog_metadata` test kind; ignored by all others. catalog_input: CatalogInput | None = None + # Only used by the `agentic_knowledge` test kind; ignored by all others. + knowledge_input: KnowledgeInput | None = None # Relayed verbatim as the chat request's `userContext`. Deliberately opaque: gen-ai owns # that schema (a discriminated union of view/widget descriptors), so re-modelling it here # would only create a second copy to keep in sync. diff --git a/packages/gooddata-eval/tests/fixtures/knowledge_calibration.json b/packages/gooddata-eval/tests/fixtures/knowledge_calibration.json new file mode 100644 index 000000000..c71102cde --- /dev/null +++ b/packages/gooddata-eval/tests/fixtures/knowledge_calibration.json @@ -0,0 +1,98 @@ +[ + { + "label": "How do we define an active customer?", + "question": "How do we define an active customer?", + "kind": "include", + "criterion": "Gives the company definition: a customer with at least one order in the last 12 months", + "text": "An active customer is defined as a customer with at least one order in the last 12 months.", + "want": "pass" + }, + { + "label": "How do we define an active customer?", + "question": "How do we define an active customer?", + "kind": "include", + "criterion": "Gives the company definition: a customer with at least one order in the last 12 months", + "text": "An active customer is defined as a customer who has transacted. The count follows the current filters and time range.", + "want": "fail" + }, + { + "label": "Who must approve a 30 % discount?", + "question": "Who must approve a 30 % discount?", + "kind": "exclude", + "criterion": "Says that the CFO must approve the discount", + "text": "A 30% discount must be approved by the regional sales manager. Discounts above 40% are not allowed.", + "want": "pass" + }, + { + "label": "Who must approve a 30 % discount?", + "question": "Who must approve a 30 % discount?", + "kind": "exclude", + "criterion": "Says that the CFO must approve the discount", + "text": "Discounts above 25 % need approval of the CFO, so a 30 % discount must be approved by the CFO.", + "want": "fail" + }, + { + "label": "Who is the category manager for Clothing?", + "question": "Who is the category manager for Clothing?", + "kind": "exclude", + "criterion": "Names a person as the Clothing category manager", + "text": "I couldn’t find a category manager for Clothing in the available data or knowledge sources.", + "want": "pass" + }, + { + "label": "Who is the category manager for Clothing?", + "question": "Who is the category manager for Clothing?", + "kind": "exclude", + "criterion": "Names a person as the Clothing category manager", + "text": "The Clothing category manager is Eva Dvořák.", + "want": "fail" + }, + { + "label": "Kdo musí schválit slevu 20 %?", + "question": "Kdo musí schválit slevu 20 %?", + "kind": "include", + "criterion": "Answers in Czech", + "text": "Slevu 20 % musí schválit regionální obchodní manažer.", + "want": "pass" + }, + { + "label": "Kdo musí schválit slevu 20 %?", + "question": "Kdo musí schválit slevu 20 %?", + "kind": "include", + "criterion": "Answers in Czech", + "text": "A 20 % discount must be approved by the regional sales manager.", + "want": "fail" + }, + { + "label": "What is the Furniture team's discount rule?", + "question": "What is the Furniture team's discount rule?", + "kind": "include", + "criterion": "Says that a discount above 15 % needs the approval of the regional sales manager.", + "text": "The Furniture team follows the general discount policy:\n\n- **Up to 15%:** the sales representative can approve it.\n- **Above 15% and up to 40%:** regional sales manager approval is required.\n- **Above 40%:** not allowed.\n\nThe policy is valid from **1 January 2024** and applies to all sales teams.", + "want": "pass" + }, + { + "label": "What is the Furniture team's discount rule?", + "question": "What is the Furniture team's discount rule?", + "kind": "include", + "criterion": "Says that a discount above 15 % needs the approval of the regional sales manager.", + "text": "The Furniture team needs approval of the Furniture category manager for discounts above 10 %.", + "want": "fail" + }, + { + "label": "What is our approval rule for free shipping?", + "question": "What is our approval rule for free shipping?", + "kind": "include", + "criterion": "Says the documents do not mention free shipping (even if it then applies another rule).", + "text": "I found a discount approval rule, but it does not explicitly mention free shipping. If free shipping is treated as a discount, the rule is:\n\n- Up to **15%**: the sales representative can approve it.\n- Above **15%**: approval from the regional sales manager is required.\n- Above **40%**: not allowed.\n\nThe policy is valid from **1 January 2024** for all sales teams.", + "want": "pass" + }, + { + "label": "What is our approval rule for free shipping?", + "question": "What is our approval rule for free shipping?", + "kind": "include", + "criterion": "Says the documents do not mention free shipping (even if it then applies another rule).", + "text": "Free shipping must be approved by the regional sales manager when the order is below 50 USD.", + "want": "fail" + } +] diff --git a/packages/gooddata-eval/tests/test_agentic_knowledge.py b/packages/gooddata-eval/tests/test_agentic_knowledge.py new file mode 100644 index 000000000..cfa2d2bf7 --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_knowledge.py @@ -0,0 +1,295 @@ +# (C) 2026 GoodData Corporation +import json +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic.knowledge import ( + KnowledgeAssertionError, + KnowledgeRunResult, + _failure_reason, + evaluate_agentic_knowledge, + run_agentic_knowledge, +) +from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError +from gooddata_eval.core.evaluators.base import ItemEvaluation +from gooddata_eval.core.models import AgenticAssertionError, ChatResult, ToolCallEvent + +_MODULE = "gooddata_eval.core.agentic.knowledge" +_Q = "Who must approve a 30 % discount?" +_FOLLOW_UP = "And who approves 45 %?" +_EXPECTED = {"expected_documents": ["Discount Policy"], "must_include": ["Names the regional sales manager."]} + + +def _turn(text: str, *, search: bool = True, title: str = "Discount Policy") -> ChatResult: + rows = [{"document_id": "d1", "filename": "", "title": title, "content": "...", "source": "the parent workspace"}] + tools = ( + [ToolCallEvent(function_name="search_knowledge", function_arguments="{}", result=json.dumps({"results": rows}))] + if search + else [] + ) + return ChatResult(text_response=text, tool_call_events=tools, response_id=f"resp-{text[:5]}") + + +def _client(*turn_results: Any) -> MagicMock: + client = MagicMock() + client.create_conversation.side_effect = [f"conv-{i}" for i in range(1, 10)] + client.send_message.side_effect = list(turn_results) + return client + + +def _run(client: MagicMock, *, judge: bool = True, **kwargs: Any) -> Any: + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch("openai.OpenAI"), + patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}), + patch("gooddata_eval.core.evaluators._llm_judge.LLMJudge.score", return_value=(judge, "ok")) as score, + ): + params: dict[str, Any] = { + "host": "https://h", + "token": "tok", + "workspace_id": "ws1", + "question": _Q, + "expected_output": _EXPECTED, + **kwargs, + } + return run_agentic_knowledge(**params), score + + +# ── runs ──────────────────────────────────────────────────────────────────── + + +def test_turns_are_sent_in_order_on_one_conversation() -> None: + client = _client(_turn("Regional sales manager."), _turn("Not allowed above 40 %.")) + summary, score = _run(client, turns=[_Q, _FOLLOW_UP]) + + assert [c.args for c in client.send_message.call_args_list] == [("conv-1", _Q), ("conv-1", _FOLLOW_UP)] + assert summary.pass_at_k is True + assert summary.best.evaluation.detail["turn_count"] == 2 + # The judge reads the earlier answer and grades the last one. + assert "Regional sales manager." in score.call_args.kwargs["input"] + assert score.call_args.kwargs["actual_output"] == "Not allowed above 40 %." + client.delete_conversation.assert_called_once_with("conv-1") + + +def test_each_k_run_gets_its_own_conversation() -> None: + client = _client(_turn("A"), _turn("B")) + summary, _ = _run(client, k=2) + assert [r.conversation_id for r in summary.run_results] == ["conv-1", "conv-2"] + assert [c.args[0] for c in client.delete_conversation.call_args_list] == ["conv-1", "conv-2"] + client.close.assert_called_once() + + +def test_a_chat_error_in_run_two_keeps_run_one() -> None: + client = _client(_turn("Regional sales manager."), ChatError("gen-ai fell over")) + summary, _ = _run(client, k=2) + + assert len(summary.run_results) == 2 + assert summary.run_results[0].chat_error is None + assert "gen-ai fell over" in (summary.run_results[1].chat_error or "") + assert summary.pass_at_k is True + assert summary.pass_power_k is False + assert summary.best.conversation_id == "conv-1" + client.delete_conversation.assert_any_call("conv-2") + + +def test_a_chat_error_on_a_later_turn_names_the_turn() -> None: + client = _client(_turn("Regional sales manager."), ChatError("boom")) + summary, score = _run(client, turns=[_Q, _FOLLOW_UP]) + run = summary.run_results[0] + assert "turn 2/2" in (run.chat_error or "") + assert run.passed is False + score.assert_not_called() + + +def test_a_failed_conversation_creation_does_not_discard_completed_runs() -> None: + client = _client(_turn("A"), _turn("B")) + client.create_conversation.side_effect = ["conv-1", RuntimeError("no conversation"), "conv-3"] + summary, _ = _run(client, k=3) + assert [r.chat_error is None for r in summary.run_results] == [True, False, True] + assert "conversation creation failed" in (summary.run_results[1].chat_error or "") + assert summary.pass_at_k is True + assert summary.pass_power_k is False + + +def test_an_initial_conversation_is_used_for_run_zero_and_kept() -> None: + client = _client(_turn("A")) + _run(client, initial_conversation_id="conv-given") + assert client.send_message.call_args.args[0] == "conv-given" + client.create_conversation.assert_not_called() + client.delete_conversation.assert_not_called() + + +def test_an_invalid_item_raises_before_any_request() -> None: + client = _client() + with patch(f"{_MODULE}.ChatClient", return_value=client) as client_cls, pytest.raises(ValueError, match="unknown"): + run_agentic_knowledge("https://h", "tok", "ws1", _Q, {"expected_docs": ["x"]}) + client_cls.assert_not_called() + + +def test_tool_events_of_every_turn_are_kept_for_the_latency_breakdown() -> None: + client = _client(_turn("A"), _turn("B")) + summary, _ = _run(client, turns=[_Q, _FOLLOW_UP]) + assert [e.function_name for e in summary.best.tool_call_events] == ["search_knowledge", "search_knowledge"] + + +# ── evaluate_* wrapper ────────────────────────────────────────────────────── + + +def _evaluate(client: MagicMock, *, judge: Any = True, **kwargs: Any) -> Any: + """`judge`: the verdict for every criterion, or a list of verdicts / exceptions in call order.""" + verdicts = judge if isinstance(judge, list) else None + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch("openai.OpenAI"), + patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}), + patch( + "gooddata_eval.core.evaluators._llm_judge.LLMJudge.score", + return_value=(judge, "ok"), + side_effect=verdicts, + ), + ): + params: dict[str, Any] = { + "host": "https://h", + "token": "tok", + "workspace_id": "ws1", + "question": _Q, + "expected_output": _EXPECTED, + "langfuse": None, + **kwargs, + } + with patch(f"{_MODULE}.open_item_trace", return_value=(params["langfuse"], MagicMock(), None)): + return evaluate_agentic_knowledge(**params) + + +def test_a_passing_item_returns_the_detail_with_the_latency_breakdown() -> None: + outcome = _evaluate(_client(_turn("Regional sales manager."))) + assert outcome.detail["expected_documents_found"] is True + assert outcome.detail["documents_retrieved"] == ["Discount Policy"] + assert "latency_breakdown" in outcome.detail + assert outcome.runs_passed == 1 + assert outcome.run_latencies_s + + +def test_a_retrieval_failure_is_named_in_the_error() -> None: + client = _client(_turn("Regional sales manager.", title="Customer Glossary")) + with pytest.raises(KnowledgeAssertionError) as exc_info: + _evaluate(client) + error = exc_info.value + assert isinstance(error, AgenticAssertionError) + assert "expected_documents_found=False" in str(error) + assert "include_0" not in str(error) + assert error.detail["include_0"] is True + assert error.runs_passed == 0 + assert error.runs_effective == 1 + assert error.conversation_id == "conv-1" + assert error.failed_runs + + +def test_a_reasoning_failure_is_named_in_the_error() -> None: + with pytest.raises(KnowledgeAssertionError) as exc_info: + _evaluate(_client(_turn("The CFO approves it.")), judge=False) + message = str(exc_info.value) + assert "include_0=False" in message + assert "expected_documents_found" not in message + + +def test_power_gate_fails_on_one_chat_error() -> None: + client = _client(_turn("A"), ChatError("boom")) + with pytest.raises(KnowledgeAssertionError) as exc_info: + _evaluate(client, k=2, gate="power") + assert exc_info.value.runs_passed == 1 + assert exc_info.value.runs_effective == 2 + + +_NO_VERDICT = JudgeResponseError("empty body twice") + + +def test_a_judge_without_verdict_is_an_error_not_a_fail() -> None: + """Code checks held and the judge returned nothing readable: the harness failed, not the agent.""" + with pytest.raises(JudgeResponseError) as exc_info: + _evaluate(_client(_turn("Regional sales manager.")), judge=[_NO_VERDICT]) + error = exc_info.value + assert error.harness_error["reason"] == "judge_failed" + assert error.runs_passed == 0 + assert error.detail["unscored_runs"] == 1 + assert error.failed_runs[0]["outcome"]["status"] == "error" + + +def test_a_failed_code_check_stays_a_fail_when_the_judge_has_no_verdict() -> None: + client = _client(_turn("Regional sales manager.", title="Customer Glossary")) + with pytest.raises(KnowledgeAssertionError, match="expected_documents_found=False"): + _evaluate(client, judge=[_NO_VERDICT]) + + +def test_a_judge_failure_on_one_run_leaves_the_graded_run_deciding() -> None: + client = _client(_turn("A"), _turn("B")) + outcome = _evaluate(client, k=2, judge=[(True, "ok"), _NO_VERDICT]) + assert outcome.runs_passed == 1 + assert outcome.detail["unscored_runs"] == 1 + with pytest.raises(KnowledgeAssertionError): + _evaluate(_client(_turn("A"), _turn("B")), k=2, gate="power", judge=[(True, "ok"), _NO_VERDICT]) + + +def _scores_written(summary_turns: list[Any], **kwargs: Any) -> MagicMock: + """Run with a fake Langfuse and replay the deferred scoring against a mock context.""" + captured: dict[str, Any] = {} + + def _submit(_link: Any, _identity: Any, **kw: Any) -> None: + captured.update(kw) + + with patch(f"{_MODULE}.submit_trace_scoring", side_effect=_submit): + try: + _evaluate(_client(*summary_turns), langfuse=MagicMock(), dataset_item_id="item-1", **kwargs) + except (KnowledgeAssertionError, JudgeResponseError): + pass + ctx = MagicMock() + ctx.run_metadata = {} + ctx.trace.return_value = None + captured["write_scores"](ctx) + return ctx + + +def test_langfuse_scores_split_retrieval_from_reasoning() -> None: + ctx = _scores_written([_turn("Regional sales manager.", title="Customer Glossary")]) + scores = {c.kwargs["name"]: c.kwargs["value"] for c in ctx.score.call_args_list} + assert scores["knowledge_pass"] == 0.0 + assert scores["expected_documents_found"] == 0.0 + assert scores["knowledge_searched"] == 1.0 + assert scores["answered"] == 1.0 + assert "include_0" not in scores + # Quality reads the same bools as rank_key: code checks and judge verdicts, no knowledge_pass. + strict = ctx.quality.call_args.kwargs["strict_checks"] + assert strict == { + "knowledge_searched": True, + "expected_documents_found": False, + "answered": True, + "include_0": True, + } + + +def test_langfuse_marks_a_run_without_judge_verdict_as_an_error() -> None: + ctx = _scores_written([_turn("Regional sales manager.")], judge=[_NO_VERDICT]) + names = [c.kwargs["name"] for c in ctx.score.call_args_list] + assert names == ["eval_error"] + assert ctx.score.call_args.kwargs["comment"] == "judge_failed" + ctx.quality.assert_not_called() + + +def test_langfuse_skips_a_run_with_a_chat_error() -> None: + ctx = _scores_written([_turn("A"), ChatError("boom")], k=2) + assert ctx.observe.call_count == 1 + assert ctx.observe.call_args.args[1] == 0 + + +@pytest.mark.parametrize( + ("detail", "error", "reason"), + [ + ({"turn_count": 1}, None, "criteria not satisfied"), + ({"answered": False}, "chat failed: timeout", "answered=False; chat failed: timeout"), + ], +) +def test_a_failed_run_names_its_failed_checks_and_error(detail: dict, error: str | None, reason: str) -> None: + evaluation = ItemEvaluation(passed=False, rank_key=(0, 0.0), detail=detail, error=error) + assert _failure_reason(KnowledgeRunResult(conversation_id="conv-1", evaluation=evaluation)) == reason diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 26abd44e2..0a5fb3d5c 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -7,7 +7,7 @@ from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime from typing import Any -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest from gooddata_eval.cli import agentic_runner @@ -22,7 +22,13 @@ from gooddata_eval.core.agentic.alert_skill import AlertSkillAssertionError from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.langfuse.item_scope import ItemTraceScope, current_scope -from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome, CreatedVisualization, DatasetItem +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + CreatedVisualization, + DatasetItem, + KnowledgeInput, +) from gooddata_eval.core.reporting.json_report import build_json_report from gooddata_eval.core.timing import PhaseTimings, note_trace_latency @@ -107,6 +113,7 @@ def test_dispatch_agentic_omits_agent_id_by_default(): ), ("agentic_anomaly_detection", {"metric": "metric/spend"}, "evaluate_agentic_anomaly_detection"), ("agentic_what_if", {"metric_id": "spend"}, "evaluate_agentic_what_if"), + ("agentic_knowledge", {"expected_documents": ["Discount Policy"]}, "evaluate_agentic_knowledge"), ] # The one kind whose dispatch needs more than question/expected_output: the dashboard to @@ -894,6 +901,7 @@ def test_an_errored_item_without_timings_keeps_its_zero_defaults(): ("agentic_document_skill", "evaluate_agentic_document_skill", {}), ("agentic_forecasting", "evaluate_agentic_forecasting", {}), ("agentic_dashboard_summary", "evaluate_agentic_dashboard_summary", {}), + ("agentic_knowledge", "evaluate_agentic_knowledge", {}), ("agentic_conversation", "evaluate_agentic_conversation", {"id": "c1", "expected_skills": [], "turns": []}), ] @@ -972,6 +980,7 @@ class _Stop(Exception): ("what_if", "run_agentic_what_if", "evaluate_agentic_what_if", {}), ("anomaly_detection", "run_agentic_anomaly_detection", "evaluate_agentic_anomaly_detection", {}), ("document_skill", "run_agentic_document_skill", "evaluate_agentic_document_skill", {}), + ("knowledge", "run_agentic_knowledge", "evaluate_agentic_knowledge", {}), ] @@ -1410,3 +1419,45 @@ def evaluate(*args: Any, **kwargs: Any) -> AgenticEvalOutcome: assert {i.id: i.trace_latencies_s for i in report.items} == { f"g{n}": [n * 100.0, n * 100.0 + 1, n * 100.0 + 2] for n in range(1, 5) } + + +def _dispatch_knowledge(workspace: str | None) -> MagicMock: + item = DatasetItem( + id="kn-1", + dataset_name="ds", + test_kind="agentic_knowledge", + question="q", + expected_output={}, + knowledge_input=KnowledgeInput(turns=["q", "q2"], workspace=workspace), + ) + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_knowledge") as mock_eval: + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + ) + return mock_eval + + +@pytest.mark.parametrize("workspace", [None, "parent", "Parent"]) +def test_dispatch_runs_a_knowledge_item_in_the_cli_workspace(workspace: str | None) -> None: + mock_eval = _dispatch_knowledge(workspace) + assert mock_eval.call_args.kwargs["workspace_id"] == "ws1" + assert mock_eval.call_args.kwargs["turns"] == ["q", "q2"] + + +@pytest.mark.parametrize("workspace", ["child", "sibling"]) +def test_dispatch_refuses_a_knowledge_item_whose_workspace_role_it_cannot_map(workspace: str) -> None: + """Running a child-workspace item in --workspace would grade the wrong knowledge sources.""" + with pytest.raises(ValueError, match="maps it to a workspace id"): + _dispatch_knowledge(workspace) + + +def test_agentic_knowledge_runs_serially_in_the_cli() -> None: + # Fixed turns are not proof of read-only behavior: an item could confirm an object creation. + assert not runs_in_parallel("agentic_knowledge") diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py index 4265612ff..443692251 100644 --- a/packages/gooddata-eval/tests/test_cli.py +++ b/packages/gooddata-eval/tests/test_cli.py @@ -1216,3 +1216,46 @@ def test_routing_backend_closes_every_client() -> None: for client in (chat, summary, takeaway): client.close.assert_called_once_with() + + +def test_cli_runs_knowledge_items_on_the_agentic_path(monkeypatch, tmp_path) -> None: + monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) + + class _FakeController: + def __init__(self, *a, **k): ... + def get_active(self): + return ActiveLlmProvider(provider_id="prov", default_model_id="gpt-5.2") + + def resolve_and_activate(self, requested, provider=None): + return ResolvedModel(provider_id="prov", model_id="gpt-5.2", switched=False, provider_name="P") + + def restore(self, original): ... + def close(self): ... + + raw = { + "id": "lf-kn-1", + "datasetName": "agent_knowledge", + "input": "Who must approve a 30 % discount?", + "metadata": {"knowledge_input": {"turns": ["Who must approve a 30 % discount?", "And 45 %?"]}}, + "expectedOutput": {"expected_documents": ["Discount Policy"]}, + } + item = _item_from_raw(raw, dataset_name="agent_knowledge", test_kind="visualization") + monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController) + monkeypatch.setattr(cli_main, "ChatClient", lambda **k: object()) + monkeypatch.setattr(cli_main, "load_local_dataset", lambda folder: [item]) + single_shot: list = [] + agentic: list = [] + monkeypatch.setattr( + cli_main, "run_items", lambda items, backend, **kw: single_shot.extend(items) or EvalReport(model="m") + ) + monkeypatch.setattr( + cli_main, "run_agentic_items", lambda items, **kw: agentic.extend(items) or EvalReport(model="m") + ) + + exit_code = cli_main.main( + ["run", "--host", "https://h", "--token", "tok", "--workspace", "ws1", "--dataset", str(tmp_path)] + ) + + assert exit_code == 0 + assert agentic == [item] + assert single_shot == [] diff --git a/packages/gooddata-eval/tests/test_knowledge_evaluator.py b/packages/gooddata-eval/tests/test_knowledge_evaluator.py new file mode 100644 index 000000000..89686dc39 --- /dev/null +++ b/packages/gooddata-eval/tests/test_knowledge_evaluator.py @@ -0,0 +1,386 @@ +# (C) 2026 GoodData Corporation +import json +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.evaluators.knowledge import ( + AgenticKnowledgeEvaluator, + judge_input, + parse_sources_cited, +) +from gooddata_eval.core.models import ChatResult, DatasetItem, KnowledgeInput, ToolCallEvent + +_Q = "Who must approve a 30 % discount?" + + +def _make_evaluator() -> AgenticKnowledgeEvaluator: + with patch("openai.OpenAI"), patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}): + return AgenticKnowledgeEvaluator() + + +def _judges(ev: AgenticKnowledgeEvaluator, positive: bool = True, violation: bool = False) -> MagicMock: + """Script both judges; returns the positive judge's mock.""" + positive_score = MagicMock(return_value=(positive, "ok")) + ev._positive_judge.score = positive_score + ev._violation_judge.score = MagicMock(return_value=(violation, "ok")) + return positive_score + + +def _item(expected: Any = None, turns: list[str] | None = None, question: str = _Q) -> DatasetItem: + return DatasetItem( + id="k1", + dataset_name="d", + test_kind="agentic_knowledge", + question=question, + expected_output=expected if expected is not None else {}, + knowledge_input=KnowledgeInput(turns=turns) if turns else None, + ) + + +def _row(title: str, source: str = "this workspace", filename: str = "") -> dict: + return {"document_id": title.lower(), "filename": filename, "title": title, "content": "...", "source": source} + + +def _tool(name: str, rows: list[dict]) -> ToolCallEvent: + """A `search_knowledge` call returning `rows`.""" + return ToolCallEvent(function_name=name, function_arguments="{}", result=json.dumps({"results": rows})) + + +def _chunks(title: str | None, source: str = "this workspace", filename: str = "doc.pdf") -> ToolCallEvent: + """A `get_knowledge_chunks` call in gen-ai's real shape: document fields top-level, chunks untitled.""" + result = { + "filename": filename, + "title": title, + "source": source, + "chunks": [{"chunk_index": 2, "total_chunks": 5, "content": "...", "page_numbers": [3], "scopes": []}], + } + return ToolCallEvent( + function_name="get_knowledge_chunks", function_arguments='{"document_id": "d1"}', result=json.dumps(result) + ) + + +def _turn(text: str, *tools: ToolCallEvent) -> ChatResult: + return ChatResult(text_response=text, tool_call_events=list(tools)) + + +_SEARCH_DISCOUNT = _tool("search_knowledge", [_row("Discount Policy", "the parent workspace")]) + + +# ── retrieval checks ──────────────────────────────────────────────────────── + + +def test_expected_document_found_passes_and_records_diagnostics() -> None: + ev = _make_evaluator() + _judges(ev) + res = ev.evaluate( + _item({"expected_documents": ["Discount Policy"], "must_include": ["Names the regional sales manager."]}), + [_turn("The regional sales manager approves it.", _SEARCH_DISCOUNT)], + ) + assert res.passed is True + assert res.detail["knowledge_searched"] is True + assert res.detail["expected_documents_found"] is True + assert res.detail["documents_retrieved"] == ["Discount Policy"] + assert res.detail["turn_count"] == 1 + assert "forbidden_documents_absent" not in res.detail + assert "expected_source_found" not in res.detail + + +def test_title_match_is_case_insensitive_and_trimmed() -> None: + ev = _make_evaluator() + _judges(ev) + res = ev.evaluate(_item({"expected_documents": [" discount POLICY "]}), [_turn("Answer.", _SEARCH_DISCOUNT)]) + assert res.detail["expected_documents_found"] is True + + +def test_a_row_without_title_matches_on_its_filename() -> None: + ev = _make_evaluator() + _judges(ev) + tool = _tool("search_knowledge", [_row("", filename="discount_policy.md")]) + res = ev.evaluate(_item({"expected_documents": ["discount_policy.md"]}), [_turn("Answer.", tool)]) + assert res.detail["expected_documents_found"] is True + assert res.detail["documents_retrieved"] == ["discount_policy.md"] + + +def test_a_missing_document_fails_without_the_judge_deciding_it() -> None: + """Retrieval failure: the answer may read right, the item still fails on retrieval.""" + ev = _make_evaluator() + _judges(ev, positive=True) + res = ev.evaluate( + _item({"expected_documents": ["Customer Glossary"], "must_include": ["x"]}), + [_turn("Answer.", _SEARCH_DISCOUNT)], + ) + assert res.passed is False + assert res.detail["expected_documents_found"] is False + assert res.detail["include_0"] is True + + +def test_a_document_found_only_in_get_knowledge_chunks_counts() -> None: + ev = _make_evaluator() + _judges(ev) + chunks = _chunks("Customer Glossary") + res = ev.evaluate( + _item({"expected_documents": ["Discount Policy", "Customer Glossary"]}), + [_turn("Answer.", _SEARCH_DISCOUNT, chunks)], + ) + assert res.detail["expected_documents_found"] is True + + +def test_a_forbidden_document_found_only_in_get_knowledge_chunks_fails() -> None: + ev = _make_evaluator() + _judges(ev) + chunks = _chunks("Old Discount Policy 2023") + res = ev.evaluate( + _item({"forbidden_documents": ["old discount policy 2023"]}), + [_turn("Answer.", _SEARCH_DISCOUNT, chunks)], + ) + assert res.passed is False + assert res.detail["forbidden_documents_absent"] is False + assert res.detail["documents_retrieved"] == ["Discount Policy", "Old Discount Policy 2023"] + + +def test_a_chunks_result_without_title_matches_on_its_filename() -> None: + ev = _make_evaluator() + _judges(ev) + res = ev.evaluate( + _item({"forbidden_documents": ["old_policy.pdf"]}), + [_turn("Answer.", _SEARCH_DISCOUNT, _chunks(None, filename="old_policy.pdf"))], + ) + assert res.detail["forbidden_documents_absent"] is False + + +def test_a_chunks_result_for_a_document_not_found_adds_no_document() -> None: + """gen-ai answers an unknown or out-of-scope id with an empty filename, no title, source "unknown".""" + ev = _make_evaluator() + _judges(ev) + res = ev.evaluate(_item({}), [_turn("Answer.", _SEARCH_DISCOUNT, _chunks(None, source="unknown", filename=""))]) + assert res.detail["documents_retrieved"] == ["Discount Policy"] + + +def test_expected_source_found_only_in_a_chunks_result() -> None: + ev = _make_evaluator() + _judges(ev) + search = _tool("search_knowledge", [_row("Customer Glossary", "this workspace")]) + res = ev.evaluate( + _item({"expected_documents": ["Discount Policy"], "expected_source": "an ancestor workspace"}), + [_turn("Answer.", search, _chunks("Discount Policy", source="an ancestor workspace"))], + ) + assert res.detail["expected_documents_found"] is True + assert res.detail["expected_source_found"] is True + + +def test_documents_from_an_earlier_turn_count() -> None: + ev = _make_evaluator() + _judges(ev) + res = ev.evaluate( + _item({"expected_documents": ["Discount Policy"]}, turns=[_Q, "And for 45 %?"]), + [_turn("Regional manager.", _SEARCH_DISCOUNT), _turn("Not allowed.")], + ) + assert res.detail["knowledge_searched"] is True + assert res.detail["expected_documents_found"] is True + assert res.detail["turn_count"] == 2 + + +def test_no_search_fails_when_a_search_is_expected() -> None: + ev = _make_evaluator() + _judges(ev) + res = ev.evaluate(_item({}), [_turn("Answer.", _chunks("Discount Policy"))]) + assert res.detail["knowledge_searched"] is False + assert res.passed is False + + +def test_expect_search_false_drops_the_check() -> None: + ev = _make_evaluator() + _judges(ev) + res = ev.evaluate(_item({"expect_search": False}), [_turn("Answer.")]) + assert "knowledge_searched" not in res.detail + assert res.passed is True + + +@pytest.mark.parametrize( + ("source", "expected"), + [("the parent workspace", True), (" The Parent Workspace ", True), ("this workspace", False)], +) +def test_expected_source_must_label_an_expected_document(source: str, expected: bool) -> None: + ev = _make_evaluator() + _judges(ev) + res = ev.evaluate( + _item({"expected_documents": ["Discount Policy"], "expected_source": source}), + [_turn("Answer.", _SEARCH_DISCOUNT)], + ) + assert res.detail["expected_source_found"] is expected + + +def test_expected_source_on_another_document_does_not_count() -> None: + ev = _make_evaluator() + _judges(ev) + tool = _tool( + "search_knowledge", [_row("Discount Policy", "this workspace"), _row("Glossary", "the parent workspace")] + ) + res = ev.evaluate( + _item({"expected_documents": ["Discount Policy"], "expected_source": "the parent workspace"}), + [_turn("Answer.", tool)], + ) + assert res.detail["expected_source_found"] is False + + +def test_an_unreadable_tool_result_is_ignored() -> None: + ev = _make_evaluator() + _judges(ev) + bad = ToolCallEvent(function_name="search_knowledge", function_arguments="{}", result="not json") + res = ev.evaluate(_item({"expected_documents": ["Discount Policy"]}), [_turn("Answer.", bad)]) + assert res.detail["expected_documents_found"] is False + assert res.detail["documents_retrieved"] == [] + + +def test_an_empty_last_answer_fails_without_a_judge_call() -> None: + ev = _make_evaluator() + positive = _judges(ev) + res = ev.evaluate( + _item({"must_include": ["x"]}, turns=[_Q, "Why?"]), [_turn("Answer.", _SEARCH_DISCOUNT), _turn("")] + ) + assert res.passed is False + assert res.detail["answered"] is False + positive.assert_not_called() + + +# ── Sources section ───────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("answer", "titles"), + [ + ( + "A 30 % discount needs the regional manager.\n\n---\n**Sources**\n" + "- *Discount Policy* (discount_policy.pdf), pp. 1-2 — from *the parent workspace*\n" + "- *Customer Glossary* (`glossary.md`) — from *this workspace*", + ["Discount Policy", "Customer Glossary"], + ), + ("Text.\n\n### Sources\n\n- **Discount Policy** (parent)\n", ["Discount Policy"]), + ("Text.\n\nSources:\n- Discount Policy (parent)\nThanks.", ["Discount Policy"]), + ("No sources here.", []), + ("**Sources**\n- *Old*\n\nMore text.\n\n**Sources**\n- *New* (x)", ["New"]), + ], +) +def test_parse_sources_cited(answer: str, titles: list[str]) -> None: + assert parse_sources_cited(answer) == titles + + +def test_sources_cited_are_diagnostics_and_never_gate() -> None: + """A list, not bools: quality_score counts bools, and a cited source decides nothing.""" + ev = _make_evaluator() + _judges(ev) + answer = "Regional manager.\n\n---\n**Sources**\n- *Some Other Doc* (other.pdf), p. 4 — from *this workspace*" + res = ev.evaluate(_item({"expected_documents": ["Discount Policy"]}), [_turn(answer, _SEARCH_DISCOUNT)]) + assert res.passed is True + assert res.detail["sources_cited"] == ["Some Other Doc"] + assert not any(isinstance(res.detail[key], bool) for key in ("sources_cited", "documents_retrieved", "turn_count")) + + +# ── judge input ───────────────────────────────────────────────────────────── + + +def test_judge_reads_the_conversation_and_grades_the_last_answer() -> None: + ev = _make_evaluator() + positive = _judges(ev) + turns = [_Q, "And who approves 45 %?"] + ev.evaluate( + _item({"must_include": ["Says 45 % is not allowed."]}, turns=turns), + [_turn("The regional sales manager.", _SEARCH_DISCOUNT), _turn("Discounts above 40 % are not allowed.")], + ) + kwargs = positive.call_args.kwargs + assert _Q in kwargs["input"] + assert "The regional sales manager." in kwargs["input"] + assert kwargs["input"].index("The regional sales manager.") < kwargs["input"].index("And who approves 45 %?") + assert kwargs["actual_output"] == "Discounts above 40 % are not allowed." + + +def test_single_turn_judge_input_is_the_question() -> None: + assert judge_input([_Q], ["answer"]) == _Q + + +def test_earlier_turns_and_answers_are_trimmed_for_the_judge() -> None: + text = judge_input(["q" * 5000, "THE-LAST-QUESTION"], ["x" * 5000, "THE-LAST-ANSWER"]) + assert "x" * 1500 in text + assert "x" * 1501 not in text + assert "q" * 1500 in text + assert "q" * 1501 not in text + assert "THE-LAST-QUESTION" in text + # The last answer is the ACTUAL OUTPUT, not part of the INPUT. + assert "THE-LAST-ANSWER" not in text + + +def test_the_last_answer_keeps_its_sources_section() -> None: + ev = _make_evaluator() + positive = _judges(ev) + answer = "Regional manager.\n\n---\n**Sources**\n- *Discount Policy* (`policy.pdf`) — from *the parent workspace*" + ev.evaluate(_item({"must_include": ["x"]}), [_turn(answer, _SEARCH_DISCOUNT)]) + assert positive.call_args.kwargs["actual_output"] == answer + + +def test_a_violation_is_inverted() -> None: + ev = _make_evaluator() + _judges(ev, violation=True) + res = ev.evaluate(_item({"must_not_include": ["Says the CFO must approve."]}), [_turn("CFO.", _SEARCH_DISCOUNT)]) + assert res.detail["exclude_0"] is False + assert res.passed is False + + +# ── validation ────────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("expected", "message"), + [ + ({"expected_document": ["x"]}, "unknown expected_output key"), + ({"expect_search": "yes"}, "expect_search must be true or false"), + ({"expected_documents": "Discount Policy"}, "expected_documents must be a JSON array"), + ({"forbidden_documents": [""]}, "forbidden_documents must be a JSON array of non-empty strings"), + ({"must_include": [1]}, "must_include must be a JSON array"), + ({"expected_documents": ["x"], "expected_source": ""}, "expected_source must be a non-empty string"), + ({"expected_source": "the parent workspace"}, "needs expected_documents"), + ({"expected_documents": ["x"], "expected_source": "parent"}, "expected_source must be one of"), + ({"expected_documents": ["x"], "expected_source": "unknown"}, "expected_source must be one of"), + ("Names the manager.", "expected_output must be a JSON object"), + ], +) +def test_validate_rejects_a_bad_expected_output(expected: Any, message: str) -> None: + with pytest.raises(ValueError, match=message): + AgenticKnowledgeEvaluator.validate(_item(expected)) + + +def test_validate_rejects_turns_that_do_not_start_with_the_question() -> None: + with pytest.raises(ValueError, match=r"turns\[0\] must equal the question"): + AgenticKnowledgeEvaluator.validate(_item({}, turns=["Something else", "Then?"])) + + +def test_validate_rejects_a_blank_question() -> None: + with pytest.raises(ValueError, match="question must be a non-empty string"): + AgenticKnowledgeEvaluator.validate(_item({}, question=" ")) + + +@pytest.mark.parametrize("label", ["this workspace", "The Parent Workspace", "an ancestor workspace", "ai hub"]) +def test_validate_accepts_every_gen_ai_source_label(label: str) -> None: + AgenticKnowledgeEvaluator.validate(_item({"expected_documents": ["x"], "expected_source": label})) + + +def test_validate_accepts_a_full_expected_output() -> None: + AgenticKnowledgeEvaluator.validate( + _item( + { + "expect_search": True, + "expected_documents": ["Discount Policy"], + "forbidden_documents": ["Old Policy"], + "expected_source": "the parent workspace", + "must_include": ["a"], + "must_not_include": ["b"], + "rubric": ["c"], + }, + turns=[_Q, "And 45 %?"], + ) + ) + + +def test_validate_accepts_the_test_kind_routing_key() -> None: + # langfuse_source routes on expectedOutput.test_kind and keeps it in expected_output. + AgenticKnowledgeEvaluator.validate(_item({"test_kind": "agentic_knowledge", "expected_documents": ["x"]})) diff --git a/packages/gooddata-eval/tests/test_langfuse_source.py b/packages/gooddata-eval/tests/test_langfuse_source.py index 2a4ded41f..ff6839383 100644 --- a/packages/gooddata-eval/tests/test_langfuse_source.py +++ b/packages/gooddata-eval/tests/test_langfuse_source.py @@ -1,4 +1,5 @@ # (C) 2026 GoodData Corporation +from typing import Any from unittest.mock import MagicMock, patch import pytest @@ -375,3 +376,64 @@ def test_langfuse_item_explicit_test_kind_beats_catalog_input() -> None: "expectedOutput": {}, } assert _item_from_raw(raw, dataset_name="ds", test_kind="visualization").test_kind == "general_question" + + +def _knowledge_raw(**overrides: Any) -> dict: + return { + "id": "lf-kn-1", + "datasetName": "agent_knowledge", + "input": "Who must approve a 30 % discount?", + "metadata": { + "knowledge_input": { + "turns": ["Who must approve a 30 % discount?", "And 45 %?"], + "workspace": "parent", + } + }, + "expectedOutput": {"expected_documents": ["Discount Policy"]}, + **overrides, + } + + +def test_langfuse_item_knowledge_input_from_metadata_sets_the_kind() -> None: + item = _item_from_raw(_knowledge_raw(), dataset_name="ds", test_kind="visualization") + assert item.test_kind == "agentic_knowledge" + assert item.knowledge_input is not None + assert item.knowledge_input.turns == ["Who must approve a 30 % discount?", "And 45 %?"] + assert item.knowledge_input.workspace == "parent" + + +def test_langfuse_item_knowledge_input_that_is_not_an_object_raises() -> None: + with pytest.raises(ValueError, match="knowledge_input must be a JSON object"): + _item_from_raw(_knowledge_raw(metadata={"knowledge_input": ["q"]}), dataset_name="ds", test_kind="x") + + +@pytest.mark.parametrize( + "knowledge_input", + [{"turns": []}, {"turns": ["q", " "]}, {"turns": "q"}, {"workspace": ""}, {"turn": ["q"]}], +) +def test_langfuse_item_invalid_knowledge_input_raises(knowledge_input: dict) -> None: + raw = _knowledge_raw(metadata={"knowledge_input": knowledge_input}) + with pytest.raises(ValueError, match="invalid knowledge_input"): + _item_from_raw(raw, dataset_name="ds", test_kind="visualization") + + +def test_langfuse_item_explicit_test_kind_beats_knowledge_input() -> None: + raw = _knowledge_raw(metadata={"test_kind": "agentic_general_question", "knowledge_input": {"turns": ["q"]}}) + assert _item_from_raw(raw, dataset_name="ds", test_kind="visualization").test_kind == "agentic_general_question" + + +def test_langfuse_item_catalog_input_beats_knowledge_input() -> None: + raw = _knowledge_raw( + metadata={ + "knowledge_input": {"turns": ["q"]}, + "catalog_input": {"object_type": "Metric", "object_id": "arpu", "output": "title"}, + } + ) + assert _item_from_raw(raw, dataset_name="ds", test_kind="visualization").test_kind == "catalog_metadata" + + +def test_langfuse_item_without_knowledge_input_keeps_the_default_kind() -> None: + raw = _knowledge_raw(metadata={}) + item = _item_from_raw(raw, dataset_name="ds", test_kind="agentic_general_question") + assert item.knowledge_input is None + assert item.test_kind == "agentic_general_question" diff --git a/packages/gooddata-eval/tests/test_models.py b/packages/gooddata-eval/tests/test_models.py index 2a79507fa..7a03bb12b 100644 --- a/packages/gooddata-eval/tests/test_models.py +++ b/packages/gooddata-eval/tests/test_models.py @@ -5,6 +5,7 @@ ChatResult, CreatedVisualization, DatasetItem, + KnowledgeInput, ReasoningStepEvent, TakeawayInput, TakeawayVisualization, @@ -229,3 +230,18 @@ def test_catalog_input_rejects_unknown_keys_and_types() -> None: CatalogInput.model_validate({"object_type": "Metric", "object_id": "a", "output": "title", "x": 1}) with pytest.raises(ValueError): CatalogInput.model_validate({"object_type": "Parameter", "object_id": "a", "output": "title"}) + + +def test_knowledge_input_defaults_and_rejects_bad_values() -> None: + assert KnowledgeInput.model_validate({}).turns is None + assert KnowledgeInput.model_validate({"workspace": "child"}).workspace == "child" + for bad in ({"turns": []}, {"turns": [" "]}, {"turns": [1]}, {"workspace": " "}, {"x": 1}): + with pytest.raises(ValueError): + KnowledgeInput.model_validate(bad) + + +def test_dataset_item_knowledge_input_defaults_to_none() -> None: + item = DatasetItem.model_validate( + {"id": "q1", "dataset_name": "d1", "test_kind": "agentic_knowledge", "question": "q", "expected_output": {}} + ) + assert item.knowledge_input is None diff --git a/packages/gooddata-eval/tests/test_trace_linker.py b/packages/gooddata-eval/tests/test_trace_linker.py index cf60029f9..647f1ffe0 100644 --- a/packages/gooddata-eval/tests/test_trace_linker.py +++ b/packages/gooddata-eval/tests/test_trace_linker.py @@ -122,6 +122,7 @@ def test_run_trace_link_inline_runs_the_task_on_the_calling_thread(): ("conversation", "evaluate_agentic_conversation"), ("forecasting", "evaluate_agentic_forecasting"), ("dashboard_summary", "evaluate_agentic_dashboard_summary"), + ("knowledge", "evaluate_agentic_knowledge"), ("anomaly_detection", "evaluate_agentic_anomaly_detection"), ("what_if", "evaluate_agentic_what_if"), ]