From 5d70179e10a688ab70d11339f7785395154d4530 Mon Sep 17 00:00:00 2001 From: duongthimy Date: Fri, 20 Nov 2026 16:59:28 +0100 Subject: [PATCH 1/5] feat(gooddata-eval): add agentic_summary and agentic_describe_object agentic_summary evaluates the chat summary of a widget or a dashboard the way the UI asks for it. An item stores the UI's userContext without result ids and one execution (AFM + result spec, filters included) per widget; every run re-executes them, because result ids expire, and puts the fresh ids back. Follow-up questions run in the same conversation. Code checks: shape by scope (chart, KPI, dashboard, no data, or the user's own format), no ids, no figures on no data, no tool calls on a follow-up that should answer from the conversation. Judge criteria are graded one by one with summary-specific steps. agentic_describe_object asks to describe or explain an analytics object with no context; the answer must rest on a workspace lookup unless the item expects the agent to ask back. Both kinds only read, so they run in parallel. jira: QA-30179 risk: low --- packages/gooddata-eval/AGENTS.md | 4 + .../src/gooddata_eval/cli/agentic_runner.py | 40 ++ .../core/agentic/chat_summary.py | 511 ++++++++++++++++++ .../core/agentic/describe_object.py | 212 ++++++++ .../core/dataset/langfuse_source.py | 18 +- .../core/evaluators/chat_summary.py | 216 ++++++++ .../core/evaluators/describe_object.py | 87 +++ .../src/gooddata_eval/core/interaction.py | 3 + .../src/gooddata_eval/core/models.py | 45 ++ .../tests/test_agentic_chat_summary.py | 141 +++++ .../tests/test_chat_summary_evaluator.py | 201 +++++++ 11 files changed, 1477 insertions(+), 1 deletion(-) create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/agentic/chat_summary.py create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/agentic/describe_object.py create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/evaluators/chat_summary.py create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/evaluators/describe_object.py create mode 100644 packages/gooddata-eval/tests/test_agentic_chat_summary.py create mode 100644 packages/gooddata-eval/tests/test_chat_summary_evaluator.py diff --git a/packages/gooddata-eval/AGENTS.md b/packages/gooddata-eval/AGENTS.md index 3923fd891..f24839d46 100644 --- a/packages/gooddata-eval/AGENTS.md +++ b/packages/gooddata-eval/AGENTS.md @@ -64,6 +64,10 @@ its own shape. `test_kind` on the item is what labels the result, not the evalua which is why `knowledge_question` can reuse `GeneralQuestionEvaluator` verbatim. `dashboard_summary` items additionally need `summary_input`, `document_takeaway` items `takeaway_input`, `catalog_metadata` items `catalog_input`. +`agentic_summary` items need `agentic_summary_input` (scope, one stored execution per widget, +follow-ups) plus `user_context`: the UI's context without result ids, which expire, so the kind +re-executes the stored executions on every run. Their `question` is a readable scenario line the +judge reads; the text sent to the agent is `agentic_summary_input.question` or the UI's own. ## Running the pipeline diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index fb2984dce..3a3d9bac9 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -15,9 +15,11 @@ from gooddata_eval.core.agentic._trace_linker import BackgroundTraceLinker, SubmitTraceLink, run_trace_link_inline from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill from gooddata_eval.core.agentic.anomaly_detection import evaluate_agentic_anomaly_detection +from gooddata_eval.core.agentic.chat_summary import evaluate_agentic_summary from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation from gooddata_eval.core.agentic.dashboard_skill import evaluate_agentic_dashboard_skill from gooddata_eval.core.agentic.dashboard_summary import evaluate_agentic_dashboard_summary +from gooddata_eval.core.agentic.describe_object import evaluate_agentic_describe_object from gooddata_eval.core.agentic.document_skill import evaluate_agentic_document_skill from gooddata_eval.core.agentic.forecasting import evaluate_agentic_forecasting from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question @@ -63,6 +65,8 @@ class _LfKw(TypedDict, total=False): "agentic_dashboard_summary", "agentic_anomaly_detection", "agentic_what_if", + "agentic_summary", + "agentic_describe_object", } ) @@ -87,6 +91,9 @@ class _LfKw(TypedDict, total=False): # vis_agentic, agentic_visualization visualizations come back as AAC proposals # in the chat response; nothing is persisted # and neither module has cleanup code +# agentic_summary executes stored AFMs (a read) and asks for a +# summary and follow-ups about the data shown +# agentic_describe_object one question answered from workspace lookups PARALLEL_SAFE_TEST_KINDS = frozenset( { "agentic_general_question", @@ -94,6 +101,8 @@ class _LfKw(TypedDict, total=False): "agentic_search", "vis_agentic", "agentic_visualization", + "agentic_summary", + "agentic_describe_object", } ) @@ -359,6 +368,37 @@ def _dispatch_agentic( agent_id=agent_id, **lf_kw, ) + elif kind == "agentic_summary": + agentic_summary_input = item.agentic_summary_input + if agentic_summary_input is None or item.user_context is None: + raise ValueError(f"agentic_summary item '{item.id}' needs both 'agentic_summary_input' and 'user_context'.") + return evaluate_agentic_summary( + host=host, + token=token, + workspace_id=workspace_id, + # The item's question is its readable scenario line, read by the judge; the text + # sent to the agent is agentic_summary_input.question or the UI's pre-filled one. + scenario=item.question, + summary_input=agentic_summary_input, + user_context=item.user_context, + expected_output=eo if isinstance(eo, dict) else {}, + k=k, + gate=gate, + agent_id=agent_id, + **lf_kw, + ) + elif kind == "agentic_describe_object": + return evaluate_agentic_describe_object( + host=host, + token=token, + workspace_id=workspace_id, + question=item.question, + expected_output=eo if isinstance(eo, dict) else {}, + k=k, + gate=gate, + agent_id=agent_id, + **lf_kw, + ) elif kind == "agentic_what_if": return evaluate_agentic_what_if( host=host, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/chat_summary.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/chat_summary.py new file mode 100644 index 000000000..6f150196a --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/chat_summary.py @@ -0,0 +1,511 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Chat summary of a widget or a dashboard, sent the way the UI sends it. + +The UI's "Summarize" items (widget menu, dashboard or tab menu) open the assistant with a +fixed question and a `userContext` that describes the dashboard on screen: its definition, +the live filter bar, the active tab, and every widget with the result id it rendered and +the filters that reached it. The skills read the data through those result ids. + +A result id is a key into a result cache and expires, so an item cannot store one. It +stores the UI's `userContext` without result ids, plus each widget's execution (the AFM +and result spec the UI ran, filters included). Before each conversation the executions +run again and the fresh result ids go back into the context. Nothing here resolves +filters: the stored execution already carries the filters the widget rendered with, which +is the only way a KPI's comparison measures, a chart's dimensions and a widget that +ignores the dashboard date filter all stay as the user saw them. +""" + +from __future__ import annotations + +import copy +import time +from dataclasses import dataclass, field +from typing import Any, cast + +import httpx + +from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) +from gooddata_eval.core.chat.render import render_answer_text +from gooddata_eval.core.chat.sse_client import ChatClient, ChatError +from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.evaluators.base import ItemEvaluation +from gooddata_eval.core.evaluators.chat_summary import SummaryScope, evaluate_turn, make_judges +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + AgenticSummaryInput, + ChatResult, + WidgetExecution, + build_latency_breakdown, +) +from gooddata_eval.core.timing import PhaseTimings, log_timer + +# The questions the UI pre-fills. An item's own question wins. +WIDGET_QUESTION = 'Summarize the visualization "{title}"' +DASHBOARD_QUESTION = "Summarize this dashboard" + +_EXECUTE_TIMEOUT_S = 120.0 + + +def execute_widget(client: httpx.Client, workspace_id: str, execution: WidgetExecution) -> str: + """Run one stored execution and return the id of the result it produced.""" + body: dict[str, Any] = {"execution": execution.afm, "resultSpec": execution.result_spec} + resp = client.post(f"/api/v1/actions/workspaces/{workspace_id}/execution/afm/execute", json=body) + resp.raise_for_status() + return str(resp.json()["executionResponse"]["links"]["executionResult"]) + + +def execute_widgets( + host: str, token: str, workspace_id: str, executions: dict[str, WidgetExecution] +) -> dict[str, str | None]: + """Fresh result ids per widget id; None for a widget whose execution failed. + + A failed widget is kept with None rather than raised: the UI sends a widget that did + not render without a result id, and the summary is still asked for. + """ + result_ids: dict[str, str | None] = {} + with httpx.Client( + base_url=host.rstrip("/"), headers={"Authorization": f"Bearer {token}"}, timeout=_EXECUTE_TIMEOUT_S + ) as client: + for widget_id, execution in executions.items(): + try: + result_ids[widget_id] = execute_widget(client, workspace_id, execution) + except (httpx.HTTPError, KeyError) as exc: # noqa: PERF203 -- one dead widget must not lose the item + log_timer(f"[chat_summary] widget '{widget_id}' failed to execute: {exc}") + result_ids[widget_id] = None + return result_ids + + +def _has_values(data: Any) -> bool: + if isinstance(data, list): + return any(_has_values(v) for v in data) + return data is not None + + +def widgets_with_data(host: str, token: str, workspace_id: str, result_ids: dict[str, str | None]) -> set[str]: + """Widget ids whose result holds at least one value; a result that cannot be read counts as empty.""" + with_data: set[str] = set() + with httpx.Client( + base_url=host.rstrip("/"), headers={"Authorization": f"Bearer {token}"}, timeout=_EXECUTE_TIMEOUT_S + ) as client: + for widget_id, result_id in result_ids.items(): + if not result_id: + continue + try: + resp = client.get(f"/api/v1/actions/workspaces/{workspace_id}/execution/afm/execute/result/{result_id}") + resp.raise_for_status() + except httpx.HTTPError as exc: # noqa: PERF203 -- read the next widget anyway + log_timer(f"[chat_summary] result of widget '{widget_id}' unreadable: {exc}") + continue + if _has_values(resp.json().get("data")): + with_data.add(widget_id) + return with_data + + +def referenced_widget_ids(user_context: dict[str, Any]) -> list[str]: + return [ + str(o.get("id")) + for group in user_context.get("referencedObjects") or [] + if isinstance(group, dict) + for o in group.get("objects") or [] + if isinstance(o, dict) and o.get("type") == "WIDGET" + ] + + +def summary_scope( + summary_input: AgenticSummaryInput, user_context: dict[str, Any], with_data: set[str] +) -> SummaryScope: + """What the summarized widgets are: the referenced ones for a widget summary, all for a dashboard. + + A KPI is an execution without attributes: one value, no breakdown. + """ + executions = summary_input.executions + in_scope = referenced_widget_ids(user_context) if summary_input.scope == "widget" else list(executions) + in_scope = [w for w in in_scope if w in executions] + kpi_only = bool(in_scope) and all(not executions[w].afm.get("attributes") for w in in_scope) + return SummaryScope( + scope=summary_input.scope, + kpi_only=kpi_only, + all_empty=not any(w in with_data for w in in_scope), + ) + + +def _widgets(user_context: dict[str, Any]) -> list[dict[str, Any]]: + view = user_context.get("view") + dashboard = view.get("dashboard") if isinstance(view, dict) else None + widgets = dashboard.get("widgets") if isinstance(dashboard, dict) else None + if not isinstance(widgets, list): + return [] + return [cast(dict[str, Any], w) for w in widgets if isinstance(w, dict)] + + +def with_result_ids(user_context: dict[str, Any], result_ids: dict[str, str | None]) -> dict[str, Any]: + """A copy of the stored context with each widget's fresh result id. + + A widget with no fresh result id is sent without the key, as the UI sends a widget + that did not render. A stored result id is always dropped: it has expired. + """ + context = copy.deepcopy(user_context) + for widget in _widgets(context): + widget.pop("resultId", None) + result_id = result_ids.get(str(widget.get("widgetId", ""))) + if result_id: + widget["resultId"] = result_id + return context + + +def missing_executions(user_context: dict[str, Any], executions: dict[str, WidgetExecution]) -> list[str]: + """Insight widgets of the context with no stored execution: they would reach the agent without data.""" + return [ + str(w.get("widgetId")) + for w in _widgets(user_context) + if w.get("widgetType", "insight") != "richText" and str(w.get("widgetId")) not in executions + ] + + +def default_question(summary_input: AgenticSummaryInput, user_context: dict[str, Any]) -> str: + """The UI's pre-filled question: the referenced widget's title, or the dashboard one.""" + if summary_input.scope == "dashboard": + return DASHBOARD_QUESTION + referenced = set(referenced_widget_ids(user_context)) + titles = [str(w.get("title")) for w in _widgets(user_context) if str(w.get("widgetId")) in referenced] + if len(titles) != 1: + raise ValueError( + f"A widget summary needs exactly one referenced widget to phrase the UI question, found {len(titles)}; " + "set the item question instead." + ) + return WIDGET_QUESTION.format(title=titles[0]) + + +@dataclass +class SummaryTurn: + """One question and what the agent answered to it.""" + + question: str + result: ChatResult | None = None + error: str | None = None + agent_s: float = 0.0 + + +@dataclass +class SummaryConversation: + """One run: the summary turn and its follow-ups, in order.""" + + conversation_id: str + turns: list[SummaryTurn] = field(default_factory=list) + widgets_total: int = 0 + widgets_executed: int = 0 + scope: SummaryScope | None = None + # One verdict per answered turn, set by evaluate_chat_summary. + evaluations: list[ItemEvaluation] = field(default_factory=list) + judge_s: float = 0.0 + + @property + def chat_error(self) -> str | None: + return next((t.error for t in self.turns if t.error), None) + + +def run_summary_conversation( + client: ChatClient, + conversation_id: str, + questions: list[str], + user_context: dict[str, Any], +) -> list[SummaryTurn]: + """Send the summary question and each follow-up in one conversation, with the same context. + + The UI keeps the dashboard context attached for every message of the conversation, so + the follow-ups carry it too. A failed turn ends the conversation: a later answer would + rest on a turn the agent never answered. + """ + turns: list[SummaryTurn] = [] + for question in questions: + started = time.monotonic() + try: + result = client.send_message(conversation_id, question, user_context=user_context) + except ChatError as exc: + turns.append(SummaryTurn(question, error=f"chat failed: {exc}", agent_s=time.monotonic() - started)) + break + turns.append(SummaryTurn(question, result=result, agent_s=time.monotonic() - started)) + return turns + + +def run_chat_summary( + host: str, + token: str, + workspace_id: str, + summary_input: AgenticSummaryInput, + user_context: dict[str, Any], + question: str | None = None, + k: int = 1, + reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, +) -> list[SummaryConversation]: + """Run the item K times; each run executes the widgets again and opens its own conversation. + + The executions run per run, not once per item: K runs that share result ids would all + read one cached result, and a result that expires between runs would fail the later ones + for a reason unrelated to the agent. + """ + missing = missing_executions(user_context, summary_input.executions) + if missing: + raise ValueError(f"No stored execution for widgets {missing}; capture them with the item.") + first = question or default_question(summary_input, user_context) + questions = [first] + [f.question for f in summary_input.follow_ups] + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) + runs: list[SummaryConversation] = [] + try: + for _ in range(k): + result_ids = execute_widgets(host, token, workspace_id, summary_input.executions) + scope = summary_scope(summary_input, user_context, widgets_with_data(host, token, workspace_id, result_ids)) + context = with_result_ids(user_context, result_ids) + try: + conversation_id = client.create_conversation() + except Exception as exc: # noqa: BLE001 -- a lost conversation must not discard completed runs + runs.append(SummaryConversation("", [SummaryTurn(first, error=f"conversation creation failed: {exc}")])) + continue + try: + turns = run_summary_conversation(client, conversation_id, questions, context) + finally: + client.delete_conversation(conversation_id) + runs.append( + SummaryConversation( + conversation_id, + turns, + widgets_total=len(result_ids), + widgets_executed=sum(1 for r in result_ids.values() if r), + scope=scope, + ) + ) + finally: + client.close() + return runs + + +def _tool_names(result: ChatResult | None) -> list[str]: + return [e.function_name for e in (result.tool_call_events if result else [])] + + +def grade_conversation( + run: SummaryConversation, + *, + scenario: str, + expected_outputs: list[dict[str, Any]], + judge_model: str | None = None, +) -> bool: + """Grade every turn of one run; the run passes when every turn passes. + + `expected_outputs[0]` grades the summary, the rest the follow-ups in order. A run whose + chat failed has no verdict: its error is recorded and nothing is graded. + """ + if run.chat_error is not None: + return False + positive, violation = make_judges(judge_model) + started = time.monotonic() + for index, turn in enumerate(run.turns): + expected = expected_outputs[index] if index < len(expected_outputs) else {} + run.evaluations.append( + evaluate_turn( + judge_input=f"{scenario}\nQuestion of this turn: {turn.question}", + answer=render_answer_text(turn.result) if turn.result else "", + tool_names=_tool_names(turn.result), + expected=expected, + scope=run.scope if index == 0 else None, + positive_judge=positive, + violation_judge=violation, + ) + ) + run.judge_s = time.monotonic() - started + return run_passed(run) + + +def run_passed(run: SummaryConversation) -> bool: + return run.chat_error is None and bool(run.evaluations) and all(e.passed for e in run.evaluations) + + +def _run_detail(run: SummaryConversation) -> dict[str, Any]: + """One run's diagnosis: each turn's checks, plus how much of the dashboard was executed.""" + turns: list[dict[str, Any]] = [] + for index, turn in enumerate(run.turns): + entry: dict[str, Any] = {"question": turn.question} + if index < len(run.evaluations): + evaluation = run.evaluations[index] + entry.update(evaluation.detail) + if evaluation.error: + entry["error"] = evaluation.error + elif turn.error: + entry["error"] = turn.error + turns.append(entry) + detail: dict[str, Any] = { + "widgets_total": run.widgets_total, + "widgets_executed": run.widgets_executed, + "turns": turns, + } + if run.scope is not None: + detail["scope"] = {"scope": run.scope.scope, "kpi_only": run.scope.kpi_only, "all_empty": run.scope.all_empty} + if run.chat_error is not None: + detail["chat_error"] = run.chat_error + events = [ev for t in run.turns if t.result for ev in t.result.tool_call_events] + steps = [ev for t in run.turns if t.result for ev in t.result.reasoning_step_events] + detail["latency_breakdown"] = build_latency_breakdown(events, steps) + return detail + + +def _run_latency(run: SummaryConversation) -> float: + return sum(t.agent_s for t in run.turns) + run.judge_s + + +def _last_result(run: SummaryConversation) -> ChatResult | None: + return next((t.result for t in reversed(run.turns) if t.result), None) + + +class ChatSummaryAssertionError(AgenticAssertionError): + """Raised when an agentic chat-summary evaluation fails.""" + + +def evaluate_agentic_summary( + host: str, + token: str, + workspace_id: str, + scenario: str, + summary_input: AgenticSummaryInput, + user_context: dict[str, Any], + expected_output: dict[str, Any], + k: int = 1, + gate: EvalGate = DEFAULT_GATE, + agent_id: str | None = None, + langfuse: object | None = None, + dataset_item_id: str = "", + dataset_name: str = "agent_summary", + run_timestamp: str | None = None, + model_version_override: str | None = None, + run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, + judge_model: str | None = None, +) -> AgenticEvalOutcome: + """Run the item K times, grade every turn, log to Langfuse, raise ChatSummaryAssertionError on failure. + + `scenario` is the item's readable line (its `question`), which the judge reads as INPUT; + the text sent to the agent is `summary_input.question` or the UI's pre-filled one. + """ + langfuse, window_start = open_trace_window(langfuse) + runs = run_chat_summary( + host, + token, + workspace_id, + summary_input, + user_context, + question=summary_input.question, + k=k, + reasoning_effort=reasoning_effort, + agent_id=agent_id, + ) + expected_outputs = [expected_output] + [ + f.expected_output if isinstance(f.expected_output, dict) else {} for f in summary_input.follow_ups + ] + for run in runs: + grade_conversation(run, scenario=scenario, expected_outputs=expected_outputs, judge_model=judge_model) + + scored = [r for r in runs if r.chat_error is None] + pass_at_k = any(run_passed(r) for r in scored) + pass_power_k = bool(scored) and len(scored) == len(runs) and all(run_passed(r) for r in scored) + + if langfuse is not None and dataset_item_id: + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(runs), gate=gate) + for run_idx, run in enumerate(runs): + if run.chat_error is not None: + # No answer, so no verdict: a 0.0 would publish a failure nobody graded. + continue + pt = ctx.trace(run.conversation_id) + with ctx.observe(pt, run_idx) as tid: + ctx.score(tid, name="summary_pass", value=float(run_passed(run)), data_type="BOOLEAN") + ctx.quality( + tid, + strict_checks={"summary_pass": run_passed(run)}, + latency_sec=pt.latency if pt else None, + cost_usd=pt.total_cost if pt else None, + ) + log_gate_scores(ctx, tid, gate=gate, pass_at_k=pass_at_k, pass_power_k=pass_power_k) + + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + conversation_ids=[r.conversation_id for r in scored], + window_start=window_start, + window_end=window_end, + suffix_runs=len(runs) > 1, + write_scores=_write_scores, + item_input=scenario, + ) + + best = max(runs, key=lambda r: (run_passed(r), sum(e.passed for e in r.evaluations))) + best_result = _last_result(best) + timings = PhaseTimings() + for run in runs: + timings = timings + PhaseTimings(agent_s=sum(t.agent_s for t in run.turns), judge_s=run.judge_s) + runs_passed = sum(1 for r in runs if run_passed(r)) + failed_runs = build_failed_runs(runs, passed=run_passed, detail=_run_detail) + reasoning = list(best_result.reasoning_steps or []) if best_result else [] + + if not gate_passed(gate, pass_at_k=pass_at_k, pass_power_k=pass_power_k): + failing = next((e for e in best.evaluations if not e.passed), None) + reason = best.chat_error or (failing.error if failing and failing.error else "criteria not satisfied") + error = ChatSummaryAssertionError( + f"Chat summary failed for '{scenario}'. {gate_failure_note(gate, runs_passed, len(runs))} " + f"({best.widgets_executed}/{best.widgets_total} widgets executed): {reason}" + ) + error.reasoning_steps = reasoning + error.conversation_id = best.conversation_id + error.response_id = best_result.response_id if best_result else None + error.detail = _run_detail(best) + error.timings = timings + error.runs_passed = runs_passed + error.runs_effective = len(runs) + error.failed_runs = failed_runs + error.run_latencies_s = [_run_latency(r) for r in runs] + raise error + + return AgenticEvalOutcome( + reasoning_steps=reasoning, + conversation_id=best.conversation_id, + response_id=best_result.response_id if best_result else None, + detail=_run_detail(best), + timings=timings, + runs_passed=runs_passed, + runs_effective=len(runs), + failed_runs=failed_runs, + run_latencies_s=[_run_latency(r) for r in runs], + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/describe_object.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/describe_object.py new file mode 100644 index 000000000..1479a6280 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/describe_object.py @@ -0,0 +1,212 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Describe or explain an analytics object in the chat, as a user asks on the Analyze page. + +One question, no `userContext`: the agent has to find the object in the workspace itself. +Each of the K runs opens its own conversation; the gate decides the item. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any + +from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) +from gooddata_eval.core.chat.render import render_answer_text +from gooddata_eval.core.chat.sse_client import ChatClient, ChatError +from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.evaluators.base import ItemEvaluation +from gooddata_eval.core.evaluators.describe_object import evaluate_answer, make_judges +from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome, ChatResult, build_latency_breakdown +from gooddata_eval.core.timing import PhaseTimings + + +@dataclass +class DescribeRun: + conversation_id: str + result: ChatResult | None = None + error: str | None = None + evaluation: ItemEvaluation | None = None + agent_s: float = 0.0 + judge_s: float = 0.0 + tool_names: list[str] = field(default_factory=list) + + @property + def passed(self) -> bool: + return self.error is None and self.evaluation is not None and self.evaluation.passed + + +def _run_once(client: ChatClient, question: str, expected: dict[str, Any], judge_model: str | None) -> DescribeRun: + try: + conversation_id = client.create_conversation() + except Exception as exc: # noqa: BLE001 -- a lost conversation must not discard completed runs + return DescribeRun("", error=f"conversation creation failed: {exc}") + run = DescribeRun(conversation_id) + started = time.monotonic() + try: + run.result = client.send_message(conversation_id, question) + except ChatError as exc: + run.error = f"chat failed: {exc}" + finally: + run.agent_s = time.monotonic() - started + client.delete_conversation(conversation_id) + if run.result is not None: + run.tool_names = [e.function_name for e in run.result.tool_call_events] + positive, violation = make_judges(judge_model) + judged = time.monotonic() + run.evaluation = evaluate_answer( + question=question, + answer=render_answer_text(run.result), + tool_names=run.tool_names, + expected=expected, + positive_judge=positive, + violation_judge=violation, + ) + run.judge_s = time.monotonic() - judged + return run + + +def _run_detail(run: DescribeRun) -> dict[str, Any]: + detail: dict[str, Any] = dict(run.evaluation.detail) if run.evaluation else {} + detail["tools"] = run.tool_names + if run.error: + detail["chat_error"] = run.error + if run.result is not None: + detail["latency_breakdown"] = build_latency_breakdown( + run.result.tool_call_events, run.result.reasoning_step_events + ) + return detail + + +class DescribeObjectAssertionError(AgenticAssertionError): + """Raised when an agentic describe-object evaluation fails.""" + + +def evaluate_agentic_describe_object( + host: str, + token: str, + workspace_id: str, + question: str, + expected_output: dict[str, Any], + k: int = 1, + gate: EvalGate = DEFAULT_GATE, + agent_id: str | None = None, + langfuse: object | None = None, + dataset_item_id: str = "", + dataset_name: str = "agent_describe_objects", + run_timestamp: str | None = None, + model_version_override: str | None = None, + run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, + judge_model: str | None = None, +) -> AgenticEvalOutcome: + """Ask K times, grade each answer, log to Langfuse, raise DescribeObjectAssertionError on failure.""" + langfuse, window_start = open_trace_window(langfuse) + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) + try: + runs = [_run_once(client, question, expected_output, judge_model) for _ in range(k)] + finally: + client.close() + + scored = [r for r in runs if r.error is None] + pass_at_k = any(r.passed for r in scored) + pass_power_k = bool(scored) and len(scored) == len(runs) and all(r.passed for r in scored) + + if langfuse is not None and dataset_item_id: + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(runs), gate=gate) + for run_idx, run in enumerate(runs): + if run.error is not None: + continue + pt = ctx.trace(run.conversation_id) + with ctx.observe(pt, run_idx) as tid: + ctx.score(tid, name="describe_pass", value=float(run.passed), data_type="BOOLEAN") + ctx.quality( + tid, + strict_checks={"describe_pass": run.passed}, + latency_sec=pt.latency if pt else None, + cost_usd=pt.total_cost if pt else None, + ) + log_gate_scores(ctx, tid, gate=gate, pass_at_k=pass_at_k, pass_power_k=pass_power_k) + + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + conversation_ids=[r.conversation_id for r in scored], + window_start=window_start, + window_end=window_end, + suffix_runs=len(runs) > 1, + write_scores=_write_scores, + item_input=question, + ) + + best = max(runs, key=lambda r: (r.passed, r.evaluation.rank_key if r.evaluation else (-2, 0.0))) + timings = PhaseTimings() + for run in runs: + timings = timings + PhaseTimings(agent_s=run.agent_s, judge_s=run.judge_s) + runs_passed = sum(1 for r in runs if r.passed) + failed_runs = build_failed_runs(runs, passed=lambda r: r.passed, detail=_run_detail) + reasoning = list(best.result.reasoning_steps or []) if best.result else [] + response_id = best.result.response_id if best.result else None + + if not gate_passed(gate, pass_at_k=pass_at_k, pass_power_k=pass_power_k): + reason = best.error or (best.evaluation.error if best.evaluation and best.evaluation.error else None) + error = DescribeObjectAssertionError( + f"Describe object failed for {question!r}. {gate_failure_note(gate, runs_passed, len(runs))}: " + f"{reason or 'criteria not satisfied'}" + ) + error.reasoning_steps = reasoning + error.conversation_id = best.conversation_id + error.response_id = response_id + error.detail = _run_detail(best) + error.timings = timings + error.runs_passed = runs_passed + error.runs_effective = len(runs) + error.failed_runs = failed_runs + error.run_latencies_s = [r.agent_s + r.judge_s for r in runs] + raise error + + return AgenticEvalOutcome( + reasoning_steps=reasoning, + conversation_id=best.conversation_id, + response_id=response_id, + detail=_run_detail(best), + timings=timings, + runs_passed=runs_passed, + runs_effective=len(runs), + failed_runs=failed_runs, + run_latencies_s=[r.agent_s + r.judge_s for r in runs], + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py b/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py index 878bac8e7..c6fc70179 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py @@ -18,7 +18,7 @@ from pydantic import ValidationError from gooddata_eval.core.langfuse._env import make_http_client -from gooddata_eval.core.models import CatalogInput, DatasetItem, SummaryInput, TakeawayInput +from gooddata_eval.core.models import AgenticSummaryInput, CatalogInput, DatasetItem, SummaryInput, TakeawayInput _PAGE_SIZE = 100 @@ -60,6 +60,21 @@ def _summary_input_from_raw(raw: dict, expected_output: Any) -> SummaryInput | N return SummaryInput.model_validate(candidate) if candidate is not None else None +def _agentic_summary_input_from_raw(raw: dict) -> AgenticSummaryInput | None: + """Locate an agentic_summary item's `agentic_summary_input`, from the item input or its metadata. + + Raises: + ValueError: The value found is not a valid `agentic_summary_input` object. + """ + candidate = _first_of(dict, "agentic_summary_input", raw.get("input"), raw.get("metadata")) + if candidate is None: + return None + try: + return AgenticSummaryInput.model_validate(candidate) + except ValidationError as exc: + raise ValueError(f"Item {raw.get('id')!r} has an invalid `agentic_summary_input`: {exc}") from exc + + def _catalog_input_from_raw(raw: dict, expected_output: Any) -> CatalogInput | None: """Locate a catalog_metadata item's `catalog_input`. @@ -177,6 +192,7 @@ def _item_from_raw(raw: dict, *, dataset_name: str, test_kind: str) -> DatasetIt summary_input=_summary_input_from_raw(raw, expected_output), takeaway_input=takeaway_input, catalog_input=catalog_input, + agentic_summary_input=_agentic_summary_input_from_raw(raw), user_context=_user_context_from_raw(raw), ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/chat_summary.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/chat_summary.py new file mode 100644 index 000000000..ec34b8483 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/chat_summary.py @@ -0,0 +1,216 @@ +# (C) 2026 GoodData Corporation +"""Checks and judge steps for `agentic_summary`: the chat summary of a widget or a dashboard. + +Each turn's `expected_output` (the item's for the summary, a follow-up's own for that turn): + + { + "format": {...} | false, + "no_tool_calls": false, + "must_include": ["...criteria the answer must satisfy..."], + "must_not_include": ["...characteristics the answer must not show..."], + "rubric": ["...soft quality dimensions..."] + } + +Every key is optional. The code checks gate the verdict and need no judge: + +- shape (summary turn only, unless `format` is false): a chart widget gets the four + headings of the visualization summary in order, a KPI widget none and at most three + sentences, a dashboard the three headings of the dashboard summary in order, and a + dashboard whose widgets all returned no data one paragraph without headings; an empty + widget has no fixed shape. A `format` + object replaces the default: `max_sentences`, `no_headings`, `no_bullets`, `headings`. +- no ids: no `{visualization/...}` reference tag and no raw object id in the answer. +- no figures on no data: when every widget in scope returned no data, the answer quotes no + amount, share or count. +- `no_tool_calls`: the turn answers from the conversation, without running a tool. + +Judge criteria are graded one by one, as for `dashboard_summary`; the INPUT the judge reads +is the item's scenario line plus the question of the turn. Criteria give a direction, never +a value. +""" + +from __future__ import annotations + +import re +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any + +from gooddata_eval.core.evaluators._llm_judge import LLMJudge +from gooddata_eval.core.evaluators._rubric import Criterion, grade_rubric +from gooddata_eval.core.evaluators.base import ItemEvaluation + +CHART_HEADINGS = ("Summary", "Key Findings", "Trends", "Areas to Watch") +DASHBOARD_HEADINGS = ("Summary", "Key Insights", "Areas to Watch") +KPI_MAX_SENTENCES = 3 + +# Tools gen-ai runs to manage the conversation, not to answer it. +_BOOKKEEPING_TOOLS = frozenset({"set_skills"}) + +_POSITIVE_STEPS = [ + "Read the INPUT: the dashboard or widget the user summarizes, its filters, and the question of this turn.", + "Read the EXPECTED OUTPUT: one criterion the answer must satisfy.", + "Read the ACTUAL OUTPUT: the assistant's answer to that question.", + "Score 1 if the actual output clearly satisfies the criterion.", + "Score 0 if the criterion is missing, contradicted, or only partially addressed.", +] + +# A presence question, inverted in code, as in summary.py: the INPUT names filters and data, +# which pulls the judge towards grading correctness; the third step keeps it on presence. +_VIOLATION_STEPS = [ + "Read the CHARACTERISTIC described in EXPECTED OUTPUT, including any exceptions it states.", + "Read the ACTUAL OUTPUT: the assistant's answer.", + "Decide only whether the actual output exhibits the characteristic, not whether it is correct or wanted.", + "Score 1 if the actual output clearly exhibits the described characteristic.", + "Score 0 if it does not exhibit it.", +] + +_HEADING = re.compile(r"^\s{0,3}#{1,6}\s+(.+?)\s*#*\s*$", re.MULTILINE) +_BULLET = re.compile(r"^\s*(?:[-*•]|\d+[.)])\s", re.MULTILINE) +_REFERENCE_TAG = re.compile(r"\{(?:visualization|dashboard|metric|attribute|label|fact|widget)/[^}]*\}") +_RAW_ID = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b|\b[0-9a-f]{32}\b") +# An amount, a share or a count: $1.2 M, 45%, 1,205, 3.5 K. Years, days and hours are dates. +_FIGURE = re.compile( + r"[$€£]\s?\d[\d,.]*(?:\s?[KMB]\b)?" + r"|\d[\d,.]*\s?%" + r"|\b\d{1,3}(?:,\d{3})+(?:\.\d+)?\b" + r"|\b\d+(?:\.\d+)?\s?[KMB]\b" + r"|\b\d+\.\d+\b", +) +_SENTENCE_END = re.compile(r"(?<=[.!?])\s+(?=[A-Z*])") + + +@dataclass(frozen=True) +class SummaryScope: + """What the summarized widgets returned, as the checks need it.""" + + scope: str # "widget" or "dashboard" + kpi_only: bool # the referenced widgets are single-value KPIs + all_empty: bool # every widget in scope returned no data + + +def headings(text: str) -> list[str]: + return [h.strip(" *_") for h in _HEADING.findall(text)] + + +def count_sentences(text: str) -> int: + plain = _HEADING.sub(" ", text).strip() + return len([s for s in _SENTENCE_END.split(plain) if s.strip()]) if plain else 0 + + +def shape_problem(text: str, scope: SummaryScope, fmt: dict[str, Any] | None) -> str | None: + """Why the answer's shape is wrong, or None. + + `fmt` replaces the default shape (an item where the user asked for another format). + """ + found = headings(text) + if fmt: + if fmt.get("no_headings") and found: + return f"headings {found} although the user asked for none" + if fmt.get("no_bullets") and _BULLET.search(text): + return "bullet points although the user asked for none" + limit = fmt.get("max_sentences") + if isinstance(limit, int) and count_sentences(text) > limit: + return f"{count_sentences(text)} sentences, more than the {limit} the user asked for" + wanted = fmt.get("headings") + if isinstance(wanted, list) and found != wanted: + return f"headings {found}, expected {wanted}" + return None + if scope.scope == "dashboard": + if scope.all_empty: + return f"headings {found} on a dashboard without data" if found else None + return None if found == list(DASHBOARD_HEADINGS) else f"headings {found}, expected {list(DASHBOARD_HEADINGS)}" + if scope.all_empty: + # An empty widget comes back with all four headings or with a single one: the prompt + # sets no shape for it, so only the no-figures check applies. + return None + if scope.kpi_only: + if found: + return f"headings {found} on a KPI" + n = count_sentences(text) + return f"{n} sentences on a KPI, more than {KPI_MAX_SENTENCES}" if n > KPI_MAX_SENTENCES else None + return None if found == list(CHART_HEADINGS) else f"headings {found}, expected {list(CHART_HEADINGS)}" + + +def id_problem(text: str) -> str | None: + tag = _REFERENCE_TAG.search(text) + if tag: + return f"reference tag {tag.group(0)!r} in the answer" + raw = _RAW_ID.search(text) + return f"raw id {raw.group(0)!r} in the answer" if raw else None + + +def figures(text: str) -> list[str]: + return [m.group(0) for m in _FIGURE.finditer(text)] + + +def answer_tools(tool_names: Sequence[str]) -> list[str]: + return [t for t in tool_names if t and t not in _BOOKKEEPING_TOOLS] + + +def _criteria(expected: dict[str, Any], key: str, prefix: str) -> list[Criterion]: + raw = expected.get(key) or [] + items = raw if isinstance(raw, list) else [raw] + return [(f"{prefix}{i}", str(c)) for i, c in enumerate(items) if str(c).strip()] + + +def evaluate_turn( + *, + judge_input: str, + answer: str, + tool_names: Sequence[str], + expected: dict[str, Any], + scope: SummaryScope | None, + positive_judge: LLMJudge, + violation_judge: LLMJudge, +) -> ItemEvaluation: + """Grade one turn. `scope` is None for a follow-up: only the summary turn has a default shape.""" + detail: dict[str, Any] = {"actual_output": answer} + passed = True + if not answer.strip(): + detail["error"] = "empty answer" + return ItemEvaluation(passed=False, rank_key=(-1, 0.0), detail=detail, error="empty answer") + + fmt = expected.get("format") + if scope is not None and fmt is not False: + problem = shape_problem(answer, scope, fmt if isinstance(fmt, dict) else None) + detail["shape_ok"] = problem is None + if problem: + detail["shape_problem"] = problem + passed = False + + problem = id_problem(answer) + detail["no_ids"] = problem is None + if problem: + detail["id_problem"] = problem + passed = False + + if scope is not None and scope.all_empty: + quoted = figures(answer) + detail["no_figures_on_no_data"] = not quoted + if quoted: + detail["figures_on_no_data"] = quoted[:5] + passed = False + + if expected.get("no_tool_calls"): + ran = answer_tools(tool_names) + detail["no_tool_calls"] = not ran + if ran: + detail["tools_run"] = ran + passed = False + + return grade_rubric( + positive_judge, + violation_judge, + question=judge_input, + actual=answer, + must_include=_criteria(expected, "must_include", "must_include_"), + must_not_include=_criteria(expected, "must_not_include", "must_not_include_"), + rubric=_criteria(expected, "rubric", "rubric_"), + detail=detail, + passed=passed, + ) + + +def make_judges(model: str | None = None) -> tuple[LLMJudge, LLMJudge]: + return LLMJudge(_POSITIVE_STEPS, model=model), LLMJudge(_VIOLATION_STEPS, model=model) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/describe_object.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/describe_object.py new file mode 100644 index 000000000..549d08bb9 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/describe_object.py @@ -0,0 +1,87 @@ +# (C) 2026 GoodData Corporation +"""Checks and judge steps for `agentic_describe_object`: "describe / explain " in the chat. + +`expected_output`: + + { + "search": true, + "must_include": ["...criteria the answer must satisfy..."], + "must_not_include": ["...characteristics the answer must not show..."], + "rubric": ["...soft quality dimensions..."] + } + +`search` (default true) gates the verdict without a judge: the answer must rest on a +workspace lookup (`search_objects` or `get_object_with_children`), never on general +knowledge. An item where the right answer is to ask back ("Describe this" with nothing on +screen) sets it to false. Judge criteria are graded one by one; the INPUT the judge reads +is the item's question. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from gooddata_eval.core.evaluators._llm_judge import LLMJudge +from gooddata_eval.core.evaluators._rubric import Criterion, grade_rubric +from gooddata_eval.core.evaluators.base import ItemEvaluation + +LOOKUP_TOOLS = frozenset({"search_objects", "get_object_with_children"}) + +_POSITIVE_STEPS = [ + "Read the INPUT: the user's request to describe or explain an analytics object of their workspace.", + "Read the EXPECTED OUTPUT: one criterion the answer must satisfy.", + "Read the ACTUAL OUTPUT: the assistant's answer.", + "Score 1 if the actual output clearly satisfies the criterion.", + "Score 0 if the criterion is missing, contradicted, or only partially addressed.", +] + +_VIOLATION_STEPS = [ + "Read the CHARACTERISTIC described in EXPECTED OUTPUT, including any exceptions it states.", + "Read the ACTUAL OUTPUT: the assistant's answer.", + "Decide only whether the actual output exhibits the characteristic, not whether it is correct or wanted.", + "Score 1 if the actual output clearly exhibits the described characteristic.", + "Score 0 if it does not exhibit it.", +] + + +def _criteria(expected: dict[str, Any], key: str) -> list[Criterion]: + raw = expected.get(key) or [] + items = raw if isinstance(raw, list) else [raw] + return [(f"{key}_{i}", str(c)) for i, c in enumerate(items) if str(c).strip()] + + +def evaluate_answer( + *, + question: str, + answer: str, + tool_names: Sequence[str], + expected: dict[str, Any], + positive_judge: LLMJudge, + violation_judge: LLMJudge, +) -> ItemEvaluation: + detail: dict[str, Any] = {"actual_output": answer} + if not answer.strip(): + return ItemEvaluation(passed=False, rank_key=(-1, 0.0), detail=detail, error="empty answer") + passed = True + if expected.get("search", True): + looked_up = sorted(LOOKUP_TOOLS.intersection(tool_names)) + detail["workspace_lookup"] = bool(looked_up) + if not looked_up: + detail["lookup_problem"] = f"answered without a workspace lookup (tools: {list(tool_names)})" + passed = False + return grade_rubric( + positive_judge, + violation_judge, + question=question, + actual=answer, + must_include=_criteria(expected, "must_include"), + must_not_include=_criteria(expected, "must_not_include"), + rubric=_criteria(expected, "rubric"), + detail=detail, + passed=passed, + ) + + +def make_judges(model: str | None = None) -> tuple[LLMJudge, LLMJudge]: + return LLMJudge(_POSITIVE_STEPS, model=model), LLMJudge(_VIOLATION_STEPS, model=model) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/interaction.py b/packages/gooddata-eval/src/gooddata_eval/core/interaction.py index 645625697..3acb8fc0f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/interaction.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/interaction.py @@ -29,6 +29,7 @@ "agentic_guardrail", "agentic_search", "agentic_dashboard_summary", + "agentic_describe_object", } ) @@ -46,6 +47,8 @@ "agentic_anomaly_detection", "agentic_what_if", "agentic_forecasting", + # One summary turn plus the item's follow-ups, in one conversation. + "agentic_summary", } ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index eceda9f54..ef47a7537 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -579,6 +579,49 @@ class CatalogInput(BaseModel): output: Literal["title", "description"] +class WidgetExecution(BaseModel): + """The execution a dashboard widget renders with, as the UI sent it. + + `afm` and `result_spec` are the body of `POST .../execution/afm/execute`, filters + included: captured from the result the UI computed, so measures (a KPI's comparison), + dimensions and sorting match what the user sees. Executed again on every run, because + a result id expires. + """ + + model_config = ConfigDict(extra="forbid") + + afm: dict[str, Any] + result_spec: dict[str, Any] = Field(default_factory=dict) + + +class SummaryFollowUp(BaseModel): + """A question sent after the summary, in the same conversation.""" + + model_config = ConfigDict(extra="forbid") + + question: str = Field(min_length=1) + expected_output: Any = None + + +class AgenticSummaryInput(BaseModel): + """Structured input for the `agentic_summary` test kind. + + The chat summary of a widget or a dashboard as the UI asks for it. The item's + `user_context` is the UI's `userContext` without result ids; `executions` holds one + execution per widget id, run before each conversation to put fresh result ids back. + A widget without an execution is sent without a result id. + """ + + model_config = ConfigDict(extra="forbid") + + scope: Literal["widget", "dashboard"] + # The message sent to the agent. The item's `question` is a readable scenario line the + # judge reads as INPUT, so the sent text lives here; None sends the UI's pre-filled one. + question: str | None = None + executions: dict[str, WidgetExecution] = Field(default_factory=dict) + follow_ups: list[SummaryFollowUp] = Field(default_factory=list) + + class DatasetItem(BaseModel): """Common dataset envelope. `expected_output` stays raw; each evaluator parses its own shape.""" @@ -595,6 +638,8 @@ class DatasetItem(BaseModel): takeaway_input: TakeawayInput | None = None # Only used by the `catalog_metadata` test kind; ignored by all others. catalog_input: CatalogInput | None = None + # Only used by the `agentic_summary` test kind; ignored by all others. + agentic_summary_input: AgenticSummaryInput | None = None # Relayed verbatim as the chat request's `userContext`. Deliberately opaque: gen-ai owns # that schema (a discriminated union of view/widget descriptors), so re-modelling it here # would only create a second copy to keep in sync. diff --git a/packages/gooddata-eval/tests/test_agentic_chat_summary.py b/packages/gooddata-eval/tests/test_agentic_chat_summary.py new file mode 100644 index 000000000..df9614fb2 --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_chat_summary.py @@ -0,0 +1,141 @@ +# (C) 2026 GoodData Corporation +from __future__ import annotations + +import json +from typing import Any, cast + +import httpx +import pytest +from gooddata_eval.core.agentic import chat_summary +from gooddata_eval.core.dataset.langfuse_source import _item_from_raw +from gooddata_eval.core.models import AgenticSummaryInput, WidgetExecution + +_AFM = { + "attributes": [], + "measures": [{"localIdentifier": "m", "definition": {"measure": {"item": {}}}}], + "filters": [], +} +_SPEC = {"dimensions": [{"localIdentifier": "dim_0", "itemIdentifiers": ["measureGroup"]}]} + + +def _context(*widget_ids: str, referenced: list[str] | None = None) -> dict[str, Any]: + widgets: list[dict[str, Any]] = [ + {"widgetId": w, "title": f"Title {w}", "widgetType": "insight", "resultId": f"stale-{w}"} for w in widget_ids + ] + context: dict[str, Any] = {"view": {"dashboard": {"id": "d", "widgets": widgets}}} + if referenced is not None: + context["referencedObjects"] = [ + {"context": {"type": "DASHBOARD", "id": "d"}, "objects": [{"type": "WIDGET", "id": r} for r in referenced]} + ] + return context + + +def _input(scope: str = "dashboard", *widget_ids: str) -> AgenticSummaryInput: + return AgenticSummaryInput.model_validate( + {"scope": scope, "executions": {w: {"afm": _AFM, "result_spec": _SPEC} for w in widget_ids}} + ) + + +def test_fresh_result_ids_replace_the_stored_ones_and_a_failed_widget_has_none() -> None: + stored = _context("a", "b") + sent = chat_summary.with_result_ids(stored, {"a": "fresh-a", "b": None}) + widgets = sent["view"]["dashboard"]["widgets"] + assert widgets[0]["resultId"] == "fresh-a" + assert "resultId" not in widgets[1] + # the stored item is not changed: the next run starts from it again + assert stored["view"]["dashboard"]["widgets"][0]["resultId"] == "stale-a" + + +def test_a_widget_without_a_stored_execution_is_reported() -> None: + context = _context("a", "b") + context["view"]["dashboard"]["widgets"].append({"widgetId": "note", "widgetType": "richText"}) + assert chat_summary.missing_executions(context, _input("dashboard", "a").executions) == ["b"] + + +def test_the_ui_question_names_the_referenced_widget() -> None: + context = _context("a", "b", referenced=["b"]) + assert chat_summary.default_question(_input("widget", "a", "b"), context) == 'Summarize the visualization "Title b"' + assert chat_summary.default_question(_input("dashboard", "a", "b"), context) == "Summarize this dashboard" + + +@pytest.mark.parametrize("referenced", [[], ["a", "b"]]) +def test_a_widget_summary_without_exactly_one_referenced_widget_needs_a_question(referenced: list[str]) -> None: + with pytest.raises(ValueError, match="exactly one referenced widget"): + chat_summary.default_question(_input("widget", "a", "b"), _context("a", "b", referenced=referenced)) + + +def test_execute_widget_posts_the_stored_execution_and_returns_the_result_id() -> None: + seen: list[dict[str, Any]] = [] + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/v1/actions/workspaces/ws/execution/afm/execute" + seen.append(json.loads(request.content)) + return httpx.Response(200, json={"executionResponse": {"links": {"executionResult": "rid-1"}}}) + + with httpx.Client(base_url="https://h", transport=httpx.MockTransport(handler)) as client: + rid = chat_summary.execute_widget(client, "ws", WidgetExecution(afm=_AFM, result_spec=_SPEC)) + assert rid == "rid-1" + assert seen == [{"execution": _AFM, "resultSpec": _SPEC}] + + +def test_execute_widgets_keeps_a_failed_widget_as_none(monkeypatch: pytest.MonkeyPatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + if body["execution"]["measures"][0]["localIdentifier"] == "bad": + return httpx.Response(400, json={"detail": "bad filter"}) + return httpx.Response(200, json={"executionResponse": {"links": {"executionResult": "rid-ok"}}}) + + real_client = httpx.Client + monkeypatch.setattr( + chat_summary.httpx, "Client", lambda **kw: real_client(transport=httpx.MockTransport(handler), **kw) + ) + bad = {**_AFM, "measures": [{"localIdentifier": "bad", "definition": {}}]} + executions = {"ok": WidgetExecution(afm=_AFM), "bad": WidgetExecution(afm=bad)} + assert chat_summary.execute_widgets("https://h", "t", "ws", executions) == {"ok": "rid-ok", "bad": None} + + +def test_a_failed_turn_ends_the_conversation() -> None: + class _Client: + def __init__(self) -> None: + self.sent: list[str] = [] + + def send_message(self, conversation_id: str, question: str, *, user_context: dict) -> Any: + self.sent.append(question) + if question == "second": + raise chat_summary.ChatError("boom") + return object() + + client = _Client() + turns = chat_summary.run_summary_conversation( + cast(chat_summary.ChatClient, client), "c", ["first", "second", "third"], {} + ) + assert client.sent == ["first", "second"] + assert [t.error is None for t in turns] == [True, False] + + +def test_the_langfuse_item_carries_the_summary_input_from_its_metadata() -> None: + raw = { + "id": "i1", + "input": "[dashboard] 2. Sales · all time", + "expectedOutput": {"must_include": ["x"]}, + "metadata": { + "test_kind": "agentic_summary", + "user_context": _context("a"), + "agentic_summary_input": { + "scope": "dashboard", + "executions": {"a": {"afm": _AFM, "result_spec": _SPEC}}, + "follow_ups": [{"question": "Why?", "expected_output": {"must_not_include": ["a cause"]}}], + }, + }, + } + item = _item_from_raw(raw, dataset_name="agent_summary", test_kind="visualization") + assert item.test_kind == "agentic_summary" + assert item.agentic_summary_input is not None + assert item.agentic_summary_input.follow_ups[0].question == "Why?" + assert item.user_context == _context("a") + + +def test_an_invalid_summary_input_fails_loudly() -> None: + raw = {"id": "i2", "input": "q", "expectedOutput": {}, "metadata": {"agentic_summary_input": {"scope": "page"}}} + with pytest.raises(ValueError, match="invalid `agentic_summary_input`"): + _item_from_raw(raw, dataset_name="agent_summary", test_kind="visualization") diff --git a/packages/gooddata-eval/tests/test_chat_summary_evaluator.py b/packages/gooddata-eval/tests/test_chat_summary_evaluator.py new file mode 100644 index 000000000..69c4769b4 --- /dev/null +++ b/packages/gooddata-eval/tests/test_chat_summary_evaluator.py @@ -0,0 +1,201 @@ +# (C) 2026 GoodData Corporation +from __future__ import annotations + +from typing import Any, cast + +import pytest +from gooddata_eval.core.agentic import chat_summary as kind +from gooddata_eval.core.evaluators import chat_summary as cs +from gooddata_eval.core.evaluators import describe_object as do +from gooddata_eval.core.evaluators._llm_judge import LLMJudge +from gooddata_eval.core.models import AgenticSummaryInput, ChatResult + +CHART = """## Summary +Net sales grew in 2024. + +## Key Findings +- Audio & Video Accessories led with **$19.5 M**. + +## Trends +- Sales rose through the year. + +## Areas to Watch +- Furniture declined by **21.9%**.""" + +DASHBOARD = """## Summary +For **2024**, total sales reached **$67.5 M**. + +## Key Insights +- Gross profit fell **0.7%**. + +## Areas to Watch +- Orders declined slightly.""" + +CHART_SCOPE = cs.SummaryScope("widget", kpi_only=False, all_empty=False) +KPI_SCOPE = cs.SummaryScope("widget", kpi_only=True, all_empty=False) +DASHBOARD_SCOPE = cs.SummaryScope("dashboard", kpi_only=False, all_empty=False) + + +class _Judge: + """Answers every criterion with the verdict it was built with, and records the criteria it saw.""" + + def __init__(self, verdict: bool) -> None: + self.verdict = verdict + self.seen: list[tuple[str, str]] = [] + + def score(self, input: str, expected_output: str, actual_output: str) -> tuple[bool, str]: # noqa: A002 + self.seen.append((input, expected_output)) + return self.verdict, "stub" + + +def _grade(answer: str, expected: dict[str, Any], scope: cs.SummaryScope | None, tools: list[str] | None = None): + positive, violation = _Judge(True), _Judge(False) + return cs.evaluate_turn( + judge_input="[widget] 2. Sales", + answer=answer, + tool_names=tools or [], + expected=expected, + scope=scope, + positive_judge=cast(LLMJudge, positive), + violation_judge=cast(LLMJudge, violation), + ) + + +def test_a_chart_summary_needs_the_four_headings_in_order() -> None: + assert _grade(CHART, {}, CHART_SCOPE).passed + swapped = ( + CHART.replace("## Trends", "## Tmp") + .replace("## Key Findings", "## Trends") + .replace("## Tmp", "## Key Findings") + ) + result = _grade(swapped, {}, CHART_SCOPE) + assert not result.passed + assert "expected ['Summary', 'Key Findings', 'Trends', 'Areas to Watch']" in result.detail["shape_problem"] + + +def test_a_dashboard_summary_needs_its_three_headings() -> None: + assert _grade(DASHBOARD, {}, DASHBOARD_SCOPE).passed + assert not _grade(CHART, {}, DASHBOARD_SCOPE).passed + + +def test_a_kpi_summary_has_no_headings_and_few_sentences() -> None: + kpi = "For **2024**, Total Sales was **$67.5 M**, up **0.27%** from the previous period." + assert _grade(kpi, {}, KPI_SCOPE).passed + assert not _grade(DASHBOARD, {}, KPI_SCOPE).passed + long_kpi = " ".join(["Sales grew."] * 4) + assert "4 sentences on a KPI" in _grade(long_kpi, {}, KPI_SCOPE).detail["shape_problem"] + + +def test_a_dashboard_without_data_is_one_paragraph_without_figures() -> None: + empty = cs.SummaryScope("dashboard", kpi_only=False, all_empty=True) + no_data = "No data is available for the current dashboard filters. Please check the filter configuration." + assert _grade(no_data, {}, empty).passed + with_figures = "No data for 2020, but last year sales were $1.9 M and grew 12%." + result = _grade(with_figures, {}, empty) + assert not result.passed + assert result.detail["figures_on_no_data"] == ["$1.9 M", "12%"] + + +def test_an_empty_widget_has_no_fixed_shape() -> None: + empty = cs.SummaryScope("widget", kpi_only=False, all_empty=True) + assert _grade("## Summary\n- The visualization returned no data for January 1–December 31, 2020.", {}, empty).passed + + +def test_a_users_own_format_replaces_the_default_shape() -> None: + brief = {"format": {"max_sentences": 3, "no_headings": True, "no_bullets": True}} + three = "Tracy Carter leads at $1.3 M. The top three are close. The tenth is at $985.3 K." + assert _grade(three, brief, CHART_SCOPE).passed + assert "headings" in _grade(CHART, brief, CHART_SCOPE).detail["shape_problem"] + four = three + " Spending is concentrated." + assert "4 sentences" in _grade(four, brief, CHART_SCOPE).detail["shape_problem"] + assert _grade(CHART, {"format": False}, CHART_SCOPE).detail.get("shape_ok") is None + + +@pytest.mark.parametrize( + "leak", + ["See {visualization/top_10_spenders} for details.", "Widget 0f1e2d3c-4b5a-6978-8a9b-0c1d2e3f4a5b is empty."], +) +def test_ids_never_reach_the_answer(leak: str) -> None: + result = _grade(f"{CHART}\n{leak}", {}, CHART_SCOPE) + assert not result.passed + assert result.detail["no_ids"] is False + + +def test_a_follow_up_without_tool_calls_has_no_default_shape() -> None: + answer = "Tracy Carter leads, about $15.7 K ahead of the second place." + assert _grade(answer, {"no_tool_calls": True}, None, tools=["set_skills"]).passed + ran = _grade(answer, {"no_tool_calls": True}, None, tools=["execute_existing_visualizations"]) + assert not ran.passed + assert ran.detail["tools_run"] == ["execute_existing_visualizations"] + + +def test_judge_criteria_gate_the_verdict() -> None: + result = _grade(CHART, {"must_include": ["names the period"], "must_not_include": ["invents a cause"]}, CHART_SCOPE) + assert result.passed + assert result.detail["must_include_0"] is True + assert result.detail["must_not_include_0"] is True + + +def _turn(answer: str, tools: list[str]) -> kind.SummaryTurn: + result = ChatResult.model_validate( + { + "textResponse": answer, + "toolCallEvents": [{"functionName": t, "functionArguments": "{}"} for t in tools], + } + ) + return kind.SummaryTurn("q", result=result) + + +def test_each_turn_is_graded_with_its_own_expectation(monkeypatch: pytest.MonkeyPatch) -> None: + judges = (_Judge(True), _Judge(False)) + monkeypatch.setattr(kind, "make_judges", lambda model=None: judges) + run = kind.SummaryConversation( + "c", + turns=[_turn(CHART, ["execute_existing_visualizations"]), _turn("Tracy Carter leads.", [])], + scope=CHART_SCOPE, + ) + passed = kind.grade_conversation( + run, + scenario="[widget] Top 10 Spenders", + expected_outputs=[ + {"must_include": ["summary criterion"]}, + {"no_tool_calls": True, "must_include": ["follow-up"]}, + ], + ) + assert passed + assert [e.passed for e in run.evaluations] == [True, True] + assert [c for _, c in judges[0].seen] == ["summary criterion", "follow-up"] + # the follow-up keeps no default shape: only the summary turn is shape-checked + assert "shape_ok" not in run.evaluations[1].detail + + +def test_the_scope_reads_kpis_and_empty_results_from_the_referenced_widgets() -> None: + kpi_afm = {"attributes": [], "measures": [{"localIdentifier": "m"}], "filters": []} + chart_afm = {"attributes": [{"localIdentifier": "a"}], "measures": [{"localIdentifier": "m"}], "filters": []} + inp = AgenticSummaryInput.model_validate( + {"scope": "widget", "executions": {"k": {"afm": kpi_afm}, "c": {"afm": chart_afm}}} + ) + context = {"referencedObjects": [{"objects": [{"type": "WIDGET", "id": "k"}]}]} + assert kind.summary_scope(inp, context, with_data={"k"}) == cs.SummaryScope("widget", True, False) + assert kind.summary_scope(inp, context, with_data={"c"}).all_empty + dashboard = inp.model_copy(update={"scope": "dashboard"}) + assert kind.summary_scope(dashboard, {}, with_data={"c"}) == cs.SummaryScope("dashboard", False, False) + + +def test_a_describe_answer_must_rest_on_a_workspace_lookup() -> None: + def grade(tools: list[str], expected: dict[str, Any]): + return do.evaluate_answer( + question="Explain the ARPU metric", + answer="ARPU is Net Sales divided by Active Customers.", + tool_names=tools, + expected=expected, + positive_judge=cast(LLMJudge, _Judge(True)), + violation_judge=cast(LLMJudge, _Judge(False)), + ) + + assert grade(["set_skills", "search_objects"], {}).passed + no_lookup = grade([], {}) + assert not no_lookup.passed + assert no_lookup.detail["workspace_lookup"] is False + # an item whose right answer is to ask back needs no lookup + assert grade([], {"search": False}).passed From 604861f0927434d34414e5ccda9ead782b1c924a Mon Sep 17 00:00:00 2001 From: duongthimy Date: Fri, 20 Nov 2026 18:28:56 +0100 Subject: [PATCH 2/5] refactor(gooddata-eval): flatten agentic_summary run detail - summary turn keys at the top level (actual_output, shape_ok, must_include_0, ...), like the other kinds' detail - follow-up i keys prefixed followup__ - tools that read detail[k] is False now see follow-up fails too jira: QA-30179 risk: low --- .../core/agentic/chat_summary.py | 41 +++++++++++-------- .../tests/test_chat_summary_evaluator.py | 7 ++++ 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/chat_summary.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/chat_summary.py index 6f150196a..3181c5a04 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/chat_summary.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/chat_summary.py @@ -341,24 +341,31 @@ def run_passed(run: SummaryConversation) -> bool: return run.chat_error is None and bool(run.evaluations) and all(e.passed for e in run.evaluations) +def _turn_detail(run: SummaryConversation, index: int) -> dict[str, Any]: + turn = run.turns[index] + entry: dict[str, Any] = {"question": turn.question} + if index < len(run.evaluations): + evaluation = run.evaluations[index] + entry.update(evaluation.detail) + if evaluation.error: + entry["error"] = evaluation.error + elif turn.error: + entry["error"] = turn.error + return entry + + def _run_detail(run: SummaryConversation) -> dict[str, Any]: - """One run's diagnosis: each turn's checks, plus how much of the dashboard was executed.""" - turns: list[dict[str, Any]] = [] - for index, turn in enumerate(run.turns): - entry: dict[str, Any] = {"question": turn.question} - if index < len(run.evaluations): - evaluation = run.evaluations[index] - entry.update(evaluation.detail) - if evaluation.error: - entry["error"] = evaluation.error - elif turn.error: - entry["error"] = turn.error - turns.append(entry) - detail: dict[str, Any] = { - "widgets_total": run.widgets_total, - "widgets_executed": run.widgets_executed, - "turns": turns, - } + """One run's diagnosis, flat like the other kinds' detail. + + - summary turn: its keys at the top level (`actual_output`, `shape_ok`, `must_include_0`, ...) + - follow-up i (from 1): the same keys prefixed `followup__` + """ + detail: dict[str, Any] = {} + for index in range(len(run.turns)): + prefix = f"followup_{index}_" if index else "" + detail.update({f"{prefix}{key}": value for key, value in _turn_detail(run, index).items()}) + detail["widgets_total"] = run.widgets_total + detail["widgets_executed"] = run.widgets_executed if run.scope is not None: detail["scope"] = {"scope": run.scope.scope, "kpi_only": run.scope.kpi_only, "all_empty": run.scope.all_empty} if run.chat_error is not None: diff --git a/packages/gooddata-eval/tests/test_chat_summary_evaluator.py b/packages/gooddata-eval/tests/test_chat_summary_evaluator.py index 69c4769b4..920c80484 100644 --- a/packages/gooddata-eval/tests/test_chat_summary_evaluator.py +++ b/packages/gooddata-eval/tests/test_chat_summary_evaluator.py @@ -167,6 +167,13 @@ def test_each_turn_is_graded_with_its_own_expectation(monkeypatch: pytest.Monkey assert [c for _, c in judges[0].seen] == ["summary criterion", "follow-up"] # the follow-up keeps no default shape: only the summary turn is shape-checked assert "shape_ok" not in run.evaluations[1].detail + # the run detail is flat: the summary turn at the top, follow-ups prefixed + detail = kind._run_detail(run) + assert detail["actual_output"] == CHART + assert detail["must_include_0"] is True + assert detail["followup_1_actual_output"] == "Tracy Carter leads." + assert detail["followup_1_no_tool_calls"] is True + assert "followup_1_shape_ok" not in detail def test_the_scope_reads_kpis_and_empty_results_from_the_referenced_widgets() -> None: From 5e00ea6e9d5b2034729fa2a5066cdcd450081eab Mon Sep 17 00:00:00 2001 From: duongthimy Date: Fri, 20 Nov 2026 18:34:20 +0100 Subject: [PATCH 3/5] test(gooddata-eval): cover agentic_summary and describe in dispatch checks - add both kinds to the registry coverage lists (dispatch, user context relay, trace linker) - agentic_summary items carry their summary input and context; a missing context is rejected before the run - agentic_describe_object asks with no context, like the Analyze page jira: QA-30179 risk: nonprod --- .../tests/test_agentic_runner.py | 46 ++++++++++++++++--- .../gooddata-eval/tests/test_trace_linker.py | 2 + 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 26abd44e2..18083230c 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -107,12 +107,27 @@ def test_dispatch_agentic_omits_agent_id_by_default(): ), ("agentic_anomaly_detection", {"metric": "metric/spend"}, "evaluate_agentic_anomaly_detection"), ("agentic_what_if", {"metric_id": "spend"}, "evaluate_agentic_what_if"), + ("agentic_summary", {"must_include": ["Names the period."]}, "evaluate_agentic_summary"), + ("agentic_describe_object", {"must_include": ["Names the chart type."]}, "evaluate_agentic_describe_object"), ] # The one kind whose dispatch needs more than question/expected_output: the dashboard to # summarize is named in summary_input, exactly as for the single-shot dashboard_summary. _SUMMARY_INPUT_KINDS = {"agentic_dashboard_summary"} +# agentic_summary needs its stored executions and the UI's context: without either the item +# cannot be sent the way the UI sends it. +_AGENTIC_SUMMARY_FIELDS: dict[str, Any] = { + "agentic_summary_input": {"scope": "widget"}, + "user_context": {"view": {"dashboard": {"widgets": []}}}, +} + + +def _kind_fields(kind: str) -> dict[str, Any]: + if kind in _SUMMARY_INPUT_KINDS: + return {"summary_input": {"dashboard_id": "dash-1"}} + return _AGENTIC_SUMMARY_FIELDS if kind == "agentic_summary" else {} + def test_all_agentic_kind_cases_covers_every_registered_kind(): """Guards the two parametrized tests below against silently going stale: a kind added @@ -130,7 +145,7 @@ def test_dispatch_agentic_passes_agent_id_through_for_every_kind(kind, expected_ test_kind=kind, question="q", expected_output=expected_output, - summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None, + **_kind_fields(kind), ) with patch(f"gooddata_eval.cli.agentic_runner.{target}") as mock_eval: _dispatch_agentic( @@ -272,7 +287,7 @@ def test_dispatch_agentic_returns_a_real_outcome_for_every_kind(kind, expected_o test_kind=kind, question="q", expected_output=expected_output, - summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None, + **_kind_fields(kind), ) canned = AgenticEvalOutcome(reasoning_steps=["x"], conversation_id="c1", response_id="r1", detail={"k": "v"}) with patch(f"gooddata_eval.cli.agentic_runner.{target}", return_value=canned) as mock_eval: @@ -895,6 +910,8 @@ def test_an_errored_item_without_timings_keeps_its_zero_defaults(): ("agentic_forecasting", "evaluate_agentic_forecasting", {}), ("agentic_dashboard_summary", "evaluate_agentic_dashboard_summary", {}), ("agentic_conversation", "evaluate_agentic_conversation", {"id": "c1", "expected_skills": [], "turns": []}), + ("agentic_summary", "evaluate_agentic_summary", {}), + ("agentic_describe_object", "evaluate_agentic_describe_object", {}), ] @@ -917,7 +934,21 @@ def test_dispatch_agentic_passes_user_context_through_to_every_kind( expected_output=expected_output, user_context=user_context, summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None, + agentic_summary_input={"scope": "widget"} if kind == "agentic_summary" else None, ) + if kind == "agentic_summary" and user_context is None: + with pytest.raises(ValueError, match="needs both"): + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + ) + return with patch(f"gooddata_eval.cli.agentic_runner.{evaluator}") as mock_eval: _dispatch_agentic( item, @@ -937,8 +968,9 @@ def test_dispatch_agentic_passes_user_context_through_to_every_kind( # agentic_dashboard_summary builds its own context from the dashboard it summarises and # takes its scope from summary_input, so relaying the item's context would override the -# thing under test rather than add to it. Excluded on purpose, not by omission. -_KINDS_WITHOUT_USER_CONTEXT = {"agentic_dashboard_summary"} +# thing under test rather than add to it. agentic_describe_object asks with no context, as +# the Analyze-page chat does. Excluded on purpose, not by omission. +_KINDS_WITHOUT_USER_CONTEXT = {"agentic_dashboard_summary", "agentic_describe_object"} class _Stop(Exception): @@ -1120,7 +1152,7 @@ def test_failed_runs_reaches_the_report_for_every_kind(kind, expected_output, ta test_kind=kind, question="q", expected_output=expected_output, - summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None, + **_kind_fields(kind), ) canned = AgenticEvalOutcome(reasoning_steps=["x"], detail={"k": "v"}, failed_runs=[_A_FAILED_RUN]) with patch(f"gooddata_eval.cli.agentic_runner.{target}", return_value=canned): @@ -1269,7 +1301,7 @@ def test_run_latencies_reach_the_report_for_every_kind(kind, expected_output, ta test_kind=kind, question="q", expected_output=expected_output, - summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None, + **_kind_fields(kind), ) canned = AgenticEvalOutcome(reasoning_steps=["x"], detail={"k": "v"}, run_latencies_s=[4.5, 6.0]) with patch(f"gooddata_eval.cli.agentic_runner.{target}", return_value=canned): @@ -1340,7 +1372,7 @@ def test_every_kind_gets_its_chat_turns_recorded_as_trace_latencies(kind, expect test_kind=kind, question="q", expected_output=expected_output, - summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None, + **_kind_fields(kind), ) with patch(f"gooddata_eval.cli.agentic_runner.{target}", side_effect=_noting(1.5, 2.5, 0.5)): report = run_agentic_items([item], host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01") diff --git a/packages/gooddata-eval/tests/test_trace_linker.py b/packages/gooddata-eval/tests/test_trace_linker.py index cf60029f9..7b4ee62c2 100644 --- a/packages/gooddata-eval/tests/test_trace_linker.py +++ b/packages/gooddata-eval/tests/test_trace_linker.py @@ -124,6 +124,8 @@ def test_run_trace_link_inline_runs_the_task_on_the_calling_thread(): ("dashboard_summary", "evaluate_agentic_dashboard_summary"), ("anomaly_detection", "evaluate_agentic_anomaly_detection"), ("what_if", "evaluate_agentic_what_if"), + ("chat_summary", "evaluate_agentic_summary"), + ("describe_object", "evaluate_agentic_describe_object"), ] From 696dbf89bd7d2c049ccf59ddc4df61de91d17e72 Mon Sep 17 00:00:00 2001 From: duongthimy Date: Fri, 20 Nov 2026 21:22:34 +0100 Subject: [PATCH 4/5] fix(gooddata-eval): tell a KPI from a measures-only chart - a KPI (headline) has one result dimension holding only measureGroup - a donut, waterfall or bullet of measures only has no attributes either, but a second dimension; it gets the chart shape check - no result spec: no attributes still means KPI jira: QA-30179 risk: low --- .../core/agentic/chat_summary.py | 21 ++++++++++++++----- .../tests/test_chat_summary_evaluator.py | 10 +++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/chat_summary.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/chat_summary.py index 3181c5a04..642a1c958 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/chat_summary.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/chat_summary.py @@ -129,17 +129,28 @@ def referenced_widget_ids(user_context: dict[str, Any]) -> list[str]: ] +def is_kpi(execution: WidgetExecution) -> bool: + """A KPI (headline): no attributes and one result dimension holding only the measures. + + A chart of measures only (donut, waterfall, bullet) has no attributes either, but lays the + measures out on a second dimension. Without a result spec, no attributes is all there is. + """ + if execution.afm.get("attributes"): + return False + dims = execution.result_spec.get("dimensions") + if not isinstance(dims, list) or not dims: + return True + return len(dims) == 1 and isinstance(dims[0], dict) and dims[0].get("itemIdentifiers") == ["measureGroup"] + + def summary_scope( summary_input: AgenticSummaryInput, user_context: dict[str, Any], with_data: set[str] ) -> SummaryScope: - """What the summarized widgets are: the referenced ones for a widget summary, all for a dashboard. - - A KPI is an execution without attributes: one value, no breakdown. - """ + """What the summarized widgets are: the referenced ones for a widget summary, all for a dashboard.""" executions = summary_input.executions in_scope = referenced_widget_ids(user_context) if summary_input.scope == "widget" else list(executions) in_scope = [w for w in in_scope if w in executions] - kpi_only = bool(in_scope) and all(not executions[w].afm.get("attributes") for w in in_scope) + kpi_only = bool(in_scope) and all(is_kpi(executions[w]) for w in in_scope) return SummaryScope( scope=summary_input.scope, kpi_only=kpi_only, diff --git a/packages/gooddata-eval/tests/test_chat_summary_evaluator.py b/packages/gooddata-eval/tests/test_chat_summary_evaluator.py index 920c80484..4cd9c5fd8 100644 --- a/packages/gooddata-eval/tests/test_chat_summary_evaluator.py +++ b/packages/gooddata-eval/tests/test_chat_summary_evaluator.py @@ -185,6 +185,16 @@ def test_the_scope_reads_kpis_and_empty_results_from_the_referenced_widgets() -> context = {"referencedObjects": [{"objects": [{"type": "WIDGET", "id": "k"}]}]} assert kind.summary_scope(inp, context, with_data={"k"}) == cs.SummaryScope("widget", True, False) assert kind.summary_scope(inp, context, with_data={"c"}).all_empty + # a chart of measures only (donut, waterfall) puts the measures on a second dimension: not a KPI + breakdown = { + "afm": {"attributes": [], "measures": [{"localIdentifier": "m"}, {"localIdentifier": "n"}], "filters": []}, + "result_spec": {"dimensions": [{"itemIdentifiers": []}, {"itemIdentifiers": ["measureGroup"]}]}, + } + headline = {"afm": kpi_afm, "result_spec": {"dimensions": [{"itemIdentifiers": ["measureGroup"]}]}} + shapes = AgenticSummaryInput.model_validate({"scope": "widget", "executions": {"k": headline, "b": breakdown}}) + assert kind.summary_scope(shapes, context, with_data={"k"}).kpi_only + donut = {"referencedObjects": [{"objects": [{"type": "WIDGET", "id": "b"}]}]} + assert not kind.summary_scope(shapes, donut, with_data={"b"}).kpi_only dashboard = inp.model_copy(update={"scope": "dashboard"}) assert kind.summary_scope(dashboard, {}, with_data={"c"}) == cs.SummaryScope("dashboard", False, False) From 636fdcccac4cd85963a0eadbe4438f5ac6333a45 Mon Sep 17 00:00:00 2001 From: duongthimy Date: Fri, 20 Nov 2026 22:56:14 +0100 Subject: [PATCH 5/5] fix(gooddata-eval): check ids on the summary turn only - the summary prompt forbids reference tags; a follow-up is an ordinary chat answer whose tags the UI renders as links - follow-ups keep the empty-answer, tool-call and judge checks jira: QA-30179 risk: low --- .../core/evaluators/chat_summary.py | 16 +++++++++------- .../tests/test_chat_summary_evaluator.py | 2 ++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/chat_summary.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/chat_summary.py index ec34b8483..040a8117f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/chat_summary.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/chat_summary.py @@ -19,7 +19,8 @@ dashboard whose widgets all returned no data one paragraph without headings; an empty widget has no fixed shape. A `format` object replaces the default: `max_sentences`, `no_headings`, `no_bullets`, `headings`. -- no ids: no `{visualization/...}` reference tag and no raw object id in the answer. +- no ids (summary turn only): no `{visualization/...}` reference tag and no raw object id. A + follow-up is answered as an ordinary chat message, whose reference tags the UI renders as links. - no figures on no data: when every widget in scope returned no data, the answer quotes no amount, share or count. - `no_tool_calls`: the turn answers from the conversation, without running a tool. @@ -164,7 +165,7 @@ def evaluate_turn( positive_judge: LLMJudge, violation_judge: LLMJudge, ) -> ItemEvaluation: - """Grade one turn. `scope` is None for a follow-up: only the summary turn has a default shape.""" + """Grade one turn. `scope` is None for a follow-up: only the summary turn has a shape and the no-ids rule.""" detail: dict[str, Any] = {"actual_output": answer} passed = True if not answer.strip(): @@ -179,11 +180,12 @@ def evaluate_turn( detail["shape_problem"] = problem passed = False - problem = id_problem(answer) - detail["no_ids"] = problem is None - if problem: - detail["id_problem"] = problem - passed = False + if scope is not None: + problem = id_problem(answer) + detail["no_ids"] = problem is None + if problem: + detail["id_problem"] = problem + passed = False if scope is not None and scope.all_empty: quoted = figures(answer) diff --git a/packages/gooddata-eval/tests/test_chat_summary_evaluator.py b/packages/gooddata-eval/tests/test_chat_summary_evaluator.py index 4cd9c5fd8..07ab3d25b 100644 --- a/packages/gooddata-eval/tests/test_chat_summary_evaluator.py +++ b/packages/gooddata-eval/tests/test_chat_summary_evaluator.py @@ -124,6 +124,8 @@ def test_ids_never_reach_the_answer(leak: str) -> None: def test_a_follow_up_without_tool_calls_has_no_default_shape() -> None: answer = "Tracy Carter leads, about $15.7 K ahead of the second place." assert _grade(answer, {"no_tool_calls": True}, None, tools=["set_skills"]).passed + # a follow-up may link objects: the UI renders its reference tags + assert _grade("Driven by {metric/active_customers}.", {}, None).detail.get("no_ids") is None ran = _grade(answer, {"no_tool_calls": True}, None, tools=["execute_existing_visualizations"]) assert not ran.passed assert ran.detail["tools_run"] == ["execute_existing_visualizations"]