Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions packages/gooddata-eval/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,10 @@ its own shape. `test_kind` on the item is what labels the result, not the evalua
which is why `knowledge_question` can reuse `GeneralQuestionEvaluator` verbatim.
`dashboard_summary` items additionally need `summary_input`, `document_takeaway` items
`takeaway_input`, `catalog_metadata` items `catalog_input`.
`agentic_summary` items need `agentic_summary_input` (scope, one stored execution per widget,
follow-ups) plus `user_context`: the UI's context without result ids, which expire, so the kind
re-executes the stored executions on every run. Their `question` is a readable scenario line the
judge reads; the text sent to the agent is `agentic_summary_input.question` or the UI's own.

## Running the pipeline

Expand Down
40 changes: 40 additions & 0 deletions packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,11 @@
from gooddata_eval.core.agentic._trace_linker import BackgroundTraceLinker, SubmitTraceLink, run_trace_link_inline
from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill
from gooddata_eval.core.agentic.anomaly_detection import evaluate_agentic_anomaly_detection
from gooddata_eval.core.agentic.chat_summary import evaluate_agentic_summary
from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation
from gooddata_eval.core.agentic.dashboard_skill import evaluate_agentic_dashboard_skill
from gooddata_eval.core.agentic.dashboard_summary import evaluate_agentic_dashboard_summary
from gooddata_eval.core.agentic.describe_object import evaluate_agentic_describe_object
from gooddata_eval.core.agentic.document_skill import evaluate_agentic_document_skill
from gooddata_eval.core.agentic.forecasting import evaluate_agentic_forecasting
from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question
Expand Down Expand Up @@ -63,6 +65,8 @@ class _LfKw(TypedDict, total=False):
"agentic_dashboard_summary",
"agentic_anomaly_detection",
"agentic_what_if",
"agentic_summary",
"agentic_describe_object",
}
)

Expand All @@ -87,13 +91,18 @@ class _LfKw(TypedDict, total=False):
# vis_agentic, agentic_visualization visualizations come back as AAC proposals
# in the chat response; nothing is persisted
# and neither module has cleanup code
# agentic_summary executes stored AFMs (a read) and asks for a
# summary and follow-ups about the data shown
# agentic_describe_object one question answered from workspace lookups
PARALLEL_SAFE_TEST_KINDS = frozenset(
{
"agentic_general_question",
"agentic_guardrail",
"agentic_search",
"vis_agentic",
"agentic_visualization",
"agentic_summary",
"agentic_describe_object",
}
)

Expand Down Expand Up @@ -359,6 +368,37 @@ def _dispatch_agentic(
agent_id=agent_id,
**lf_kw,
)
elif kind == "agentic_summary":
agentic_summary_input = item.agentic_summary_input
if agentic_summary_input is None or item.user_context is None:
raise ValueError(f"agentic_summary item '{item.id}' needs both 'agentic_summary_input' and 'user_context'.")
return evaluate_agentic_summary(
host=host,
token=token,
workspace_id=workspace_id,
# The item's question is its readable scenario line, read by the judge; the text
# sent to the agent is agentic_summary_input.question or the UI's pre-filled one.
scenario=item.question,
summary_input=agentic_summary_input,
user_context=item.user_context,
expected_output=eo if isinstance(eo, dict) else {},
k=k,
gate=gate,
agent_id=agent_id,
**lf_kw,
)
elif kind == "agentic_describe_object":
return evaluate_agentic_describe_object(
host=host,
token=token,
workspace_id=workspace_id,
question=item.question,
expected_output=eo if isinstance(eo, dict) else {},
k=k,
gate=gate,
agent_id=agent_id,
**lf_kw,
)
elif kind == "agentic_what_if":
return evaluate_agentic_what_if(
host=host,
Expand Down
Loading
Loading