Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion packages/gooddata-eval/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -752,7 +752,10 @@ Item-level keys, all optional except the candidates:
- `format_intents` — per metric uri, how the metric's numbers must read, for example
`{"metric/net_sales": {"divide_by": 1000000, "decimals": 1, "suffix": "M"}}`.
- `requires_visualization_part` — the chart must come back as a part of the answer, not only as
the arguments of the chart tool call.
the arguments of the chart tool call. Without it, an answer with no chart part is scored on the
arguments of the last chart tool call, unless the backend rejected that definition as
unbuildable (not comparable to the dimensionality, or no witness) or the call was a lookup
(`show_to_user: false`): then the answer has no chart.
- `date_filter_from_context` — `true` or `{"dataset": "<date dataset>"}`: a candidate without a
date filter also accepts the chart carrying a date filter of the dashboard in the item's
`user_context`. See *Date filter carried over from the dashboard* below.
Expand Down
69 changes: 53 additions & 16 deletions packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
normalize_reasoning_effort,
)
from gooddata_eval.core.langfuse.item_scope import JoinedRun, baggage_entries, current_scope, mark_joined, trace_ids_for
from gooddata_eval.core.models import ChatResult, DatasetItem
from gooddata_eval.core.models import ChatResult, DatasetItem, ToolCallEvent
from gooddata_eval.core.timing import note_trace_latency

_log = logging.getLogger(__name__)
Expand All @@ -58,6 +58,12 @@
# Stands in for the `id` a persisted visualization would carry, on the fallback path
# where the agent's create_adhoc_visualization call failed and only its arguments survive.
ADHOC_VIZ_ID = "adhoc-visualization-not-persisted"
# Messages of a create_adhoc_visualization error that reject the chart itself, not the
# environment: the backend found the definition unbuildable against the workspace's model.
# Kept equal to gen-ai's `_COMPATIBILITY_REJECTIONS` in
# app/infrastructure/services/tools/create_adhoc_visualization_tool.py, which raises them.
# A rejected call's arguments are no answer, so the fallback below never scores them.
_UNBUILDABLE_VIZ_MARKERS: tuple[str, ...] = ("is not comparable to the dimensionality", "has no witness")

_KNOWN_PART_TYPES: frozenset[str] = frozenset(
{
Expand Down Expand Up @@ -210,7 +216,9 @@ class _SseAccumulator:
tool_call_events: list[dict[str, Any]] = field(default_factory=list)
call_id_to_event_index: dict[str, int] = field(default_factory=dict)
reasoning_steps: list[dict[str, Any]] = field(default_factory=list)
adhoc_viz_args: list[dict[str, Any]] = field(default_factory=list)
# (tool_call_events index, visualization argument, lookup) of each create_adhoc_visualization
# call; lookup is `show_to_user: false`, a chart the agent built only to read a value itself.
adhoc_viz_args: list[tuple[int, dict[str, Any], bool]] = field(default_factory=list)
response_id: str | None = None
trace_id: str | None = None
stream_ended: bool = False
Expand Down Expand Up @@ -277,9 +285,10 @@ def _handle_tool_call(content: dict[str, Any], acc: _SseAccumulator) -> None:
# Stash visualization definition from create_adhoc_visualization so we can
# evaluate the agent's intended answer even when the data source call fails.
if content.get("name") == "create_adhoc_visualization":
viz = (content.get("arguments") or {}).get("visualization")
arguments = content.get("arguments") or {}
viz = arguments.get("visualization")
if viz and isinstance(viz, dict):
acc.adhoc_viz_args.append(viz)
acc.adhoc_viz_args.append((idx, viz, arguments.get("show_to_user") is False))


def _handle_tool_result(content: dict[str, Any], acc: _SseAccumulator) -> None:
Expand All @@ -290,6 +299,15 @@ def _handle_tool_result(content: dict[str, Any], acc: _SseAccumulator) -> None:
acc.tool_call_events[idx]["result_ts"] = round(time.monotonic() - acc.t0, 3)


def _rejected_as_unbuildable(event: dict[str, Any]) -> bool:
"""Whether a create_adhoc_visualization result is an error rejecting the chart's definition."""
result = ToolCallEvent.model_validate(event).parsed_result()
if not isinstance(result, dict) or result.get("status") != "error":
return False
message = str(result.get("message") or "")
return any(marker in message for marker in _UNBUILDABLE_VIZ_MARKERS)


def _build_chat_result(acc: _SseAccumulator) -> ChatResult:
if not acc.text_parts and (acc.search_results or acc.unhandled_parts):
_log.warning(
Expand All @@ -315,24 +333,43 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult:
"reasoning": "\n".join(acc.viz_reasoning_parts),
}
elif acc.adhoc_viz_args:
# Fallback: the agent produced a correct visualization definition via
# create_adhoc_visualization but the call failed (e.g. data source not
# accessible). The last attempt is the agent's best answer.
# Fallback: the agent produced a visualization definition via
# create_adhoc_visualization but the answer carries no chart part, e.g. because the
# call failed for an environment reason (data source not accessible). The LAST call is
# the agent's answer; an earlier one may be a lookup it never meant to show. If the
# backend rejected that last definition as unbuildable, the turn has no chart at all --
# never fall back to an earlier call. Nor when that last call is itself a lookup
# (`show_to_user: false`, gen-ai's flag for a chart built only to read a value): the user
# never saw it, so it is no answer either.
#
# These are raw tool-call arguments, so they carry no `id` -- the server
# mints it. The sentinel marks the chart as never persisted, and goes last so
# that an `id` in the arguments cannot hide that. Logged because
# a turn that called the tool and answered without a chart is otherwise
# invisible: the user saw no chart either.
_log.warning(
"create_adhoc_visualization was called %d time(s) but the answer has no visualization part; "
"scoring the last call's arguments",
len(acc.adhoc_viz_args),
)
payload["createdVisualizations"] = {
"objects": [{**acc.adhoc_viz_args[-1], "id": ADHOC_VIZ_ID}],
"reasoning": "\n".join(acc.viz_reasoning_parts),
}
last_index, last_viz, last_is_lookup = acc.adhoc_viz_args[-1]
if last_is_lookup:
_log.warning(
"create_adhoc_visualization was called %d time(s) and the last call was a lookup "
"(show_to_user: false); scoring no chart",
len(acc.adhoc_viz_args),
)
elif _rejected_as_unbuildable(acc.tool_call_events[last_index]):
_log.warning(
"create_adhoc_visualization was called %d time(s) and the backend rejected the last definition "
"as unbuildable; scoring no chart",
len(acc.adhoc_viz_args),
)
else:
_log.warning(
"create_adhoc_visualization was called %d time(s) but the answer has no visualization part; "
"scoring the last call's arguments",
len(acc.adhoc_viz_args),
)
payload["createdVisualizations"] = {
"objects": [{**last_viz, "id": ADHOC_VIZ_ID}],
"reasoning": "\n".join(acc.viz_reasoning_parts),
}
result = ChatResult.model_validate(payload)
result.response_id = acc.response_id
result.trace_id = acc.trace_id
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,9 @@ class ExpectationOptions:
to match and no ``must_not_match`` pattern may.
- ``format_intents``: per metric uri, how its numbers must read (see ``number_format``).
- ``requires_visualization_part``: the chart must come back as a part of the answer. Without
it, a chart that exists only in the tool arguments is scored like one the user saw.
it, a chart that exists only in the tool arguments is scored like one the user saw -- the
last call's arguments, unless the backend rejected that definition as unbuildable or the
call was a lookup (``show_to_user: false``), in which case the answer has no chart.
- ``date_filter_from_context``: the chart may carry the date filter of the dashboard in the
item's ``user_context`` instead of none (see ``ContextDateFilter``). For a question that
names no period asked from an open dashboard, where carrying the dashboard's period over
Expand Down
72 changes: 71 additions & 1 deletion packages/gooddata-eval/tests/test_agentic_visualization.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
All tests mock ChatClient so no network is needed.
"""

import json
from collections.abc import Callable
from dataclasses import replace
from typing import Any
Expand All @@ -18,7 +19,7 @@
evaluate_agentic_visualization,
run_agentic_visualization,
)
from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError
from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError, parse_sse_lines
from gooddata_eval.core.evaluators.visualization import ExpectationOptions
from gooddata_eval.core.models import ChatResult, CreatedVisualization, LoopExit

Expand Down Expand Up @@ -116,6 +117,75 @@ def test_execute_single_run_clarification_then_viz(monkeypatch):
assert client.send_message.call_args_list[1] == call("conv-1", "Revenue please")


def test_execute_single_run_continues_after_a_turn_whose_chart_the_backend_rejected(monkeypatch):
"""A turn whose only chart the backend rejected as unbuildable produced no chart, so the run
goes on to a simulated reply instead of ending as if one was built."""
rejected = {
"type": "column_chart",
"query": {"fields": {"m": {"using": "metric/revenue_preagg"}, "d": {"using": "label/merchant"}}},
"metrics": ["m"],
"view_by": ["d"],
}
first_turn = parse_sse_lines(
[
"data: "
+ json.dumps(
{
"item": {
"role": "assistant",
"content": {
"type": "toolCall",
"callId": "c1",
"name": "create_adhoc_visualization",
"arguments": {"visualization": rejected},
},
}
}
),
"data: "
+ json.dumps(
{
"item": {
"role": "tool",
"content": {
"type": "toolResult",
"callId": "c1",
"result": json.dumps(
{"status": "error", "message": "... is not comparable to the dimensionality ..."}
),
},
}
}
),
"data: "
+ json.dumps(
{
"item": {
"role": "assistant",
"content": {
"type": "multipart",
"parts": [{"type": "text", "text": "I could not chart that."}],
},
}
}
),
]
)
client = MagicMock()
client.send_message.side_effect = [first_turn, _chat_with_viz()]
monkeypatch.setattr(
"gooddata_eval.core.agentic.visualization.generate_simulated_response",
lambda msg, exp: "Use the transaction-level metric",
)

result = _execute_single_run(client, "conv-1", "Revenue by merchant", [_expected()])

assert client.send_message.call_count == 2
assert client.send_message.call_args_list[1] == call("conv-1", "Use the transaction-level metric")
assert result.total_turns == 2
assert result.eval_result.visualization_created is True


def test_execute_single_run_no_viz_no_text():
"""Agent returns nothing — visualization_created is False."""
client = MagicMock()
Expand Down
117 changes: 116 additions & 1 deletion packages/gooddata-eval/tests/test_sse_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -373,7 +373,7 @@ def test_parse_sse_lines_falls_back_to_adhoc_viz_when_multipart_viz_is_null():
# agent calls create_adhoc_visualization — stash the viz
f'data: {{"item": {{"role": "assistant", "content": {{"type": "toolCall", "callId": "c1", "name": "create_adhoc_visualization", "arguments": {{"visualization": {json.dumps(viz_def)}}}}}}}}}',
# data source fails
'data: {"item": {"role": "tool", "content": {"type": "toolResult", "callId": "c1", "result": "{"status": "error", "message": "Data source does not exist"}"}}}',
'data: {"item": {"role": "tool", "content": {"type": "toolResult", "callId": "c1", "result": "{\\"status\\": \\"error\\", \\"message\\": \\"Data source does not exist\\"}"}}}',
# final multipart — visualization is null
'data: {"item": {"role": "assistant", "content": {"type": "multipart", "parts": [{"type": "text", "text": "Could not create"}, {"type": "visualization", "visualization": null}]}}}',
]
Expand All @@ -383,6 +383,121 @@ def test_parse_sse_lines_falls_back_to_adhoc_viz_when_multipart_viz_is_null():
assert result.created_visualizations.objects[0].metrics == ["m"]


def _sse_item(role: str, content: dict[str, Any]) -> str:
return "data: " + json.dumps({"item": {"role": role, "content": content}})


def _adhoc_call(call_id: str, viz: dict[str, Any]) -> str:
return _sse_item(
"assistant",
{
"type": "toolCall",
"callId": call_id,
"name": "create_adhoc_visualization",
"arguments": {"visualization": viz},
},
)


def _adhoc_result(call_id: str, result: dict[str, Any]) -> str:
return _sse_item("tool", {"type": "toolResult", "callId": call_id, "result": json.dumps(result)})


_NOT_COMPARABLE = {
"status": "error",
"message": "Aggregation dimension='[attribute/industry_name]' is not comparable to the "
"dimensionality='[dataset/fact_cuta_daily_summable_metrics]' of the subtree",
}
_NO_WITNESS = {"status": "error", "message": "Attribute 'label/merchant_name' has no witness in the subtree"}
_TEXT_ONLY_ANSWER = _sse_item(
"assistant", {"type": "multipart", "parts": [{"type": "text", "text": "Here is the data."}]}
)


def _viz(metric: str) -> dict[str, Any]:
return {
"type": "bar_chart",
"query": {"fields": {"m": {"using": metric}, "d": {"using": "label/industry_name"}}, "filter_by": {}},
"metrics": ["m"],
"view_by": ["d"],
}


@pytest.mark.parametrize("rejection", [_NOT_COMPARABLE, _NO_WITNESS], ids=["not-comparable", "no-witness"])
def test_parse_sse_lines_adhoc_fallback_scores_no_chart_for_a_rejected_definition(rejection):
"""A definition the backend refused as unbuildable is no answer, so the turn has no chart."""
lines = [_adhoc_call("c1", _viz("metric/cnp_share_-_cuta")), _adhoc_result("c1", rejection), _TEXT_ONLY_ANSWER]

assert parse_sse_lines(lines).created_visualizations is None


def test_parse_sse_lines_adhoc_fallback_scores_no_chart_after_two_rejections():
lines = [
_adhoc_call("c1", _viz("metric/cnp_share_-_cuta")),
_adhoc_result("c1", _NOT_COMPARABLE),
_adhoc_call("c2", _viz("metric/cnp_share_-_cutcgco")),
_adhoc_result("c2", _NO_WITNESS),
_TEXT_ONLY_ANSWER,
]

assert parse_sse_lines(lines).created_visualizations is None


def test_parse_sse_lines_adhoc_fallback_never_scores_an_earlier_accepted_lookup():
"""Only the last call is the agent's answer: an accepted lookup before a rejected chart is not."""
lines = [
_adhoc_call("c1", _viz("metric/transaction_count_-_txn_-_cuta")),
_adhoc_result("c1", {"status": "success", "ref": "viz-lookup"}),
_adhoc_call("c2", _viz("metric/cnp_share_-_cuta")),
_adhoc_result("c2", _NOT_COMPARABLE),
_TEXT_ONLY_ANSWER,
]

assert parse_sse_lines(lines).created_visualizations is None


def test_parse_sse_lines_adhoc_fallback_still_scores_a_definition_an_environment_error_stopped():
"""An error about the environment, not the definition, leaves the arguments as the answer."""
lines = [
_adhoc_call("c1", _viz("metric/cnp_share_-_cuta")),
_adhoc_result("c1", {"status": "error", "message": "Data source does not exist"}),
_TEXT_ONLY_ANSWER,
]
result = parse_sse_lines(lines)

assert result.created_visualizations is not None
assert result.created_visualizations.objects[0].query.fields["m"].using == "metric/cnp_share_-_cuta"


def test_parse_sse_lines_adhoc_fallback_scores_no_chart_when_the_last_call_is_a_lookup():
"""A last call with `show_to_user: false` is a chart the agent built for itself, not an answer."""
lookup = _sse_item(
"assistant",
{
"type": "toolCall",
"callId": "c1",
"name": "create_adhoc_visualization",
"arguments": {"visualization": _viz("metric/latest_date"), "show_to_user": False},
},
)
lines = [lookup, _adhoc_result("c1", {"status": "success", "ref": "viz-lookup"}), _TEXT_ONLY_ANSWER]

assert parse_sse_lines(lines).created_visualizations is None


def test_parse_sse_lines_adhoc_fallback_matches_out_of_order_results_by_call_id():
"""Parallel calls whose results arrive in reverse order: the last call's own result decides."""
lines = [
_adhoc_call("c1", _viz("metric/transaction_count_-_txn_-_cuta")),
_adhoc_call("c2", _viz("metric/cnp_share_-_cuta")),
_adhoc_result("c2", _NOT_COMPARABLE),
_adhoc_result("c1", {"status": "success", "ref": "viz-1"}),
_TEXT_ONLY_ANSWER,
]

assert parse_sse_lines(lines).created_visualizations is None


def test_parse_sse_lines_adhoc_fallback_synthesizes_id_when_args_have_none(caplog):
"""A create_adhoc_visualization definition carries no `id` -- the server mints it.

Expand Down
Loading