diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index 9749259d1..449fac2d2 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -752,7 +752,10 @@ Item-level keys, all optional except the candidates: - `format_intents` — per metric uri, how the metric's numbers must read, for example `{"metric/net_sales": {"divide_by": 1000000, "decimals": 1, "suffix": "M"}}`. - `requires_visualization_part` — the chart must come back as a part of the answer, not only as - the arguments of the chart tool call. + the arguments of the chart tool call. Without it, an answer with no chart part is scored on the + arguments of the last chart tool call, unless the backend rejected that definition as + unbuildable (not comparable to the dimensionality, or no witness) or the call was a lookup + (`show_to_user: false`): then the answer has no chart. - `date_filter_from_context` — `true` or `{"dataset": ""}`: a candidate without a date filter also accepts the chart carrying a date filter of the dashboard in the item's `user_context`. See *Date filter carried over from the dashboard* below. diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index 78a7388aa..dd773dd0a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -33,7 +33,7 @@ normalize_reasoning_effort, ) from gooddata_eval.core.langfuse.item_scope import JoinedRun, baggage_entries, current_scope, mark_joined, trace_ids_for -from gooddata_eval.core.models import ChatResult, DatasetItem +from gooddata_eval.core.models import ChatResult, DatasetItem, ToolCallEvent from gooddata_eval.core.timing import note_trace_latency _log = logging.getLogger(__name__) @@ -58,6 +58,12 @@ # Stands in for the `id` a persisted visualization would carry, on the fallback path # where the agent's create_adhoc_visualization call failed and only its arguments survive. ADHOC_VIZ_ID = "adhoc-visualization-not-persisted" +# Messages of a create_adhoc_visualization error that reject the chart itself, not the +# environment: the backend found the definition unbuildable against the workspace's model. +# Kept equal to gen-ai's `_COMPATIBILITY_REJECTIONS` in +# app/infrastructure/services/tools/create_adhoc_visualization_tool.py, which raises them. +# A rejected call's arguments are no answer, so the fallback below never scores them. +_UNBUILDABLE_VIZ_MARKERS: tuple[str, ...] = ("is not comparable to the dimensionality", "has no witness") _KNOWN_PART_TYPES: frozenset[str] = frozenset( { @@ -210,7 +216,9 @@ class _SseAccumulator: tool_call_events: list[dict[str, Any]] = field(default_factory=list) call_id_to_event_index: dict[str, int] = field(default_factory=dict) reasoning_steps: list[dict[str, Any]] = field(default_factory=list) - adhoc_viz_args: list[dict[str, Any]] = field(default_factory=list) + # (tool_call_events index, visualization argument, lookup) of each create_adhoc_visualization + # call; lookup is `show_to_user: false`, a chart the agent built only to read a value itself. + adhoc_viz_args: list[tuple[int, dict[str, Any], bool]] = field(default_factory=list) response_id: str | None = None trace_id: str | None = None stream_ended: bool = False @@ -277,9 +285,10 @@ def _handle_tool_call(content: dict[str, Any], acc: _SseAccumulator) -> None: # Stash visualization definition from create_adhoc_visualization so we can # evaluate the agent's intended answer even when the data source call fails. if content.get("name") == "create_adhoc_visualization": - viz = (content.get("arguments") or {}).get("visualization") + arguments = content.get("arguments") or {} + viz = arguments.get("visualization") if viz and isinstance(viz, dict): - acc.adhoc_viz_args.append(viz) + acc.adhoc_viz_args.append((idx, viz, arguments.get("show_to_user") is False)) def _handle_tool_result(content: dict[str, Any], acc: _SseAccumulator) -> None: @@ -290,6 +299,15 @@ def _handle_tool_result(content: dict[str, Any], acc: _SseAccumulator) -> None: acc.tool_call_events[idx]["result_ts"] = round(time.monotonic() - acc.t0, 3) +def _rejected_as_unbuildable(event: dict[str, Any]) -> bool: + """Whether a create_adhoc_visualization result is an error rejecting the chart's definition.""" + result = ToolCallEvent.model_validate(event).parsed_result() + if not isinstance(result, dict) or result.get("status") != "error": + return False + message = str(result.get("message") or "") + return any(marker in message for marker in _UNBUILDABLE_VIZ_MARKERS) + + def _build_chat_result(acc: _SseAccumulator) -> ChatResult: if not acc.text_parts and (acc.search_results or acc.unhandled_parts): _log.warning( @@ -315,24 +333,43 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult: "reasoning": "\n".join(acc.viz_reasoning_parts), } elif acc.adhoc_viz_args: - # Fallback: the agent produced a correct visualization definition via - # create_adhoc_visualization but the call failed (e.g. data source not - # accessible). The last attempt is the agent's best answer. + # Fallback: the agent produced a visualization definition via + # create_adhoc_visualization but the answer carries no chart part, e.g. because the + # call failed for an environment reason (data source not accessible). The LAST call is + # the agent's answer; an earlier one may be a lookup it never meant to show. If the + # backend rejected that last definition as unbuildable, the turn has no chart at all -- + # never fall back to an earlier call. Nor when that last call is itself a lookup + # (`show_to_user: false`, gen-ai's flag for a chart built only to read a value): the user + # never saw it, so it is no answer either. # # These are raw tool-call arguments, so they carry no `id` -- the server # mints it. The sentinel marks the chart as never persisted, and goes last so # that an `id` in the arguments cannot hide that. Logged because # a turn that called the tool and answered without a chart is otherwise # invisible: the user saw no chart either. - _log.warning( - "create_adhoc_visualization was called %d time(s) but the answer has no visualization part; " - "scoring the last call's arguments", - len(acc.adhoc_viz_args), - ) - payload["createdVisualizations"] = { - "objects": [{**acc.adhoc_viz_args[-1], "id": ADHOC_VIZ_ID}], - "reasoning": "\n".join(acc.viz_reasoning_parts), - } + last_index, last_viz, last_is_lookup = acc.adhoc_viz_args[-1] + if last_is_lookup: + _log.warning( + "create_adhoc_visualization was called %d time(s) and the last call was a lookup " + "(show_to_user: false); scoring no chart", + len(acc.adhoc_viz_args), + ) + elif _rejected_as_unbuildable(acc.tool_call_events[last_index]): + _log.warning( + "create_adhoc_visualization was called %d time(s) and the backend rejected the last definition " + "as unbuildable; scoring no chart", + len(acc.adhoc_viz_args), + ) + else: + _log.warning( + "create_adhoc_visualization was called %d time(s) but the answer has no visualization part; " + "scoring the last call's arguments", + len(acc.adhoc_viz_args), + ) + payload["createdVisualizations"] = { + "objects": [{**last_viz, "id": ADHOC_VIZ_ID}], + "reasoning": "\n".join(acc.viz_reasoning_parts), + } result = ChatResult.model_validate(payload) result.response_id = acc.response_id result.trace_id = acc.trace_id diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py index a0265ad12..c9d00cf1f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py @@ -287,7 +287,9 @@ class ExpectationOptions: to match and no ``must_not_match`` pattern may. - ``format_intents``: per metric uri, how its numbers must read (see ``number_format``). - ``requires_visualization_part``: the chart must come back as a part of the answer. Without - it, a chart that exists only in the tool arguments is scored like one the user saw. + it, a chart that exists only in the tool arguments is scored like one the user saw -- the + last call's arguments, unless the backend rejected that definition as unbuildable or the + call was a lookup (``show_to_user: false``), in which case the answer has no chart. - ``date_filter_from_context``: the chart may carry the date filter of the dashboard in the item's ``user_context`` instead of none (see ``ContextDateFilter``). For a question that names no period asked from an open dashboard, where carrying the dashboard's period over diff --git a/packages/gooddata-eval/tests/test_agentic_visualization.py b/packages/gooddata-eval/tests/test_agentic_visualization.py index 53108b52f..9859a66be 100644 --- a/packages/gooddata-eval/tests/test_agentic_visualization.py +++ b/packages/gooddata-eval/tests/test_agentic_visualization.py @@ -4,6 +4,7 @@ All tests mock ChatClient so no network is needed. """ +import json from collections.abc import Callable from dataclasses import replace from typing import Any @@ -18,7 +19,7 @@ evaluate_agentic_visualization, run_agentic_visualization, ) -from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError, parse_sse_lines from gooddata_eval.core.evaluators.visualization import ExpectationOptions from gooddata_eval.core.models import ChatResult, CreatedVisualization, LoopExit @@ -116,6 +117,75 @@ def test_execute_single_run_clarification_then_viz(monkeypatch): assert client.send_message.call_args_list[1] == call("conv-1", "Revenue please") +def test_execute_single_run_continues_after_a_turn_whose_chart_the_backend_rejected(monkeypatch): + """A turn whose only chart the backend rejected as unbuildable produced no chart, so the run + goes on to a simulated reply instead of ending as if one was built.""" + rejected = { + "type": "column_chart", + "query": {"fields": {"m": {"using": "metric/revenue_preagg"}, "d": {"using": "label/merchant"}}}, + "metrics": ["m"], + "view_by": ["d"], + } + first_turn = parse_sse_lines( + [ + "data: " + + json.dumps( + { + "item": { + "role": "assistant", + "content": { + "type": "toolCall", + "callId": "c1", + "name": "create_adhoc_visualization", + "arguments": {"visualization": rejected}, + }, + } + } + ), + "data: " + + json.dumps( + { + "item": { + "role": "tool", + "content": { + "type": "toolResult", + "callId": "c1", + "result": json.dumps( + {"status": "error", "message": "... is not comparable to the dimensionality ..."} + ), + }, + } + } + ), + "data: " + + json.dumps( + { + "item": { + "role": "assistant", + "content": { + "type": "multipart", + "parts": [{"type": "text", "text": "I could not chart that."}], + }, + } + } + ), + ] + ) + client = MagicMock() + client.send_message.side_effect = [first_turn, _chat_with_viz()] + monkeypatch.setattr( + "gooddata_eval.core.agentic.visualization.generate_simulated_response", + lambda msg, exp: "Use the transaction-level metric", + ) + + result = _execute_single_run(client, "conv-1", "Revenue by merchant", [_expected()]) + + assert client.send_message.call_count == 2 + assert client.send_message.call_args_list[1] == call("conv-1", "Use the transaction-level metric") + assert result.total_turns == 2 + assert result.eval_result.visualization_created is True + + def test_execute_single_run_no_viz_no_text(): """Agent returns nothing — visualization_created is False.""" client = MagicMock() diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index 6f28a767a..485ff698e 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -373,7 +373,7 @@ def test_parse_sse_lines_falls_back_to_adhoc_viz_when_multipart_viz_is_null(): # agent calls create_adhoc_visualization — stash the viz f'data: {{"item": {{"role": "assistant", "content": {{"type": "toolCall", "callId": "c1", "name": "create_adhoc_visualization", "arguments": {{"visualization": {json.dumps(viz_def)}}}}}}}}}', # data source fails - 'data: {"item": {"role": "tool", "content": {"type": "toolResult", "callId": "c1", "result": "{"status": "error", "message": "Data source does not exist"}"}}}', + 'data: {"item": {"role": "tool", "content": {"type": "toolResult", "callId": "c1", "result": "{\\"status\\": \\"error\\", \\"message\\": \\"Data source does not exist\\"}"}}}', # final multipart — visualization is null 'data: {"item": {"role": "assistant", "content": {"type": "multipart", "parts": [{"type": "text", "text": "Could not create"}, {"type": "visualization", "visualization": null}]}}}', ] @@ -383,6 +383,121 @@ def test_parse_sse_lines_falls_back_to_adhoc_viz_when_multipart_viz_is_null(): assert result.created_visualizations.objects[0].metrics == ["m"] +def _sse_item(role: str, content: dict[str, Any]) -> str: + return "data: " + json.dumps({"item": {"role": role, "content": content}}) + + +def _adhoc_call(call_id: str, viz: dict[str, Any]) -> str: + return _sse_item( + "assistant", + { + "type": "toolCall", + "callId": call_id, + "name": "create_adhoc_visualization", + "arguments": {"visualization": viz}, + }, + ) + + +def _adhoc_result(call_id: str, result: dict[str, Any]) -> str: + return _sse_item("tool", {"type": "toolResult", "callId": call_id, "result": json.dumps(result)}) + + +_NOT_COMPARABLE = { + "status": "error", + "message": "Aggregation dimension='[attribute/industry_name]' is not comparable to the " + "dimensionality='[dataset/fact_cuta_daily_summable_metrics]' of the subtree", +} +_NO_WITNESS = {"status": "error", "message": "Attribute 'label/merchant_name' has no witness in the subtree"} +_TEXT_ONLY_ANSWER = _sse_item( + "assistant", {"type": "multipart", "parts": [{"type": "text", "text": "Here is the data."}]} +) + + +def _viz(metric: str) -> dict[str, Any]: + return { + "type": "bar_chart", + "query": {"fields": {"m": {"using": metric}, "d": {"using": "label/industry_name"}}, "filter_by": {}}, + "metrics": ["m"], + "view_by": ["d"], + } + + +@pytest.mark.parametrize("rejection", [_NOT_COMPARABLE, _NO_WITNESS], ids=["not-comparable", "no-witness"]) +def test_parse_sse_lines_adhoc_fallback_scores_no_chart_for_a_rejected_definition(rejection): + """A definition the backend refused as unbuildable is no answer, so the turn has no chart.""" + lines = [_adhoc_call("c1", _viz("metric/cnp_share_-_cuta")), _adhoc_result("c1", rejection), _TEXT_ONLY_ANSWER] + + assert parse_sse_lines(lines).created_visualizations is None + + +def test_parse_sse_lines_adhoc_fallback_scores_no_chart_after_two_rejections(): + lines = [ + _adhoc_call("c1", _viz("metric/cnp_share_-_cuta")), + _adhoc_result("c1", _NOT_COMPARABLE), + _adhoc_call("c2", _viz("metric/cnp_share_-_cutcgco")), + _adhoc_result("c2", _NO_WITNESS), + _TEXT_ONLY_ANSWER, + ] + + assert parse_sse_lines(lines).created_visualizations is None + + +def test_parse_sse_lines_adhoc_fallback_never_scores_an_earlier_accepted_lookup(): + """Only the last call is the agent's answer: an accepted lookup before a rejected chart is not.""" + lines = [ + _adhoc_call("c1", _viz("metric/transaction_count_-_txn_-_cuta")), + _adhoc_result("c1", {"status": "success", "ref": "viz-lookup"}), + _adhoc_call("c2", _viz("metric/cnp_share_-_cuta")), + _adhoc_result("c2", _NOT_COMPARABLE), + _TEXT_ONLY_ANSWER, + ] + + assert parse_sse_lines(lines).created_visualizations is None + + +def test_parse_sse_lines_adhoc_fallback_still_scores_a_definition_an_environment_error_stopped(): + """An error about the environment, not the definition, leaves the arguments as the answer.""" + lines = [ + _adhoc_call("c1", _viz("metric/cnp_share_-_cuta")), + _adhoc_result("c1", {"status": "error", "message": "Data source does not exist"}), + _TEXT_ONLY_ANSWER, + ] + result = parse_sse_lines(lines) + + assert result.created_visualizations is not None + assert result.created_visualizations.objects[0].query.fields["m"].using == "metric/cnp_share_-_cuta" + + +def test_parse_sse_lines_adhoc_fallback_scores_no_chart_when_the_last_call_is_a_lookup(): + """A last call with `show_to_user: false` is a chart the agent built for itself, not an answer.""" + lookup = _sse_item( + "assistant", + { + "type": "toolCall", + "callId": "c1", + "name": "create_adhoc_visualization", + "arguments": {"visualization": _viz("metric/latest_date"), "show_to_user": False}, + }, + ) + lines = [lookup, _adhoc_result("c1", {"status": "success", "ref": "viz-lookup"}), _TEXT_ONLY_ANSWER] + + assert parse_sse_lines(lines).created_visualizations is None + + +def test_parse_sse_lines_adhoc_fallback_matches_out_of_order_results_by_call_id(): + """Parallel calls whose results arrive in reverse order: the last call's own result decides.""" + lines = [ + _adhoc_call("c1", _viz("metric/transaction_count_-_txn_-_cuta")), + _adhoc_call("c2", _viz("metric/cnp_share_-_cuta")), + _adhoc_result("c2", _NOT_COMPARABLE), + _adhoc_result("c1", {"status": "success", "ref": "viz-1"}), + _TEXT_ONLY_ANSWER, + ] + + assert parse_sse_lines(lines).created_visualizations is None + + def test_parse_sse_lines_adhoc_fallback_synthesizes_id_when_args_have_none(caplog): """A create_adhoc_visualization definition carries no `id` -- the server mints it.