From e3510647888647f4fd82f36a614837bb37641dd5 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:04:01 +0000 Subject: [PATCH] feat(providers/anthropic): support the computer_toolset_20260801 computer use toolset --- providers/anthropic/anthropic.go | 52 ++- providers/anthropic/computer_toolset.go | 123 +++++++ providers/anthropic/computer_toolset_test.go | 352 +++++++++++++++++++ providers/anthropic/computer_use.go | 113 +++++- providers/anthropic/computer_use_test.go | 9 + 5 files changed, 632 insertions(+), 17 deletions(-) create mode 100644 providers/anthropic/computer_toolset.go create mode 100644 providers/anthropic/computer_toolset_test.go diff --git a/providers/anthropic/anthropic.go b/providers/anthropic/anthropic.go index 6d5d591c2..4e6962457 100644 --- a/providers/anthropic/anthropic.go +++ b/providers/anthropic/anthropic.go @@ -563,6 +563,9 @@ func (a languageModel) prepareParams(call fantasy.Call) ( sendReasoning = *providerOptions.SendReasoning } systemBlocks, messages, warnings := toPrompt(call.Prompt, sendReasoning) + if declaresComputerToolset(call.Tools) { + warnings = append(warnings, replayComputerToolsetCalls(messages)...) + } if call.FrequencyPenalty != nil { warnings = append(warnings, fantasy.CallWarning{ @@ -1081,7 +1084,9 @@ func (a languageModel) toTools(tools []fantasy.Tool, toolChoice *fantasy.ToolCho }) continue } - betaFlags = append(betaFlags, flag) + if flag != "" { + betaFlags = append(betaFlags, flag) + } } rawTools = append(rawTools, raw) continue @@ -1657,10 +1662,15 @@ func (a languageModel) Generate(ctx context.Context, call fantasy.Call) (*fantas if !ok { continue } + toolName, input := toolUse.Name, string(toolUse.Input) + if isComputerToolsetMember(toolUse) { + toolName = computerUseAPIName + input = computerToolsetCallInput(toolUse.Name, input) + } content = append(content, fantasy.ToolCallContent{ ToolCallID: toolUse.ID, - ToolName: toolUse.Name, - Input: string(toolUse.Input), + ToolName: toolName, + Input: input, ProviderExecuted: false, }) case "server_tool_use": @@ -1786,12 +1796,19 @@ func (a languageModel) Stream(ctx context.Context, call fantasy.Call) (fantasy.S } case "content_block_start": contentBlockType := chunk.ContentBlock.Type + toolName := chunk.ContentBlock.Name if contentBlockType == "tool_use" || contentBlockType == "server_tool_use" { - openToolBlocks[chunk.Index] = &openToolBlock{ + open := &openToolBlock{ id: chunk.ContentBlock.ID, - name: chunk.ContentBlock.Name, + name: toolName, providerExecuted: contentBlockType == "server_tool_use", } + if contentBlockType == "tool_use" && isComputerToolsetMember(chunk.ContentBlock.AsToolUse()) { + open.toolsetMember = toolName + open.name = computerUseAPIName + toolName = computerUseAPIName + } + openToolBlocks[chunk.Index] = open } switch contentBlockType { case "text": @@ -1820,7 +1837,7 @@ func (a languageModel) Stream(ctx context.Context, call fantasy.Call) (fantasy.S if !yield(fantasy.StreamPart{ Type: fantasy.StreamPartTypeToolInputStart, ID: chunk.ContentBlock.ID, - ToolCallName: chunk.ContentBlock.Name, + ToolCallName: toolName, ToolCallInput: "", }) { return @@ -1969,6 +1986,11 @@ func (a languageModel) Stream(ctx context.Context, call fantasy.Call) (fantasy.S continue } open.input.WriteString(chunk.Delta.PartialJSON) + if open.toolsetMember != "" { + // The member input lacks the action field, so it + // is sent once, merged, when the block closes. + continue + } if !yield(fantasy.StreamPart{ Type: fantasy.StreamPartTypeToolInputDelta, ID: open.id, @@ -2066,12 +2088,26 @@ type openToolBlock struct { id string name string providerExecuted bool - input strings.Builder + // toolsetMember is the member name of a computer use toolset call, + // which is reported as the computer tool with an action input. + toolsetMember string + input strings.Builder } // close emits the end-of-input and tool call parts for the block. It reports // whether the consumer wants more parts. func (o *openToolBlock) close(acc anthropic.Message, index int64, yield func(fantasy.StreamPart) bool) bool { + arguments := o.arguments(acc, index) + if o.toolsetMember != "" { + arguments = computerToolsetCallInput(o.toolsetMember, arguments) + if !yield(fantasy.StreamPart{ + Type: fantasy.StreamPartTypeToolInputDelta, + ID: o.id, + ToolCallInput: arguments, + }) { + return false + } + } if !yield(fantasy.StreamPart{ Type: fantasy.StreamPartTypeToolInputEnd, ID: o.id, @@ -2083,7 +2119,7 @@ func (o *openToolBlock) close(acc anthropic.Message, index int64, yield func(fan Type: fantasy.StreamPartTypeToolCall, ID: o.id, ToolCallName: o.name, - ToolCallInput: o.arguments(acc, index), + ToolCallInput: arguments, ProviderExecuted: o.providerExecuted, }) } diff --git a/providers/anthropic/computer_toolset.go b/providers/anthropic/computer_toolset.go new file mode 100644 index 000000000..8e5788882 --- /dev/null +++ b/providers/anthropic/computer_toolset.go @@ -0,0 +1,123 @@ +package anthropic + +import ( + "encoding/json" + "fmt" + "maps" + "strings" + + "charm.land/fantasy" + "github.com/charmbracelet/anthropic-sdk-go" +) + +// computerToolsetName is the toolset_name that marks tool_use and +// tool_result blocks as members of the computer use toolset. +const computerToolsetName = "computer" + +// computerToolsetReplayFallbackMember is the member a replayed +// computer call is sent as when its input has no usable action. +const computerToolsetReplayFallbackMember = string(ActionScreenshot) + +// isComputerToolsetMember reports whether a tool_use block is a +// computer use toolset member call. The SDK has no typed +// toolset_name field, so it is read from the block's extra fields. +func isComputerToolsetMember(block anthropic.ToolUseBlock) bool { + return block.JSON.ExtraFields["toolset_name"].Raw() == `"`+computerToolsetName+`"` +} + +// computerToolsetCallInput returns a member call's input in the +// legacy computer tool shape: the member input object with an +// "action" field naming the member. Input that is not a JSON object, +// such as arguments truncated at max_tokens, is returned unchanged +// so the call is still reported as invalid instead of being guessed. +func computerToolsetCallInput(member, input string) string { + fields := map[string]any{} + if strings.TrimSpace(input) != "" { + var raw map[string]json.RawMessage + if err := json.Unmarshal([]byte(input), &raw); err != nil { + return input + } + for k, v := range raw { + fields[k] = v + } + } + fields["action"] = member + merged, err := json.Marshal(fields) + if err != nil { + return input + } + return string(merged) +} + +// declaresComputerToolset reports whether tools include the computer +// use tool at version ComputerUseToolset20260801. +func declaresComputerToolset(tools []fantasy.Tool) bool { + for _, tool := range tools { + pdt, ok := asProviderDefinedTool(tool) + if !ok || pdt.ID != computerUseToolID { + continue + } + if version, ok := getComputerUseVersion(pdt); ok && version == ComputerUseToolset20260801 { + return true + } + } + return false +} + +// replayComputerToolsetCalls rewrites replayed computer calls from +// the legacy shape (name "computer", input with "action") into +// toolset member calls, and tags every tool_result answering one +// with toolset_name. The API rejects a member call or result that +// lacks toolset_name. It edits messages in place. +func replayComputerToolsetCalls(messages []anthropic.MessageParam) []fantasy.CallWarning { + var warnings []fantasy.CallWarning + memberCalls := map[string]struct{}{} + for _, msg := range messages { + for _, block := range msg.Content { + switch { + case block.OfToolUse != nil && block.OfToolUse.Name == computerUseAPIName: + toolUse := block.OfToolUse + member, input, ok := computerToolsetMemberCall(toolUse.Input) + if !ok { + // Every tool_result must answer a tool_use with the + // same ID, so the call cannot be dropped. screenshot + // takes no input, so it is a valid member call for + // any history; the result keeps its pairing. + member, input = computerToolsetReplayFallbackMember, map[string]any{} + warnings = append(warnings, fantasy.CallWarning{ + Type: fantasy.CallWarningTypeOther, + Message: fmt.Sprintf( + "computer tool call %q has no string action; replaying it as the %s toolset member to preserve tool_use and tool_result pairing", + toolUse.ID, computerToolsetReplayFallbackMember, + ), + }) + } + toolUse.Name = member + toolUse.Input = input + toolUse.SetExtraFields(map[string]any{"toolset_name": computerToolsetName}) + memberCalls[toolUse.ID] = struct{}{} + case block.OfToolResult != nil: + if _, ok := memberCalls[block.OfToolResult.ToolUseID]; ok { + block.OfToolResult.SetExtraFields(map[string]any{"toolset_name": computerToolsetName}) + } + } + } + } + return warnings +} + +// computerToolsetMemberCall splits a legacy computer call input into +// the member name (its action) and the member input (the rest). +func computerToolsetMemberCall(input any) (member string, memberInput map[string]any, ok bool) { + fields, ok := input.(map[string]any) + if !ok { + return "", nil, false + } + action, ok := fields["action"].(string) + if !ok || action == "" { + return "", nil, false + } + memberInput = maps.Clone(fields) + delete(memberInput, "action") + return action, memberInput, true +} diff --git a/providers/anthropic/computer_toolset_test.go b/providers/anthropic/computer_toolset_test.go new file mode 100644 index 000000000..216c52191 --- /dev/null +++ b/providers/anthropic/computer_toolset_test.go @@ -0,0 +1,352 @@ +package anthropic + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "charm.land/fantasy" + "github.com/stretchr/testify/require" +) + +func TestComputerUseToolVersionForModel(t *testing.T) { + t.Parallel() + + tests := []struct { + model string + want ComputerUseToolVersion + }{ + {"claude-opus-5-5", ComputerUseToolset20260801}, + {"claude-sonnet-5-5", ComputerUseToolset20260801}, + {"claude-haiku-5-5", ComputerUseToolset20260801}, + {"claude-fable-5-1", ComputerUseToolset20260801}, + {"claude-mythos-5", ComputerUseToolset20260801}, + {"claude-mythos-preview", ComputerUseToolset20260801}, + {"claude-opus-4-8", ComputerUseToolset20260801}, + {"anthropic/claude-opus-5-5", ComputerUseToolset20260801}, + {"my-custom-alias", ComputerUseToolset20260801}, + {"claude-opus-4-7", ComputerUse20251124}, + {"claude-opus-4-7@20260101", ComputerUse20251124}, + {"claude-opus-4-6", ComputerUse20251124}, + {"claude-sonnet-4-6", ComputerUse20251124}, + {"us.anthropic.claude-sonnet-4-6", ComputerUse20251124}, + {"claude-opus-4-5-20251101", ComputerUse20251124}, + {"claude-sonnet-4-5-20250929", ComputerUse20250124}, + {"claude-haiku-4-5-20251001", ComputerUse20250124}, + {"claude-opus-4-1", ComputerUse20250124}, + {"claude-sonnet-4-20250514", ComputerUse20250124}, + {"claude-3-7-sonnet-20250219", ComputerUse20250124}, + } + for _, tt := range tests { + t.Run(tt.model, func(t *testing.T) { + t.Parallel() + require.Equal(t, tt.want, ComputerUseToolVersionForModel(tt.model)) + }) + } +} + +func newComputerToolset(t *testing.T, opts ComputerUseToolOptions) fantasy.ProviderDefinedTool { + t.Helper() + opts.ToolVersion = ComputerUseToolset20260801 + return jsonRoundTripTool(t, NewComputerUseTool(opts, noopComputerRun)) +} + +func TestComputerToolsetToolJSON(t *testing.T) { + t.Parallel() + + zoomOff := false + displayNumber := int64(1) + tests := []struct { + name string + opts ComputerUseToolOptions + want string + }{ + { + name: "defaults omit display fields and configs", + opts: ComputerUseToolOptions{DisplayWidthPx: 1920, DisplayHeightPx: 1080, DisplayNumber: &displayNumber}, + want: `{"type":"computer_toolset_20260801"}`, + }, + { + name: "zoom disabled and cache control", + opts: ComputerUseToolOptions{EnableZoom: &zoomOff, CacheControl: &CacheControl{Type: "ephemeral"}}, + want: `{"type":"computer_toolset_20260801","configs":{"zoom":{"enabled":false}},"cache_control":{"type":"ephemeral"}}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + lm := languageModel{options: options{}} + rawTools, _, warnings, betaFlags := lm.toTools([]fantasy.Tool{newComputerToolset(t, tt.opts)}, nil, false) + require.Empty(t, warnings) + require.Empty(t, betaFlags) + require.Len(t, rawTools, 1) + require.JSONEq(t, tt.want, string(rawTools[0])) + }) + } +} + +func computerToolsetMessageResponse(content []any) map[string]any { + resp := mockAnthropicGenerateResponse() + resp["content"] = content + resp["stop_reason"] = "tool_use" + return resp +} + +type capturedRequest struct { + header http.Header + rawQuery string + body map[string]any +} + +func newCapturingServer(t *testing.T, response map[string]any) (*httptest.Server, <-chan capturedRequest) { + t.Helper() + requests := make(chan capturedRequest, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + requests <- capturedRequest{header: r.Header.Clone(), rawQuery: r.URL.RawQuery, body: body} + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(response); err != nil { + t.Errorf("encode response: %v", err) + } + })) + t.Cleanup(server.Close) + return server, requests +} + +func generateWithServer(t *testing.T, serverURL string, call fantasy.Call) *fantasy.Response { + t.Helper() + provider, err := New(WithAPIKey("test-api-key"), WithBaseURL(serverURL)) + require.NoError(t, err) + model, err := provider.LanguageModel(context.Background(), "claude-opus-5-5") + require.NoError(t, err) + resp, err := model.Generate(context.Background(), call) + require.NoError(t, err) + return resp +} + +func TestGenerate_ComputerToolset(t *testing.T) { + t.Parallel() + + server, requests := newCapturingServer(t, computerToolsetMessageResponse([]any{ + map[string]any{ + "type": "tool_use", + "id": "toolu_click", + "name": "left_click", + "toolset_name": "computer", + "input": map[string]any{"coordinate": []int{640, 60}}, + }, + map[string]any{ + "type": "tool_use", + "id": "toolu_key", + "name": "key", + "toolset_name": "computer", + "input": map[string]any{"text": "Tab", "repeat": 4}, + }, + map[string]any{ + "type": "tool_use", + "id": "toolu_weather", + "name": "weather", + "input": map[string]any{"location": "Paris"}, + }, + })) + + resp := generateWithServer(t, server.URL, fantasy.Call{ + Prompt: testPrompt(), + Tools: []fantasy.Tool{ + newComputerToolset(t, ComputerUseToolOptions{DisplayWidthPx: 1920, DisplayHeightPx: 1080}), + fantasy.FunctionTool{Name: "weather", InputSchema: map[string]any{"properties": map[string]any{}}}, + }, + }) + + req := <-requests + require.Empty(t, req.header.Values("Anthropic-Beta")) + require.NotContains(t, req.rawQuery, "beta") + tools, err := json.Marshal(req.body["tools"].([]any)[0]) + require.NoError(t, err) + require.JSONEq(t, `{"type":"computer_toolset_20260801"}`, string(tools)) + + toolCalls := resp.Content.ToolCalls() + require.Len(t, toolCalls, 3) + require.Equal(t, "toolu_click", toolCalls[0].ToolCallID) + require.Equal(t, "computer", toolCalls[0].ToolName) + require.JSONEq(t, `{"action":"left_click","coordinate":[640,60]}`, toolCalls[0].Input) + require.Equal(t, "computer", toolCalls[1].ToolName) + parsed, err := ParseComputerUseInput(toolCalls[1].Input) + require.NoError(t, err) + require.Equal(t, ComputerUseInput{Action: ActionKey, Text: "Tab", Repeat: 4}, parsed) + require.Equal(t, "weather", toolCalls[2].ToolName) + require.JSONEq(t, `{"location":"Paris"}`, toolCalls[2].Input) +} + +func TestStream_ComputerToolsetBatch(t *testing.T) { + t.Parallel() + + chunks := []string{ + anthropicSSEEvent("message_start", `{"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","model":"claude-opus-5-5","content":[],"stop_reason":null,"usage":{"input_tokens":1,"output_tokens":0}}}`), + anthropicSSEEvent("content_block_start", `{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_click","name":"left_click","toolset_name":"computer","input":{}}}`), + anthropicSSEEvent("content_block_delta", `{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"coordinate\":"}}`), + anthropicSSEEvent("content_block_delta", `{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"[640,60]}"}}`), + anthropicSSEEvent("content_block_stop", `{"type":"content_block_stop","index":0}`), + anthropicSSEEvent("content_block_start", `{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_shot","name":"screenshot","toolset_name":"computer","input":{}}}`), + anthropicSSEEvent("content_block_stop", `{"type":"content_block_stop","index":1}`), + anthropicSSEEvent("content_block_start", `{"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_weather","name":"weather","input":{}}}`), + anthropicSSEEvent("content_block_delta", `{"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"location\":\"Paris\"}"}}`), + anthropicSSEEvent("content_block_stop", `{"type":"content_block_stop","index":2}`), + anthropicSSEEvent("message_delta", `{"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":20}}`), + anthropicSSEEvent("message_stop", `{"type":"message_stop"}`), + } + + parts := streamAnthropicParts(t, chunks, + newComputerToolset(t, ComputerUseToolOptions{}), + fantasy.FunctionTool{Name: "weather", InputSchema: map[string]any{"properties": map[string]any{}}}, + ) + + starts := streamPartsByType(parts, fantasy.StreamPartTypeToolInputStart) + require.Len(t, starts, 3) + require.Equal(t, []string{"computer", "computer", "weather"}, + []string{starts[0].ToolCallName, starts[1].ToolCallName, starts[2].ToolCallName}) + + deltas := map[string]string{} + for _, part := range streamPartsByType(parts, fantasy.StreamPartTypeToolInputDelta) { + deltas[part.ID] += part.ToolCallInput + } + require.Len(t, streamPartsByType(parts, fantasy.StreamPartTypeToolInputEnd), 3) + + calls := streamPartsByType(parts, fantasy.StreamPartTypeToolCall) + require.Len(t, calls, 3) + want := []struct{ id, name, input string }{ + {"toolu_click", "computer", `{"action":"left_click","coordinate":[640,60]}`}, + {"toolu_shot", "computer", `{"action":"screenshot"}`}, + {"toolu_weather", "weather", `{"location":"Paris"}`}, + } + for i, w := range want { + require.Equal(t, w.id, calls[i].ID) + require.Equal(t, w.name, calls[i].ToolCallName) + require.JSONEq(t, w.input, calls[i].ToolCallInput) + // Streamed deltas must add up to the reported input. + require.Equal(t, calls[i].ToolCallInput, deltas[w.id]) + } +} + +// computerBatchPrompt is a turn where the model ran a two-action +// computer batch next to a regular tool call, followed by the results. +func computerBatchPrompt(clickInput string) fantasy.Prompt { + return fantasy.Prompt{ + {Role: fantasy.MessageRoleUser, Content: []fantasy.MessagePart{fantasy.TextPart{Text: "Search for cats"}}}, + {Role: fantasy.MessageRoleAssistant, Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ToolCallID: "toolu_click", ToolName: "computer", Input: clickInput}, + fantasy.ToolCallPart{ToolCallID: "toolu_shot", ToolName: "computer", Input: `{"action":"screenshot"}`}, + fantasy.ToolCallPart{ToolCallID: "toolu_weather", ToolName: "weather", Input: `{"location":"Paris"}`}, + }}, + {Role: fantasy.MessageRoleTool, Content: []fantasy.MessagePart{ + NewComputerUseTextResult("toolu_click", "OK"), + NewComputerUseScreenshotResultWithMediaType("toolu_shot", "aW1n", "image/png"), + fantasy.ToolResultPart{ToolCallID: "toolu_weather", Output: fantasy.ToolResultOutputContentText{Text: "sunny"}}, + }}, + } +} + +func replayedBlocks(t *testing.T, body map[string]any) (assistant, user string) { + t.Helper() + messages := body["messages"].([]any) + require.Len(t, messages, 3) + a, err := json.Marshal(messages[1].(map[string]any)["content"]) + require.NoError(t, err) + u, err := json.Marshal(messages[2].(map[string]any)["content"]) + require.NoError(t, err) + return string(a), string(u) +} + +func TestReplay_ComputerToolset(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + version ComputerUseToolVersion + clickInput string + wantAssistant string + wantUser string + wantWarning bool + }{ + { + name: "toolset members carry toolset_name", + version: ComputerUseToolset20260801, + clickInput: `{"action":"left_click","coordinate":[640,60]}`, + wantAssistant: `[ + {"type":"tool_use","id":"toolu_click","name":"left_click","toolset_name":"computer","input":{"coordinate":[640,60]}}, + {"type":"tool_use","id":"toolu_shot","name":"screenshot","toolset_name":"computer","input":{}}, + {"type":"tool_use","id":"toolu_weather","name":"weather","input":{"location":"Paris"}} + ]`, + wantUser: `[ + {"type":"tool_result","tool_use_id":"toolu_click","toolset_name":"computer","content":[{"type":"text","text":"OK"}]}, + {"type":"tool_result","tool_use_id":"toolu_shot","toolset_name":"computer","content":[{"type":"image","source":{"type":"base64","media_type":"image/png","data":"aW1n"}}]}, + {"type":"tool_result","tool_use_id":"toolu_weather","content":[{"type":"text","text":"sunny"}]} + ]`, + }, + { + name: "toolset call without action falls back to screenshot", + version: ComputerUseToolset20260801, + clickInput: `{"coordinate":[640,60]}`, + wantWarning: true, + wantAssistant: `[ + {"type":"tool_use","id":"toolu_click","name":"screenshot","toolset_name":"computer","input":{}}, + {"type":"tool_use","id":"toolu_shot","name":"screenshot","toolset_name":"computer","input":{}}, + {"type":"tool_use","id":"toolu_weather","name":"weather","input":{"location":"Paris"}} + ]`, + wantUser: `[ + {"type":"tool_result","tool_use_id":"toolu_click","toolset_name":"computer","content":[{"type":"text","text":"OK"}]}, + {"type":"tool_result","tool_use_id":"toolu_shot","toolset_name":"computer","content":[{"type":"image","source":{"type":"base64","media_type":"image/png","data":"aW1n"}}]}, + {"type":"tool_result","tool_use_id":"toolu_weather","content":[{"type":"text","text":"sunny"}]} + ]`, + }, + { + name: "computer_20251124 replay is unchanged", + version: ComputerUse20251124, + clickInput: `{"action":"left_click","coordinate":[640,60]}`, + wantAssistant: `[ + {"type":"tool_use","id":"toolu_click","name":"computer","input":{"action":"left_click","coordinate":[640,60]}}, + {"type":"tool_use","id":"toolu_shot","name":"computer","input":{"action":"screenshot"}}, + {"type":"tool_use","id":"toolu_weather","name":"weather","input":{"location":"Paris"}} + ]`, + wantUser: `[ + {"type":"tool_result","tool_use_id":"toolu_click","content":[{"type":"text","text":"OK"}]}, + {"type":"tool_result","tool_use_id":"toolu_shot","content":[{"type":"image","source":{"type":"base64","media_type":"image/png","data":"aW1n"}}]}, + {"type":"tool_result","tool_use_id":"toolu_weather","content":[{"type":"text","text":"sunny"}]} + ]`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + server, requests := newCapturingServer(t, mockAnthropicGenerateResponse()) + computer := jsonRoundTripTool(t, NewComputerUseTool(ComputerUseToolOptions{ + DisplayWidthPx: 1920, DisplayHeightPx: 1080, ToolVersion: tt.version, + }, noopComputerRun)) + resp := generateWithServer(t, server.URL, fantasy.Call{ + Prompt: computerBatchPrompt(tt.clickInput), + Tools: []fantasy.Tool{ + computer, + fantasy.FunctionTool{Name: "weather", InputSchema: map[string]any{"properties": map[string]any{}}}, + }, + }) + + assistant, user := replayedBlocks(t, (<-requests).body) + require.JSONEq(t, tt.wantAssistant, assistant) + require.JSONEq(t, tt.wantUser, user) + + if !tt.wantWarning { + require.Empty(t, resp.Warnings) + return + } + require.Len(t, resp.Warnings, 1) + require.Contains(t, resp.Warnings[0].Message, `"toolu_click" has no string action`) + }) + } +} diff --git a/providers/anthropic/computer_use.go b/providers/anthropic/computer_use.go index f4526e9ae..5aec77fc7 100644 --- a/providers/anthropic/computer_use.go +++ b/providers/anthropic/computer_use.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "strings" "charm.land/fantasy" anthropicsdk "github.com/charmbracelet/anthropic-sdk-go" @@ -25,6 +26,13 @@ const computerUseAPIName = "computer" type ComputerUseToolVersion string const ( + // ComputerUseToolset20260801 selects the August 2026 computer use + // toolset, the generally available version. It needs no beta + // header. On the wire each action is a member tool (screenshot, + // left_click, ...) tagged with toolset_name "computer"; this + // package maps those calls to and from the "computer" tool with an + // "action" input field, so callers handle every version alike. + ComputerUseToolset20260801 ComputerUseToolVersion = "computer_toolset_20260801" // ComputerUse20251124 selects the November 2025 version of the // computer use tool. ComputerUse20251124 ComputerUseToolVersion = "computer_20251124" @@ -36,14 +44,20 @@ const ( // ComputerUseToolOptions holds the configuration for creating a // computer use tool instance. type ComputerUseToolOptions struct { - // DisplayWidthPx is the width of the display in pixels. + // DisplayWidthPx is the width of the display in pixels. The + // ComputerUseToolset20260801 version does not send it: toolset + // coordinates are in the pixel space of the returned screenshots. DisplayWidthPx int64 - // DisplayHeightPx is the height of the display in pixels. + // DisplayHeightPx is the height of the display in pixels. The + // ComputerUseToolset20260801 version does not send it. DisplayHeightPx int64 - // DisplayNumber is an optional X11 display number. + // DisplayNumber is an optional X11 display number. The + // ComputerUseToolset20260801 version does not send it. DisplayNumber *int64 - // EnableZoom enables zoom support. Only used with the - // ComputerUse20251124 version. + // EnableZoom controls zoom support. ComputerUse20251124 sends it + // as enable_zoom (off when nil). ComputerUseToolset20260801 sends + // it as configs.zoom.enabled (on when nil). ComputerUse20250124 + // ignores it. EnableZoom *bool // ToolVersion selects which computer use tool version to use. ToolVersion ComputerUseToolVersion @@ -106,10 +120,43 @@ func getComputerUseVersion(tool fantasy.ProviderDefinedTool) (ComputerUseToolVer return ComputerUseToolVersion(s), true } +// ComputerUseToolVersionForModel returns the computer use tool +// version that the Claude API accepts for modelID. Claude 4.8 and +// later get the toolset, which Claude 5.5 and later accept +// exclusively. Opus 4.5 through 4.7 and Sonnet 4.6 get +// ComputerUse20251124. Sonnet 4.5, Haiku 4.5, and older models get +// ComputerUse20250124. IDs without a recognizable Claude version +// (for example claude-mythos-preview or a custom alias) get the +// toolset, because the newest models accept only it. +// +// The mapping is for the Claude API. Amazon Bedrock and some other +// platforms do not offer the toolset. +func ComputerUseToolVersionForModel(modelID string) ComputerUseToolVersion { + major, minor, ok := claudeVersion(modelID) + if !ok { + return ComputerUseToolset20260801 + } + switch { + case major > 4 || (major == 4 && minor >= 8): + return ComputerUseToolset20260801 + case major == 4 && minor >= 5: + model := strings.ToLower(modelID) + if minor == 5 && (strings.Contains(model, "sonnet") || strings.Contains(model, "haiku")) { + return ComputerUse20250124 + } + return ComputerUse20251124 + default: + return ComputerUse20250124 + } +} + // computerUseBetaFlag returns the Anthropic beta header value for -// the given computer use tool version. +// the given computer use tool version. The toolset is generally +// available and returns an empty flag. func computerUseBetaFlag(version ComputerUseToolVersion) (string, error) { switch version { + case ComputerUseToolset20260801: + return "", nil case ComputerUse20251124: // TODO: Replace with SDK constant when available. return "computer-use-2025-11-24", nil @@ -130,6 +177,9 @@ func computerUseToolJSON(pdt fantasy.ProviderDefinedTool) (json.RawMessage, erro if !ok { return nil, fmt.Errorf("computerUseToolJSON: tool_version arg is missing") } + if version == ComputerUseToolset20260801 { + return computerToolsetJSON(pdt) + } h, hOK := anyToInt64(pdt.Args["display_height_px"]) w, wOK := anyToInt64(pdt.Args["display_width_px"]) @@ -179,6 +229,44 @@ func computerUseToolJSON(pdt fantasy.ProviderDefinedTool) (json.RawMessage, erro } } +// computerToolsetMemberConfig is the per-member entry of a toolset's +// configs object. +type computerToolsetMemberConfig struct { + Enabled bool `json:"enabled"` +} + +// computerToolsetCacheControl is the toolset entry's cache_control +// breakpoint. +type computerToolsetCacheControl struct { + Type string `json:"type"` +} + +// computerToolsetParam is the tools entry for the computer use +// toolset. The API rejects name, display_width_px, +// display_height_px, display_number, and enable_zoom on it. +type computerToolsetParam struct { + Type string `json:"type"` + Configs map[string]computerToolsetMemberConfig `json:"configs,omitempty"` + CacheControl *computerToolsetCacheControl `json:"cache_control,omitempty"` +} + +func computerToolsetJSON(pdt fantasy.ProviderDefinedTool) (json.RawMessage, error) { + tool := computerToolsetParam{Type: string(ComputerUseToolset20260801)} + if v, ok := pdt.Args["enable_zoom"]; ok { + enabled, ok := v.(bool) + if !ok { + return nil, fmt.Errorf("computer use tool has invalid enable_zoom") + } + tool.Configs = map[string]computerToolsetMemberConfig{ + string(ActionZoom): {Enabled: enabled}, + } + } + if _, ok := pdt.Args["cache_control"]; ok { + tool.CacheControl = &computerToolsetCacheControl{Type: "ephemeral"} + } + return json.Marshal(tool) +} + // ComputerAction identifies the action Claude wants to perform. // // Unless noted otherwise on a specific action, respond by returning a @@ -232,6 +320,8 @@ const ( // ActionKey presses a key combination. // // - Text: key combo string (e.g. "ctrl+c", "Return"). + // - Repeat: optional press count from 1 to 100 + // (ComputerUseToolset20260801 only); absent means once. ActionKey ComputerAction = "key" // ActionScroll scrolls the screen (v20250124+). // @@ -261,8 +351,9 @@ const ( // No additional fields are populated. ActionWait ComputerAction = "wait" // ActionZoom views a specific screen region at full - // resolution (v20251124 only). Requires enable_zoom in the - // tool definition. + // resolution (v20251124 and the toolset). v20251124 needs + // EnableZoom set to true; the toolset offers zoom unless + // EnableZoom is set to false. // // - Region: [x1, y1, x2, y2] top-left and bottom-right. // @@ -293,8 +384,12 @@ type ComputerUseInput struct { // Duration is how long to hold the key in seconds // (ActionHoldKey). Duration int64 `json:"duration,omitempty"` + // Repeat is how many times to press the key combination + // (ActionKey, ComputerUseToolset20260801 only). Zero means the + // field was absent; press once. + Repeat int64 `json:"repeat,omitempty"` // Region is [x1, y1, x2, y2] defining the zoom area - // (ActionZoom, v20251124 only). + // (ActionZoom, v20251124 and the toolset). Region [4]int64 `json:"region,omitempty"` } diff --git a/providers/anthropic/computer_use_test.go b/providers/anthropic/computer_use_test.go index 6278327a1..ecbf4802e 100644 --- a/providers/anthropic/computer_use_test.go +++ b/providers/anthropic/computer_use_test.go @@ -86,6 +86,15 @@ func TestParseComputerUseInput(t *testing.T) { require.Equal(t, "ctrl+c", input.Text) }) + t.Run("key with repeat", func(t *testing.T) { + t.Parallel() + input, err := ParseComputerUseInput(`{"action":"key","text":"Tab","repeat":4}`) + require.NoError(t, err) + require.Equal(t, ActionKey, input.Action) + require.Equal(t, "Tab", input.Text) + require.Equal(t, int64(4), input.Repeat) + }) + t.Run("scroll with coordinate direction and amount", func(t *testing.T) { t.Parallel() input, err := ParseComputerUseInput(`{"action":"scroll","coordinate":[960,540],"scroll_direction":"down","scroll_amount":3}`)