Skip to content

fix(providers/openai): replay web searches and computer calls without item IDs #77

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ethanndickson wants to merge 1 commit into from
+83 −425
Open

fix(providers/openai): replay web searches and computer calls without item IDs #77

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 18 additions & 44 deletions providers/openai/openai_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4713,7 +4713,7 @@ func TestResponsesToPrompt_WebSearchProviderExecutedToolResults(t *testing.T) {
require.Nil(t, input[1].OfItemReference)
})

t.Run("store true skips unreferenced search without metadata", func(t *testing.T) {
t.Run("store true skips search without metadata", func(t *testing.T) {
t.Parallel()

input, warnings, err := toResponsesPrompt(prompt, "system instructions", true)
Expand Down Expand Up @@ -4892,6 +4892,13 @@ func TestResponsesToPrompt_ReasoningWithWebSearchCombined(t *testing.T) {
fantasy.ToolResultPart{
ToolCallID: webSearchItemID,
ProviderExecuted: true,
ProviderOptions: fantasy.ProviderOptions{
Name: &WebSearchCallMetadata{
ItemID: webSearchItemID,
Status: "completed",
Action: &WebSearchAction{Type: "search", Query: "weather in San Francisco"},
},
},
},
fantasy.TextPart{Text: "Sunny."},
},
Expand All @@ -4904,77 +4911,43 @@ func TestResponsesToPrompt_ReasoningWithWebSearchCombined(t *testing.T) {
},
}

t.Run("store true pairs reasoning and web search", func(t *testing.T) {
t.Run("store true references reasoning before the search", func(t *testing.T) {
t.Parallel()

input, warnings, err := toResponsesPrompt(prompt, "system", true)

require.NoError(t, err)
require.Empty(t, warnings)

// Order: user, item_reference(rs_*), item_reference(ws_*),
// Order: user, item_reference(rs_*), web_search_call,
// assistant text, user.
require.Len(t, input, 5)

require.NotNil(t, input[1].OfItemReference)
require.Equal(t, reasoningItemID, input[1].OfItemReference.ID,
"reasoning item_reference must precede web_search item_reference")
require.Equal(t, reasoningItemID, input[1].OfItemReference.ID)

require.NotNil(t, input[2].OfItemReference)
require.Equal(t, webSearchItemID, input[2].OfItemReference.ID)
require.NotNil(t, input[2].OfWebSearchCall)
})

t.Run("store false skips both reasoning and provider tool call", func(t *testing.T) {
t.Run("store false replays the search without reasoning", func(t *testing.T) {
t.Parallel()

input, warnings, err := toResponsesPrompt(prompt, "system", false)

require.NoError(t, err)
require.Equal(t, []fantasy.CallWarning{{
Type: fantasy.CallWarningTypeOther,
Message: `dropping web search "ws_002": it has no web search metadata`,
}}, warnings)
require.Empty(t, warnings)

// Reasoning without encrypted content is skipped under
// store=false, and so is the web_search_call, whose result has
// no metadata to send inline. user, assistant text, user.
require.Len(t, input, 3)
// store=false. user, web_search_call, assistant text, user.
require.Len(t, input, 4)
require.NotNil(t, input[1].OfWebSearchCall)
for _, item := range input {
require.Nil(t, item.OfItemReference)
require.Nil(t, item.OfReasoning)
}
})
}

func TestResponsesToPrompt_WebSearchRequiresReasoningReference(t *testing.T) {
t.Parallel()

input, warnings, err := toResponsesPrompt(fantasy.Prompt{
{
Role: fantasy.MessageRoleAssistant,
Content: []fantasy.MessagePart{
fantasy.ReasoningPart{
Text: "I should search.",
ProviderOptions: fantasy.ProviderOptions{
Name: &ResponsesReasoningMetadata{Summary: []string{}},
},
},
fantasy.ToolCallPart{
ToolCallID: "ws_missing_reasoning",
ToolName: "web_search",
ProviderExecuted: true,
},
fantasy.TextPart{Text: "Search completed."},
},
},
}, "system", true)

require.NoError(t, err)
require.Empty(t, warnings)
require.Len(t, input, 1)
require.NotNil(t, input[0].OfMessage)
}

func TestResponsesToPrompt_ReasoningWithFunctionCallCombined(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -5583,6 +5556,7 @@ func TestComputerUseGenerateRoundTrip(t *testing.T) {
case "computer_call":
foundComputerCall = true
require.Equal(t, "call_01", itemMap["call_id"])
require.NotContains(t, itemMap, "id", "a replayed computer_call must not carry its item ID")
case "computer_call_output":
foundComputerCallOutput = true
require.Equal(t, toolCall.ToolCallID, itemMap["call_id"])
Expand Down
104 changes: 32 additions & 72 deletions providers/openai/responses_language_model.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -491,8 +491,8 @@ func toResponsesPromptWithValidation(prompt fantasy.Prompt, systemMessageMode st
}
}

// A search's call and result parts can each replay it, possibly from
// different messages; only the first one does.
// A search's result part can appear more than once, possibly in
// different messages; only the first one replays it.
replayedSearches := make(map[string]bool)
var dropped droppedSearches
for _, msg := range prompt {
Expand Down Expand Up @@ -613,14 +613,12 @@ func toResponsesPromptWithValidation(prompt fantasy.Prompt, systemMessageMode st
input = append(input, responses.ResponseInputItemParamOfMessage(contentParts, responses.EasyInputMessageRoleUser))

case fantasy.MessageRoleAssistant:
// Replayed output items other than reasoning carry no item ID.
// The API pairs an item that has a stored ID with the reasoning
// item right before it, and rejects the request if that
// reasoning is missing. It never pairs an item without an ID.
startIdx := len(input)
// referenceAt is the index of the part that can reference a
// stored search: the part right after a reasoning part from a
// stored response. It counts parts, not input items, because a
// skipped part means the search did not directly follow its
// reasoning in the response.
referenceAt := -1
for i, c := range msg.Content {
for _, c := range msg.Content {
switch c.GetType() {
case fantasy.ContentTypeText:
textPart, ok := fantasy.AsContentType[fantasy.TextPart](c)
Expand Down Expand Up @@ -649,21 +647,15 @@ func toResponsesPromptWithValidation(prompt fantasy.Prompt, systemMessageMode st
continue
}

// A provider-executed call is replayed from its result.
if toolCallPart.ProviderExecuted {
if store && referenceAt == i &&
isResponsesWebSearchToolCall(toolCallPart) &&
toolCallPart.ToolCallID != "" &&
!replayedSearches[toolCallPart.ToolCallID] {
input = append(input, responses.ResponseInputItemParamOfItemReference(toolCallPart.ToolCallID))
replayedSearches[toolCallPart.ToolCallID] = true
}
continue
}

// Computer calls are round-tripped via param.Override
// using the raw wire-format JSON from the original response.
if rawJSON, ok := computerCallRawJSON[toolCallPart.ToolCallID]; ok {
overridden := param.Override[responses.ResponseComputerToolCallParam](json.RawMessage(rawJSON))
overridden := param.Override[responses.ResponseComputerToolCallParam](dropItemID(rawJSON))
input = append(input, responses.ResponseInputItemUnionParam{
OfComputerCall: &overridden,
})
Expand Down Expand Up @@ -697,10 +689,6 @@ func toResponsesPromptWithValidation(prompt fantasy.Prompt, systemMessageMode st
// content cannot be resolved by the API.
continue
}
// Item-reference replay predates source storage metadata.
if !meta.Finalized || meta.SourceStoreEnabled {
referenceAt = i + 1
}
continue
case fantasy.ContentTypeToolResult:
toolResultPart, ok := fantasy.AsContentType[fantasy.ToolResultPart](c)
Expand Down Expand Up @@ -909,12 +897,11 @@ func responsesReasoningInputItem(meta *ResponsesReasoningMetadata) (responses.Re
return item, true
}

func finalResponsesReasoningMetadata(item responses.ResponseOutputItemUnion, sourceStoreEnabled bool) *ResponsesReasoningMetadata {
func finalResponsesReasoningMetadata(item responses.ResponseOutputItemUnion) *ResponsesReasoningMetadata {
metadata := &ResponsesReasoningMetadata{
ItemID: item.ID,
Summary: make([]string, 0, len(item.Summary)),
Finalized: true,
SourceStoreEnabled: sourceStoreEnabled,
ItemID: item.ID,
Summary: make([]string, 0, len(item.Summary)),
Finalized: true,
}
if item.EncryptedContent != "" {
metadata.EncryptedContent = &item.EncryptedContent
Expand All @@ -925,22 +912,12 @@ func finalResponsesReasoningMetadata(item responses.ResponseOutputItemUnion, sou
return metadata
}

// OpenAI echoes store=false for zero data retention organizations.
func effectiveResponseStore(requested bool, response *responses.Response) bool {
return requested && response.JSON.ExtraFields["store"].Raw() != "false"
}

func hasResponsesWebSearchTool(tools []responses.ToolUnionParam) bool {
return slices.ContainsFunc(tools, func(tool responses.ToolUnionParam) bool {
return tool.OfWebSearch != nil || tool.OfWebSearchPreview != nil
})
}

func isResponsesWebSearchToolCall(toolCallPart fantasy.ToolCallPart) bool {
return toolCallPart.ToolName == "web_search" ||
toolCallPart.ToolName == "web_search_preview"
}

// droppedSearches records why search results were not replayed, so that
// each search the whole prompt never replays gets one warning. Another
// part of the prompt can still replay a search, and a reason from an
Expand Down Expand Up @@ -1001,10 +978,9 @@ func webSearchResult(part fantasy.ToolResultPart) (*WebSearchCallMetadata, strin

// inlineWebSearchCallItem rebuilds a web search call from its result
// metadata, which unlike the call input keeps the status and the
// deprecated single query. The item has no id: with an id, a stored
// request resolves the stored item instead, which fails without its
// reasoning item right before it and can resolve another credential's
// item.
// deprecated single query. The item has no id: with an id, the API
// resolves the stored item instead, which fails without its reasoning
// item right before it and can resolve another credential's item.
func inlineWebSearchCallItem(meta *WebSearchCallMetadata) (responses.ResponseInputItemUnionParam, error) {
if meta.Action == nil {
return responses.ResponseInputItemUnionParam{}, errors.New("metadata has no action")
Expand Down Expand Up @@ -1042,18 +1018,25 @@ func inlineWebSearchCallItem(meta *WebSearchCallMetadata) (responses.ResponseInp
return responses.ResponseInputItemUnionParam{OfWebSearchCall: &item}, nil
}

func validateResponsesInput(input responses.ResponseInputParam) error {
if err := validateResponsesFunctionCallOutputs(input); err != nil {
return err
// dropItemID removes the item ID from a raw output item.
func dropItemID(raw string) json.RawMessage {
var fields map[string]json.RawMessage
if err := json.Unmarshal([]byte(raw), &fields); err != nil {
return json.RawMessage(raw)
}
delete(fields, "id")
stripped, err := json.Marshal(fields)
if err != nil {
return json.RawMessage(raw)
}
return validateResponsesItemReferences(input)
return stripped
}

func validateResponsesInputWithOptions(input responses.ResponseInputParam, allowUnpairedFunctionCallOutputs bool) error {
if !allowUnpairedFunctionCallOutputs {
return validateResponsesInput(input)
if allowUnpairedFunctionCallOutputs {
return nil
}
return validateResponsesItemReferences(input)
return validateResponsesFunctionCallOutputs(input)
}

func validateResponsesFunctionCallOutputs(input responses.ResponseInputParam) error {
Expand Down Expand Up @@ -1130,27 +1113,6 @@ func validateResponsesFunctionCallOutputs(input responses.ResponseInputParam) er
return nil
}

func validateResponsesItemReferences(input responses.ResponseInputParam) error {
previousReferenceID := ""
for _, item := range input {
if item.OfReasoning != nil {
previousReferenceID = item.OfReasoning.ID
continue
}
if item.OfItemReference == nil {
previousReferenceID = ""
continue
}

itemID := item.OfItemReference.ID
if strings.HasPrefix(itemID, "ws_") && !strings.HasPrefix(previousReferenceID, "rs_") {
return fmt.Errorf("openai responses prompt has web_search_call item_reference without preceding reasoning item_reference for item_id %q", itemID)
}
previousReferenceID = itemID
}
return nil
}

func hasVisibleResponsesUserContent(content responses.ResponseInputMessageContentListParam) bool {
return len(content) > 0
}
Expand Down Expand Up @@ -1383,7 +1345,7 @@ func (o responsesLanguageModel) Generate(ctx context.Context, call fantasy.Call)
},
})
case "reasoning":
metadata := finalResponsesReasoningMetadata(outputItem, effectiveResponseStore(params.Store.Value, response))
metadata := finalResponsesReasoningMetadata(outputItem)
if len(metadata.Summary) == 0 && metadata.EncryptedContent == nil {
continue
}
Expand Down Expand Up @@ -1481,7 +1443,6 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) (
ongoingToolCalls := make(map[int64]*ongoingToolCall)
hasFunctionCall := false
activeReasoning := make(map[string]*reasoningState)
sourceStoreEnabled := params.Store.Value

return func(yield func(fantasy.StreamPart) bool) {
if len(warnings) > 0 {
Expand All @@ -1500,7 +1461,6 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) (
case "response.created":
created := event.AsResponseCreated()
responseID = created.Response.ID
sourceStoreEnabled = effectiveResponseStore(sourceStoreEnabled, &created.Response)

case "response.output_item.added":
added := event.AsResponseOutputItemAdded()
Expand Down Expand Up @@ -1691,7 +1651,7 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) (
Type: fantasy.StreamPartTypeReasoningEnd,
ID: done.Item.ID,
ProviderMetadata: fantasy.ProviderMetadata{
Name: finalResponsesReasoningMetadata(done.Item, sourceStoreEnabled),
Name: finalResponsesReasoningMetadata(done.Item),
},
}) {
return
Expand Down
7 changes: 1 addition & 6 deletions providers/openai/responses_options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,9 +153,6 @@ type ResponsesReasoningMetadata struct {
// item. Only finalized encrypted content is replayed inline; streaming
// placeholders and metadata persisted before this field existed are not.
Finalized bool `json:"finalized,omitempty"`
// SourceStoreEnabled records the source request's store setting, so
// unstored search items are not referenced after storage is enabled.
SourceStoreEnabled bool `json:"source_store_enabled,omitempty"`
}

// Options implements the ProviderOptions interface.
Expand Down Expand Up @@ -467,9 +464,7 @@ type WebSearchAction struct {

// WebSearchCallMetadata stores structured data from a web_search_call
// output item for round-tripping through multi-turn conversations.
// The ItemID is used with item_reference for efficient round-tripping
// when response storage is enabled. When the call cannot be referenced,
// Status and Action rebuild it inline.
// Status and Action rebuild the call inline, without its ItemID.
type WebSearchCallMetadata struct {
// ItemID is the server-side ID of the web_search_call output item.
ItemID string `json:"item_id"`
Expand Down
Loading
Loading