From d73396ab3d4936352fe09d2ccf44728de597318c Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:57:22 -0500 Subject: [PATCH] fix(ai): preserve Gemini thought signatures on OpenAI Chat tool calls (#51768) --- packages/ai/src/protocols/openai-chat.ts | 35 ++++-- .../ai/src/protocols/utils/tool-stream.ts | 9 +- .../gemini-parallel-tool-signatures.json | 54 +++++++++ .../gemini-openai-chat.recorded.test.ts | 68 +++++++++++ packages/ai/test/provider/openai-chat.test.ts | 111 ++++++++++++++++++ 5 files changed, 266 insertions(+), 11 deletions(-) create mode 100644 packages/ai/test/fixtures/recordings/openai-compatible-chat/gemini-parallel-tool-signatures.json create mode 100644 packages/ai/test/provider/gemini-openai-chat.recorded.test.ts diff --git a/packages/ai/src/protocols/openai-chat.ts b/packages/ai/src/protocols/openai-chat.ts index b72864abbcb..10a6a0db173 100644 --- a/packages/ai/src/protocols/openai-chat.ts +++ b/packages/ai/src/protocols/openai-chat.ts @@ -64,6 +64,14 @@ const OpenAIChatTool = Schema.Struct({ }) type OpenAIChatTool = Schema.Schema.Type +// Gemini's OpenAI-compatible surface carries thought signatures in tool call +// `extra_content` and rejects replayed parallel calls without them: +// https://ai.google.dev/gemini-api/docs/thinking#signatures +const ExtraContent = Schema.Struct({ + google: Schema.Struct({ thought_signature: Schema.String }), +}) +const decodeExtraContent = (value: unknown) => Option.getOrUndefined(Schema.decodeUnknownOption(ExtraContent)(value)) + const OpenAIChatAssistantToolCall = Schema.Struct({ id: Schema.String, type: Schema.tag("function"), @@ -71,6 +79,7 @@ const OpenAIChatAssistantToolCall = Schema.Struct({ name: Schema.String, arguments: Schema.String, }), + extra_content: Schema.optional(ExtraContent), }) type OpenAIChatAssistantToolCall = Schema.Schema.Type @@ -112,12 +121,6 @@ const decodeReasoningDetail = Schema.decodeUnknownOption(ReasoningDetail) const knownReasoningDetails = (details: ReadonlyArray) => details.flatMap((detail) => Option.toArray(decodeReasoningDetail(detail))) -// Intentionally omit Gemini's provider-specific `extra_content.google.thought_signature` -// extension until direct Google OpenAI-compatible routing is supported here: -// https://github.com/vercel/ai/issues/11590 -// https://github.com/vercel/ai/pull/11745 -// https://ai.google.dev/gemini-api/docs/thought-signatures#openai - const OpenAIChatUserContent = Schema.Union([ Schema.Struct({ type: Schema.Literal("text"), @@ -242,6 +245,7 @@ const OpenAIChatToolCallDelta = Schema.Struct({ index: optionalNull(Schema.Number), id: optionalNull(Schema.String), function: optionalNull(OpenAIChatToolCallDeltaFunction), + extra_content: optionalNull(Schema.Unknown), }) type OpenAIChatToolCallDelta = Schema.Schema.Type @@ -294,6 +298,7 @@ interface PendingToolDelta { readonly id?: string readonly name?: string readonly input: string + readonly extraContent?: Schema.Schema.Type } export interface ParserState { @@ -347,13 +352,17 @@ const lowerToolChoice = (toolChoice: NonNullable) => tool: (name) => ({ type: "function" as const, function: { name } }), }) -const lowerToolCall = (part: ToolCallPart, options: LoweringOptions): OpenAIChatAssistantToolCall => ({ +const lowerToolCall = ( + part: ToolCallPart, + options: LoweringOptions & { readonly providerMetadataKey: string }, +): OpenAIChatAssistantToolCall => ({ id: options.toolCallID?.(part.id) ?? part.id, type: "function", function: { name: part.name, arguments: ProviderShared.encodeJson(part.input === undefined ? {} : part.input), }, + extra_content: decodeExtraContent(part.providerMetadata?.[options.providerMetadataKey]?.extraContent), }) const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (part: MediaPart) { @@ -721,7 +730,9 @@ const detectSupportsStore = (provider: string, baseURL: string | undefined): boo p === "vercel-ai-gateway" || url.includes("ai-gateway.vercel.sh") || url.includes("vercel.sh") const isAntLing = p === "ant-ling" || url.includes("api.ant-ling.com") const isOpencode = p === "opencode" || url.includes("opencode.ai") + const isGemini = url.includes("generativelanguage.googleapis.com") const isNonStandard = + isGemini || isNvidia || isCerebras || isXai || @@ -1114,12 +1125,13 @@ const step = (state: ParserState, event: OpenAIChatEvent) => const id = current?.id ?? pending?.id ?? (tool.id || undefined) const name = current?.name ?? pending?.name ?? (tool.function?.name || undefined) const text = `${pending?.input ?? ""}${tool.function?.arguments ?? ""}` + const extraContent = pending?.extraContent ?? decodeExtraContent(tool.extra_content) latestToolIndex = index nextToolIndex = Math.max(nextToolIndex, index + 1) if (!current && (!id || !name)) { pendingTools = { ...pendingTools, - [index]: { id: id || undefined, name: name || undefined, input: text }, + [index]: { id: id || undefined, name: name || undefined, input: text, extraContent }, } continue } @@ -1131,7 +1143,12 @@ const step = (state: ParserState, event: OpenAIChatEvent) => ADAPTER, tools, index, - { id: id || undefined, name: name || undefined, text }, + { + id: id || undefined, + name: name || undefined, + text, + providerMetadata: extraContent && { [state.providerMetadataKey]: { extraContent } }, + }, "OpenAI Chat tool call delta is missing id or name", ) if (ToolStream.isError(result)) diff --git a/packages/ai/src/protocols/utils/tool-stream.ts b/packages/ai/src/protocols/utils/tool-stream.ts index 0aa522c5f8d..2640077411c 100644 --- a/packages/ai/src/protocols/utils/tool-stream.ts +++ b/packages/ai/src/protocols/utils/tool-stream.ts @@ -147,7 +147,12 @@ export const appendOrStart = ( route: string, tools: State, key: K, - delta: { readonly id?: string; readonly name?: string; readonly text: string }, + delta: { + readonly id?: string + readonly name?: string + readonly text: string + readonly providerMetadata?: ProviderMetadata + }, missingToolMessage: string, ): AppendOutcome | AIError => { const current = tools[key] @@ -161,7 +166,7 @@ export const appendOrStart = ( namespace: current?.namespace, input: `${current?.input ?? ""}${delta.text}`, providerExecuted: current?.providerExecuted, - providerMetadata: current?.providerMetadata, + providerMetadata: current?.providerMetadata ?? delta.providerMetadata, } if (current && delta.text.length === 0 && current.id === id && current.name === name) return { tools, tool: current, events: [] } diff --git a/packages/ai/test/fixtures/recordings/openai-compatible-chat/gemini-parallel-tool-signatures.json b/packages/ai/test/fixtures/recordings/openai-compatible-chat/gemini-parallel-tool-signatures.json new file mode 100644 index 00000000000..b07a647f59a --- /dev/null +++ b/packages/ai/test/fixtures/recordings/openai-compatible-chat/gemini-parallel-tool-signatures.json @@ -0,0 +1,54 @@ +{ + "version": 1, + "metadata": { + "model": "gemini-3.8-flash", + "tags": [ + "prefix:openai-compatible-chat", + "provider:google", + "protocol:openai-chat", + "tool", + "tool-loop", + "continuation" + ], + "name": "gemini-parallel-tool-signatures", + "recordedAt": "2026-09-28T03:12:05.083Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gemini-3.8-flash\",\"messages\":[{\"role\":\"system\",\"content\":\"Call get_weather for every requested city in parallel, then answer in one short sentence.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris and in Tokyo?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}}],\"stream\":true,\"stream_options\":{\"include_usage\":true}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": "data: {\"choices\":[{\"delta\":{\"role\":\"assistant\",\"tool_calls\":[{\"extra_content\":{\"google\":{\"thought_signature\":\"ErYCCrMCAWkUfRNGh+/Zbc8YUSzk1yfWAfROcjA4HyF1x69jz6167w8zd4n6kZQQ5FDeBZ5HZMEbEkQ4ENOpzsQL8roCR6wONkhXpiduWrTD6XwbP8KGNkf6D1tX/JlBh7G5Cl+0rdjiSOl/mdY1lcjbkfyCRFs5T8odNWMG7WD3rCrXJDFQ/5QfOl+tqVTceKGz2yyXBhOhvsQDU33ulR9tHQJo/Fmx3HNyDdwvmyKUXm+kgqHsYZnkdv6y6xZwy9zBXGUO4QwxelMw3Rrc24Mp2rNbEDZS3YEeP72Jn/hIP5a8XWOx6+zop/4/CjYxxSfN/tJRY5d48NAKRNFzUN1Az8SvAs/N5X2I3/5ZMaDnQWW/BVdS9fm2KFYJ2baqtBiRRnJxMq4ELArkKjGZzHpd97XG8YjV6g==\"}},\"function\":{\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"name\":\"get_weather\"},\"id\":\"call_723181\",\"type\":\"function\"}]},\"index\":0}],\"created\":1790565123,\"id\":\"A9u5aoufBp3rz7IPke_3oAo\",\"model\":\"gemini-3.8-flash\",\"object\":\"chat.completion.chunk\",\"usage\":{\"completion_tokens\":16,\"prompt_tokens\":74,\"total_tokens\":132}}\n\ndata: {\"choices\":[{\"delta\":{\"role\":\"assistant\",\"tool_calls\":[{\"function\":{\"arguments\":\"{\\\"city\\\":\\\"Tokyo\\\"}\",\"name\":\"get_weather\"},\"id\":\"call_723184\",\"type\":\"function\"}]},\"index\":0}],\"created\":1790565123,\"id\":\"A9u5aoufBp3rz7IPke_3oAo\",\"model\":\"gemini-3.8-flash\",\"object\":\"chat.completion.chunk\",\"usage\":{\"completion_tokens\":32,\"prompt_tokens\":74,\"total_tokens\":148}}\n\ndata: {\"choices\":[{\"delta\":{\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"index\":0}],\"created\":1790565124,\"id\":\"A9u5aoufBp3rz7IPke_3oAo\",\"model\":\"gemini-3.8-flash\",\"object\":\"chat.completion.chunk\",\"usage\":{\"completion_tokens\":32,\"prompt_tokens\":74,\"total_tokens\":148}}\n\ndata: [DONE]\n\n" + } + }, + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gemini-3.8-flash\",\"messages\":[{\"role\":\"system\",\"content\":\"Call get_weather for every requested city in parallel, then answer in one short sentence.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris and in Tokyo?\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_723181\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"},\"extra_content\":{\"google\":{\"thought_signature\":\"ErYCCrMCAWkUfRNGh+/Zbc8YUSzk1yfWAfROcjA4HyF1x69jz6167w8zd4n6kZQQ5FDeBZ5HZMEbEkQ4ENOpzsQL8roCR6wONkhXpiduWrTD6XwbP8KGNkf6D1tX/JlBh7G5Cl+0rdjiSOl/mdY1lcjbkfyCRFs5T8odNWMG7WD3rCrXJDFQ/5QfOl+tqVTceKGz2yyXBhOhvsQDU33ulR9tHQJo/Fmx3HNyDdwvmyKUXm+kgqHsYZnkdv6y6xZwy9zBXGUO4QwxelMw3Rrc24Mp2rNbEDZS3YEeP72Jn/hIP5a8XWOx6+zop/4/CjYxxSfN/tJRY5d48NAKRNFzUN1Az8SvAs/N5X2I3/5ZMaDnQWW/BVdS9fm2KFYJ2baqtBiRRnJxMq4ELArkKjGZzHpd97XG8YjV6g==\"}}},{\"id\":\"call_723184\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Tokyo\\\"}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_723181\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"},{\"role\":\"tool\",\"tool_call_id\":\"call_723184\",\"content\":\"{\\\"temperature\\\":0,\\\"condition\\\":\\\"unknown\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}}],\"stream\":true,\"stream_options\":{\"include_usage\":true}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": "data: {\"choices\":[{\"delta\":{\"content\":\"It is currently sunny and 22°C in Paris,\",\"role\":\"assistant\"},\"index\":0}],\"created\":1790565125,\"id\":\"BNu5auWeB-2fz7IPxY6l4QY\",\"model\":\"gemini-3.8-flash\",\"object\":\"chat.completion.chunk\",\"usage\":{\"completion_tokens\":13,\"prompt_tokens\":150,\"total_tokens\":226}}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\" while Tokyo is 0°C.\",\"role\":\"assistant\"},\"index\":0}],\"created\":1790565125,\"id\":\"BNu5auWeB-2fz7IPxY6l4QY\",\"model\":\"gemini-3.8-flash\",\"object\":\"chat.completion.chunk\",\"usage\":{\"completion_tokens\":21,\"prompt_tokens\":150,\"total_tokens\":234}}\n\ndata: {\"choices\":[{\"delta\":{\"extra_content\":{\"google\":{\"thought_signature\":\"EpcDCpQDAWkUfRM325TAUtfFiOeQIEWn/TsCU9oi2js4VPHFeeLfAu+2k8PJm0fN/OaF0y4ovau7S9QIAsuOPI2w2aIyQ2kMGj1XvUyRTvz30DOZgtq1km6W6YGzZyyTCNSeBcpwtJtziHZVVWq9xEI/HHB8Ta1Ot215xnFyDL7iUGEwgGu45/mInpk+SOCYBy9biDddpDxcDi14BGoleArY9XEFAzYLxXssl7HMWjpfee5095im7gD125Nripq1Jf3nGY/2TxqjgQAdJpQybwct63p74O1szGHQxrkBt7AwphDgbOWtLpUP/QJBRdl8qhrozqRe611NQ6V5lMSwpO7OhQ/IDRtWMwOyrrKblZfmMnnPl2/9xDfZRsYnfmWq+7PeAptJl1cDRlMBKhj5iRn31xvN43EiuvWwWPsSndiWvxrMvVBd89TR4u0+z0pYCYZcsFkYKFlPA7pZsdnh6CON24AA5WUkO4dQNoqYdik6aO5hE9jLHG5FHTdr0W69qgPNozbxnO0ptRcOtkGpB9xYUVoTxcSXXZE=\"}},\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"index\":0}],\"created\":1790565125,\"id\":\"BNu5auWeB-2fz7IPxY6l4QY\",\"model\":\"gemini-3.8-flash\",\"object\":\"chat.completion.chunk\",\"usage\":{\"completion_tokens\":21,\"prompt_tokens\":192,\"total_tokens\":276}}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/ai/test/provider/gemini-openai-chat.recorded.test.ts b/packages/ai/test/provider/gemini-openai-chat.recorded.test.ts new file mode 100644 index 00000000000..c9eea21fdd9 --- /dev/null +++ b/packages/ai/test/provider/gemini-openai-chat.recorded.test.ts @@ -0,0 +1,68 @@ +import { describe, expect } from "bun:test" +import { Effect } from "effect" +import { LLM, LLMEvent, LLMRequest, Message, ToolRuntime, toDefinitions } from "../../src/index.js" +import * as OpenAICompatible from "../../src/providers/openai-compatible.js" +import { LLMClient } from "../../src/route.js" +import { compileRequest } from "../../src/route/client.js" +import { recordedTests } from "../recorded-test.js" +import { weatherRuntimeTool, weatherToolName } from "../recorded-scenarios.js" + +const model = OpenAICompatible.configure({ + provider: "google", + baseURL: "https://generativelanguage.googleapis.com/v1beta/openai", + apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? "fixture", +}).model("gemini-3.8-flash") + +const recorded = recordedTests({ + prefix: "openai-compatible-chat", + provider: "google", + protocol: "openai-chat", + requires: ["GOOGLE_GENERATIVE_AI_API_KEY"], + tags: ["tool", "tool-loop", "continuation"], + metadata: { model: model.id }, +}) + +describe("Gemini OpenAI-compatible Chat recorded", () => { + recorded.effect.with( + "replays thought signatures through a parallel tool loop", + { cassette: "openai-compatible-chat/gemini-parallel-tool-signatures" }, + () => + Effect.gen(function* () { + const tools = { [weatherToolName]: weatherRuntimeTool } + const request = LLM.request({ + model, + system: "Call get_weather for every requested city in parallel, then answer in one short sentence.", + prompt: "What is the weather in Paris and in Tokyo?", + tools: toDefinitions(tools), + cache: "none", + }) + const first = yield* LLMClient.generate(request) + const calls = first.events.filter(LLMEvent.is.toolCall) + expect(calls.map((call) => call.input)).toEqual([{ city: "Paris" }, { city: "Tokyo" }]) + const extraContent = calls[0]?.providerMetadata?.google?.extraContent + expect(extraContent).toEqual({ google: { thought_signature: expect.any(String) } }) + + const results = yield* Effect.forEach(calls, (call) => ToolRuntime.dispatch(tools, call)) + const continuation = LLMRequest.update(request, { + messages: [ + ...request.messages, + first.message, + ...calls.map((call, index) => + Message.tool({ id: call.id, name: call.name, result: results[index]!.result }), + ), + ], + }) + const prepared = yield* compileRequest(continuation) + const assistant = prepared.body.messages.find((message) => message.role === "assistant") + expect(assistant?.role === "assistant" ? assistant.tool_calls?.[0]?.extra_content : undefined).toEqual( + extraContent, + ) + + const second = yield* LLMClient.generate(continuation) + expect(second.events.filter(LLMEvent.is.toolCall)).toHaveLength(0) + expect(second.text).toMatch(/Paris/) + expect(second.text).toMatch(/Tokyo/) + }), + 60_000, + ) +}) diff --git a/packages/ai/test/provider/openai-chat.test.ts b/packages/ai/test/provider/openai-chat.test.ts index 99445904878..bd7cc2893a5 100644 --- a/packages/ai/test/provider/openai-chat.test.ts +++ b/packages/ai/test/provider/openai-chat.test.ts @@ -472,6 +472,45 @@ describe("OpenAI Chat route", () => { }), ) + it.effect("replays Gemini thought signatures as tool call extra content", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model, + messages: [ + Message.user("Weather in Paris and Tokyo?"), + Message.assistant([ + ToolCallPart.make({ + id: "call_1", + name: "lookup", + input: { city: "Paris" }, + providerMetadata: { openai: { extraContent: { google: { thought_signature: "sig_1" } } } }, + }), + ToolCallPart.make({ id: "call_2", name: "lookup", input: { city: "Tokyo" } }), + ]), + Message.tool({ id: "call_1", name: "lookup", result: "Sunny" }), + Message.tool({ id: "call_2", name: "lookup", result: "Rainy" }), + ], + }), + ) + + const assistant = prepared.body.messages[1] + expect(assistant?.role === "assistant" ? assistant.tool_calls : undefined).toEqual([ + { + id: "call_1", + type: "function", + function: { name: "lookup", arguments: encodeJson({ city: "Paris" }) }, + extra_content: { google: { thought_signature: "sig_1" } }, + }, + { + id: "call_2", + type: "function", + function: { name: "lookup", arguments: encodeJson({ city: "Tokyo" }) }, + }, + ]) + }), + ) + it.effect("limits OpenAI and Azure Chat tool call IDs to 40 characters", () => Effect.gen(function* () { const id = `call_${"a".repeat(48)}` @@ -1805,6 +1844,78 @@ describe("OpenAI Chat route", () => { }), ) + it.effect("preserves Gemini thought signatures on streamed parallel tool calls", () => + Effect.gen(function* () { + // Gemini's OpenAI-compatible endpoint omits `index`, streams each call whole, + // and signs only the first call of a parallel batch. + const body = sseEvents( + deltaChunk({ + role: "assistant", + tool_calls: [ + { + extra_content: { google: { thought_signature: "sig_1" } }, + id: "call_1", + type: "function", + function: { name: "lookup", arguments: '{"city":"Paris"}' }, + }, + ], + }), + deltaChunk({ + role: "assistant", + tool_calls: [{ id: "call_2", type: "function", function: { name: "lookup", arguments: '{"city":"Tokyo"}' } }], + }), + deltaChunk({}, "stop"), + ) + const response = yield* LLMClient.generate( + LLMRequest.update(request, { + tools: [ToolDefinition.make({ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } })], + }), + ).pipe(Effect.provide(fixedResponse(body))) + + expect(response.events.filter(LLMEvent.is.toolCall)).toEqual([ + { + type: "tool-call", + id: "call_1", + name: "lookup", + input: { city: "Paris" }, + providerExecuted: undefined, + providerMetadata: { openai: { extraContent: { google: { thought_signature: "sig_1" } } } }, + }, + { + type: "tool-call", + id: "call_2", + name: "lookup", + input: { city: "Tokyo" }, + providerExecuted: undefined, + providerMetadata: undefined, + }, + ]) + }), + ) + + it.effect("keeps extra content that arrives before the tool identity", () => + Effect.gen(function* () { + const body = sseEvents( + deltaChunk({ + tool_calls: [ + { index: 0, extra_content: { google: { thought_signature: "sig_1" } }, function: { arguments: "{" } }, + ], + }), + deltaChunk({ tool_calls: [{ index: 0, id: "call_1", function: { name: "lookup", arguments: "}" } }] }), + deltaChunk({}, "tool_calls"), + ) + const response = yield* LLMClient.generate( + LLMRequest.update(request, { + tools: [ToolDefinition.make({ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } })], + }), + ).pipe(Effect.provide(fixedResponse(body))) + + expect(response.events.filter(LLMEvent.is.toolCall).map((event) => event.providerMetadata)).toEqual([ + { openai: { extraContent: { google: { thought_signature: "sig_1" } } } }, + ]) + }), + ) + it.effect("does not finalize streamed tool calls when content is filtered", () => Effect.gen(function* () { const body = sseEvents(