From 74dbc509d74df46a2523676dd4068225c4f0c9b0 Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Tue, 29 Sep 2026 18:07:49 -0500 Subject: [PATCH] fix(ai): place prompt cache breakpoints on OpenRouter Anthropic and Qwen requests (#52110) --- packages/ai/src/cache-policy.ts | 18 +- packages/ai/src/protocols/openai-chat.ts | 87 ++++----- .../recordings/openrouter-reasoning.json | 6 +- .../openai-chat-reasoning.recorded.test.ts | 9 +- packages/ai/test/provider/openrouter.test.ts | 168 +++++++++++++++++- 5 files changed, 237 insertions(+), 51 deletions(-) diff --git a/packages/ai/src/cache-policy.ts b/packages/ai/src/cache-policy.ts index 2eb5c220720..ff111e77df9 100644 --- a/packages/ai/src/cache-policy.ts +++ b/packages/ai/src/cache-policy.ts @@ -51,6 +51,17 @@ const RESPECTS_INLINE_HINTS = new Set([ "openrouter", ]) +// OpenRouter upstreams other than Anthropic and Alibaba Qwen cache without breakpoints. Gemini uses only the last +// breakpoint, so a conversation-tail breakpoint writes a new cache every step and costs more than none. Qwen ignores +// breakpoints on tool definitions and caches tools with the system prompt. +const openRouterPolicy = (modelID: string): CachePolicyObject => { + // `~anthropic/claude-sonnet-latest` style IDs are OpenRouter aliases for the latest model in a family. + const id = modelID.replace(/^~/, "") + if (id.startsWith("anthropic/")) return AUTO + if (id.startsWith("qwen/")) return { system: true, messages: { tail: 1 } } + return NONE +} + const makeHint = (ttlSeconds: number | undefined): CacheHint => ttlSeconds !== undefined ? new CacheHint({ type: "ephemeral", ttlSeconds }) : new CacheHint({ type: "ephemeral" }) @@ -156,9 +167,10 @@ const countHints = (request: LLMRequest) => export const applyCachePolicy = (request: LLMRequest): LLMRequest => { if (!RESPECTS_INLINE_HINTS.has(request.model.route.id)) return request - if (request.model.route.id === "openrouter" && (request.cache === undefined || request.cache === "auto")) - return request - const policy = resolve(request.cache) + const policy = + request.model.route.id === "openrouter" && (request.cache === undefined || request.cache === "auto") + ? openRouterPolicy(request.model.id) + : resolve(request.cache) if (!policy.tools && !policy.system && !policy.messages) return request const hint = makeHint(policy.ttlSeconds) diff --git a/packages/ai/src/protocols/openai-chat.ts b/packages/ai/src/protocols/openai-chat.ts index 10a6a0db173..8f1794b1e34 100644 --- a/packages/ai/src/protocols/openai-chat.ts +++ b/packages/ai/src/protocols/openai-chat.ts @@ -20,7 +20,6 @@ import { type LLMRequest, type MediaPart, type ReasoningPart, - type TextPart, type ToolCallPart, type ToolDefinition, } from "../schema/index.js" @@ -45,6 +44,7 @@ const OpenAIChatCacheControl = Schema.Struct({ type: Schema.Literal("ephemeral"), ttl: Schema.optional(Schema.String), }) +type OpenAIChatCacheControl = Schema.Schema.Type const OpenAIChatFunction = Schema.Struct({ name: Schema.String, @@ -121,12 +121,15 @@ const decodeReasoningDetail = Schema.decodeUnknownOption(ReasoningDetail) const knownReasoningDetails = (details: ReadonlyArray) => details.flatMap((detail) => Option.toArray(decodeReasoningDetail(detail))) +const OpenAIChatTextContent = Schema.Struct({ + type: Schema.Literal("text"), + text: Schema.String, + cache_control: Schema.optional(OpenAIChatCacheControl), +}) +type OpenAIChatTextContent = Schema.Schema.Type + const OpenAIChatUserContent = Schema.Union([ - Schema.Struct({ - type: Schema.Literal("text"), - text: Schema.String, - cache_control: Schema.optional(OpenAIChatCacheControl), - }), + OpenAIChatTextContent, Schema.Struct({ type: Schema.Literal("image_url"), image_url: Schema.Struct({ url: Schema.String }), @@ -136,6 +139,7 @@ const OpenAIChatUserContent = Schema.Union([ file: Schema.Struct({ filename: Schema.String, file_data: Schema.String }), }), ]) +type OpenAIChatUserContent = Schema.Schema.Type const OpenAIChatMessage = Schema.Union([ Schema.Struct({ @@ -149,21 +153,19 @@ const OpenAIChatMessage = Schema.Union([ Schema.StructWithRest( Schema.Struct({ role: Schema.Literal("assistant"), - content: Schema.NullOr(Schema.String), + content: Schema.NullOr(Schema.Union([Schema.String, Schema.Array(OpenAIChatTextContent)])), tool_calls: optionalArray(OpenAIChatAssistantToolCall), reasoning_content: Schema.optional(Schema.String), reasoning: Schema.optional(Schema.String), reasoning_text: Schema.optional(Schema.String), reasoning_details: Schema.optional(Schema.Unknown), - cache_control: Schema.optional(OpenAIChatCacheControl), }), [Schema.Record(Schema.String, Schema.Unknown)], ), Schema.Struct({ role: Schema.Literal("tool"), tool_call_id: Schema.String, - content: Schema.String, - cache_control: Schema.optional(OpenAIChatCacheControl), + content: Schema.Union([Schema.String, Schema.Array(OpenAIChatTextContent)]), }), ]).pipe(Schema.toTaggedUnion("role")) type OpenAIChatMessage = Schema.Schema.Type @@ -327,9 +329,7 @@ export interface ParserState { // OpenAI Chat wire format. Keep provider quirks here instead of leaking native // fields into `LLMRequest`. interface LoweringOptions { - readonly cacheControl?: ( - cache: CacheHint | undefined, - ) => Schema.Schema.Type | undefined + readonly cacheControl?: (cache: CacheHint | undefined) => OpenAIChatCacheControl | undefined readonly toolCallID?: (id: string) => string } @@ -415,7 +415,7 @@ const lowerUserMessage = Effect.fn("OpenAIChat.lowerUserMessage")(function* ( message: OpenAIChatRequestMessage, options: LoweringOptions, ) { - const content: Array> = [] + const content: OpenAIChatUserContent[] = [] for (const part of message.content) { if (part.type === "text") { content.push({ type: "text", text: part.text, cache_control: options.cacheControl?.(part.cache) }) @@ -441,14 +441,14 @@ const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(func requireReasoning: boolean, options: LoweringOptions & { readonly providerMetadataKey: string }, ) { - const content: TextPart[] = [] + const content: OpenAIChatTextContent[] = [] const reasoning: ReasoningPart[] = [] const toolCalls: OpenAIChatAssistantToolCall[] = [] for (const part of message.content) { if (!ProviderShared.supportsContent(part, ["text", "reasoning", "tool-call"])) return yield* ProviderShared.unsupportedContent("OpenAI Chat", "assistant", ["text", "reasoning", "tool-call"]) if (part.type === "text") { - content.push(part) + content.push({ type: "text", text: part.text, cache_control: options.cacheControl?.(part.cache) }) continue } if (part.type === "reasoning") { @@ -486,14 +486,15 @@ const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(func if (reasoning.length === 0) return nativeReasoning ?? (requireReasoning ? "" : undefined) return text })() - const cached = message.content.findLast((part) => "cache" in part && part.cache !== undefined) - const cacheControl = options.cacheControl?.(cached && "cache" in cached ? cached.cache : undefined) const result = { role: "assistant" as const, - content: content.length > 0 ? content.map((part) => part.text).join("") : toolCalls.length > 0 ? null : "", + content: (() => { + if (content.some((part) => part.cache_control !== undefined)) return content + if (content.length === 0 && toolCalls.length > 0) return null + return content.map((part) => part.text).join("") + })(), ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}), ...(details !== undefined ? { reasoning_details: details } : {}), - ...(cacheControl !== undefined ? { cache_control: cacheControl } : {}), } if (field === undefined || reasoningText === undefined) return result return { ...result, [field]: reasoningText } @@ -504,33 +505,38 @@ const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* ( options: LoweringOptions, ) { const messages: OpenAIChatMessage[] = [] - const attachments: Array> = [] + const attachments: OpenAIChatUserContent[] = [] for (const part of message.content) { if (!ProviderShared.supportsContent(part, ["tool-result"])) return yield* ProviderShared.unsupportedContent("OpenAI Chat", "tool", ["tool-result"]) if (part.result.type !== "content") { - messages.push({ - role: "tool", - tool_call_id: options.toolCallID?.(part.id) ?? part.id, - content: ProviderShared.toolResultText(part), - cache_control: options.cacheControl?.(part.cache), - }) + messages.push( + toolMessage( + options.toolCallID?.(part.id) ?? part.id, + ProviderShared.toolResultText(part), + options.cacheControl?.(part.cache), + ), + ) continue } const content: ReadonlyArray = part.result.value const text = content.filter((item) => item.type === "text").map((item) => item.text) - messages.push({ - role: "tool", - tool_call_id: options.toolCallID?.(part.id) ?? part.id, - content: text.join("\n"), - cache_control: options.cacheControl?.(part.cache), - }) + messages.push( + toolMessage(options.toolCallID?.(part.id) ?? part.id, text.join("\n"), options.cacheControl?.(part.cache)), + ) const files = content.filter((item) => item.type === "file") attachments.push(...(yield* Effect.forEach(files, (item) => lowerMedia(ProviderShared.toolFileMedia(item))))) } return { messages, attachments } }) +// Chat cache breakpoints belong on text content parts, not on the message itself. +const toolMessage = (toolCallID: string, text: string, cacheControl: OpenAIChatCacheControl | undefined) => ({ + role: "tool" as const, + tool_call_id: toolCallID, + content: cacheControl === undefined ? text : [{ type: "text" as const, text, cache_control: cacheControl }], +}) + const lowerMessage = Effect.fn("OpenAIChat.lowerMessage")(function* ( message: OpenAIChatRequestMessage, reasoningField: string | undefined, @@ -589,7 +595,7 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request: if (requireAssistantAfterTool && messages.at(-1)?.role === "tool") messages.push({ role: "assistant", content: "Done." }) } - const pendingAttachments: Array> = [] + const pendingAttachments: OpenAIChatUserContent[] = [] const flushAttachments = () => { if (pendingAttachments.length === 0) return bridgeTools() @@ -599,24 +605,25 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request: if (message.role === "user") bridgeTools() if (message.role === "system") { const part = yield* ProviderShared.wrappedSystemUpdate("OpenAI Chat", message) + const cacheControl = options.cacheControl?.(part.cache) if (pendingAttachments.length > 0) { messages.push({ role: "user", content: [ ...pendingAttachments.splice(0), - { type: "text", text: part.text, cache_control: options.cacheControl?.(part.cache) }, + { type: "text", text: part.text, cache_control: cacheControl }, ], }) continue } const previous = messages.at(-1) if (previous?.role === "user" && typeof previous.content === "string") - messages[messages.length - 1] = options.cacheControl?.(part.cache) + messages[messages.length - 1] = cacheControl ? { role: "user", content: [ { type: "text", text: previous.content }, - { type: "text", text: part.text, cache_control: options.cacheControl(part.cache) }, + { type: "text", text: part.text, cache_control: cacheControl }, ], } : { role: "user", content: `${previous.content}\n${part.text}` } @@ -625,15 +632,15 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request: role: "user", content: [ ...previous.content, - { type: "text", text: part.text, cache_control: options.cacheControl?.(part.cache) }, + { type: "text", text: part.text, cache_control: cacheControl }, ], } else messages.push( - options.cacheControl?.(part.cache) + cacheControl ? { role: "user", - content: [{ type: "text", text: part.text, cache_control: options.cacheControl(part.cache) }], + content: [{ type: "text", text: part.text, cache_control: cacheControl }], } : { role: "user", content: part.text }, ) diff --git a/packages/ai/test/fixtures/recordings/openrouter-reasoning.json b/packages/ai/test/fixtures/recordings/openrouter-reasoning.json index 852a6f6b9f3..d819ffd064c 100644 --- a/packages/ai/test/fixtures/recordings/openrouter-reasoning.json +++ b/packages/ai/test/fixtures/recordings/openrouter-reasoning.json @@ -4,7 +4,7 @@ "model": "anthropic/claude-sonnet-4.6", "tags": ["prefix:openai-compatible-chat", "provider:openrouter", "protocol:openai-chat", "reasoning"], "name": "openrouter-reasoning", - "recordedAt": "2026-07-18T11:28:39.267Z" + "recordedAt": "2026-09-29T16:19:02.835Z" }, "interactions": [ { @@ -15,14 +15,14 @@ "headers": { "content-type": "application/json" }, - "body": "{\"model\":\"anthropic/claude-sonnet-4.6\",\"messages\":[{\"role\":\"system\",\"content\":\"Think through the arithmetic, then reply with only the final integer.\"},{\"role\":\"user\",\"content\":\"What is 173 multiplied by 219?\"}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"temperature\":0,\"reasoning\":{\"max_tokens\":1024},\"max_completion_tokens\":1536,\"store\":false,\"usage\":{\"include\":true}}" + "body": "{\"model\":\"anthropic/claude-sonnet-4.6\",\"messages\":[{\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"Think through the arithmetic, then reply with only the final integer.\",\"cache_control\":{\"type\":\"ephemeral\"}}]},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is 173 multiplied by 219?\",\"cache_control\":{\"type\":\"ephemeral\"}}]}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"store\":false,\"max_completion_tokens\":1536,\"temperature\":0,\"usage\":{\"include\":true},\"reasoning\":{\"max_tokens\":1024}}" }, "response": { "status": 200, "headers": { "content-type": "text/event-stream" }, - "body": ": OPENROUTER PROCESSING\n\n: OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1784374117-AXXPsQRoclZeQGx2uHeK\",\"object\":\"chat.completion.chunk\",\"created\":1784374117,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":\"173\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"173\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\n: OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1784374117-AXXPsQRoclZeQGx2uHeK\",\"object\":\"chat.completion.chunk\",\"created\":1784374117,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":\" × 219\\n\\n173 × 200 = 34,600\\n173 × 19 = 173 × 20 - 173 = 3,460 - 173 = 3,287\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\" × 219\\n\\n173 × 200 = 34,600\\n173 × 19 = 173 × 20 - 173 = 3,460 - 173 = 3,287\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784374117-AXXPsQRoclZeQGx2uHeK\",\"object\":\"chat.completion.chunk\",\"created\":1784374117,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":\"\\n\\n34,600 + 3,287 = 37,887\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"\\n\\n34,600 + 3,287 = 37,887\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784374117-AXXPsQRoclZeQGx2uHeK\",\"object\":\"chat.completion.chunk\",\"created\":1784374117,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"signature\":\"EtgCCosBCA8YAipA0W4viH3kgBs43Cl5ewwVBPXTQElvzfbA2TLF4iSbKy9ZZDCSDjjAlF3Bs4ELEnP3vrrTuTioC6OB380lXQdyIDIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJDRjMGYwNDZmLTI1ZmQtNDVmYi1iZmIzLWEwOGE4ZTI0OWNhNxIMMiUlJC3x/5p5PuTwGgwlc8eipZyoM94BHwMiMO45uQx/ymeOjbugi7RDVPZ4jZXSIiEbVi2CD7zPjAK5fFQoVGP1HD55v9CER823JCp6Dg5Xb7Lrk6NUd1XN2KTKrttK7mATE+IBrDTFmor/1cNeg+9gjIbxM/jn/6L5HPmh3/esEVu24Q0IGLZVoE7cTgGgxsrceKMD71Jp2XQgIWD8ltsPfWw3gSc4p+z18UuPN6LuR0mHHENTnClHrAPnOrxbDIl4ZwZgMX8YAQ==\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784374117-AXXPsQRoclZeQGx2uHeK\",\"object\":\"chat.completion.chunk\",\"created\":1784374117,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"37887\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784374117-AXXPsQRoclZeQGx2uHeK\",\"object\":\"chat.completion.chunk\",\"created\":1784374117,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":null},\"finish_reason\":\"stop\",\"native_finish_reason\":\"end_turn\"}]}\n\ndata: {\"id\":\"gen-1784374117-AXXPsQRoclZeQGx2uHeK\",\"object\":\"chat.completion.chunk\",\"created\":1784374117,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"service_tier\":\"default\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"end_turn\"}],\"usage\":{\"prompt_tokens\":61,\"completion_tokens\":80,\"total_tokens\":141,\"cost\":0.001383,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.001383,\"upstream_inference_prompt_cost\":0.000183,\"upstream_inference_completions_cost\":0.0012},\"completion_tokens_details\":{\"reasoning_tokens\":29,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n" + "body": "data: {\"id\":\"gen-1790698739-Acm5E9n2uVTO14sqrick\",\"object\":\"chat.completion.chunk\",\"created\":1790698739,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Claude Platform on AWS\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":\"173\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"173\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\n: OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1790698739-Acm5E9n2uVTO14sqrick\",\"object\":\"chat.completion.chunk\",\"created\":1790698739,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Claude Platform on AWS\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":\" × 219\\n\\n173 × 200 = 34,600\\n173 × 19 = 173 × 20 - 173 = 3,460 - 173 = 3,287\\n\\n34,600 + 3,287 = 37,887\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\" × 219\\n\\n173 × 200 = 34,600\\n173 × 19 = 173 × 20 - 173 = 3,460 - 173 = 3,287\\n\\n34,600 + 3,287 = 37,887\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1790698739-Acm5E9n2uVTO14sqrick\",\"object\":\"chat.completion.chunk\",\"created\":1790698739,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Claude Platform on AWS\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"signature\":\"Ev8CCrIBCBIYAipA0W4viH3kgBs43Cl5ewwVBPXTQElvzfbA2TLF4iSbKy9ZZDCSDjjAlF3Bs4ELEnP3vrrTuTioC6OB380lXQdyIDIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJDRjMGYwNDZmLTI1ZmQtNDVmYi1iZmIzLWEwOGE4ZTI0OWNhN3oedXByb2ZfMDExQ2VkSE1TN0tEcTM4Z2JkUkp0bXN3qAH2ye/VBhIMhjcPOnE2wLhs0P05GgwA3z9luwbhnSyRMSQiMDYeIIThirdkJb/n2s2TxUk5EOMrSc1xejAb0RN7BIl66P5KhtwXvcZTiyCkBS8+7Cp6MOinWi+I8U7DhWskSA0pHZqaAncbV7qCIHkoZ5aT8jiPWplAOnlciDo2tLkGsLnzXrnfhI11GTF0kcfyhbrt3tjEDSiPstBbT2u19LJ5IqwV2VKl0pWVdl4PTsDwC09ClbnJj1y0A3AM9gEh4B9mEo4BvT0XVEncux4YAQ==.eyJlbmRwb2ludF9zbHVnIjoiYW50aHJvcGljL2NsYXVkZS00LjYtc29ubmV0LTIwMjYwMjE3fGNsYXVkZS1vbi1hd3MifQ\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1790698739-Acm5E9n2uVTO14sqrick\",\"object\":\"chat.completion.chunk\",\"created\":1790698739,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Claude Platform on AWS\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"37\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1790698739-Acm5E9n2uVTO14sqrick\",\"object\":\"chat.completion.chunk\",\"created\":1790698739,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Claude Platform on AWS\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"887\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1790698739-Acm5E9n2uVTO14sqrick\",\"object\":\"chat.completion.chunk\",\"created\":1790698739,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Claude Platform on AWS\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":null},\"finish_reason\":\"stop\",\"native_finish_reason\":\"end_turn\"}]}\n\ndata: {\"id\":\"gen-1790698739-Acm5E9n2uVTO14sqrick\",\"object\":\"chat.completion.chunk\",\"created\":1790698739,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Claude Platform on AWS\",\"service_tier\":\"default\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"end_turn\"}],\"usage\":{\"prompt_tokens\":61,\"completion_tokens\":80,\"total_tokens\":141,\"cost\":0.001383,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.001383,\"upstream_inference_prompt_cost\":0.000183,\"upstream_inference_completions_cost\":0.0012},\"completion_tokens_details\":{\"reasoning_tokens\":73,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n" } } ] diff --git a/packages/ai/test/provider/openai-chat-reasoning.recorded.test.ts b/packages/ai/test/provider/openai-chat-reasoning.recorded.test.ts index 76059ef8bf6..89429916ce6 100644 --- a/packages/ai/test/provider/openai-chat-reasoning.recorded.test.ts +++ b/packages/ai/test/provider/openai-chat-reasoning.recorded.test.ts @@ -89,7 +89,14 @@ for (const item of cases) { const replay = yield* compileRequest(LLM.request({ model: item.model, messages: [response.message] })) expect(replay.body.messages).toMatchObject([ - { role: "assistant", content: response.text, reasoning: response.reasoning }, + { + role: "assistant", + content: + item.model.route.id === "openrouter" + ? [{ type: "text", text: response.text, cache_control: { type: "ephemeral" } }] + : response.text, + reasoning: response.reasoning, + }, ]) const replayDetails = replay.body.messages[0]?.role === "assistant" ? replay.body.messages[0].reasoning_details : undefined diff --git a/packages/ai/test/provider/openrouter.test.ts b/packages/ai/test/provider/openrouter.test.ts index 7e717b96bde..a4b446b17e7 100644 --- a/packages/ai/test/provider/openrouter.test.ts +++ b/packages/ai/test/provider/openrouter.test.ts @@ -32,6 +32,125 @@ describe("OpenRouter", () => { }), ) + it.effect("places default cache breakpoints on tools, system boundaries, and the conversation tail", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("anthropic/claude-sonnet-4.6"), + system: [ + { type: "text", text: "Base agent" }, + { type: "text", text: "Model details" }, + { type: "text", text: "Project instructions" }, + ], + tools: [ + { name: "read", description: "Read", inputSchema: { type: "object", properties: {} } }, + { name: "lookup", description: "Lookup", inputSchema: { type: "object", properties: {} } }, + ], + prompt: "Hello", + }), + ) + + expect(prepared.body.tools?.map((tool) => tool.cache_control)).toEqual([undefined, { type: "ephemeral" }]) + expect(prepared.body.messages).toMatchObject([ + { + role: "system", + content: [ + { text: "Base agent", cache_control: { type: "ephemeral" } }, + { text: "Model details" }, + { text: "Project instructions", cache_control: { type: "ephemeral" } }, + ], + }, + { role: "user", content: [{ text: "Hello", cache_control: { type: "ephemeral" } }] }, + ]) + expect(prepared.body.messages[0]?.content).not.toContainEqual( + expect.objectContaining({ text: "Model details", cache_control: expect.anything() }), + ) + }), + ) + + it.effect("places default cache breakpoints on OpenRouter latest-model aliases", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("~anthropic/claude-sonnet-latest"), + system: "Base agent", + tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object", properties: {} } }], + prompt: "Hello", + }), + ) + + expect(prepared.body.tools?.[0]?.cache_control).toEqual({ type: "ephemeral" }) + expect(prepared.body.messages).toMatchObject([ + { role: "system", content: [{ text: "Base agent", cache_control: { type: "ephemeral" } }] }, + { role: "user", content: [{ text: "Hello", cache_control: { type: "ephemeral" } }] }, + ]) + }), + ) + + it.effect("skips the tool breakpoint for Qwen, which caches tools with the system prompt", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("qwen/qwen3-coder-plus"), + system: "Base agent", + tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object", properties: {} } }], + prompt: "Hello", + }), + ) + + expect(prepared.body.tools?.[0]?.cache_control).toBeUndefined() + expect(prepared.body.messages).toMatchObject([ + { role: "system", content: [{ text: "Base agent", cache_control: { type: "ephemeral" } }] }, + { role: "user", content: [{ text: "Hello", cache_control: { type: "ephemeral" } }] }, + ]) + }), + ) + + it.effect("places the default Qwen conversation-tail breakpoint inside tool-result text", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("qwen/qwen3-coder-plus"), + messages: [ + Message.user("Call the tool"), + Message.assistant([{ type: "tool-call", id: "call_1", name: "lookup", input: {} }]), + Message.tool({ id: "call_1", name: "lookup", result: "Done" }), + ], + }), + ) + + expect(prepared.body.messages.at(-1)).toMatchObject({ + role: "tool", + tool_call_id: "call_1", + content: [{ type: "text", text: '"Done"', cache_control: { type: "ephemeral" } }], + }) + }), + ) + + it.effect("sends no default breakpoints to upstreams that cache without them", () => + Effect.gen(function* () { + const openrouter = OpenRouter.configure({ apiKey: "test-key" }) + const bodies = yield* Effect.forEach(["google/gemini-2.5-flash", "openai/gpt-5-mini"], (id) => + compileRequest( + LLM.request({ + model: openrouter.model(id), + system: "Base agent", + tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object", properties: {} } }], + prompt: "Hello", + }), + ).pipe(Effect.map((prepared) => prepared.body)), + ) + + bodies.forEach((body) => { + expect(body.tools?.[0]?.cache_control).toBeUndefined() + expect(body.messages).toMatchObject([ + { role: "system", content: "Base agent" }, + { role: "user", content: "Hello" }, + ]) + }) + }), + ) + it.effect("lowers the native cache policy to OpenRouter cache controls", () => Effect.gen(function* () { const prepared = yield* compileRequest( @@ -74,6 +193,8 @@ describe("OpenRouter", () => { cache: "none", messages: [ Message.user("Call the tool"), + Message.assistant("Unmarked reply"), + Message.user("Call again"), Message.assistant([ { type: "text", text: "Calling", cache: new CacheHint({ type: "ephemeral" }) }, { type: "tool-call", id: "call_1", name: "lookup", input: {} }, @@ -90,8 +211,17 @@ describe("OpenRouter", () => { expect(prepared.body.messages).toMatchObject([ { role: "user", content: "Call the tool" }, - { role: "assistant", content: "Calling", cache_control: { type: "ephemeral" } }, - { role: "tool", content: '"Done"', cache_control: { type: "ephemeral", ttl: "1h" } }, + { role: "assistant", content: "Unmarked reply" }, + { role: "user", content: "Call again" }, + { + role: "assistant", + content: [{ type: "text", text: "Calling", cache_control: { type: "ephemeral" } }], + tool_calls: [{ id: "call_1", type: "function", function: { name: "lookup", arguments: "{}" } }], + }, + { + role: "tool", + content: [{ type: "text", text: '"Done"', cache_control: { type: "ephemeral", ttl: "1h" } }], + }, ]) }), ) @@ -118,7 +248,7 @@ describe("OpenRouter", () => { }), ) - it.effect("preserves cache policy hints on reasoning-only assistant messages", () => + it.effect("does not emit text cache markers on reasoning-only assistant messages", () => Effect.gen(function* () { const prepared = yield* compileRequest( LLM.request({ @@ -130,8 +260,38 @@ describe("OpenRouter", () => { expect(prepared.body.messages).toMatchObject([ { role: "user", content: "Think" }, - { role: "assistant", cache_control: { type: "ephemeral" } }, + { role: "assistant", content: "" }, ]) + expect(prepared.body.messages[1]).not.toHaveProperty("cache_control") + }), + ) + + it.effect("counts wrapped system-update markers once so all four default breakpoints survive", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("anthropic/claude-sonnet-4.6"), + system: [ + { type: "text", text: "Base agent" }, + { type: "text", text: "Project instructions" }, + ], + tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object", properties: {} } }], + messages: [Message.user("Start"), Message.system("Updated instructions")], + }), + ) + + expect(prepared.body.tools?.[0]?.cache_control).toEqual({ type: "ephemeral" }) + expect(prepared.body.messages.at(-1)).toMatchObject({ + role: "user", + content: [ + { type: "text", text: "Start" }, + { + type: "text", + text: "\nUpdated instructions\n", + cache_control: { type: "ephemeral" }, + }, + ], + }) }), )