From c14b662da705137fbbd9e4c25f22fc6bc01021c3 Mon Sep 17 00:00:00 2001 From: nexxeln <95541290+nexxeln@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:43:51 +0000 Subject: [PATCH] fix(ai): guard Sonnet 5.5 Bedrock Converse requests --- packages/ai/src/protocols/bedrock-converse.ts | 110 +++++++-- .../bedrock-converse-sonnet55.test.ts | 223 ++++++++++++++++++ 2 files changed, 318 insertions(+), 15 deletions(-) create mode 100644 packages/ai/test/provider/bedrock-converse-sonnet55.test.ts diff --git a/packages/ai/src/protocols/bedrock-converse.ts b/packages/ai/src/protocols/bedrock-converse.ts index 52a411e966e..a665df219e8 100644 --- a/packages/ai/src/protocols/bedrock-converse.ts +++ b/packages/ai/src/protocols/bedrock-converse.ts @@ -6,6 +6,7 @@ import { AIError, LLMEvent, Usage, + mergeJsonRecords, type CacheHint, type FinishReason, type FinishReasonDetails, @@ -23,7 +24,7 @@ import { JsonObject, optionalArray, ProviderShared } from "./shared.js" import { BedrockAuth } from "./utils/bedrock-auth.js" import { BedrockCache } from "./utils/bedrock-cache.js" import { BedrockMedia } from "./utils/bedrock-media.js" -import { supportsThinkingBlockBinding, THINKING_BINDING_BETA } from "./utils/claude-model.js" +import { claudeVersion, supportsThinkingBlockBinding, THINKING_BINDING_BETA } from "./utils/claude-model.js" import { Lifecycle } from "./utils/lifecycle.js" import { MistralToolID } from "./utils/mistral-tool-id.js" import { ToolStream } from "./utils/tool-stream.js" @@ -428,6 +429,10 @@ const lowerSystem = (breakpoints: BedrockCache.Breakpoints, system: ReadonlyArra // Nova 2 rejects `maxTokens` at high reasoning effort, where its output can exceed the field's maximum. Other models // that take `reasoningConfig`, such as Grok on Bedrock, accept it. const isNova2 = (model: LanguageModel) => /\bamazon\.nova-2-/.test(model.id) +const isSonnet55 = (model: LanguageModel) => { + const version = claudeVersion(model.id) + return version?.family === "sonnet" && version.major === 5 && version.minor === 5 +} const isHighReasoningEffort = Schema.is( Schema.Struct({ additionalModelRequestFields: Schema.Struct({ @@ -437,23 +442,31 @@ const isHighReasoningEffort = Schema.is( ) const Options = Schema.Struct({ - thinking: Schema.optional(Schema.Struct({ type: Schema.Literal("enabled"), budgetTokens: Schema.Number })), + thinking: Schema.optional( + Schema.Union([ + Schema.Struct({ type: Schema.Literal("enabled"), budgetTokens: Schema.Number }), + Schema.Struct({ type: Schema.Literal("between_tools") }), + ]), + ), }) export type OptionsInput = typeof Options.Type const decodeOptions = ProviderShared.validateWith(Schema.decodeUnknownEffect(Options)) // Claude on Bedrock requires the thinking budget below `maxTokens`, with a minimum of 1,024. const MIN_THINKING_BUDGET = 1_024 -const isThinkingDisabled = Schema.is( +const isThinkingWithoutBinding = Schema.is( Schema.Struct({ - additionalModelRequestFields: Schema.Struct({ thinking: Schema.Struct({ type: Schema.Literal("disabled") }) }), + additionalModelRequestFields: Schema.Struct({ + thinking: Schema.Struct({ type: Schema.Literals(["disabled", "between_tools"]) }), + }), }), ) // Claude 5.1+ binds each thinking signature to the prefix above it. Ask Bedrock to drop the affected blocks instead of -// failing when that prefix changes. `http.body` overlays this field by field, so callers can still override it. +// failing when that prefix changes. `http.body` overlays this field by field; between_tools cannot carry binding. const applyThinkingBindingDefault = (request: LLMRequest, thinking: Readonly> | undefined) => { - if (isThinkingDisabled(request.http?.body)) return thinking + if (thinking?.type === "between_tools") return thinking + if (isThinkingWithoutBinding(request.http?.body)) return thinking if (!supportsThinkingBlockBinding(request.model)) return thinking return { ...(thinking ?? { type: "adaptive" as const }), @@ -466,20 +479,24 @@ const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request: const flattened = ProviderShared.flattenToolRequest(request) const generation = request.generation const options = yield* decodeOptions(request.providerOptions ?? {}) + if (options.thinking?.type === "between_tools" && !isSonnet55(request.model)) + return yield* ProviderShared.invalidRequest("Bedrock Converse between_tools thinking requires Claude Sonnet 5.5") const maxTokens = isNova2(request.model) && isHighReasoningEffort(request.http?.body) ? undefined : generation?.maxTokens const thinking = applyThinkingBindingDefault( request, options.thinking === undefined ? undefined - : { - type: "enabled", - budget_tokens: ProviderShared.fitThinkingBudget( - options.thinking.budgetTokens, - maxTokens, - MIN_THINKING_BUDGET, - ), - }, + : options.thinking.type === "between_tools" + ? { type: "between_tools" } + : { + type: "enabled", + budget_tokens: ProviderShared.fitThinkingBudget( + options.thinking.budgetTokens, + maxTokens, + MIN_THINKING_BUDGET, + ), + }, ) // Bedrock-Claude shares Anthropic's 4-breakpoint cap. Spend the budget in // tools → system → messages order to favour the highest-impact prefixes. @@ -515,7 +532,7 @@ const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request: stopSequences: generation?.stop, } })() - return { + const body = { modelId: request.model.id, messages, system, @@ -533,6 +550,69 @@ const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request: ...(thinking?.block_binding === undefined ? {} : { anthropic_beta: [THINKING_BINDING_BETA] }), }, } + if (isSonnet55(request.model)) yield* validateSonnet55Body(request, body) + return body +}) + +const validateSonnet55Body = Effect.fn("BedrockConverse.validateSonnet55Body")(function* ( + request: LLMRequest, + body: Record, +) { + // The HTTP body overlay is merged after protocol lowering; check the request that will actually be sent. + const effective = mergeJsonRecords(body, request.http?.body) + const config = effective?.toolConfig + const choice = ProviderShared.isRecord(config) ? config.toolChoice : undefined + if (ProviderShared.isRecord(choice) && ("any" in choice || "tool" in choice)) + return yield* ProviderShared.invalidRequest( + "Bedrock Converse Claude Sonnet 5.5 does not support forced tool choice; use auto instead", + ) + const additional = effective?.additionalModelRequestFields + const thinking = ProviderShared.isRecord(additional) ? additional.thinking : undefined + if (ProviderShared.isRecord(thinking) && thinking.type === "enabled") + return yield* ProviderShared.invalidRequest( + "Bedrock Converse Claude Sonnet 5.5 does not support manual thinking budgets; use adaptive thinking instead", + ) + if (ProviderShared.isRecord(thinking) && thinking.type === "disabled") + return yield* ProviderShared.invalidRequest( + "Bedrock Converse Claude Sonnet 5.5 does not support disabled thinking; use between_tools instead", + ) + if ( + ProviderShared.isRecord(thinking) && + thinking.type === "between_tools" && + ("budget_tokens" in thinking || "block_binding" in thinking || "display" in thinking) + ) + return yield* ProviderShared.invalidRequest( + "Bedrock Converse Claude Sonnet 5.5 between_tools does not accept a budget, block binding, or display", + ) + const output = ProviderShared.isRecord(additional) ? additional.output_config : undefined + if ( + ProviderShared.isRecord(thinking) && + thinking.type === "between_tools" && + ProviderShared.isRecord(output) && + (output.effort === "xhigh" || output.effort === "max") + ) + return yield* ProviderShared.invalidRequest( + "Bedrock Converse Claude Sonnet 5.5 between_tools does not support xhigh or max effort; use adaptive thinking", + ) + const inference = effective?.inferenceConfig + if (ProviderShared.isRecord(inference) && inference.temperature !== undefined && inference.temperature !== 1) + return yield* ProviderShared.invalidRequest( + "Bedrock Converse Claude Sonnet 5.5 does not support non-default temperature; omit temperature instead", + ) + // AWS's Claude parameter reference documents 0.999 for top_p, while Anthropic's API uses 1. + if ( + ProviderShared.isRecord(inference) && + inference.topP !== undefined && + inference.topP !== 1 && + inference.topP !== 0.999 + ) + return yield* ProviderShared.invalidRequest( + "Bedrock Converse Claude Sonnet 5.5 does not support non-default topP; omit topP instead", + ) + if (ProviderShared.isRecord(additional) && additional.top_k !== undefined && additional.top_k !== 0) + return yield* ProviderShared.invalidRequest( + "Bedrock Converse Claude Sonnet 5.5 does not support non-default top_k; omit topK instead", + ) }) // ============================================================================= diff --git a/packages/ai/test/provider/bedrock-converse-sonnet55.test.ts b/packages/ai/test/provider/bedrock-converse-sonnet55.test.ts new file mode 100644 index 00000000000..d0cba3143c9 --- /dev/null +++ b/packages/ai/test/provider/bedrock-converse-sonnet55.test.ts @@ -0,0 +1,223 @@ +import { EventStreamCodec } from "@smithy/eventstream-codec" +import { fromUtf8, toUtf8 } from "@smithy/util-utf8" +import { describe, expect } from "bun:test" +import { Effect } from "effect" +import { HttpClientRequest } from "effect/unstable/http" +import { LLM, LLMRequest, ToolChoice, ToolDefinition } from "../../src/index.js" +import { AmazonBedrock } from "../../src/providers.js" +import { LLMClient } from "../../src/route.js" +import { compileRequest } from "../../src/route/client.js" +import { it } from "../lib/effect.js" +import { dynamicResponse } from "../lib/http.js" + +const bedrock = (id: string, settings: Parameters[1] = {}) => + AmazonBedrock.model(id, { baseURL: "https://bedrock-runtime.test", apiKey: "test-bearer", ...settings }) + +const request = LLM.request({ + model: bedrock("global.anthropic.claude-sonnet-5-5"), + prompt: "What is the weather?", + cache: "none", + tools: [ToolDefinition.make({ name: "lookup", description: "Weather lookup", inputSchema: { type: "object" } })], +}) + +describe("Bedrock Converse Claude Sonnet 5.5", () => { + it.effect("rejects required tool choice rather than silently allowing a different tool policy", () => + Effect.gen(function* () { + const error = yield* compileRequest( + LLMRequest.update(request, { toolChoice: ToolChoice.make({ type: "required" }) }), + ).pipe(Effect.flip) + + expect(error.reason._tag).toBe("InvalidRequest") + expect(error.message).toContain("tool choice") + }), + ) + + it.effect("rejects named tool choice supplied by a body overlay", () => + Effect.gen(function* () { + const error = yield* compileRequest( + LLMRequest.update(request, { http: { body: { toolConfig: { toolChoice: { tool: { name: "lookup" } } } } } }), + ).pipe(Effect.flip) + + expect(error.reason._tag).toBe("InvalidRequest") + expect(error.message).toContain("tool choice") + }), + ) + + it.effect("rejects manual thinking budgets instead of silently dropping the budget", () => + Effect.gen(function* () { + const error = yield* compileRequest( + LLMRequest.update(request, { + model: bedrock("global.anthropic.claude-sonnet-5-5", { + thinking: { type: "enabled", budgetTokens: 4_000 }, + }), + }), + ).pipe(Effect.flip) + + expect(error.reason._tag).toBe("InvalidRequest") + expect(error.message).toContain("manual thinking") + }), + ) + + it.effect("rejects explicitly disabled thinking with an actionable alternative", () => + Effect.gen(function* () { + const error = yield* compileRequest( + LLMRequest.update(request, { + http: { body: { additionalModelRequestFields: { thinking: { type: "disabled" } } } }, + }), + ).pipe(Effect.flip) + + expect(error.reason._tag).toBe("InvalidRequest") + expect(error.message).toContain("between_tools") + }), + ) + + it.effect("rejects non-default temperature without discarding the caller's sampling preference", () => + Effect.gen(function* () { + const error = yield* compileRequest(LLMRequest.update(request, { generation: { temperature: 0 } })).pipe( + Effect.flip, + ) + + expect(error.reason._tag).toBe("InvalidRequest") + expect(error.message).toContain("temperature") + }), + ) + + it.effect("rejects non-default topP configured through the Bedrock provider", () => + Effect.gen(function* () { + const error = yield* compileRequest( + LLMRequest.update(request, { model: bedrock("global.anthropic.claude-sonnet-5-5", { topP: 0.8 }) }), + ).pipe(Effect.flip) + + expect(error.reason._tag).toBe("InvalidRequest") + expect(error.message).toContain("topP") + }), + ) + + it.effect("rejects non-default topK in additional model request fields", () => + Effect.gen(function* () { + const error = yield* compileRequest(LLMRequest.update(request, { generation: { topK: 40 } })).pipe(Effect.flip) + + expect(error.reason._tag).toBe("InvalidRequest") + expect(error.message).toContain("top_k") + }), + ) + + it.effect("does not add adaptive thinking binding to an explicit between_tools request", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLMRequest.update(request, { + http: { body: { additionalModelRequestFields: { thinking: { type: "between_tools" } } } }, + }), + ) + + expect(prepared.body.additionalModelRequestFields).toBeUndefined() + }), + ) + + it.effect("accepts between_tools as a typed Bedrock model setting", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLMRequest.update(request, { + model: bedrock("global.anthropic.claude-sonnet-5-5", { thinking: { type: "between_tools" } }), + }), + ) + + expect(prepared.body.additionalModelRequestFields).toEqual({ thinking: { type: "between_tools" } }) + }), + ) + + it.effect("rejects a between_tools overlay that retains a manual budget", () => + Effect.gen(function* () { + const error = yield* compileRequest( + LLMRequest.update(request, { + model: bedrock("global.anthropic.claude-sonnet-5-5", { + thinking: { type: "enabled", budgetTokens: 4_000 }, + }), + http: { body: { additionalModelRequestFields: { thinking: { type: "between_tools" } } } }, + }), + ).pipe(Effect.flip) + + expect(error.reason._tag).toBe("InvalidRequest") + expect(error.message).toContain("between_tools") + }), + ) + + it.effect("rejects between_tools at unsupported max effort", () => + Effect.gen(function* () { + const error = yield* compileRequest( + LLMRequest.update(request, { + model: bedrock("global.anthropic.claude-sonnet-5-5", { thinking: { type: "between_tools" } }), + http: { body: { additionalModelRequestFields: { output_config: { effort: "max" } } } }, + }), + ).pipe(Effect.flip) + + expect(error.reason._tag).toBe("InvalidRequest") + expect(error.message).toContain("max") + }), + ) + + it.effect("sends between_tools without block binding and preserves auto tool choice on the wire", () => + Effect.gen(function* () { + const bodies: Array> = [] + const response = new EventStreamCodec(toUtf8, fromUtf8).encode({ + headers: { + ":message-type": { type: "string", value: "event" }, + ":event-type": { type: "string", value: "messageStop" }, + ":content-type": { type: "string", value: "application/json" }, + }, + body: new TextEncoder().encode('{"stopReason":"end_turn"}'), + }) + yield* LLMClient.generate( + LLMRequest.update(request, { + toolChoice: ToolChoice.make({ type: "auto" }), + http: { body: { additionalModelRequestFields: { thinking: { type: "between_tools" } } } }, + }), + ).pipe( + Effect.provide( + dynamicResponse((input) => + Effect.gen(function* () { + const sent = yield* HttpClientRequest.toWeb(input.request) + bodies.push(JSON.parse(yield* Effect.promise(() => sent.text()))) + return input.respond(response, { headers: { "content-type": "application/vnd.amazon.eventstream" } }) + }), + ), + ), + ) + + expect(bodies[0]?.additionalModelRequestFields).toEqual({ thinking: { type: "between_tools" } }) + expect(bodies[0]?.toolConfig).toMatchObject({ toolChoice: { auto: {} } }) + }), + ) + + it.effect("keeps Sonnet 5.5 defaults and supported sampling defaults", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLMRequest.update(request, { + toolChoice: ToolChoice.make({ type: "none" }), + generation: { temperature: 1, topP: 0.999, topK: 0 }, + }), + ) + + expect(prepared.body.toolConfig?.toolChoice).toBeUndefined() + expect(prepared.body.inferenceConfig).toMatchObject({ temperature: 1, topP: 0.999 }) + expect(prepared.body.additionalModelRequestFields).toMatchObject({ top_k: 0, thinking: { type: "adaptive" } }) + }), + ) + + it.effect("keeps earlier Claude and other Bedrock models' request behavior", () => + Effect.gen(function* () { + for (const id of ["us.anthropic.claude-sonnet-4-6", "us.amazon.nova-2-lite-v1:0"]) { + const prepared = yield* compileRequest( + LLMRequest.update(request, { + model: bedrock(id), + toolChoice: ToolChoice.make({ type: "required" }), + generation: { temperature: 0 }, + }), + ) + + expect(prepared.body.toolConfig?.toolChoice).toEqual({ any: {} }) + expect(prepared.body.inferenceConfig).toEqual({ temperature: 0 }) + } + }), + ) +})