From c33de198e3b7ebd205d93bad4cb7b070f5442bff Mon Sep 17 00:00:00 2001 From: Shoubhit Dash Date: Sun, 27 Sep 2026 14:29:36 +0530 Subject: [PATCH] fix(ai): surface truncated Gemini speech and transcription results (#51379) --- packages/ai/src/protocols/gemini.ts | 14 +------ packages/ai/src/protocols/google-speech.ts | 9 +++- .../ai/src/protocols/google-transcription.ts | 4 ++ .../utils/gemini-generate-content.ts | 38 +++++++++++++++++ .../ai/src/protocols/utils/speech-stream.ts | 2 + packages/ai/test/speech.test.ts | 42 ++++++++++++++++++- packages/ai/test/transcription.test.ts | 31 ++++++++++++++ 7 files changed, 123 insertions(+), 17 deletions(-) diff --git a/packages/ai/src/protocols/gemini.ts b/packages/ai/src/protocols/gemini.ts index 1d8e502dbb6..8c02f11cc9f 100644 --- a/packages/ai/src/protocols/gemini.ts +++ b/packages/ai/src/protocols/gemini.ts @@ -526,19 +526,7 @@ const mapFinishReason = (finishReason: string | undefined, hasToolCalls: boolean if (finishReason === undefined) return hasToolCalls ? "tool-calls" : "unknown" if (finishReason === "STOP") return hasToolCalls ? "tool-calls" : "stop" if (finishReason === "MAX_TOKENS") return "length" - if ( - finishReason === "IMAGE_SAFETY" || - finishReason === "RECITATION" || - finishReason === "SAFETY" || - finishReason === "BLOCKLIST" || - finishReason === "PROHIBITED_CONTENT" || - finishReason === "SPII" || - finishReason === "MODEL_ARMOR" || - finishReason === "IMAGE_PROHIBITED_CONTENT" || - finishReason === "IMAGE_RECITATION" || - finishReason === "LANGUAGE" - ) - return "content-filter" + if (GeminiGenerateContent.contentFiltered(finishReason)) return "content-filter" if ( finishReason === "MALFORMED_FUNCTION_CALL" || finishReason === "UNEXPECTED_TOOL_CALL" || diff --git a/packages/ai/src/protocols/google-speech.ts b/packages/ai/src/protocols/google-speech.ts index 02534d3ca76..6a25890de47 100644 --- a/packages/ai/src/protocols/google-speech.ts +++ b/packages/ai/src/protocols/google-speech.ts @@ -102,10 +102,14 @@ const step = Effect.fn("GoogleSpeech.step")(function* (state: State, frame: stri part.inlineData === undefined ? [] : [part.inlineData], ) const next: State = { ...GeminiGenerateContent.track(state, chunk), mimeType: state.mimeType ?? audio[0]?.mimeType } - return [next, audio.flatMap((part) => SpeechStream.delta(next, part.data)[1])] as const + const events = audio.flatMap((part) => SpeechStream.delta(next, part.data)[1]) + const withheld = next.chunks.length === 0 ? GeminiGenerateContent.withheld(route.name, chunk, frame) : undefined + if (withheld !== undefined) return yield* withheld + return [next, events] as const }) const finish = (state: State, context: MediaProtocol.ResponseContext) => { + if (state.finishReason === undefined) return Effect.fail(route.incomplete()) const sampleRate = SpeechStream.sampleRate(state.mimeType) ?? DEFAULT_SAMPLE_RATE const output = state.mimeType?.split(";")[0]?.toLowerCase() === "audio/wav" @@ -118,8 +122,9 @@ const finish = (state: State, context: MediaProtocol.ResponseContext) = return SpeechStream.finish(route, state, { ...output, usage: GeminiGenerateContent.usage(state.usage), + notices: GeminiGenerateContent.notices(route.name, state), providerMetadata: GeminiGenerateContent.providerMetadata(state), - detail: state.finishReason === undefined ? undefined : `finish reason: ${state.finishReason}`, + detail: `finish reason: ${state.finishReason}`, }) } diff --git a/packages/ai/src/protocols/google-transcription.ts b/packages/ai/src/protocols/google-transcription.ts index e9cca057454..77a66ad3ec4 100644 --- a/packages/ai/src/protocols/google-transcription.ts +++ b/packages/ai/src/protocols/google-transcription.ts @@ -154,6 +154,9 @@ const step = Effect.fn("GoogleTranscription.step")(function* (state: State, fram .filter((item) => item.length > 0) .join(" ") const delta = text.length === 0 || state.text.length === 0 ? text : ` ${text}` + const withheld = + state.text.length + delta.length === 0 ? GeminiGenerateContent.withheld(route.name, chunk, frame) : undefined + if (withheld !== undefined) return yield* withheld const events: ReadonlyArray = [ ...(delta.length === 0 ? [] : [TranscriptionTextDeltaEvent.make({ delta })]), ...segments.map((segment) => TranscriptionSegmentEvent.make({ segment })), @@ -169,6 +172,7 @@ const finish = (state: State) => { segments: state.segments.length === 0 ? undefined : state.segments, words: state.words.length === 0 ? undefined : state.words, usage: GeminiGenerateContent.usage(state.usage), + notices: GeminiGenerateContent.notices(route.name, state), providerMetadata: GeminiGenerateContent.providerMetadata(state), }), ]) diff --git a/packages/ai/src/protocols/utils/gemini-generate-content.ts b/packages/ai/src/protocols/utils/gemini-generate-content.ts index 9ccd02d5362..ea83b74cc7c 100644 --- a/packages/ai/src/protocols/utils/gemini-generate-content.ts +++ b/packages/ai/src/protocols/utils/gemini-generate-content.ts @@ -72,6 +72,44 @@ export const blocked = (name: string, chunk: Chunk, frame: string) => { }) } +const CONTENT_FILTER_REASONS = new Set([ + "IMAGE_SAFETY", + "RECITATION", + "SAFETY", + "BLOCKLIST", + "PROHIBITED_CONTENT", + "SPII", + "MODEL_ARMOR", + "IMAGE_PROHIBITED_CONTENT", + "IMAGE_RECITATION", + "LANGUAGE", +]) + +/** Finish reasons for which Gemini stops output on safety or policy grounds. */ +export const contentFiltered = (finishReason: string | undefined) => + finishReason !== undefined && CONTENT_FILTER_REASONS.has(finishReason) + +/** Callers check that the response produced no output: a policy stop after output is a partial result instead. */ +export const withheld = (name: string, chunk: Chunk, frame: string) => { + const finishReason = chunk.candidates?.[0]?.finishReason + if (!contentFiltered(finishReason)) return undefined + return new AIError({ + reason: new ContentPolicyError({ message: `${name} withheld its output (${finishReason})`, body: frame }), + }) +} + +/** Any finish reason other than `STOP` means the output may be cut short, so it is surfaced rather than dropped. */ +export const notices = (name: string, state: Metadata): ReadonlyArray | undefined => + state.finishReason === undefined || state.finishReason === "STOP" + ? undefined + : [ + { + type: contentFiltered(state.finishReason) ? "filtered" : "other", + message: `${name} finished with ${state.finishReason}`, + providerMetadata: { google: { finishReason: state.finishReason } }, + }, + ] + export const usage = (usage: UsageMetadata | undefined): MediaUsage | undefined => usage === undefined ? undefined diff --git a/packages/ai/src/protocols/utils/speech-stream.ts b/packages/ai/src/protocols/utils/speech-stream.ts index c9cb6fe9753..2b0188e4311 100644 --- a/packages/ai/src/protocols/utils/speech-stream.ts +++ b/packages/ai/src/protocols/utils/speech-stream.ts @@ -85,6 +85,7 @@ export const finish = ( readonly mediaType: string | undefined readonly info?: Media.Info readonly usage?: MediaUsage + readonly notices?: ReadonlyArray readonly providerMetadata?: ProviderMetadata readonly detail?: string }, @@ -97,6 +98,7 @@ export const finish = ( SpeechFinishEvent.make({ audio: Media.bytes(concatBytes(state.chunks), output.mediaType, { info: output.info }), usage: output.usage, + notices: output.notices, providerMetadata: output.providerMetadata, }), ]) diff --git a/packages/ai/test/speech.test.ts b/packages/ai/test/speech.test.ts index 78205c8e4a7..6d88a7aea22 100644 --- a/packages/ai/test/speech.test.ts +++ b/packages/ai/test/speech.test.ts @@ -46,7 +46,10 @@ describe("Speech", () => { respond( JSON.stringify({ candidates: [ - { content: { parts: [{ inlineData: { mimeType: "audio/wav", data: Encoding.encodeBase64(bytes) } }] } }, + { + content: { parts: [{ inlineData: { mimeType: "audio/wav", data: Encoding.encodeBase64(bytes) } }] }, + finishReason: "STOP", + }, ], }), "application/json", @@ -141,7 +144,10 @@ describe("Speech", () => { const bytes = Uint8Array.from([1, 2, 3]) const gemini = JSON.stringify({ candidates: [ - { content: { parts: [{ inlineData: { mimeType: "audio/L16;codec=pcm;rate=24000", data: "AQID" } }] } }, + { + content: { parts: [{ inlineData: { mimeType: "audio/L16;codec=pcm;rate=24000", data: "AQID" } }] }, + finishReason: "STOP", + }, ], }) const responses = yield* Effect.all([ @@ -202,6 +208,38 @@ describe("Speech", () => { }), ) + it.effect("surfaces Gemini speech that ended without STOP instead of returning it as complete", () => + Effect.gen(function* () { + const document = (finishReason?: string) => + JSON.stringify({ + candidates: [ + { + content: { parts: [{ inlineData: { mimeType: "audio/L16;codec=pcm;rate=24000", data: "AQI=" } }] }, + finishReason, + }, + ], + }) + const withheld = JSON.stringify({ candidates: [{ finishReason: "SAFETY" }] }) + const generate = (body: string) => + Speech.generate({ model: google, text: "Hi" }).pipe(Effect.provide(respond(body, "application/json"))) + + const truncated = yield* generate(document()).pipe(Effect.flip) + const partial = yield* generate(document("MAX_TOKENS")) + const policy = yield* generate(withheld).pipe(Effect.flip) + + expect(truncated.reason).toMatchObject({ _tag: "InvalidProviderOutput", classification: "incomplete-stream" }) + expect(yield* partial.audio.bytes()).toEqual(Uint8Array.from([1, 2])) + expect(partial.notices).toEqual([ + { + type: "other", + message: "Google Speech finished with MAX_TOKENS", + providerMetadata: { google: { finishReason: "MAX_TOKENS" } }, + }, + ]) + expect(policy.reason).toMatchObject({ _tag: "ContentPolicy", body: withheld }) + }), + ) + it.effect("parses ElevenLabs timestamped records split across network chunks", () => Effect.gen(function* () { const record = (bytes: ReadonlyArray, character: string, start: number) => diff --git a/packages/ai/test/transcription.test.ts b/packages/ai/test/transcription.test.ts index 1efdd9a79e5..544e505616d 100644 --- a/packages/ai/test/transcription.test.ts +++ b/packages/ai/test/transcription.test.ts @@ -271,4 +271,35 @@ describe("Transcription", () => { ), ), ) + + it.effect("surfaces Gemini transcripts that ended without STOP instead of returning them as complete", () => + Effect.gen(function* () { + const document = (finishReason?: string) => + JSON.stringify({ + candidates: [{ content: { parts: [{ audioTranscription: { text: "Hello" } }] }, finishReason }], + }) + const withheld = JSON.stringify({ candidates: [{ finishReason: "SAFETY" }] }) + const generate = (body: string) => + Transcription.generate({ model: google, audio }).pipe( + Effect.provide( + layer((input) => Effect.succeed(input.respond(body, { headers: { "content-type": "application/json" } }))), + ), + ) + + const truncated = yield* generate(document()).pipe(Effect.flip) + const partial = yield* generate(document("MAX_TOKENS")) + const policy = yield* generate(withheld).pipe(Effect.flip) + + expect(truncated.reason).toMatchObject({ _tag: "InvalidProviderOutput", classification: "incomplete-stream" }) + expect(partial.text).toBe("Hello") + expect(partial.notices).toEqual([ + { + type: "other", + message: "Google Transcription finished with MAX_TOKENS", + providerMetadata: { google: { finishReason: "MAX_TOKENS" } }, + }, + ]) + expect(policy.reason).toMatchObject({ _tag: "ContentPolicy", body: withheld }) + }), + ) })