fix(ai): surface truncated Gemini speech and transcription results (#51379)

packages/ai/src/protocols/gemini.ts

@@ -526,19 +526,7 @@ const mapFinishReason = (finishReason: string | undefined, hasToolCalls: boolean
   if (finishReason === undefined) return hasToolCalls ? "tool-calls" : "unknown"
   if (finishReason === "STOP") return hasToolCalls ? "tool-calls" : "stop"
   if (finishReason === "MAX_TOKENS") return "length"
-  if (
-    finishReason === "IMAGE_SAFETY" ||
-    finishReason === "RECITATION" ||
-    finishReason === "SAFETY" ||
-    finishReason === "BLOCKLIST" ||
-    finishReason === "PROHIBITED_CONTENT" ||
-    finishReason === "SPII" ||
-    finishReason === "MODEL_ARMOR" ||
-    finishReason === "IMAGE_PROHIBITED_CONTENT" ||
-    finishReason === "IMAGE_RECITATION" ||
-    finishReason === "LANGUAGE"
-  )
-    return "content-filter"
+  if (GeminiGenerateContent.contentFiltered(finishReason)) return "content-filter"
   if (
     finishReason === "MALFORMED_FUNCTION_CALL" ||
     finishReason === "UNEXPECTED_TOOL_CALL" ||

packages/ai/src/protocols/google-speech.ts

@@ -102,10 +102,14 @@ const step = Effect.fn("GoogleSpeech.step")(function* (state: State, frame: stri
     part.inlineData === undefined ? [] : [part.inlineData],
   )
   const next: State = { ...GeminiGenerateContent.track(state, chunk), mimeType: state.mimeType ?? audio[0]?.mimeType }
-  return [next, audio.flatMap((part) => SpeechStream.delta(next, part.data)[1])] as const
+  const events = audio.flatMap((part) => SpeechStream.delta(next, part.data)[1])
+  const withheld = next.chunks.length === 0 ? GeminiGenerateContent.withheld(route.name, chunk, frame) : undefined
+  if (withheld !== undefined) return yield* withheld
+  return [next, events] as const
 })
 
 const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
+  if (state.finishReason === undefined) return Effect.fail(route.incomplete())
   const sampleRate = SpeechStream.sampleRate(state.mimeType) ?? DEFAULT_SAMPLE_RATE
   const output =
     state.mimeType?.split(";")[0]?.toLowerCase() === "audio/wav"
@@ -118,8 +122,9 @@ const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) =
   return SpeechStream.finish(route, state, {
     ...output,
     usage: GeminiGenerateContent.usage(state.usage),
+    notices: GeminiGenerateContent.notices(route.name, state),
     providerMetadata: GeminiGenerateContent.providerMetadata(state),
-    detail: state.finishReason === undefined ? undefined : `finish reason: ${state.finishReason}`,
+    detail: `finish reason: ${state.finishReason}`,
   })
 }
 

packages/ai/src/protocols/google-transcription.ts

@@ -154,6 +154,9 @@ const step = Effect.fn("GoogleTranscription.step")(function* (state: State, fram
     .filter((item) => item.length > 0)
     .join(" ")
   const delta = text.length === 0 || state.text.length === 0 ? text : ` ${text}`
+  const withheld =
+    state.text.length + delta.length === 0 ? GeminiGenerateContent.withheld(route.name, chunk, frame) : undefined
+  if (withheld !== undefined) return yield* withheld
   const events: ReadonlyArray<TranscriptionEvent> = [
     ...(delta.length === 0 ? [] : [TranscriptionTextDeltaEvent.make({ delta })]),
     ...segments.map((segment) => TranscriptionSegmentEvent.make({ segment })),
@@ -169,6 +172,7 @@ const finish = (state: State) => {
       segments: state.segments.length === 0 ? undefined : state.segments,
       words: state.words.length === 0 ? undefined : state.words,
       usage: GeminiGenerateContent.usage(state.usage),
+      notices: GeminiGenerateContent.notices(route.name, state),
       providerMetadata: GeminiGenerateContent.providerMetadata(state),
     }),
   ])

packages/ai/src/protocols/utils/gemini-generate-content.ts

@@ -72,6 +72,44 @@ export const blocked = (name: string, chunk: Chunk, frame: string) => {
   })
 }
 
+const CONTENT_FILTER_REASONS = new Set([
+  "IMAGE_SAFETY",
+  "RECITATION",
+  "SAFETY",
+  "BLOCKLIST",
+  "PROHIBITED_CONTENT",
+  "SPII",
+  "MODEL_ARMOR",
+  "IMAGE_PROHIBITED_CONTENT",
+  "IMAGE_RECITATION",
+  "LANGUAGE",
+])
+
+/** Finish reasons for which Gemini stops output on safety or policy grounds. */
+export const contentFiltered = (finishReason: string | undefined) =>
+  finishReason !== undefined && CONTENT_FILTER_REASONS.has(finishReason)
+
+/** Callers check that the response produced no output: a policy stop after output is a partial result instead. */
+export const withheld = (name: string, chunk: Chunk, frame: string) => {
+  const finishReason = chunk.candidates?.[0]?.finishReason
+  if (!contentFiltered(finishReason)) return undefined
+  return new AIError({
+    reason: new ContentPolicyError({ message: `${name} withheld its output (${finishReason})`, body: frame }),
+  })
+}
+
+/** Any finish reason other than `STOP` means the output may be cut short, so it is surfaced rather than dropped. */
+export const notices = (name: string, state: Metadata): ReadonlyArray<Media.Notice> | undefined =>
+  state.finishReason === undefined || state.finishReason === "STOP"
+    ? undefined
+    : [
+        {
+          type: contentFiltered(state.finishReason) ? "filtered" : "other",
+          message: `${name} finished with ${state.finishReason}`,
+          providerMetadata: { google: { finishReason: state.finishReason } },
+        },
+      ]
+
 export const usage = (usage: UsageMetadata | undefined): MediaUsage | undefined =>
   usage === undefined
     ? undefined

packages/ai/src/protocols/utils/speech-stream.ts

@@ -85,6 +85,7 @@ export const finish = (
     readonly mediaType: string | undefined
     readonly info?: Media.Info
     readonly usage?: MediaUsage
+    readonly notices?: ReadonlyArray<Media.Notice>
     readonly providerMetadata?: ProviderMetadata
     readonly detail?: string
   },
@@ -97,6 +98,7 @@ export const finish = (
     SpeechFinishEvent.make({
       audio: Media.bytes(concatBytes(state.chunks), output.mediaType, { info: output.info }),
       usage: output.usage,
+      notices: output.notices,
       providerMetadata: output.providerMetadata,
     }),
   ])

packages/ai/test/speech.test.ts

@@ -46,7 +46,10 @@ describe("Speech", () => {
           respond(
             JSON.stringify({
               candidates: [
-                { content: { parts: [{ inlineData: { mimeType: "audio/wav", data: Encoding.encodeBase64(bytes) } }] } },
+                {
+                  content: { parts: [{ inlineData: { mimeType: "audio/wav", data: Encoding.encodeBase64(bytes) } }] },
+                  finishReason: "STOP",
+                },
               ],
             }),
             "application/json",
@@ -141,7 +144,10 @@ describe("Speech", () => {
       const bytes = Uint8Array.from([1, 2, 3])
       const gemini = JSON.stringify({
         candidates: [
-          { content: { parts: [{ inlineData: { mimeType: "audio/L16;codec=pcm;rate=24000", data: "AQID" } }] } },
+          {
+            content: { parts: [{ inlineData: { mimeType: "audio/L16;codec=pcm;rate=24000", data: "AQID" } }] },
+            finishReason: "STOP",
+          },
         ],
       })
       const responses = yield* Effect.all([
@@ -202,6 +208,38 @@ describe("Speech", () => {
     }),
   )
 
+  it.effect("surfaces Gemini speech that ended without STOP instead of returning it as complete", () =>
+    Effect.gen(function* () {
+      const document = (finishReason?: string) =>
+        JSON.stringify({
+          candidates: [
+            {
+              content: { parts: [{ inlineData: { mimeType: "audio/L16;codec=pcm;rate=24000", data: "AQI=" } }] },
+              finishReason,
+            },
+          ],
+        })
+      const withheld = JSON.stringify({ candidates: [{ finishReason: "SAFETY" }] })
+      const generate = (body: string) =>
+        Speech.generate({ model: google, text: "Hi" }).pipe(Effect.provide(respond(body, "application/json")))
+
+      const truncated = yield* generate(document()).pipe(Effect.flip)
+      const partial = yield* generate(document("MAX_TOKENS"))
+      const policy = yield* generate(withheld).pipe(Effect.flip)
+
+      expect(truncated.reason).toMatchObject({ _tag: "InvalidProviderOutput", classification: "incomplete-stream" })
+      expect(yield* partial.audio.bytes()).toEqual(Uint8Array.from([1, 2]))
+      expect(partial.notices).toEqual([
+        {
+          type: "other",
+          message: "Google Speech finished with MAX_TOKENS",
+          providerMetadata: { google: { finishReason: "MAX_TOKENS" } },
+        },
+      ])
+      expect(policy.reason).toMatchObject({ _tag: "ContentPolicy", body: withheld })
+    }),
+  )
+
   it.effect("parses ElevenLabs timestamped records split across network chunks", () =>
     Effect.gen(function* () {
       const record = (bytes: ReadonlyArray<number>, character: string, start: number) =>

packages/ai/test/transcription.test.ts

@@ -271,4 +271,35 @@ describe("Transcription", () => {
       ),
     ),
   )
+
+  it.effect("surfaces Gemini transcripts that ended without STOP instead of returning them as complete", () =>
+    Effect.gen(function* () {
+      const document = (finishReason?: string) =>
+        JSON.stringify({
+          candidates: [{ content: { parts: [{ audioTranscription: { text: "Hello" } }] }, finishReason }],
+        })
+      const withheld = JSON.stringify({ candidates: [{ finishReason: "SAFETY" }] })
+      const generate = (body: string) =>
+        Transcription.generate({ model: google, audio }).pipe(
+          Effect.provide(
+            layer((input) => Effect.succeed(input.respond(body, { headers: { "content-type": "application/json" } }))),
+          ),
+        )
+
+      const truncated = yield* generate(document()).pipe(Effect.flip)
+      const partial = yield* generate(document("MAX_TOKENS"))
+      const policy = yield* generate(withheld).pipe(Effect.flip)
+
+      expect(truncated.reason).toMatchObject({ _tag: "InvalidProviderOutput", classification: "incomplete-stream" })
+      expect(partial.text).toBe("Hello")
+      expect(partial.notices).toEqual([
+        {
+          type: "other",
+          message: "Google Transcription finished with MAX_TOKENS",
+          providerMetadata: { google: { finishReason: "MAX_TOKENS" } },
+        },
+      ])
+      expect(policy.reason).toMatchObject({ _tag: "ContentPolicy", body: withheld })
+    }),
+  )
 })