diff --git a/packages/ai/src/protocols/deepgram-speech.ts b/packages/ai/src/protocols/deepgram-speech.ts index 59b58db1d41..b8d3390fc72 100644 --- a/packages/ai/src/protocols/deepgram-speech.ts +++ b/packages/ai/src/protocols/deepgram-speech.ts @@ -67,6 +67,9 @@ const queryParameters = (request: Request) => { } const fromRequest = Effect.fn("DeepgramSpeech.fromRequest")(function* (request: Request) { + // Not in `unsupported`: that list would also reject `timestamps: false`, which asks for nothing. + if (request.timestamps === true) + return yield* route.unsupported("media.timestamps", `${route.name} does not return timestamps`) if ( request.format !== undefined && FORMATS[request.format] === undefined && @@ -117,7 +120,7 @@ const finish = (state: State, context: MediaProtocol.ResponseContext) = // --------------------------------------------------------------------------- export const protocol = MediaProtocol.stream(route, { - unsupported: ["voice", "language", "instructions", "timestamps"], + unsupported: ["voice", "language", "instructions"], body: { from: fromRequest }, frames: (bytes) => bytes, initial: () => ({ chunks: [] }), diff --git a/packages/ai/src/protocols/google-speech.ts b/packages/ai/src/protocols/google-speech.ts index f168f416b56..02534d3ca76 100644 --- a/packages/ai/src/protocols/google-speech.ts +++ b/packages/ai/src/protocols/google-speech.ts @@ -56,6 +56,9 @@ interface State extends SpeechStream.Audio, GeminiGenerateContent.Metadata { // --------------------------------------------------------------------------- const fromRequest = Effect.fn("GoogleSpeech.fromRequest")(function* (request: MediaProtocol.Addressed) { + // Not in `unsupported`: that list would also reject `timestamps: false`, which asks for nothing. + if (request.timestamps === true) + return yield* route.unsupported("media.timestamps", `${route.name} does not return timestamps`) if (request.format === "pcm" && request.mode === "generate" && /^gemini-3\.8-.*-tts(?:-|$)/.test(request.model.id)) return yield* route.unsupported( "media.format", @@ -125,7 +128,7 @@ const finish = (state: State, context: MediaProtocol.ResponseContext) = // --------------------------------------------------------------------------- export const protocol = MediaProtocol.stream(route, { - unsupported: ["instructions", "speed", "timestamps"], + unsupported: ["instructions", "speed"], body: { from: fromRequest }, frames: (bytes, context) => GeminiGenerateContent.frames(bytes, context.request.mode), initial: () => ({ chunks: [] }), diff --git a/packages/ai/src/protocols/openai-speech.ts b/packages/ai/src/protocols/openai-speech.ts index 7fc12dc5a0a..50ea4cbd21b 100644 --- a/packages/ai/src/protocols/openai-speech.ts +++ b/packages/ai/src/protocols/openai-speech.ts @@ -61,6 +61,9 @@ interface State extends SpeechStream.Audio { const supportsSse = (model: string) => !/^tts-1(-hd)?(-|$)/.test(model) const fromRequest = Effect.fn("OpenAISpeech.fromRequest")(function* (request: MediaProtocol.Addressed) { + // Not in `unsupported`: that list would also reject `timestamps: false`, which asks for nothing. + if (request.timestamps === true) + return yield* route.unsupported("media.timestamps", `${route.name} does not return timestamps`) return MediaProtocol.json( mergeJsonRecords( { @@ -121,7 +124,7 @@ const finish = (state: State, context: MediaProtocol.ResponseContext) = // --------------------------------------------------------------------------- export const protocol = MediaProtocol.stream(route, { - unsupported: ["language", "timestamps"], + unsupported: ["language"], body: { from: fromRequest }, frames: (bytes, context) => (isSse(context.body) ? Framing.sse.frame(bytes) : bytes), initial: () => ({ chunks: [], done: false }), diff --git a/packages/ai/src/route/media.ts b/packages/ai/src/route/media.ts index b3c278b121b..0dca4ac69a2 100644 --- a/packages/ai/src/route/media.ts +++ b/packages/ai/src/route/media.ts @@ -385,7 +385,11 @@ const encode = (body: MediaProtocol.Body | undefined, headers: Headers.Headers) } } -/** Common fields are never silently dropped: a present field the protocol declared unsupported fails typed. */ +/** + * Common fields are never silently dropped: a present field the protocol declared unsupported fails typed. `false` + * counts as present because some booleans mean something when false (video `audio`); protocols reject opt-in + * booleans such as speech `timestamps` with `=== true` in `body.from` instead of listing them. + */ const rejectUnsupported = ( route: string, provider: ProviderID, diff --git a/packages/ai/test/speech.test.ts b/packages/ai/test/speech.test.ts index a4ae078ed32..d5731b97a95 100644 --- a/packages/ai/test/speech.test.ts +++ b/packages/ai/test/speech.test.ts @@ -97,6 +97,42 @@ describe("Speech", () => { }).pipe(Effect.provide(layer(() => Effect.die("an unsupported request reached the network")))), ) + it.effect("treats timestamps: false as not asking for timestamps on routes that cannot return them", () => + Effect.gen(function* () { + const bytes = Uint8Array.from([1, 2, 3]) + const gemini = JSON.stringify({ + candidates: [ + { content: { parts: [{ inlineData: { mimeType: "audio/L16;codec=pcm;rate=24000", data: "AQID" } }] } }, + ], + }) + const responses = yield* Effect.all([ + Speech.generate({ model: openai, text: "Hi", timestamps: false }).pipe( + Effect.provide(respond(new Blob([bytes]).stream(), "audio/mpeg")), + ), + Speech.generate({ model: google, text: "Hi", timestamps: false }).pipe( + Effect.provide(respond(gemini, "application/json")), + ), + Speech.generate({ model: deepgram, text: "Hi", timestamps: false }).pipe( + Effect.provide(respond(new Blob([bytes]).stream(), "audio/mpeg")), + ), + ]) + for (const response of responses) expect(yield* response.audio.bytes()).toEqual(bytes) + + const errors = yield* Effect.all( + [openai, google, deepgram].map((model) => + Speech.generate({ model, text: "Hi", timestamps: true }).pipe(Effect.flip), + ), + ).pipe(Effect.provide(layer(() => Effect.die("an unsupported request reached the network")))) + expect(errors.map((error) => [error.reason._tag, "operation" in error.reason && error.reason.operation])).toEqual( + [ + ["UnsupportedOperation", "media.timestamps"], + ["UnsupportedOperation", "media.timestamps"], + ["UnsupportedOperation", "media.timestamps"], + ], + ) + }), + ) + it.effect("classifies stream failures and keeps the provider payload and HTTP context", () => Effect.gen(function* () { const badFrame = JSON.stringify({ type: "speech.audio.delta", audio: "not base64!" })