mirror of
https://gitee.com/zhangsken/opencode.git
synced 2026-10-16 10:30:01 +08:00
fix(ai): preserve Gemini 3.8 TTS WAV output (#51300)
Co-authored-by: nexxeln <95541290+nexxeln@users.noreply.github.com>
This commit is contained in:
committed by
GitHub
parent
beeb14e910
commit
c34ffa117e
@@ -56,10 +56,15 @@ interface State extends SpeechStream.Audio, GeminiGenerateContent.Metadata {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const fromRequest = Effect.fn("GoogleSpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
|
||||
if (request.format === "pcm" && request.mode === "generate" && /^gemini-3\.8-.*-tts(?:-|$)/.test(request.model.id))
|
||||
return yield* route.unsupported(
|
||||
"media.format",
|
||||
`${route.name} returns WAV by default for Gemini 3.8 TTS unary requests; omit the format to accept it`,
|
||||
)
|
||||
if (request.format !== undefined && request.format !== "pcm")
|
||||
return yield* route.unsupported(
|
||||
"media.format",
|
||||
`${route.name} only returns raw PCM; request format "pcm" or omit it, then wrap the samples yourself`,
|
||||
`${route.name} only accepts raw PCM as an explicit format; omit it to accept the provider's default output`,
|
||||
)
|
||||
const voiceName = SpeechStream.voiceID(request.voice)
|
||||
return MediaProtocol.json(
|
||||
@@ -97,10 +102,18 @@ const step = Effect.fn("GoogleSpeech.step")(function* (state: State, frame: stri
|
||||
return [next, audio.flatMap((part) => SpeechStream.delta(next, part.data)[1])] as const
|
||||
})
|
||||
|
||||
const finish = (state: State) => {
|
||||
const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
|
||||
const sampleRate = SpeechStream.sampleRate(state.mimeType) ?? DEFAULT_SAMPLE_RATE
|
||||
const output =
|
||||
state.mimeType?.split(";")[0]?.toLowerCase() === "audio/wav"
|
||||
? SpeechStream.container("wav", sampleRate)
|
||||
: SpeechStream.pcm("pcm_s16le", sampleRate, state.mimeType ?? `audio/L16;codec=pcm;rate=${sampleRate}`)
|
||||
if (context.request.format === "pcm" && output.info.format !== "pcm")
|
||||
return Effect.fail(
|
||||
route.frameError(`Google Speech returned ${output.info.format} instead of the requested raw PCM`),
|
||||
)
|
||||
return SpeechStream.finish(route, state, {
|
||||
...SpeechStream.pcm("pcm_s16le", sampleRate, state.mimeType ?? `audio/L16;codec=pcm;rate=${sampleRate}`),
|
||||
...output,
|
||||
usage: GeminiGenerateContent.usage(state.usage),
|
||||
providerMetadata: GeminiGenerateContent.providerMetadata(state),
|
||||
detail: state.finishReason === undefined ? undefined : `finish reason: ${state.finishReason}`,
|
||||
|
||||
@@ -28,10 +28,49 @@ const cartesia = Cartesia.configure({ apiKey: "test", baseURL: "https://cartesia
|
||||
const google = Google.configure({ apiKey: "test", baseURL: "https://google.test/v1beta" }).speech(
|
||||
"gemini-2.5-flash-preview-tts",
|
||||
)
|
||||
const google38 = Google.configure({ apiKey: "test", baseURL: "https://google.test/v1beta" }).speech(
|
||||
"gemini-3.8-flash-tts",
|
||||
)
|
||||
const google38Lite = Google.configure({ apiKey: "test", baseURL: "https://google.test/v1beta" }).speech(
|
||||
"gemini-3.8-flash-lite-tts",
|
||||
)
|
||||
const deepgram = Deepgram.configure({ apiKey: "test", baseURL: "https://deepgram.test" }).speech("aura-2-thalia-en")
|
||||
const voice = "JBFqnCBsd6RMkjVDRZzb"
|
||||
|
||||
describe("Speech", () => {
|
||||
it.effect("preserves Google's WAV output instead of describing it as raw PCM", () =>
|
||||
Effect.gen(function* () {
|
||||
const bytes = new TextEncoder().encode("RIFF....WAVEfmt ")
|
||||
const response = yield* Speech.generate({ model: google38, text: "Hi" }).pipe(
|
||||
Effect.provide(
|
||||
respond(
|
||||
JSON.stringify({
|
||||
candidates: [
|
||||
{ content: { parts: [{ inlineData: { mimeType: "audio/wav", data: Encoding.encodeBase64(bytes) } }] } },
|
||||
],
|
||||
}),
|
||||
"application/json",
|
||||
),
|
||||
),
|
||||
)
|
||||
expect(response.audio.mediaType).toBe("audio/wav")
|
||||
expect(response.audio.info?.format).toBe("wav")
|
||||
expect(response.audio.info?.encoding).toBeUndefined()
|
||||
expect(yield* response.audio.bytes()).toEqual(bytes)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects raw PCM for Gemini 3.8 unary requests before sending", () =>
|
||||
Effect.gen(function* () {
|
||||
const errors = yield* Effect.all(
|
||||
[google38, google38Lite].map((model) =>
|
||||
Speech.generate({ model, text: "Hi", format: "pcm" }).pipe(Effect.flip),
|
||||
),
|
||||
).pipe(Effect.provide(layer(() => Effect.die("An unsupported request reached the network"))))
|
||||
expect(errors.map((error) => error.reason._tag)).toEqual(["UnsupportedOperation", "UnsupportedOperation"])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects what a provider cannot produce before sending anything", () =>
|
||||
Effect.gen(function* () {
|
||||
const errors = yield* Effect.all(
|
||||
|
||||
Reference in New Issue
Block a user