From 0715b931f841b10e6d986c5ce4c9f87a04343d5c Mon Sep 17 00:00:00 2001 From: nexxeln <95541290+nexxeln@users.noreply.github.com> Date: Wed, 30 Sep 2026 08:35:27 +0000 Subject: [PATCH] fix(ai): surface ElevenLabs batch transcript edits --- .../src/protocols/elevenlabs-transcription.ts | 56 +++++++++++- packages/ai/test/transcription.test.ts | 87 +++++++++++++++++++ 2 files changed, 142 insertions(+), 1 deletion(-) diff --git a/packages/ai/src/protocols/elevenlabs-transcription.ts b/packages/ai/src/protocols/elevenlabs-transcription.ts index 626df5e8e0a..099372ceefe 100644 --- a/packages/ai/src/protocols/elevenlabs-transcription.ts +++ b/packages/ai/src/protocols/elevenlabs-transcription.ts @@ -22,6 +22,8 @@ export const PATH = "/v1/speech-to-text" // --------------------------------------------------------------------------- export type ElevenLabsTranscriptionOptions = { + /** Experimental batch edit; adds a 30% surcharge. The original timed transcript is unchanged. */ + readonly transcript_edit?: string readonly tag_audio_events?: boolean readonly timestamps_granularity?: OpenString<"none" | "word" | "character"> readonly diarization_threshold?: number @@ -60,6 +62,16 @@ const Transcript = Schema.Struct({ words: optionalNull(Schema.Array(Token)), transcription_id: optionalNull(Schema.String), audio_duration_secs: optionalNull(Schema.Number), + edited_transcript: optionalNull( + Schema.Union([ + Schema.Struct({ kind: Schema.Literal("transcript"), text: Schema.String }), + Schema.Struct({ + kind: Schema.Literal("error"), + error_type: Schema.Literal("edit_failed"), + message: Schema.String, + }), + ]), + ), }) // --------------------------------------------------------------------------- @@ -84,6 +96,22 @@ const validate = (request: Request, overlay: Record) => { // Webhook requests return 202 with no transcript; the result arrives at a configured webhook instead. if (overlay.webhook === true) return Effect.fail(route.unsupported("transcription.webhook", `${route.name} does not deliver to webhooks`)) + if (overlay.transcript_edit != null) { + if (typeof overlay.transcript_edit !== "string" || overlay.transcript_edit.length > 2000) + return Effect.fail( + route.unsupported( + "transcription.transcript_edit", + `${route.name} transcript_edit must be a string of at most 2000 characters`, + ), + ) + if (overlay.entity_detection != null || overlay.entity_redaction != null || overlay.use_multi_channel === true) + return Effect.fail( + route.unsupported( + "transcription.transcript_edit", + `${route.name} transcript_edit cannot be combined with entity_detection, entity_redaction, or use_multi_channel`, + ), + ) + } // Separate multichannel output replaces the transcript with one transcript per channel. if (overlay.use_multi_channel === true && overlay.multichannel_output_style !== "combined") return Effect.fail( @@ -172,6 +200,7 @@ const decodeResponse = Effect.fn("ElevenLabsTranscription.decodeResponse")(funct const tokens = transcript.words ?? [] const duration = transcript.audio_duration_secs ?? undefined const transcriptionID = transcript.transcription_id ?? undefined + const edited = transcript.edited_transcript return new TranscriptionResponse({ text: transcript.text, segments: diarizes(context.request) ? speakerTurns(tokens) : undefined, @@ -185,7 +214,32 @@ const decodeResponse = Effect.fn("ElevenLabsTranscription.decodeResponse")(funct language: transcript.language_code?.toLowerCase(), durationSeconds: duration, usage: duration === undefined ? undefined : { type: "seconds", seconds: duration }, - providerMetadata: transcriptionID === undefined ? undefined : { elevenlabs: { transcriptionId: transcriptionID } }, + notices: + edited?.kind === "error" + ? [ + { + type: "other", + message: `ElevenLabs transcript edit failed: ${edited.message}`, + providerMetadata: { elevenlabs: { errorType: edited.error_type } }, + }, + ] + : undefined, + providerMetadata: + transcriptionID === undefined && edited == null + ? undefined + : { + elevenlabs: { + ...(transcriptionID === undefined ? {} : { transcriptionId: transcriptionID }), + ...(edited == null + ? {} + : { + editedTranscript: + edited.kind === "transcript" + ? { kind: edited.kind, text: edited.text } + : { kind: edited.kind, errorType: edited.error_type, message: edited.message }, + }), + }, + }, }) }) diff --git a/packages/ai/test/transcription.test.ts b/packages/ai/test/transcription.test.ts index 8a980afc92a..d4ff91b1e4d 100644 --- a/packages/ai/test/transcription.test.ts +++ b/packages/ai/test/transcription.test.ts @@ -571,6 +571,93 @@ describe("Transcription", () => { }), ) + it.effect("keeps ElevenLabs transcript edits separate from the original timed transcript", () => + Effect.gen(function* () { + const calls: Array = [] + const response = yield* Transcription.generate({ + model: elevenlabs, + audio, + providerOptions: { transcript_edit: "Write dates in ISO 8601 format" }, + }).pipe( + Effect.provide( + layer((input) => + observe(calls, input).pipe( + Effect.as( + json(input, { + text: "the twelfth of July", + words: [{ text: "twelfth", type: "word", start: 0.5, end: 0.9 }], + transcription_id: "tr_2", + edited_transcript: { kind: "transcript", text: "2026-07-12" }, + }), + ), + ), + ), + ), + ) + expect((yield* formFields(calls[0])).transcript_edit).toEqual(["Write dates in ISO 8601 format"]) + expect(response.text).toBe("the twelfth of July") + expect(response.words).toMatchObject([{ text: "twelfth", startSeconds: 0.5, endSeconds: 0.9 }]) + expect(response.providerMetadata).toEqual({ + elevenlabs: { transcriptionId: "tr_2", editedTranscript: { kind: "transcript", text: "2026-07-12" } }, + }) + expect(response.notices).toBeUndefined() + }), + ) + + it.effect("exposes an ElevenLabs edit failure without losing the successful transcription", () => + Effect.gen(function* () { + const response = yield* Transcription.generate({ + model: elevenlabs, + audio, + providerOptions: { transcript_edit: "Write dates in ISO 8601 format" }, + }).pipe( + Effect.provide( + layer((input) => + Effect.succeed( + json(input, { + text: "Original transcript", + edited_transcript: { kind: "error", error_type: "edit_failed", message: "Please try again" }, + }), + ), + ), + ), + ) + expect(response.text).toBe("Original transcript") + expect(response.providerMetadata).toEqual({ + elevenlabs: { editedTranscript: { kind: "error", errorType: "edit_failed", message: "Please try again" } }, + }) + expect(response.notices).toEqual([ + { + type: "other", + message: "ElevenLabs transcript edit failed: Please try again", + providerMetadata: { elevenlabs: { errorType: "edit_failed" } }, + }, + ]) + }), + ) + + it.effect("rejects incompatible ElevenLabs edit options before sending audio", () => + Effect.gen(function* () { + const errors = yield* Effect.all( + [ + { transcript_edit: "Edit", entity_detection: ["person"] }, + { transcript_edit: "Edit", entity_redaction: ["person"] }, + { transcript_edit: "Edit", use_multi_channel: true, multichannel_output_style: "combined" }, + { transcript_edit: "x".repeat(2001) }, + ].map((providerOptions) => + Transcription.generate({ model: elevenlabs, audio, providerOptions }).pipe(Effect.flip), + ), + ) + const overlayError = yield* Transcription.generate({ + model: elevenlabs, + audio, + providerOptions: { transcript_edit: "Edit" }, + http: { body: { entity_redaction: ["person"] } }, + }).pipe(Effect.flip) + expect([...errors, overlayError].map((error) => error.reason._tag)).toEqual(Array(5).fill("UnsupportedOperation")) + }).pipe(Effect.provide(layer(() => Effect.die("an unsupported request reached the network")))), + ) + it.effect("rejects reading an AssemblyAI result before the transcript finishes", () => Effect.gen(function* () { const generation = yield* Transcription.resume(assemblyai, { transcriptID: "tr_1" })