From dbaa57a21bfeea61b2055857126d11ae7e6c2099 Mon Sep 17 00:00:00 2001 From: Shoubhit Dash Date: Wed, 23 Sep 2026 21:15:50 +0530 Subject: [PATCH] feat(ai): add transcription across inline, streamed, and queued routes (#50910) --- packages/ai/AGENTS.md | 4 +- packages/ai/README.md | 66 +++- packages/ai/docs/media-design.md | 77 ++++- packages/ai/script/setup-recording-env.ts | 14 +- packages/ai/src/generation.ts | 29 +- packages/ai/src/index.ts | 19 ++ packages/ai/src/promise.ts | 46 ++- .../src/protocols/assemblyai-transcription.ts | 211 +++++++++++++ packages/ai/src/protocols/deepgram-speech.ts | 45 ++- .../src/protocols/deepgram-transcription.ts | 191 ++++++++++++ packages/ai/src/protocols/gemini.ts | 8 +- packages/ai/src/protocols/google-images.ts | 25 +- packages/ai/src/protocols/google-speech.ts | 116 ++----- .../ai/src/protocols/google-transcription.ts | 214 +++++++++++++ packages/ai/src/protocols/openai-images.ts | 13 +- .../ai/src/protocols/openai-transcription.ts | 277 +++++++++++++++++ .../utils/gemini-generate-content.ts | 113 +++++++ .../ai/src/protocols/utils/media-input.ts | 21 ++ packages/ai/src/providers/assemblyai.ts | 43 +++ packages/ai/src/providers/deepgram.ts | 21 +- packages/ai/src/providers/google.ts | 4 + packages/ai/src/providers/index.ts | 1 + packages/ai/src/providers/openai.ts | 5 + packages/ai/src/route/media-protocol.ts | 31 +- packages/ai/src/route/media.ts | 134 ++++---- packages/ai/src/transcription-client.ts | 122 ++++++++ packages/ai/src/transcription.ts | 294 ++++++++++++++++++ packages/ai/src/utils/media-type.ts | 16 + packages/ai/src/video-client.ts | 16 +- packages/ai/src/video.ts | 20 +- packages/ai/test/exports.test.ts | 5 + packages/ai/test/fixtures/audio/dialog.mp3 | Bin 0 -> 14211 bytes packages/ai/test/fixtures/audio/hello.mp3 | Bin 0 -> 6165 bytes ...ploads-submits-and-polls-a-transcript.json | 98 ++++++ .../splits-utterances-at-speaker-changes.json | 32 ++ ...dio-with-speakers-and-word-timestamps.json | 50 +++ .../streams-speaker-turns.json | 32 ++ ...bes-with-speakers-and-word-timestamps.json | 32 ++ .../diarizes-speakers-into-segments.json | 32 ++ .../returns-whisper-word-timestamps.json | 32 ++ .../streams-text-deltas.json | 32 ++ .../transcribes-with-a-language-hint.json | 32 ++ packages/ai/test/promise.test.ts | 54 +++- .../assemblyai-transcription.recorded.test.ts | 39 +++ .../deepgram-transcription.recorded.test.ts | 47 +++ .../test/provider/fal-video.recorded.test.ts | 4 +- .../google-transcription.recorded.test.ts | 54 ++++ .../provider/google-video.recorded.test.ts | 4 +- .../openai-transcription.recorded.test.ts | 86 +++++ ...video-recording.ts => queued-recording.ts} | 4 +- .../provider/runway-video.recorded.test.ts | 4 +- .../test/provider/transcription-recording.ts | 41 +++ .../test/provider/xai-video.recorded.test.ts | 4 +- packages/ai/test/recorded-test.ts | 4 + packages/ai/test/transcription.test.ts | 174 +++++++++++ packages/ai/test/transcription.types.ts | 34 ++ 56 files changed, 2840 insertions(+), 286 deletions(-) create mode 100644 packages/ai/src/protocols/assemblyai-transcription.ts create mode 100644 packages/ai/src/protocols/deepgram-transcription.ts create mode 100644 packages/ai/src/protocols/google-transcription.ts create mode 100644 packages/ai/src/protocols/openai-transcription.ts create mode 100644 packages/ai/src/protocols/utils/gemini-generate-content.ts create mode 100644 packages/ai/src/providers/assemblyai.ts create mode 100644 packages/ai/src/transcription-client.ts create mode 100644 packages/ai/src/transcription.ts create mode 100644 packages/ai/test/fixtures/audio/dialog.mp3 create mode 100644 packages/ai/test/fixtures/audio/hello.mp3 create mode 100644 packages/ai/test/fixtures/recordings/assemblyai-transcription/uploads-submits-and-polls-a-transcript.json create mode 100644 packages/ai/test/fixtures/recordings/deepgram-transcription/splits-utterances-at-speaker-changes.json create mode 100644 packages/ai/test/fixtures/recordings/deepgram-transcription/transcribes-raw-audio-with-speakers-and-word-timestamps.json create mode 100644 packages/ai/test/fixtures/recordings/google-transcription/streams-speaker-turns.json create mode 100644 packages/ai/test/fixtures/recordings/google-transcription/transcribes-with-speakers-and-word-timestamps.json create mode 100644 packages/ai/test/fixtures/recordings/openai-transcription/diarizes-speakers-into-segments.json create mode 100644 packages/ai/test/fixtures/recordings/openai-transcription/returns-whisper-word-timestamps.json create mode 100644 packages/ai/test/fixtures/recordings/openai-transcription/streams-text-deltas.json create mode 100644 packages/ai/test/fixtures/recordings/openai-transcription/transcribes-with-a-language-hint.json create mode 100644 packages/ai/test/provider/assemblyai-transcription.recorded.test.ts create mode 100644 packages/ai/test/provider/deepgram-transcription.recorded.test.ts create mode 100644 packages/ai/test/provider/google-transcription.recorded.test.ts create mode 100644 packages/ai/test/provider/openai-transcription.recorded.test.ts rename packages/ai/test/provider/{video-recording.ts => queued-recording.ts} (63%) create mode 100644 packages/ai/test/provider/transcription-recording.ts create mode 100644 packages/ai/test/transcription.test.ts create mode 100644 packages/ai/test/transcription.types.ts diff --git a/packages/ai/AGENTS.md b/packages/ai/AGENTS.md index 52dd8a33449..b21c89a6f95 100644 --- a/packages/ai/AGENTS.md +++ b/packages/ai/AGENTS.md @@ -100,7 +100,9 @@ Media does not fit the SSE-frames-to-event-state-machine LLM route. `MediaRoute. `MediaProtocol.queued` is the submit-then-poll kind every video route uses: `start` (body + decode into `{ token, snapshot }`), `status`, `result`, and optional `cancel`, each addressed by a route-owned `token` whose `Schema.Codec` makes it serializable. `MediaRoute.inline` and `MediaRoute.queued` compose the two kinds with `Endpoint` and `Auth`; the queued route decodes the token once at the boundary (`start` output or `resume` input) and closes over it in a token-free `GenerationRoute` (`status`/`result`/`cancel` are plain Effects), so `Generation` never sees the token's shape and only carries the encoded JSON for persistence. Polls reuse the route's auth and deployment headers plus the request's `http` overlay after `start`, and resolve relative paths against the route base URL (provider-issued absolute URLs such as fal's `status_url` pass through). `result` is always its own GET even when the provider returns output inside the status document, so `Generation.await` behaves the same after `start` and after `resume`. `PollContext.auth` carries only what `Auth` added so protocols can hand download credentials to output assets as transient `Media.Asset.headers` (Veo) — never part of `source` or JSON. Status strings map through a per-protocol `STATUS` table via `MediaProtocol.status`; terminal generations without output fail through `output.ended` / `output.contentPolicy` with the provider document on `reason.body`. `Generation.AwaitOptions` (`{ poll?: Poll }`) is the one options type for `await`, `events`, `Video.generate`, and `Video.stream`. -`MediaProtocol.stream` is the incremental kind every speech route uses, with the same discipline as LLM protocols. `MediaRoute.stream` submits the caller's request as `MediaProtocol.Addressed` (`{ ...request, mode }`, `mode: "generate" | "stream"`), so one provider stays one protocol: `body.from`, the endpoint path, and `frames` read `request.mode` to pick the body, path, and framing. `frames(bytes, context)` returns frames — `Framing.sse`, `Framing.lines`, `Framing.document` (a single-document response shaped like a streamed record), or the raw `bytes` for chunked audio. `initial()` is fresh per-response parser state; `step` folds each frame into it and emits modality events; `finish(state, context)` runs once after the last frame with the request, body, and observed `http` (header-only usage lives there) and emits exactly one terminal event or fails with `MediaProtocol.incomplete`. Keep parser state to real accumulators and derive anything the request or body determines in `finish`. `generate` runs the same stream and folds it with the modality's `collect`. Request-derived URL parameters go on the JSON body's `query`, applied before route and caller `http.query`. Decode frames with `MediaProtocol.decodeFrame` and raise stream-time failures with `MediaProtocol.frameError` (the frame stays on `reason.body`); protocols never thread HTTP context, because the route fills `reason.http` on stream errors that lack it. Speech protocols share `protocols/utils/speech-stream.ts` for deltas, timestamps, voice ids, PCM and container descriptions, and the terminal asset. +`MediaProtocol.stream` is the incremental kind every speech route uses, with the same discipline as LLM protocols. `MediaRoute.stream` submits the caller's request as `MediaProtocol.Addressed` (`{ ...request, mode }`, `mode: "generate" | "stream"`), so one provider stays one protocol: `body.from`, the endpoint path, and `frames` read `request.mode` to pick the body, path, and framing. `frames(bytes, context)` returns frames — `Framing.sse`, `Framing.lines`, `Framing.document` (a single-document response shaped like a streamed record), or the raw `bytes` for chunked audio. `initial()` is fresh per-response parser state; `step` folds each frame into it and emits modality events; `finish(state, context)` runs once after the last frame with the request, body, and observed `http` (header-only usage lives there) and emits exactly one terminal event or fails with `MediaProtocol.incomplete`. Keep parser state to real accumulators and derive anything the request or body determines in `finish`. `generate` runs the same stream and folds it with the modality's `collect`. Request-derived URL parameters go on the body's `query` (array values repeat the parameter), applied before route and caller `http.query`. Decode frames with `MediaProtocol.decodeFrame` and raise stream-time failures with `MediaProtocol.frameError` (the frame stays on `reason.body`); protocols never thread HTTP context, because the route fills `reason.http` on stream errors that lack it. Speech protocols share `protocols/utils/speech-stream.ts` for deltas, timestamps, voice ids, PCM and container descriptions, and the terminal asset. + +Transcription uses all three kinds (OpenAI and Gemini stream, Deepgram is inline, AssemblyAI is queued): every route carries its `kind` and `TranscriptionClient` dispatches on it. Bodies are `json`, `multipart`, or `binary` (a raw upload), and a queued protocol that must upload media before submitting implements `start.prepare` (`MediaProtocol.Prepare`; AssemblyAI `/v2/upload`). ### URL Construction diff --git a/packages/ai/README.md b/packages/ai/README.md index 564492ece77..27c43b16bb5 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -766,6 +766,69 @@ for await (const event of ai.speech.stream({ model, text: "Hello from OpenCode." } ``` +## Transcription + +Transcription (speech-to-text) is the one modality whose providers use every route kind: OpenAI and Gemini stream, +Deepgram answers inline, and AssemblyAI is queued. `Transcription.generate` and `Transcription.stream` work on all of +them; `Transcription.start` / `resume` return a `Generation` on queued routes and fail with `UnsupportedOperation` +elsewhere. Models come from `.transcription(...)` selectors on the `OpenAI`, `Google`, `Deepgram`, and `AssemblyAI` +facades. Common fields (`language`, `prompt`, `timestamps: "none" | "segment" | "word"`, `diarize`, `speakers`) lower +natively or fail with a typed `AIError` before any network call; a route may return more than asked. + +```ts +import { Media, Transcription, TranscriptionEvent } from "@opencode/ai" +import { AssemblyAI, Deepgram, OpenAI } from "@opencode/ai/providers" + +const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY }) + +const program = Effect.gen(function* () { + const audio = yield* Media.file("./call.mp3") + + // Speaker-labelled segments; labels are provider-native strings ("A", "0", "spk:0"). + const response = yield* Transcription.generate({ + model: Deepgram.configure({ apiKey }).transcription("nova-3"), + audio, + diarize: true, + timestamps: "word", + }) + response.text // "Hello from OpenCode." + response.segments // [{ text, startSeconds, endSeconds, speaker: "0" }] + response.words // [{ text, startSeconds, endSeconds, speaker, confidence }] + response.language // the provider's own value, lowercased ("en", "english", "en_us") + + // Text deltas as the model transcribes, then one finish carrying the whole transcript. + yield* Transcription.stream({ model: openai.transcription("gpt-4o-mini-transcribe"), audio }).pipe( + Stream.tap((event) => (TranscriptionEvent.is.textDelta(event) ? Console.log(event.delta) : Effect.void)), + Stream.runDrain, + ) + + // Queued: persist the token, resume from another process, and await. + const model = AssemblyAI.configure({ apiKey }).transcription("universal-3-5-pro") + const generation = yield* Transcription.start({ model, audio }) + const resumed = yield* Transcription.resume(model, JSON.parse(JSON.stringify(generation.token))) + const transcript = yield* resumed.await({ poll: { interval: "3 seconds" } }) +}) +``` + +Inline routes emit only `finish` from `stream` (no faked deltas); queued routes emit `generation-queued` / +`generation-progress` before it. `TranscriptionClient.layer` needs `RequestExecutor.Service`. + +Provider notes: + +- **OpenAI** takes inline audio only; `diarize` needs `gpt-4o-transcribe-diarize`, timestamps need `whisper-1`, and `whisper-1` does not stream. +- **Gemini** needs a transcribe model (`gemini-3.5-transcribe`); `prompt` and `speakers` fail typed. +- **Deepgram** detects the language unless `language` is set; vocabulary goes in `providerOptions.keyterm`. +- **AssemblyAI** uploads inline audio before submitting and is the only route that accepts `speakers`. + +The promise client mirrors the Effect API: + +```ts +const text = (await ai.transcription.generate({ model, audio })).text +for await (const event of ai.transcription.stream({ model, audio })) if (event.type === "text-delta") write(event.delta) +const generation = await ai.transcription.start({ model: assemblyai, audio }) +const transcript = await generation.await({ poll: { interval: 3_000 } }) +``` + ## Public API - **`LLM.request({...})`** — build a provider-neutral `LLMRequest`. Accepts ergonomic inputs (`system: string`, `prompt: string`) that normalize into the canonical Schema classes. @@ -778,7 +841,8 @@ for await (const event of ai.speech.stream({ model, text: "Hello from OpenCode." - **`Media`** — the shared asset type (`Media.Asset`, `Media.Source`) and constructors used by messages, tool results, and media requests. - **`Generation`** — provider-neutral handle for an in-flight media generation (`await`, `refresh`, `cancel`, `events`) used by queued media routes. - **`Speech.request` / `Speech.generate` / `Speech.stream`** — text-to-speech through a provider-neutral request; `SpeechClient` is its Effect service and layer. -- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm`, `image`, `video`, and `speech` as Promise / `AsyncIterable` APIs. +- **`Transcription.request` / `generate` / `stream` / `start` / `resume`** — speech-to-text over inline, streaming, and queued routes; `TranscriptionClient` is its Effect service and layer. +- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm`, `image`, `video`, `speech`, and `transcription` as Promise / `AsyncIterable` APIs. ## Testing diff --git a/packages/ai/docs/media-design.md b/packages/ai/docs/media-design.md index 8a7a4a2559a..4d0fd121c7f 100644 --- a/packages/ai/docs/media-design.md +++ b/packages/ai/docs/media-design.md @@ -1,6 +1,6 @@ # Media generation in `@opencode/ai` — public API direction -Status: phases 1–2 implemented; phase 3 Speech implemented, Transcription pending; phases 4–5 proposal. +Status: phases 1–3 implemented (Speech and Transcription); phases 4–5 proposal. ## Goal @@ -249,23 +249,71 @@ Deferred: `Speech.session(...)` — input-streaming TTS where text arrives incre #### Transcription (STT) +Shipped as the second half of phase 3 (`src/transcription.ts`, `src/transcription-client.ts`, protocols +`openai-transcription`, `google-transcription`, `deepgram-transcription`, `assemblyai-transcription`; new `AssemblyAI` +facade). + ```ts const request = Transcription.request({ - model: openai.transcription("gpt-4o-transcribe"), - audio: Media.file("./call.wav"), - language: "en", - prompt: "Names: Shoubhit, OpenCode.", - timestamps: "word", // none | segment | word + model: openai.transcription("gpt-4o-transcribe-diarize"), + audio: yield* Media.file("./call.wav"), + language: "en", // provider-native passthrough + timestamps: "segment", // none | segment | word diarize: true, - providerOptions: { chunkingStrategy: "auto" }, + speakers: 2, // expected count, hint only (AssemblyAI) + providerOptions: { known_speaker_names: ["agent"] }, }) const response = yield* Transcription.generate(request) -response.text; response.segments; response.words; response.language; response.durationSeconds -yield* Transcription.stream(request) // Stream: text-delta | segment | finish +response.text; response.segments; response.words; response.language; response.durationSeconds; response.usage +yield* Transcription.stream(request) // Stream: generation-queued | generation-progress | text-delta | segment | finish +const generation = yield* Transcription.start(request) // queued routes only +yield* Transcription.resume(model, token) ``` -Realtime STT over WebSocket is the same future `session` shape as input-streaming TTS. +Transcription is the first modality whose providers span all three protocol kinds, and it needed no fourth kind. +Every `MediaRoute` now carries its `kind`; `TranscriptionRoute` is the union of the inline, stream, and queued routes; +`TranscriptionModel.fromRoute` is overloaded per protocol kind (arity picks the overload: ``, +``, ``) and composes through `MediaRoute.inline` / `stream` / `queued`; and +`TranscriptionClient` dispatches on `route.kind`. `generate` on a queued route is `start` then `await`; `stream` on an +inline route is the response as a single `finish`, and on a queued route it is the status observations followed by +`finish`. `start` / `resume` on a non-queued route fail with `UnsupportedOperation` (`transcription.start`). The +`finish` event carries the whole transcript (text, segments, words, language, duration, usage), so the stream route's +`collect` is just "take `finish`". + +The route layer gained a `binary` body with array-valued `query` (Deepgram) and `Queued.start.prepare` (AssemblyAI's +upload); `packages/ai/AGENTS.md` (Media Routes) describes both. + +Settled rules: + +- **Timestamps.** A granularity the selected route or model cannot produce fails as `UnsupportedOperation` + (`media.timestamps`), following Speech; a route that returns more than asked (Deepgram and AssemblyAI always return + words) is not stripped. Segments always carry start and end times: Gemini times each transcription part from its + word offsets, so segment timestamps and diarization also request word offsets there. +- **Diarization.** `diarize` means segments (and words, where the provider labels them) carry `speaker`. Labels are + provider-native strings — OpenAI `A` or a known speaker name, Deepgram `0`, Gemini `spk:0`, AssemblyAI `A` — with no + cross-provider speaker model. `speakers` is a hint; only AssemblyAI (`speakers_expected`) accepts it. +- **Language** is passed through (`language`, OpenAI `gpt-transcribe` `languages[]`, Gemini `languageCodes`, + AssemblyAI `language_code`). `response.language` is the provider's own value, lowercased but not normalized: an + ISO code on most routes, `english` from whisper-1, `en_us` from AssemblyAI. Deepgram and AssemblyAI assume English + unless asked to detect, so a missing `language` enables their detection. +- **Gemini** requires a transcribe model; other model ids fail with `UnsupportedOperation` before the call, because + general models ignore `audioTranscriptionConfig` and answer conversationally. Streamed chunks carry whole speaker + turns (one part per turn), which join with a space. +- **Streaming inline providers** emit only `finish`; deltas are never faked. +- **Units.** AssemblyAI milliseconds and Gemini protobuf durations (`"0.400s"`) are normalized to seconds at the + protocol boundary. + +| Provider | Kind | Audio input | `timestamps` | `diarize` | Unsupported | Usage | +|---|---|---|---|---|---|---| +| OpenAI | stream (`stream: true` in `stream` mode) | multipart `file` (inline only) | `whisper-1` (`verbose_json`); diarize model: `segment` | `gpt-4o-transcribe-diarize` (`diarized_json`) | `speakers`; `prompt` on the diarize model; streaming on `whisper-1` | `tokens` or `seconds` | +| Gemini | stream (`generateContent` / `streamGenerateContent`) | `inlineData` or Gemini Files `fileData` | `audioTranscriptionConfig.wordTimestamp` | `audioTranscriptionConfig.diarization` | `prompt`, `speakers` | `tokens` | +| Deepgram | inline | raw body, or JSON `{ url }` | words always; `segment` → `utterances` | `diarize_model=latest` + `utterances` | `prompt`, `speakers` | `seconds` (`metadata.duration`) | +| AssemblyAI | queued (upload → submit → poll) | `/v2/upload` then `audio_url`, or a URL | words always; `segment` → `speaker_labels` | `speaker_labels` | — | `seconds` (`audio_duration`) | + +Deferred: `Transcription.session(...)` — realtime STT over WebSocket (Deepgram live, AssemblyAI streaming, ElevenLabs +realtime, OpenAI realtime transcription) — is the same future scoped `session` shape as input-streaming TTS and ships +with the realtime work in phase 5. ElevenLabs Scribe is not implemented yet. ### `Generation` — shared async execution @@ -338,13 +386,14 @@ Existing facades gain per-modality selectors; the modality routes each facade pr | Facade | llm | image | video | speech | transcription | other | |---|---|---|---|---|---|---| | `OpenAI` | responses (default), chat | Images API | Sora (deprecated 2026-09-24) | ✓ | ✓ | | -| `Google` | Gemini | Gemini-native (default), `imagen` | Veo | Gemini TTS | Gemini transcribe | | +| `Google` | Gemini | Gemini-native (default), `imagen` | Veo | Gemini TTS | `gemini-3.5-transcribe` | | | `XAI` | ✓ | ✓ | ✓ | | | | | `ElevenLabs` | | | | ✓ | Scribe | soundEffect, music | | `Cartesia` | | | | ✓ | | | | `Deepgram` | | | | Aura | ✓ | | | `Fal` | | ✓ | ✓ | | | | -| `Replicate`, `Runway`, `Luma`, `Kling`, `MiniMax`, `AssemblyAI`, `BlackForestLabs`, `Stability` | | per provider | | | | | +| `AssemblyAI` | | | | | ✓ (queued) | | +| `Replicate`, `Runway`, `Luma`, `Kling`, `MiniMax`, `BlackForestLabs`, `Stability` | | per provider | | | | | New facades follow the existing one-file-per-provider rule. Package entrypoints are modality-specific, such as `@opencode/ai/providers/openai/images`, and return the concrete model. @@ -358,7 +407,7 @@ Media does not fit the LLM four-axis route (SSE frames → event state machine) - `MediaProtocol.queued` — `start` (body + decode to `{ token, snapshot }`), `status`, `result`, optional `cancel`, `pollHint`, and a `token` codec. `result` is always a separate GET (against the status document for Veo/xAI/Runway, fal's `response_url` otherwise) so `await` after `start` and after `resume` share one path. `PollContext.auth` hands the auth headers the route sent to the protocol for output URLs that need them (Veo downloads); they become transient `Media.Asset.headers`, never part of `source`. There is no separate `download` step: `Media.Asset.bytes()` downloads through the executor with those headers. `MediaRoute.inline(...)` / `MediaRoute.queued(...)` compose each kind with endpoint and auth; the queued route decodes the token once and hands `Generation` a token-free `{ status, result, cancel? }`. - `MediaProtocol.stream` — `body.from(request)` over the request plus its `mode`, `frames` (a function that picks the framing for the call: `Framing.sse`, `lines`, `document`, or the raw bytes), fresh per-response `initial()` state, `step` emitting modality events, and `finish(state, context)` — with the observed response for header-only usage — emitting exactly one terminal event or failing as an incomplete stream. The route fills `reason.http` on stream errors. `MediaRoute.stream(...)` exposes `stream` and `generate` (the same stream folded by the modality's `collect`). -`MediaRoute.inline` / `MediaRoute.queued` / `MediaRoute.stream` compose one protocol kind with endpoint/auth; `ImageModel`/`VideoModel`/`SpeechModel` share the `MediaModel` base (`src/media-model.ts`). +`MediaRoute.inline` / `MediaRoute.queued` / `MediaRoute.stream` compose one protocol kind with endpoint/auth and tag the route with its `kind`; `ImageModel`/`VideoModel`/`SpeechModel`/`TranscriptionModel` share the `MediaModel` base (`src/media-model.ts`). ### LLM integration @@ -388,7 +437,7 @@ Foundation + Image ship together as the reference implementation, serially. Vide 1. **Foundation** — per-modality selectors, `Media`, `Generation`, `Poll`, `Usage` union, `MediaProtocol` kinds, `@opencode/ai/promise` with `llm` + `image`. Port the five existing image protocols onto it. Unify `MediaPart` and add the `media` LLM event (fixes Gemini image output being dropped). 2. **Video** — ✅ Veo, xAI, fal, Runway shipped (`MediaProtocol.queued`, `Video.start/generate/resume/stream`, promise `ai.video`). Deferred: `Video.complete` (webhooks), Luma, Kling, MiniMax, Replicate. -3. **Speech + Transcription** — ✅ Speech: OpenAI, Gemini TTS, ElevenLabs, Cartesia, Deepgram shipped (`MediaProtocol.stream`, `Speech.generate/stream`, promise `ai.speech`). Pending: Transcription (OpenAI, ElevenLabs Scribe, Gemini, Deepgram, AssemblyAI). Deferred: `Speech.session` (WebSocket input streaming). +3. **Speech + Transcription** — ✅ Speech: OpenAI, Gemini TTS, ElevenLabs, Cartesia, Deepgram shipped (`MediaProtocol.stream`, `Speech.generate/stream`, promise `ai.speech`). ✅ Transcription: OpenAI, Gemini, Deepgram, AssemblyAI shipped across all three route kinds (`Transcription.generate/stream/start/resume`, promise `ai.transcription`). Pending: ElevenLabs Scribe. Deferred: `Speech.session` and `Transcription.session` (WebSocket streaming). 4. **Image queued routes and partials** — BFL, fal, Replicate, Stability; OpenAI `partial_images` streaming. 5. **Later** — ElevenLabs music/SFX, Lyria, `Speech.session` / `Transcription.session`, realtime. diff --git a/packages/ai/script/setup-recording-env.ts b/packages/ai/script/setup-recording-env.ts index ec47dea8e95..8e5d9be701f 100644 --- a/packages/ai/script/setup-recording-env.ts +++ b/packages/ai/script/setup-recording-env.ts @@ -161,7 +161,7 @@ const PROVIDERS: ReadonlyArray = [ id: "deepgram", label: "Deepgram", tier: "canary", - note: "Deepgram Aura text-to-speech recorded tests", + note: "Deepgram Aura text-to-speech and Nova transcription recorded tests", vars: [{ name: "DEEPGRAM_API_KEY" }], validate: (env) => HttpClientRequest.get("https://api.deepgram.com/v1/projects").pipe( @@ -169,6 +169,18 @@ const PROVIDERS: ReadonlyArray = [ executeRequest, ), }, + { + id: "assemblyai", + label: "AssemblyAI", + tier: "canary", + note: "AssemblyAI queued transcription recorded tests", + vars: [{ name: "ASSEMBLYAI_API_KEY" }], + validate: (env) => + HttpClientRequest.get("https://api.assemblyai.com/v2/transcript?limit=1").pipe( + HttpClientRequest.setHeader("authorization", Redacted.value(Redacted.make(env.ASSEMBLYAI_API_KEY))), + executeRequest, + ), + }, { id: "cloudflare-ai-gateway", label: "Cloudflare AI Gateway", diff --git a/packages/ai/src/generation.ts b/packages/ai/src/generation.ts index f1296b1473b..b2e93dac220 100644 --- a/packages/ai/src/generation.ts +++ b/packages/ai/src/generation.ts @@ -40,10 +40,21 @@ export interface AwaitOptions { export const DEFAULT_POLL_INTERVAL = Duration.seconds(5) export const DEFAULT_POLL_TIMEOUT = Duration.minutes(10) -export type Event = - | { readonly type: "generation-queued"; readonly id: string; readonly position?: number } - | { readonly type: "generation-progress"; readonly id: string; readonly progress?: number } - | { readonly type: "generation-finished"; readonly id: string; readonly status: Status } +export const QueuedEvent = Schema.Struct({ + type: Schema.tag("generation-queued"), + id: Schema.String, + position: Schema.optional(Schema.Number), +}).annotate({ identifier: "Generation.Event.Queued" }) + +export const ProgressEvent = Schema.Struct({ + type: Schema.tag("generation-progress"), + id: Schema.String, + progress: Schema.optional(Schema.Number), +}).annotate({ identifier: "Generation.Event.Progress" }) + +export type Observation = Schema.Schema.Type | Schema.Schema.Type + +export type Event = Observation | { readonly type: "generation-finished"; readonly id: string; readonly status: Status } const TERMINAL: ReadonlySet = new Set(["completed", "failed", "cancelled", "expired"]) @@ -168,3 +179,13 @@ export class Generation { ) } } + +export const resultEvents = ( + generation: Generation, + expand: (response: Response) => ReadonlyArray, + options?: AwaitOptions, +): Stream.Stream => + generation.events(options).pipe( + Stream.filter((event): event is Observation => event.type !== "generation-finished"), + Stream.concat(Stream.fromIterableEffect(Effect.map(generation.result(), expand))), + ) diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index d4907125bcf..594ae5d6d58 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -67,6 +67,25 @@ export type { SpeechRoute, } from "./speech.js" export { Speech } from "./speech.js" +export { TranscriptionClient } from "./transcription-client.js" +export { + TranscriptionEvent, + TranscriptionModel, + TranscriptionModelSchema, + TranscriptionRequest, + TranscriptionResponse, + TranscriptionSegment, + TranscriptionTimestamps, + TranscriptionWord, +} from "./transcription.js" +export type { + TranscriptionModelOptions, + TranscriptionOptions, + TranscriptionRequestFor, + TranscriptionRequestInput, + TranscriptionRoute, +} from "./transcription.js" +export { Transcription } from "./transcription.js" export { Media } from "./media.js" export { Generation } from "./generation.js" export type { diff --git a/packages/ai/src/promise.ts b/packages/ai/src/promise.ts index 703c693cc96..873e5fa2ec9 100644 --- a/packages/ai/src/promise.ts +++ b/packages/ai/src/promise.ts @@ -9,12 +9,21 @@ import { LanguageModel, LLMRequest } from "./schema/index.js" import type { RequestInput } from "./llm.js" import { Speech, SpeechModel, SpeechRequest, type SpeechRequestInput } from "./speech.js" import { SpeechClient } from "./speech-client.js" +import { + Transcription, + TranscriptionModel, + TranscriptionRequest, + type TranscriptionOptions, + type TranscriptionRequestInput, +} from "./transcription.js" +import { TranscriptionClient } from "./transcription-client.js" import { Video, VideoModel, VideoRequest, type VideoOptions, type VideoRequestInput } from "./video.js" import { VideoClient } from "./video-client.js" /** - * Promise-first entrypoint for scripts and non-Effect callers. One `ManagedRuntime` hosts the LLM, image, video, and - * speech clients over a request executor; every method runs the corresponding Effect API and rethrows `AIError` unchanged. + * Promise-first entrypoint for scripts and non-Effect callers. One `ManagedRuntime` hosts the LLM, image, video, speech, + * and transcription clients over a request executor; every method runs the corresponding Effect API and rethrows + * `AIError` unchanged. */ export interface Options { /** Executor layer; defaults to `RequestExecutor.fetchLayer`. Inject a recorder or middleware here. */ @@ -30,6 +39,7 @@ export type Services = | Layer.Success | Layer.Success | Layer.Success + | Layer.Success | RequestExecutor.Service /** Promise view of a `Generation`: its snapshot plus `await`, `refresh`, and `cancel` returning promises. */ @@ -56,9 +66,13 @@ const abortEffect = (signal: AbortSignal | undefined) => export const make = (options: Options = {}) => { const runtime = ManagedRuntime.make( - Layer.mergeAll(LLMClient.layer, ImageClient.layer, VideoClient.layer, SpeechClient.layer).pipe( - Layer.provideMerge(options.layer ?? RequestExecutor.fetchLayer), - ), + Layer.mergeAll( + LLMClient.layer, + ImageClient.layer, + VideoClient.layer, + SpeechClient.layer, + TranscriptionClient.layer, + ).pipe(Layer.provideMerge(options.layer ?? RequestExecutor.fetchLayer)), ) /** Run any package Effect (for example `asset.bytes()`) inside this runtime. */ @@ -93,6 +107,8 @@ export const make = (options: Options = {}) => { input instanceof VideoRequest ? input : Video.request(input) const speechRequest = (input: SpeechRequestInput | SpeechRequest) => input instanceof SpeechRequest ? input : Speech.request(input) + const transcriptionRequest = (input: TranscriptionRequestInput | TranscriptionRequest) => + input instanceof TranscriptionRequest ? input : Transcription.request(input) return { run, @@ -138,6 +154,26 @@ export const make = (options: Options = {}) => { options?: RunOptions, ) => iterate(Speech.stream(speechRequest(input)), options), }, + transcription: { + request: Transcription.request, + generate: ( + input: TranscriptionRequestInput | TranscriptionRequest, + options?: AwaitOptions & RunOptions, + ) => run(Transcription.generate(transcriptionRequest(input), { poll: options?.poll }), options), + stream: ( + input: TranscriptionRequestInput | TranscriptionRequest, + options?: AwaitOptions & RunOptions, + ) => iterate(Transcription.stream(transcriptionRequest(input), { poll: options?.poll }), options), + start: ( + input: TranscriptionRequestInput | TranscriptionRequest, + options?: RunOptions, + ) => run(Transcription.start(transcriptionRequest(input)), options).then(handle), + resume: ( + model: TranscriptionModel, + token: unknown, + options?: RunOptions, + ) => run(Transcription.resume(model, token), options).then(handle), + }, dispose: () => runtime.dispose(), } } diff --git a/packages/ai/src/protocols/assemblyai-transcription.ts b/packages/ai/src/protocols/assemblyai-transcription.ts new file mode 100644 index 00000000000..b3172b89980 --- /dev/null +++ b/packages/ai/src/protocols/assemblyai-transcription.ts @@ -0,0 +1,211 @@ +import { Effect, Schema } from "effect" +import type { HttpClientResponse } from "effect/unstable/http" +import type { Status } from "../generation.js" +import { Media } from "../media.js" +import { MediaProtocol } from "../route/media-protocol.js" +import { MediaRoute } from "../route/media.js" +import { ProviderID, mergeJsonRecords } from "../schema/index.js" +import { TranscriptionModel, TranscriptionResponse, type TranscriptionRequestFor } from "../transcription.js" +import { ProviderShared, optionalNull } from "./shared.js" +import { MediaInput } from "./utils/media-input.js" + +const ADAPTER = "assemblyai-transcription" +const NAME = "AssemblyAI" +const PROVIDER = ProviderID.make("assemblyai") +export const DEFAULT_BASE_URL = "https://api.assemblyai.com" +export const PATH = "/v2/transcript" +export const UPLOAD_PATH = "/v2/upload" + +// --------------------------------------------------------------------------- +// 1. Public model input +// --------------------------------------------------------------------------- + +export type AssemblyAITranscriptionOptions = { + readonly keyterms_prompt?: ReadonlyArray + readonly punctuate?: boolean + readonly format_text?: boolean + readonly disfluencies?: boolean + readonly filter_profanity?: boolean + readonly temperature?: number + readonly speaker_options?: { readonly min_speakers_expected?: number; readonly max_speakers_expected?: number } + readonly language_detection_options?: { + readonly expected_languages?: ReadonlyArray + readonly fallback_language?: string + readonly code_switching?: boolean + } + readonly speech_models?: ReadonlyArray<"universal-3-5-pro" | "universal-2" | (string & {})> +} & Record + +export type Request = TranscriptionRequestFor + +// --------------------------------------------------------------------------- +// 2. Token and response schemas +// --------------------------------------------------------------------------- + +export const Token = Schema.Struct({ transcriptID: Schema.String }) +export type Token = Schema.Schema.Type + +const Upload = Schema.Struct({ upload_url: Schema.String }) + +/** Word and utterance times are milliseconds. */ +const Transcript = Schema.Struct({ + id: Schema.String, + status: Schema.String, + text: optionalNull(Schema.String), + words: optionalNull( + Schema.Array( + Schema.Struct({ + text: Schema.String, + start: Schema.Number, + end: Schema.Number, + confidence: optionalNull(Schema.Number), + speaker: optionalNull(Schema.String), + }), + ), + ), + utterances: optionalNull( + Schema.Array( + Schema.Struct({ + text: Schema.String, + start: Schema.Number, + end: Schema.Number, + speaker: optionalNull(Schema.String), + }), + ), + ), + language_code: optionalNull(Schema.String), + audio_duration: optionalNull(Schema.Number), + speech_model_used: optionalNull(Schema.String), + error: optionalNull(Schema.String), +}) + +const STATUS = { + queued: "queued", + processing: "running", + completed: "completed", + error: "failed", +} as const satisfies Record + +// --------------------------------------------------------------------------- +// 5. Request body construction +// --------------------------------------------------------------------------- + +const decodeUpload = MediaProtocol.decodeJson(ADAPTER, NAME, Upload) + +/** `/v2/transcript` only takes a URL, so inline audio is uploaded to `/v2/upload` first. */ +const prepare = Effect.fn("AssemblyAITranscription.prepare")(function* (request: Request, send: MediaProtocol.Send) { + if (request.audio.source.type !== "bytes" && request.audio.source.type !== "base64") return request + const audio = yield* MediaInput.inlineBytes(ADAPTER, request.audio) + const uploaded = yield* send(UPLOAD_PATH, MediaProtocol.binary(audio, "application/octet-stream")).pipe( + Effect.flatMap(decodeUpload), + ) + return { ...request, audio: Media.url(uploaded.value.upload_url, { mediaType: request.audio.mediaType }) } +}) + +const fromRequest = Effect.fn("AssemblyAITranscription.fromRequest")(function* (request: Request) { + const audio = yield* ProviderShared.mediaReference(request.audio, PROVIDER, NAME) + return MediaProtocol.json( + mergeJsonRecords( + { + audio_url: audio.value, + speech_models: [request.model.id], + language_code: request.language, + language_detection: request.language === undefined ? true : undefined, + prompt: request.prompt, + // Turn-level `utterances`, the only segments AssemblyAI returns, require speaker labels. + speaker_labels: request.diarize === true || request.timestamps === "segment" ? true : undefined, + speakers_expected: request.speakers, + }, + request.providerOptions, + request.http?.body, + ) ?? {}, + ) +}) + +// --------------------------------------------------------------------------- +// 6. Response decoding +// --------------------------------------------------------------------------- + +const decodeTranscript = MediaProtocol.decodeJson(ADAPTER, NAME, Transcript) + +const decodeStart = Effect.fn("AssemblyAITranscription.decodeStart")(function* ( + response: HttpClientResponse.HttpClientResponse, +) { + const output = yield* decodeTranscript(response) + const status = yield* MediaProtocol.status(STATUS, output.value.status, output) + return { token: { transcriptID: output.value.id }, snapshot: { id: output.value.id, status } } +}) + +const decodeStatus = Effect.fn("AssemblyAITranscription.decodeStatus")(function* ( + response: HttpClientResponse.HttpClientResponse, + context: MediaProtocol.PollContext, +) { + const output = yield* decodeTranscript(response) + const status = yield* MediaProtocol.status(STATUS, output.value.status, output) + return { id: context.token.transcriptID, status } +}) + +const seconds = (milliseconds: number) => milliseconds / 1000 + +const decodeResult = Effect.fn("AssemblyAITranscription.decodeResult")(function* ( + response: HttpClientResponse.HttpClientResponse, + context: MediaProtocol.PollContext, +) { + const output = yield* decodeTranscript(response) + const transcript = output.value + const status = yield* MediaProtocol.status(STATUS, transcript.status, output) + const error = transcript.error ?? undefined + if (status === "failed") + return yield* output.ended("failed", `${NAME} transcription failed${error === undefined ? "" : `: ${error}`}`) + if (status !== "completed") + return yield* output.invalid(`${NAME} transcript ${context.token.transcriptID} has not finished`) + const duration = transcript.audio_duration ?? undefined + return new TranscriptionResponse({ + text: transcript.text ?? "", + segments: transcript.utterances?.map((utterance) => ({ + text: utterance.text, + startSeconds: seconds(utterance.start), + endSeconds: seconds(utterance.end), + speaker: utterance.speaker ?? undefined, + })), + words: transcript.words?.map((word) => ({ + text: word.text, + startSeconds: seconds(word.start), + endSeconds: seconds(word.end), + speaker: word.speaker ?? undefined, + confidence: word.confidence ?? undefined, + })), + language: transcript.language_code?.toLowerCase(), + durationSeconds: duration, + usage: duration === undefined ? undefined : { type: "seconds", seconds: duration }, + providerMetadata: { + assemblyai: { transcriptId: transcript.id, speechModel: transcript.speech_model_used ?? undefined }, + }, + }) +}) + +// --------------------------------------------------------------------------- +// 7. Protocol and route +// --------------------------------------------------------------------------- + +const transcriptPath = (token: Token) => `${PATH}/${token.transcriptID}` + +export const protocol = MediaProtocol.queued({ + id: ADAPTER, + name: NAME, + token: Token, + start: { prepare, body: { from: fromRequest }, decode: decodeStart }, + status: { path: transcriptPath, decode: decodeStatus }, + result: { path: transcriptPath, decode: decodeResult }, +}) + +export const model = (input: MediaRoute.ModelInput) => + TranscriptionModel.fromRoute( + { id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH }, + input, + ) + +export const AssemblyAITranscription = { + protocol, + model, +} as const diff --git a/packages/ai/src/protocols/deepgram-speech.ts b/packages/ai/src/protocols/deepgram-speech.ts index f58cc026cd6..15926314a09 100644 --- a/packages/ai/src/protocols/deepgram-speech.ts +++ b/packages/ai/src/protocols/deepgram-speech.ts @@ -3,6 +3,7 @@ import { MediaProtocol } from "../route/media-protocol.js" import { MediaRoute } from "../route/media.js" import { ProviderID, mergeJsonRecords } from "../schema/index.js" import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js" +import { MediaInput } from "./utils/media-input.js" import { SpeechStream } from "./utils/speech-stream.js" const ADAPTER = "deepgram-speech" @@ -49,26 +50,24 @@ const FORMATS: Readonly - typeof value === "string" || typeof value === "number" || typeof value === "boolean" ? String(value) : undefined +const audioFormat = (request: Request) => { + const format = request.format === undefined ? undefined : FORMATS[request.format] + return { + encoding: request.providerOptions?.encoding ?? format?.encoding, + container: request.providerOptions?.container ?? format?.container, + } +} const queryParameters = (request: Request) => { - const format = request.format === undefined ? undefined : FORMATS[request.format] - const { encoding, container, sampleRate, bitRate, ...native } = request.providerOptions ?? {} - return Object.fromEntries( - Object.entries({ - ...native, - model: request.model.id, - encoding: encoding ?? format?.encoding, - container: container ?? format?.container, - sample_rate: sampleRate, - bit_rate: bitRate, - speed: request.speed, - }).flatMap(([key, value]) => { - const text = queryValue(value) - return text === undefined ? [] : [[key, text]] - }), - ) + const { encoding: _encoding, container: _container, sampleRate, bitRate, ...native } = request.providerOptions ?? {} + return MediaInput.query(ADAPTER, { + ...native, + model: request.model.id, + ...audioFormat(request), + sample_rate: sampleRate, + bit_rate: bitRate, + speed: request.speed, + }) } const fromRequest = Effect.fn("DeepgramSpeech.fromRequest")(function* (request: Request) { @@ -84,7 +83,7 @@ const fromRequest = Effect.fn("DeepgramSpeech.fromRequest")(function* (request: ) return MediaProtocol.json( mergeJsonRecords({ text: request.text }, request.http?.body) ?? {}, - queryParameters(request), + yield* queryParameters(request), ) }) @@ -101,15 +100,15 @@ const HEADERLESS_ENCODINGS: Readonly> = const finish = (state: State, context: MediaProtocol.ResponseContext) => { const headers = context.http.headers const mediaType = headers["content-type"] - const query = queryParameters(context.request) - const encoding = HEADERLESS_ENCODINGS[query.encoding ?? ""] + const format = audioFormat(context.request) + const encoding = HEADERLESS_ENCODINGS[format.encoding ?? ""] const requestID = headers["dg-request-id"] const modelName = headers["dg-model-name"] return SpeechStream.finish(ADAPTER, state, { - ...(query.container === "none" && encoding !== undefined + ...(format.container === "none" && encoding !== undefined ? SpeechStream.pcm(encoding, SpeechStream.sampleRate(mediaType), mediaType) : // Deepgram's default encoding is MP3; WAV is a container around any encoding. - { mediaType, info: { format: query.container === "wav" ? "wav" : (query.encoding ?? "mp3") } }), + { mediaType, info: { format: format.container === "wav" ? "wav" : (format.encoding ?? "mp3") } }), usage: SpeechStream.headerUsage("characters", headers["dg-char-count"]), providerMetadata: requestID === undefined && modelName === undefined diff --git a/packages/ai/src/protocols/deepgram-transcription.ts b/packages/ai/src/protocols/deepgram-transcription.ts new file mode 100644 index 00000000000..f070584c48a --- /dev/null +++ b/packages/ai/src/protocols/deepgram-transcription.ts @@ -0,0 +1,191 @@ +import { Effect, Schema } from "effect" +import type { HttpClientResponse } from "effect/unstable/http" +import { MediaProtocol } from "../route/media-protocol.js" +import { MediaRoute } from "../route/media.js" +import { ProviderID, mergeJsonRecords } from "../schema/index.js" +import { TranscriptionModel, TranscriptionResponse, type TranscriptionRequestFor } from "../transcription.js" +import { ProviderShared } from "./shared.js" +import { MediaInput } from "./utils/media-input.js" + +const ADAPTER = "deepgram-transcription" +const NAME = "Deepgram" +const PROVIDER = ProviderID.make("deepgram") +export const DEFAULT_BASE_URL = "https://api.deepgram.com" +export const PATH = "/v1/listen" + +// --------------------------------------------------------------------------- +// 1. Public model input +// --------------------------------------------------------------------------- + +export type DeepgramTranscriptionOptions = { + readonly smart_format?: boolean + readonly punctuate?: boolean + readonly paragraphs?: boolean + readonly utterances?: boolean + readonly detect_language?: boolean | ReadonlyArray + readonly keyterm?: ReadonlyArray + readonly diarize_model?: "latest" | "v1" | "v2" | (string & {}) + readonly filler_words?: boolean + readonly numerals?: boolean + readonly mip_opt_out?: boolean + readonly tag?: string | ReadonlyArray +} & Record + +export type Request = TranscriptionRequestFor + +// --------------------------------------------------------------------------- +// 2. Response schema +// --------------------------------------------------------------------------- + +const Word = Schema.Struct({ + word: Schema.String, + start: Schema.Number, + end: Schema.Number, + confidence: Schema.optional(Schema.Number), + speaker: Schema.optional(Schema.Number), + punctuated_word: Schema.optional(Schema.String), +}) + +const ListenResponse = Schema.Struct({ + metadata: Schema.optional( + Schema.Struct({ request_id: Schema.optional(Schema.String), duration: Schema.optional(Schema.Number) }), + ), + results: Schema.Struct({ + channels: Schema.Array( + Schema.Struct({ + alternatives: Schema.optional( + Schema.Array(Schema.Struct({ transcript: Schema.String, words: Schema.optional(Schema.Array(Word)) })), + ), + detected_language: Schema.optional(Schema.String), + }), + ), + utterances: Schema.optional( + Schema.Array( + Schema.Struct({ + start: Schema.Number, + end: Schema.Number, + transcript: Schema.String, + speaker: Schema.optional(Schema.Number), + words: Schema.optional(Schema.Array(Word)), + }), + ), + ), + }), +}) + +// --------------------------------------------------------------------------- +// 5. Request body construction +// --------------------------------------------------------------------------- + +const query = (request: Request) => + MediaInput.query( + ADAPTER, + mergeJsonRecords( + { + model: request.model.id, + smart_format: true, + language: request.language, + // Deepgram assumes English unless asked to detect, unlike the other routes' auto-detection. + detect_language: request.language === undefined ? true : undefined, + // `diarize=true` is deprecated in favor of choosing a diarization model. + diarize_model: request.diarize === true ? "latest" : undefined, + utterances: request.diarize === true || request.timestamps === "segment" ? true : undefined, + }, + request.providerOptions, + ) ?? {}, + ) + +const fromRequest = Effect.fn("DeepgramTranscription.fromRequest")(function* (request: Request) { + const url = ProviderShared.mediaUrl(request.audio) + if (url !== undefined) + return MediaProtocol.json(mergeJsonRecords({ url }, request.http?.body) ?? {}, yield* query(request)) + if (request.http?.body !== undefined) + return yield* ProviderShared.invalidRequest(`${NAME} sends inline audio as the raw body, so http.body cannot apply`) + const audio = yield* MediaInput.inlineBytes(ADAPTER, request.audio) + return MediaProtocol.binary(audio, request.audio.mediaType, yield* query(request)) +}) + +// --------------------------------------------------------------------------- +// 6. Response decoding +// --------------------------------------------------------------------------- + +const decodeListen = MediaProtocol.decodeJson(ADAPTER, NAME, ListenResponse) + +const speaker = (value: number | undefined) => (value === undefined ? undefined : String(value)) + +const wordText = (word: typeof Word.Type) => word.punctuated_word ?? word.word + +// Utterances split on pauses, not speakers: the v2 diarizer labels a whole utterance with one speaker even when its +// words change speaker, so segments split each utterance at speaker changes. +const speakerTurns = (words: ReadonlyArray) => + words.reduce>>((turns, word) => { + const last = turns.at(-1) + if (last === undefined || last[0].speaker !== word.speaker) return [...turns, [word]] + last.push(word) + return turns + }, []) + +const decodeResponse = Effect.fn("DeepgramTranscription.decodeResponse")(function* ( + response: HttpClientResponse.HttpClientResponse, +) { + const output = yield* decodeListen(response) + const channel = output.value.results.channels[0] + const alternative = channel?.alternatives?.[0] + if (alternative === undefined) return yield* output.invalid(`${NAME} returned no transcript`) + const duration = output.value.metadata?.duration + const requestID = output.value.metadata?.request_id + return new TranscriptionResponse({ + text: alternative.transcript, + segments: output.value.results.utterances?.flatMap((utterance) => + utterance.words === undefined || utterance.words.length === 0 + ? [ + { + text: utterance.transcript, + startSeconds: utterance.start, + endSeconds: utterance.end, + speaker: speaker(utterance.speaker), + }, + ] + : speakerTurns(utterance.words).map((turn) => ({ + text: turn.map(wordText).join(" "), + startSeconds: turn[0].start, + endSeconds: turn[turn.length - 1].end, + speaker: speaker(turn[0].speaker), + })), + ), + words: alternative.words?.map((word) => ({ + text: wordText(word), + startSeconds: word.start, + endSeconds: word.end, + speaker: speaker(word.speaker), + confidence: word.confidence, + })), + language: channel?.detected_language?.toLowerCase(), + durationSeconds: duration, + usage: duration === undefined ? undefined : { type: "seconds", seconds: duration }, + providerMetadata: requestID === undefined ? undefined : { deepgram: { requestId: requestID } }, + }) +}) + +// --------------------------------------------------------------------------- +// 7. Protocol and route +// --------------------------------------------------------------------------- + +export const protocol = MediaProtocol.inline({ + id: ADAPTER, + name: NAME, + unsupported: ["prompt", "speakers"], + body: { from: fromRequest }, + response: { decode: decodeResponse }, +}) + +export const model = (input: MediaRoute.ModelInput) => + TranscriptionModel.fromRoute( + { id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH }, + input, + ) + +export const DeepgramTranscription = { + protocol, + model, +} as const diff --git a/packages/ai/src/protocols/gemini.ts b/packages/ai/src/protocols/gemini.ts index d1cdb308fea..46b7210ffa7 100644 --- a/packages/ai/src/protocols/gemini.ts +++ b/packages/ai/src/protocols/gemini.ts @@ -22,6 +22,7 @@ import { import { classifyProviderFailure } from "../provider-error.js" import { Media } from "../media.js" import { JsonObject, knownString, lenient, optionalArray, optionalNull, ProviderShared } from "./shared.js" +import { GeminiGenerateContent } from "./utils/gemini-generate-content.js" import { GeminiToolSchema } from "./utils/gemini-tool-schema.js" import { Lifecycle } from "./utils/lifecycle.js" import { ToolSchemaProjection } from "./utils/tool-schema.js" @@ -305,14 +306,9 @@ const lowerToolConfig = (toolChoice: NonNullable) => tool: (name) => ({ functionCallingConfig: { mode: "ANY" as const, allowedFunctionNames: [name] } }), }) -// Gemini does not fetch public URLs; inline payloads and Gemini Files references are the accepted inputs. const lowerContentPart = Effect.fn("Gemini.lowerContentPart")(function* (part: TextPart | MediaPart) { if (part.type === "text") return { text: part.text } - const source = part.media.source - if (source.type === "ref" && source.provider === "google") - return { fileData: { mimeType: part.media.mediaType, fileUri: source.id } } - const media = yield* ProviderShared.requireInlineMedia("Gemini", part.media) - return { inlineData: { mimeType: media.mime, data: media.base64 } } + return yield* GeminiGenerateContent.mediaPart("Gemini", part.media) }) const providerMetadata = (key: string, metadata: Record): ProviderMetadata => ({ [key]: metadata }) diff --git a/packages/ai/src/protocols/google-images.ts b/packages/ai/src/protocols/google-images.ts index d2a6885f1c6..5dd13fe2916 100644 --- a/packages/ai/src/protocols/google-images.ts +++ b/packages/ai/src/protocols/google-images.ts @@ -1,11 +1,11 @@ import { Effect, Schema } from "effect" import type { HttpClientResponse } from "effect/unstable/http" import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js" -import { Media } from "../media.js" import { MediaProtocol } from "../route/media-protocol.js" import { MediaRoute } from "../route/media.js" -import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js" +import { ProviderID, mergeJsonRecords } from "../schema/index.js" import { ProviderShared } from "./shared.js" +import { GeminiGenerateContent } from "./utils/gemini-generate-content.js" import { MediaInput } from "./utils/media-input.js" const ADAPTER = "google-images" @@ -102,25 +102,6 @@ const generationConfig = (request: Request) => { ) } -// Gemini does not fetch public URLs; inline payloads or Gemini Files references are the only accepted inputs. -const imagePart = (asset: Media.Asset): Effect.Effect, AIError> => { - const inline = asset.inline() - if (inline) return Effect.succeed({ inlineData: { mimeType: inline.mime, data: inline.base64 } }) - const id = MediaInput.refID(asset, PROVIDER) - if (id) return Effect.succeed({ fileData: { mimeType: asset.mediaType, fileUri: id } }) - if (asset.source.type === "ref") - return Effect.fail( - ProviderShared.invalidRequest( - "Google generateContent requires Gemini file references rather than other providers' file IDs", - ), - ) - return Effect.fail( - ProviderShared.invalidRequest( - "Google generateContent does not fetch public image URLs; use bytes, a data URL, or a Gemini file reference", - ), - ) -} - const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request: Request) { if (request.n !== undefined && request.n > 1) return yield* ProviderShared.unsupportedOperation({ @@ -129,7 +110,7 @@ const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request: Re route: ADAPTER, message: `${NAME} generates one image per request; call it once per image instead of n=${request.n}`, }) - const parts = yield* Effect.forEach(request.images ?? [], imagePart) + const parts = yield* Effect.forEach(request.images ?? [], (image) => GeminiGenerateContent.mediaPart(NAME, image)) return MediaProtocol.json( mergeJsonRecords( { diff --git a/packages/ai/src/protocols/google-speech.ts b/packages/ai/src/protocols/google-speech.ts index c76f736b786..7f09167bca3 100644 --- a/packages/ai/src/protocols/google-speech.ts +++ b/packages/ai/src/protocols/google-speech.ts @@ -1,10 +1,9 @@ import { Effect, Schema } from "effect" -import { Framing } from "../route/framing.js" import { MediaProtocol } from "../route/media-protocol.js" import { MediaRoute } from "../route/media.js" -import { AIError, ContentPolicyError, ProviderID, mergeJsonRecords } from "../schema/index.js" +import { ProviderID, mergeJsonRecords } from "../schema/index.js" import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js" -import { ProviderShared } from "./shared.js" +import { GeminiGenerateContent } from "./utils/gemini-generate-content.js" import { SpeechStream } from "./utils/speech-stream.js" const ADAPTER = "google-speech" @@ -37,42 +36,12 @@ export type Request = SpeechRequestFor // 3. Streaming event schema // --------------------------------------------------------------------------- -const GenerateContentChunk = Schema.Struct({ - candidates: Schema.optional( - Schema.Array( - Schema.Struct({ - content: Schema.optional( - Schema.Struct({ - parts: Schema.optional( - Schema.Array( - Schema.Struct({ - text: Schema.optional(Schema.String), - inlineData: Schema.optional( - Schema.Struct({ mimeType: Schema.String, data: Schema.Uint8ArrayFromBase64 }), - ), - }), - ), - ), - }), - ), - finishReason: Schema.optional(Schema.String), - }), - ), - ), - promptFeedback: Schema.optional( - Schema.Struct({ blockReason: Schema.optional(Schema.String), blockReasonMessage: Schema.optional(Schema.String) }), - ), - usageMetadata: Schema.optional( - Schema.Struct({ - promptTokenCount: Schema.optional(Schema.Number), - candidatesTokenCount: Schema.optional(Schema.Number), - totalTokenCount: Schema.optional(Schema.Number), - }), - ), - modelVersion: Schema.optional(Schema.String), - responseId: Schema.optional(Schema.String), -}) -type GenerateContentChunk = Schema.Schema.Type +const GenerateContentChunk = GeminiGenerateContent.chunk( + Schema.Struct({ + text: Schema.optional(Schema.String), + inlineData: Schema.optional(Schema.Struct({ mimeType: Schema.String, data: Schema.Uint8ArrayFromBase64 })), + }), +) const decodeChunk = MediaProtocol.decodeFrame(ADAPTER, NAME, GenerateContentChunk) @@ -80,12 +49,8 @@ const decodeChunk = MediaProtocol.decodeFrame(ADAPTER, NAME, GenerateContentChun // 4. Parser state // --------------------------------------------------------------------------- -interface State extends SpeechStream.Audio { +interface State extends SpeechStream.Audio, GeminiGenerateContent.Metadata { readonly mimeType?: string - readonly usage?: GenerateContentChunk["usageMetadata"] - readonly finishReason?: string - readonly modelVersion?: string - readonly responseId?: string } // --------------------------------------------------------------------------- @@ -120,65 +85,27 @@ const fromRequest = Effect.fn("GoogleSpeech.fromRequest")(function* (request: Me ) }) -// Only `gemini-3.1-flash-tts-preview` and later stream; earlier TTS models reject `streamGenerateContent`. -const path = (request: MediaProtocol.Addressed) => - request.mode === "stream" - ? `/models/${request.model.id}:streamGenerateContent?alt=sse` - : `/models/${request.model.id}:generateContent` - // --------------------------------------------------------------------------- // 6. Stream parsing // --------------------------------------------------------------------------- const step = Effect.fn("GoogleSpeech.step")(function* (state: State, frame: string) { const chunk = yield* decodeChunk(frame) - const blockReason = chunk.promptFeedback?.blockReason - if (blockReason !== undefined) - return yield* new AIError({ - reason: new ContentPolicyError({ - message: `${NAME} blocked the prompt (${blockReason})${ - chunk.promptFeedback?.blockReasonMessage === undefined ? "" : `: ${chunk.promptFeedback.blockReasonMessage}` - }`, - body: frame, - }), - }) - const candidate = chunk.candidates?.[0] - const audio = (candidate?.content?.parts ?? []).flatMap((part) => + const blocked = GeminiGenerateContent.blocked(NAME, chunk, frame) + if (blocked !== undefined) return yield* blocked + const audio = (chunk.candidates?.[0]?.content?.parts ?? []).flatMap((part) => part.inlineData === undefined ? [] : [part.inlineData], ) - const next: State = { - ...state, - mimeType: state.mimeType ?? audio[0]?.mimeType, - usage: chunk.usageMetadata ?? state.usage, - finishReason: candidate?.finishReason ?? state.finishReason, - modelVersion: chunk.modelVersion ?? state.modelVersion, - responseId: chunk.responseId ?? state.responseId, - } + const next: State = { ...GeminiGenerateContent.track(state, chunk), mimeType: state.mimeType ?? audio[0]?.mimeType } return [next, audio.flatMap((part) => SpeechStream.delta(next, part.data)[1])] as const }) const finish = (state: State) => { - const usage = state.usage const sampleRate = SpeechStream.sampleRate(state.mimeType) ?? DEFAULT_SAMPLE_RATE return SpeechStream.finish(ADAPTER, state, { ...SpeechStream.pcm("pcm_s16le", sampleRate, state.mimeType ?? `audio/L16;codec=pcm;rate=${sampleRate}`), - usage: - usage === undefined - ? undefined - : { - type: "tokens", - input: usage.promptTokenCount, - output: usage.candidatesTokenCount, - total: ProviderShared.totalTokens( - usage.promptTokenCount, - usage.candidatesTokenCount, - usage.totalTokenCount, - ), - details: { google: usage }, - }, - providerMetadata: { - google: { finishReason: state.finishReason, modelVersion: state.modelVersion, responseId: state.responseId }, - }, + usage: GeminiGenerateContent.usage(state.usage), + providerMetadata: GeminiGenerateContent.providerMetadata(state), detail: state.finishReason === undefined ? undefined : `finish reason: ${state.finishReason}`, }) } @@ -192,9 +119,7 @@ export const protocol = MediaProtocol.stream - context.request.mode === "stream" ? Framing.sse.frame(bytes) : Framing.document.frame(bytes), + frames: (bytes, context) => GeminiGenerateContent.frames(bytes, context.request.mode), initial: () => ({ chunks: [] }), step, finish, @@ -202,7 +127,14 @@ export const protocol = MediaProtocol.stream SpeechModel.fromRoute( - { id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => path(request) }, + { + id: ADAPTER, + provider: PROVIDER, + protocol, + baseURL: DEFAULT_BASE_URL, + // Only `gemini-3.1-flash-tts-preview` and later stream; earlier TTS models reject `streamGenerateContent`. + path: ({ request }) => GeminiGenerateContent.path(request.model.id, request.mode), + }, input, ) diff --git a/packages/ai/src/protocols/google-transcription.ts b/packages/ai/src/protocols/google-transcription.ts new file mode 100644 index 00000000000..a7470786169 --- /dev/null +++ b/packages/ai/src/protocols/google-transcription.ts @@ -0,0 +1,214 @@ +import { Effect, Schema, SchemaGetter } from "effect" +import { MediaProtocol } from "../route/media-protocol.js" +import { MediaRoute } from "../route/media.js" +import { ProviderID, mergeJsonRecords } from "../schema/index.js" +import { + TranscriptionFinishEvent, + TranscriptionModel, + TranscriptionSegmentEvent, + TranscriptionTextDeltaEvent, + type TranscriptionRequestFor, + type TranscriptionSegment, + type TranscriptionWord, + type TranscriptionEvent, +} from "../transcription.js" +import { ProviderShared } from "./shared.js" +import { GeminiGenerateContent } from "./utils/gemini-generate-content.js" + +const ADAPTER = "google-transcription" +const NAME = "Google Transcription" +const PROVIDER = ProviderID.make("google") +export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta" + +// --------------------------------------------------------------------------- +// 1. Public model input +// --------------------------------------------------------------------------- + +/** + * Merged into `generationConfig`. The API rejects `customVocabulary` and `mode: "SMART"` alongside diarization or word + * timestamps. + */ +export type GoogleTranscriptionOptions = { + readonly audioTranscriptionConfig?: { + readonly mode?: "VERBATIM" | "SMART" | (string & {}) + readonly customVocabulary?: ReadonlyArray + readonly languageCodes?: ReadonlyArray + } +} & Record + +export type Request = TranscriptionRequestFor + +// --------------------------------------------------------------------------- +// 3. Streaming event schema +// --------------------------------------------------------------------------- + +const Seconds = Schema.String.check(Schema.isPattern(/^\d+(\.\d+)?s$/)).pipe( + Schema.decodeTo(Schema.Number, { + decode: SchemaGetter.transform((value) => Number.parseFloat(value)), + encode: SchemaGetter.transform((value) => `${value}s`), + }), +) + +const AudioTranscription = Schema.Struct({ + text: Schema.String, + speakerLabel: Schema.optional(Schema.String), + words: Schema.optional( + Schema.Array( + Schema.Struct({ + word: Schema.String, + startOffset: Seconds, + endOffset: Seconds, + speakerLabel: Schema.optional(Schema.String), + }), + ), + ), +}) + +const decodeChunk = MediaProtocol.decodeFrame( + ADAPTER, + NAME, + GeminiGenerateContent.chunk(Schema.Struct({ audioTranscription: Schema.optional(AudioTranscription) })), +) + +// --------------------------------------------------------------------------- +// 4. Parser state +// --------------------------------------------------------------------------- + +interface State extends GeminiGenerateContent.Metadata { + readonly text: string + readonly segments: Array + readonly words: Array +} + +// --------------------------------------------------------------------------- +// 5. Request body construction +// --------------------------------------------------------------------------- + +const fromRequest = Effect.fn("GoogleTranscription.fromRequest")(function* (request: MediaProtocol.Addressed) { + // General Gemini models ignore `audioTranscriptionConfig` and answer the audio conversationally. + if (!request.model.id.includes("transcribe")) + return yield* ProviderShared.unsupportedOperation({ + operation: "transcription.model", + provider: PROVIDER, + route: ADAPTER, + message: `${request.model.id} is not a transcription model; use a transcribe model such as gemini-3.5-transcribe`, + }) + return MediaProtocol.json( + mergeJsonRecords( + { + contents: [{ role: "user", parts: [yield* GeminiGenerateContent.mediaPart(ADAPTER, request.audio)] }], + generationConfig: mergeJsonRecords( + { + audioTranscriptionConfig: { + languageCodes: request.language === undefined ? undefined : [request.language], + // Parts carry no offsets of their own, so segment times come from word offsets. + wordTimestamp: + request.diarize === true || request.timestamps === "word" || request.timestamps === "segment" + ? true + : undefined, + diarization: request.diarize === true ? true : undefined, + }, + }, + request.providerOptions, + ), + }, + request.http?.body, + ) ?? {}, + ) +}) + +// --------------------------------------------------------------------------- +// 6. Stream parsing +// --------------------------------------------------------------------------- + +const turn = (part: Schema.Schema.Type) => { + const words = (part.words ?? []).map((word) => ({ + text: word.word, + startSeconds: word.startOffset, + endSeconds: word.endOffset, + speaker: word.speakerLabel ?? part.speakerLabel, + })) + const first = words[0] + const last = words.at(-1) + return { + text: part.text, + words, + segment: + first === undefined || last === undefined + ? undefined + : { + text: part.text, + startSeconds: first.startSeconds, + endSeconds: last.endSeconds, + speaker: part.speakerLabel, + }, + } +} + +const step = Effect.fn("GoogleTranscription.step")(function* (state: State, frame: string) { + const chunk = yield* decodeChunk(frame) + const blocked = GeminiGenerateContent.blocked(NAME, chunk, frame) + if (blocked !== undefined) return yield* blocked + const turns = (chunk.candidates?.[0]?.content?.parts ?? []).flatMap((part) => + part.audioTranscription === undefined ? [] : [turn(part.audioTranscription)], + ) + const segments = turns.flatMap((item) => (item.segment === undefined ? [] : [item.segment])) + state.words.push(...turns.flatMap((item) => item.words)) + state.segments.push(...segments) + // Each part is one whole speaker turn without surrounding whitespace, so turns join with a space. + const text = turns + .map((item) => item.text) + .filter((item) => item.length > 0) + .join(" ") + const delta = text.length === 0 || state.text.length === 0 ? text : ` ${text}` + const events: ReadonlyArray = [ + ...(delta.length === 0 ? [] : [TranscriptionTextDeltaEvent.make({ delta })]), + ...segments.map((segment) => TranscriptionSegmentEvent.make({ segment })), + ] + return [{ ...GeminiGenerateContent.track(state, chunk), text: state.text + delta }, events] as const +}) + +const finish = (state: State) => { + if (state.finishReason === undefined) return Effect.fail(MediaProtocol.incomplete(ADAPTER)) + return Effect.succeed([ + TranscriptionFinishEvent.make({ + text: state.text, + segments: state.segments.length === 0 ? undefined : state.segments, + words: state.words.length === 0 ? undefined : state.words, + usage: GeminiGenerateContent.usage(state.usage), + providerMetadata: GeminiGenerateContent.providerMetadata(state), + }), + ]) +} + +// --------------------------------------------------------------------------- +// 7. Protocol and route +// --------------------------------------------------------------------------- + +export const protocol = MediaProtocol.stream({ + id: ADAPTER, + name: NAME, + unsupported: ["prompt", "speakers"], + body: { from: fromRequest }, + frames: (bytes, context) => GeminiGenerateContent.frames(bytes, context.request.mode), + initial: () => ({ text: "", segments: [], words: [] }), + step, + finish, +}) + +export const model = (input: MediaRoute.ModelInput) => + TranscriptionModel.fromRoute( + { + id: ADAPTER, + provider: PROVIDER, + protocol, + baseURL: DEFAULT_BASE_URL, + path: ({ request }) => GeminiGenerateContent.path(request.model.id, request.mode), + }, + input, + ) + +export const GoogleTranscription = { + protocol, + model, +} as const diff --git a/packages/ai/src/protocols/openai-images.ts b/packages/ai/src/protocols/openai-images.ts index 34a581b4179..0eeeb1dd835 100644 --- a/packages/ai/src/protocols/openai-images.ts +++ b/packages/ai/src/protocols/openai-images.ts @@ -72,12 +72,6 @@ const isEdit = (request: Request) => (request.images?.length ?? 0) > 0 const isInline = (asset: Media.Asset) => asset.inline() !== undefined -const blob = (data: Uint8Array, mediaType: string) => { - const buffer = new ArrayBuffer(data.byteLength) - new Uint8Array(buffer).set(data) - return new Blob([buffer], { type: mediaType }) -} - const reference = (asset: Media.Asset) => ProviderShared.mediaReference(asset, PROVIDER, NAME).pipe( Effect.map((item) => (item.type === "ref" ? { file_id: item.value } : { image_url: item.value })), @@ -104,9 +98,11 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Re form.append(key, typeof value === "string" ? value : ProviderShared.encodeJson(value)) }) const uploads = yield* Effect.forEach(images, (image) => MediaInput.inlineBytes(ADAPTER, image)) - uploads.forEach((data, index) => form.append("image[]", blob(data, images[index].mediaType), `image-${index}`)) + uploads.forEach((data, index) => + form.append("image[]", MediaInput.blob(data, images[index].mediaType), `image-${index}`), + ) if (mask !== undefined) - form.append("mask", blob(yield* MediaInput.inlineBytes(ADAPTER, mask), mask.mediaType), "mask") + form.append("mask", MediaInput.blob(yield* MediaInput.inlineBytes(ADAPTER, mask), mask.mediaType), "mask") return MediaProtocol.multipart(form) } @@ -130,6 +126,7 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Re // --------------------------------------------------------------------------- const requestedFormat = (body: MediaProtocol.Body) => { + if (body.type === "binary") return undefined const value = body.type === "json" ? body.value.output_format : body.value.get("output_format") return typeof value === "string" ? value : undefined } diff --git a/packages/ai/src/protocols/openai-transcription.ts b/packages/ai/src/protocols/openai-transcription.ts new file mode 100644 index 00000000000..a6d0ae98720 --- /dev/null +++ b/packages/ai/src/protocols/openai-transcription.ts @@ -0,0 +1,277 @@ +import { Effect, Schema, Stream } from "effect" +import { Framing } from "../route/framing.js" +import { MediaProtocol } from "../route/media-protocol.js" +import { MediaRoute } from "../route/media.js" +import { ProviderID, mergeJsonRecords, type MediaUsage } from "../schema/index.js" +import { + TranscriptionFinishEvent, + TranscriptionModel, + TranscriptionSegmentEvent, + TranscriptionTextDeltaEvent, + type TranscriptionEvent, + type TranscriptionRequestFor, + type TranscriptionSegment, +} from "../transcription.js" +import { mediaTypeExtension } from "../utils/media-type.js" +import { ProviderShared } from "./shared.js" +import { MediaInput } from "./utils/media-input.js" + +const ADAPTER = "openai-transcription" +const NAME = "OpenAI Transcription" +const PROVIDER = ProviderID.make("openai") +export const DEFAULT_BASE_URL = "https://api.openai.com/v1" +export const PATH = "/audio/transcriptions" + +// --------------------------------------------------------------------------- +// 1. Public model input +// --------------------------------------------------------------------------- + +export type OpenAITranscriptionOptions = { + readonly chunking_strategy?: + | "auto" + | { + readonly type: "server_vad" + readonly prefix_padding_ms?: number + readonly silence_duration_ms?: number + readonly threshold?: number + } + readonly include?: ReadonlyArray<"logprobs"> + readonly keywords?: ReadonlyArray + readonly known_speaker_names?: ReadonlyArray + readonly known_speaker_references?: ReadonlyArray + readonly temperature?: number +} & Record + +export type Request = TranscriptionRequestFor + +// --------------------------------------------------------------------------- +// 3. Streaming event schema +// --------------------------------------------------------------------------- + +const Segment = Schema.Struct({ + text: Schema.String, + start: Schema.Number, + end: Schema.Number, + speaker: Schema.optional(Schema.String), +}) + +const Usage = Schema.Union([ + Schema.Struct({ + type: Schema.Literal("tokens"), + input_tokens: Schema.optional(Schema.Number), + output_tokens: Schema.optional(Schema.Number), + total_tokens: Schema.optional(Schema.Number), + }), + Schema.Struct({ type: Schema.Literal("duration"), seconds: Schema.Number }), +]) + +const transcriptFields = { + text: Schema.String, + language: Schema.optional(Schema.String), + languages: Schema.optional(Schema.Array(Schema.Struct({ code: Schema.String }))), + duration: Schema.optional(Schema.Number), + segments: Schema.optional(Schema.Array(Segment)), + words: Schema.optional( + Schema.Array(Schema.Struct({ word: Schema.String, start: Schema.Number, end: Schema.Number })), + ), + usage: Schema.optional(Usage), +} + +const Event = Schema.Union([ + Schema.Struct({ type: Schema.Literal("transcript.text.delta"), delta: Schema.String }), + Schema.Struct({ type: Schema.Literal("transcript.text.segment"), ...Segment.fields }), + Schema.Struct({ type: Schema.Literal("transcript.text.done"), ...transcriptFields }), +]) +const Transcript = Schema.Struct(transcriptFields) +type Transcript = Schema.Schema.Type + +const decodeEvent = MediaProtocol.decodeFrame(ADAPTER, NAME, Event) +const decodeTranscript = MediaProtocol.decodeFrame(ADAPTER, NAME, Transcript) + +type Frame = string | { readonly document: string } + +// --------------------------------------------------------------------------- +// 4. Parser state +// --------------------------------------------------------------------------- + +interface State { + readonly segments: Array + readonly transcript?: Transcript +} + +// --------------------------------------------------------------------------- +// 5. Request body construction +// --------------------------------------------------------------------------- + +interface Capabilities { + readonly stream: boolean + readonly timestamps: ReadonlyArray<"segment" | "word"> + readonly diarize: boolean + readonly languageField: "language" | "languages" +} + +const TRANSCRIBE: Capabilities = { stream: true, timestamps: [], diarize: false, languageField: "language" } + +const capabilities = (model: string): Capabilities => { + if (model.startsWith("whisper")) return { ...TRANSCRIBE, stream: false, timestamps: ["segment", "word"] } + if (model.includes("diarize")) return { ...TRANSCRIBE, timestamps: ["segment"], diarize: true } + // `gpt-transcribe` replaces `language` with `languages[]` and rejects both together. + if (model.startsWith("gpt-transcribe")) return { ...TRANSCRIBE, languageField: "languages" } + return TRANSCRIBE +} + +const unsupported = (operation: string, message: string) => + Effect.fail(ProviderShared.unsupportedOperation({ operation, provider: PROVIDER, route: ADAPTER, message })) + +const validate = (request: MediaProtocol.Addressed, model: Capabilities) => { + const id = request.model.id + if (request.mode === "stream" && !model.stream) + return unsupported("media.stream", `${id} does not stream; use Transcription.generate`) + if (request.diarize === true && !model.diarize) + return unsupported("media.diarize", `${id} does not diarize; use gpt-4o-transcribe-diarize`) + if (request.prompt !== undefined && model.diarize) + return unsupported("media.prompt", `${id} does not accept a prompt`) + if ( + request.timestamps === undefined || + request.timestamps === "none" || + model.timestamps.includes(request.timestamps) + ) + return Effect.void + return unsupported("media.timestamps", `${id} does not return ${request.timestamps} timestamps`) +} + +const RESERVED_FORM_FIELDS = new Set([ + "file", + "model", + "prompt", + "language", + "response_format", + "timestamp_granularities", + "stream", +]) + +const appendField = (form: FormData, key: string, value: unknown) => { + if (Array.isArray(value)) return value.forEach((item) => form.append(`${key}[]`, String(item))) + form.append(key, typeof value === "object" && value !== null ? ProviderShared.encodeJson(value) : String(value)) +} + +const fromRequest = Effect.fn("OpenAITranscription.fromRequest")(function* (request: MediaProtocol.Addressed) { + const model = capabilities(request.model.id) + yield* validate(request, model) + // The API detects the audio format from the upload's filename extension. + const extension = mediaTypeExtension(request.audio.mediaType) + if (extension === undefined) + return yield* ProviderShared.invalidRequest( + `${NAME} cannot name a ${request.audio.mediaType} upload; send mp3, mp4, m4a, wav, webm, ogg, or flac audio`, + ) + const audio = yield* MediaInput.inlineBytes(ADAPTER, request.audio) + const responseFormat = model.diarize + ? "diarized_json" + : request.timestamps === undefined || request.timestamps === "none" + ? undefined + : "verbose_json" + const native = Object.entries(mergeJsonRecords(request.providerOptions, request.http?.body) ?? {}).filter( + ([key]) => !RESERVED_FORM_FIELDS.has(key), + ) + const fields = mergeJsonRecords( + { + model: request.model.id, + language: model.languageField === "language" ? request.language : undefined, + languages: model.languageField === "languages" && request.language !== undefined ? [request.language] : undefined, + prompt: request.prompt, + response_format: responseFormat, + timestamp_granularities: responseFormat === "verbose_json" ? [request.timestamps] : undefined, + // Diarizing audio longer than 30 seconds requires a chunking strategy. + chunking_strategy: model.diarize ? "auto" : undefined, + stream: request.mode === "stream" ? true : undefined, + }, + Object.fromEntries(native), + ) + const form = new FormData() + form.append("file", MediaInput.blob(audio, request.audio.mediaType), `audio.${extension}`) + Object.entries(fields ?? {}).forEach(([key, value]) => appendField(form, key, value)) + return MediaProtocol.multipart(form) +}) + +// --------------------------------------------------------------------------- +// 6. Stream parsing +// --------------------------------------------------------------------------- + +const segment = (value: Schema.Schema.Type): TranscriptionSegment => ({ + text: value.text.trim(), + startSeconds: value.start, + endSeconds: value.end, + speaker: value.speaker, +}) + +const onEvent = Effect.fn("OpenAITranscription.onEvent")(function* (state: State, frame: string) { + const event = yield* decodeEvent(frame) + if (event.type === "transcript.text.done") return [{ ...state, transcript: event }, []] as const + if (event.type === "transcript.text.delta") + return [state, event.delta.length === 0 ? [] : [TranscriptionTextDeltaEvent.make({ delta: event.delta })]] as const + const next = segment(event) + state.segments.push(next) + return [state, [TranscriptionSegmentEvent.make({ segment: next })]] as const +}) + +const step = (state: State, frame: Frame) => + typeof frame === "string" + ? onEvent(state, frame) + : decodeTranscript(frame.document).pipe(Effect.map((transcript) => [{ ...state, transcript }, []] as const)) + +const usage = (value: Transcript["usage"]): MediaUsage | undefined => { + if (value === undefined) return undefined + if (value.type === "duration") return { type: "seconds", seconds: value.seconds } + return { + type: "tokens", + input: value.input_tokens, + output: value.output_tokens, + total: value.total_tokens, + details: { openai: value }, + } +} + +const finish = (state: State) => { + const transcript = state.transcript + if (transcript === undefined) return Effect.fail(MediaProtocol.incomplete(ADAPTER)) + const segments = transcript.segments?.map(segment) ?? state.segments + return Effect.succeed([ + TranscriptionFinishEvent.make({ + text: transcript.text, + segments: segments.length === 0 ? undefined : segments, + words: transcript.words?.map((word) => ({ text: word.word, startSeconds: word.start, endSeconds: word.end })), + language: (transcript.language ?? transcript.languages?.[0]?.code)?.toLowerCase(), + durationSeconds: transcript.duration, + usage: usage(transcript.usage), + }), + ]) +} + +// --------------------------------------------------------------------------- +// 7. Protocol and route +// --------------------------------------------------------------------------- + +export const protocol = MediaProtocol.stream({ + id: ADAPTER, + name: NAME, + unsupported: ["speakers"], + body: { from: fromRequest }, + frames: (bytes, context) => + context.request.mode === "stream" + ? Framing.sse.frame(bytes) + : Framing.document.frame(bytes).pipe(Stream.map((document) => ({ document }))), + initial: () => ({ segments: [] }), + step, + finish, +}) + +export const model = (input: MediaRoute.ModelInput) => + TranscriptionModel.fromRoute( + { id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH }, + input, + ) + +export const OpenAITranscription = { + protocol, + model, +} as const diff --git a/packages/ai/src/protocols/utils/gemini-generate-content.ts b/packages/ai/src/protocols/utils/gemini-generate-content.ts new file mode 100644 index 00000000000..9ccd02d5362 --- /dev/null +++ b/packages/ai/src/protocols/utils/gemini-generate-content.ts @@ -0,0 +1,113 @@ +import { Effect, Schema, type Stream } from "effect" +import type { Media } from "../../media.js" +import { Framing } from "../../route/framing.js" +import type { MediaProtocol } from "../../route/media-protocol.js" +import { AIError, ContentPolicyError, ProviderID, type MediaUsage, type ProviderMetadata } from "../../schema/index.js" +import { ProviderShared } from "../shared.js" +import { MediaInput } from "./media-input.js" + +const PROVIDER = ProviderID.make("google") + +const UsageMetadata = Schema.Struct({ + promptTokenCount: Schema.optional(Schema.Number), + candidatesTokenCount: Schema.optional(Schema.Number), + totalTokenCount: Schema.optional(Schema.Number), +}) +type UsageMetadata = Schema.Schema.Type + +export const chunk = (part: Part) => + Schema.Struct({ + candidates: Schema.optional( + Schema.Array( + Schema.Struct({ + content: Schema.optional(Schema.Struct({ parts: Schema.optional(Schema.Array(part)) })), + finishReason: Schema.optional(Schema.String), + }), + ), + ), + promptFeedback: Schema.optional( + Schema.Struct({ + blockReason: Schema.optional(Schema.String), + blockReasonMessage: Schema.optional(Schema.String), + }), + ), + usageMetadata: Schema.optional(UsageMetadata), + modelVersion: Schema.optional(Schema.String), + responseId: Schema.optional(Schema.String), + }) + +interface Chunk { + readonly candidates?: ReadonlyArray<{ readonly finishReason?: string }> + readonly promptFeedback?: { readonly blockReason?: string; readonly blockReasonMessage?: string } + readonly usageMetadata?: UsageMetadata + readonly modelVersion?: string + readonly responseId?: string +} + +export interface Metadata { + readonly usage?: UsageMetadata + readonly finishReason?: string + readonly modelVersion?: string + readonly responseId?: string +} + +export const track = (state: State, chunk: Chunk): State => ({ + ...state, + usage: chunk.usageMetadata ?? state.usage, + finishReason: chunk.candidates?.[0]?.finishReason ?? state.finishReason, + modelVersion: chunk.modelVersion ?? state.modelVersion, + responseId: chunk.responseId ?? state.responseId, +}) + +export const blocked = (name: string, chunk: Chunk, frame: string) => { + const feedback = chunk.promptFeedback + if (feedback?.blockReason === undefined) return undefined + return new AIError({ + reason: new ContentPolicyError({ + message: `${name} blocked the request (${feedback.blockReason})${ + feedback.blockReasonMessage === undefined ? "" : `: ${feedback.blockReasonMessage}` + }`, + body: frame, + }), + }) +} + +export const usage = (usage: UsageMetadata | undefined): MediaUsage | undefined => + usage === undefined + ? undefined + : { + type: "tokens", + input: usage.promptTokenCount, + output: usage.candidatesTokenCount, + total: ProviderShared.totalTokens(usage.promptTokenCount, usage.candidatesTokenCount, usage.totalTokenCount), + details: { google: usage }, + } + +export const providerMetadata = (state: Metadata): ProviderMetadata => ({ + google: { finishReason: state.finishReason, modelVersion: state.modelVersion, responseId: state.responseId }, +}) + +export const path = (model: string, mode: MediaProtocol.Mode) => + mode === "stream" ? `/models/${model}:streamGenerateContent?alt=sse` : `/models/${model}:generateContent` + +// `generateContent` answers with one document shaped exactly like a streamed chunk, so it is a single frame. +export const frames = (bytes: Stream.Stream, mode: MediaProtocol.Mode) => + mode === "stream" ? Framing.sse.frame(bytes) : Framing.document.frame(bytes) + +// Gemini does not fetch public URLs; inline payloads and Gemini Files references are the accepted inputs. +export const mediaPart = ( + route: string, + asset: Media.Asset, +): Effect.Effect< + | { readonly fileData: { readonly mimeType: string; readonly fileUri: string } } + | { readonly inlineData: { readonly mimeType: string; readonly data: string } }, + AIError +> => { + const fileUri = MediaInput.refID(asset, PROVIDER) + if (fileUri !== undefined) return Effect.succeed({ fileData: { mimeType: asset.mediaType, fileUri } }) + return ProviderShared.requireInlineMedia(route, asset).pipe( + Effect.map((media) => ({ inlineData: { mimeType: media.mime, data: media.base64 } })), + ) +} + +export * as GeminiGenerateContent from "./gemini-generate-content.js" diff --git a/packages/ai/src/protocols/utils/media-input.ts b/packages/ai/src/protocols/utils/media-input.ts index 9a62e8e9b8d..2650ae5e669 100644 --- a/packages/ai/src/protocols/utils/media-input.ts +++ b/packages/ai/src/protocols/utils/media-input.ts @@ -1,5 +1,6 @@ import { Effect, Encoding } from "effect" import { Media } from "../../media.js" +import type { MediaProtocol } from "../../route/media-protocol.js" import type { AIError, ProviderID } from "../../schema/index.js" import { ProviderShared } from "../shared.js" @@ -13,6 +14,26 @@ export const inlineBytes = (route: string, asset: Media.Asset): Effect.Effect { + const buffer = new ArrayBuffer(data.byteLength) + new Uint8Array(buffer).set(data) + return new Blob([buffer], { type: mediaType }) +} + +const isScalar = (value: unknown): value is string | number | boolean => + typeof value === "string" || typeof value === "number" || typeof value === "boolean" + +export const query = (route: string, values: Record): Effect.Effect => { + const entries = Object.entries(values).filter(([, value]) => value !== undefined) + const invalid = entries.find(([, value]) => !isScalar(value) && !(Array.isArray(value) && value.every(isScalar))) + if (invalid !== undefined) + return Effect.fail(ProviderShared.invalidRequest(`${route} cannot send "${invalid[0]}" as a query parameter`)) + return Effect.succeed( + Object.fromEntries(entries.map(([key, value]) => [key, Array.isArray(value) ? value.map(String) : String(value)])), + ) +} + /** Provider file handle when the ref belongs to this provider; refs from other providers are never forwarded. */ export const refID = (asset: Media.Asset, provider: ProviderID) => asset.source.type === "ref" && asset.source.provider === provider ? asset.source.id : undefined diff --git a/packages/ai/src/providers/assemblyai.ts b/packages/ai/src/providers/assemblyai.ts new file mode 100644 index 00000000000..896984a1f65 --- /dev/null +++ b/packages/ai/src/providers/assemblyai.ts @@ -0,0 +1,43 @@ +import { Auth } from "../route/auth.js" +import type { ProviderAuthOption } from "../route/auth-options.js" +import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js" +import { AssemblyAITranscription, DEFAULT_BASE_URL } from "../protocols/assemblyai-transcription.js" + +export type { AssemblyAITranscriptionOptions } from "../protocols/assemblyai-transcription.js" + +export const id = ProviderID.make("assemblyai") +const baseURL = DEFAULT_BASE_URL + +export type Config = ProviderAuthOption<"optional"> & { + /** `https://api.eu.assemblyai.com` for the EU region. */ + readonly baseURL?: string + readonly headers?: Record + readonly http?: HttpOptions.Input +} + +// The key is the whole `authorization` value, without a scheme. +const auth = (options: ProviderAuthOption<"optional">) => { + if ("auth" in options && options.auth) return options.auth + return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey") + .orElse(Auth.config("ASSEMBLYAI_API_KEY")) + .pipe(Auth.header("authorization")) +} + +export const configure = (input: Config = {}) => { + const transcription = (modelID: string | ModelID) => + AssemblyAITranscription.model({ + id: modelID, + auth: auth(input), + baseURL: input.baseURL ?? baseURL, + headers: input.headers, + http: input.http === undefined ? undefined : HttpOptions.make(input.http), + }) + return { + id, + transcription, + configure, + } +} + +export const provider = configure() +export const transcription = provider.transcription diff --git a/packages/ai/src/providers/deepgram.ts b/packages/ai/src/providers/deepgram.ts index c6553c25fe4..cfdc35b39c1 100644 --- a/packages/ai/src/providers/deepgram.ts +++ b/packages/ai/src/providers/deepgram.ts @@ -2,8 +2,10 @@ import { Auth } from "../route/auth.js" import type { ProviderAuthOption } from "../route/auth-options.js" import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js" import { DEFAULT_BASE_URL, DeepgramSpeech } from "../protocols/deepgram-speech.js" +import { DeepgramTranscription } from "../protocols/deepgram-transcription.js" export type { DeepgramEncoding, DeepgramSpeechOptions } from "../protocols/deepgram-speech.js" +export type { DeepgramTranscriptionOptions } from "../protocols/deepgram-transcription.js" export const id = ProviderID.make("deepgram") const baseURL = DEFAULT_BASE_URL @@ -22,20 +24,21 @@ const auth = (options: ProviderAuthOption<"optional">) => { } export const configure = (input: Config = {}) => { - const speech = (modelID: string | ModelID) => - DeepgramSpeech.model({ - id: modelID, - auth: auth(input), - baseURL: input.baseURL ?? baseURL, - headers: input.headers, - http: input.http === undefined ? undefined : HttpOptions.make(input.http), - }) + const media = (modelID: string | ModelID) => ({ + id: modelID, + auth: auth(input), + baseURL: input.baseURL ?? baseURL, + headers: input.headers, + http: input.http === undefined ? undefined : HttpOptions.make(input.http), + }) return { id, - speech, + speech: (modelID: string | ModelID) => DeepgramSpeech.model(media(modelID)), + transcription: (modelID: string | ModelID) => DeepgramTranscription.model(media(modelID)), configure, } } export const provider = configure() export const speech = provider.speech +export const transcription = provider.transcription diff --git a/packages/ai/src/providers/google.ts b/packages/ai/src/providers/google.ts index 8fe99c58d6d..aba6636c155 100644 --- a/packages/ai/src/providers/google.ts +++ b/packages/ai/src/providers/google.ts @@ -6,10 +6,12 @@ import { HttpOptions, ProviderID, mergeHttpOptions, type ModelID } from "../sche import { Gemini } from "../protocols/gemini.js" import { GoogleImages } from "../protocols/google-images.js" import { GoogleSpeech } from "../protocols/google-speech.js" +import { GoogleTranscription } from "../protocols/google-transcription.js" import { GoogleVideo } from "../protocols/google-video.js" export type { GoogleImageOptions } from "../protocols/google-images.js" export type { GoogleSpeechOptions } from "../protocols/google-speech.js" +export type { GoogleTranscriptionOptions } from "../protocols/google-transcription.js" export type { GoogleVideoOptions } from "../protocols/google-video.js" export type GeminiOptionsInput = Gemini.OptionsInput export type GeminiProviderOptionsInput = Gemini.ProviderOptionsInput @@ -57,6 +59,7 @@ export const configure = (input: Config = {}) => { image: (modelID: string | ModelID) => GoogleImages.model(media(modelID)), video: (modelID: string | ModelID) => GoogleVideo.model(media(modelID)), speech: (modelID: string | ModelID) => GoogleSpeech.model(media(modelID)), + transcription: (modelID: string | ModelID) => GoogleTranscription.model(media(modelID)), configure, } } @@ -77,3 +80,4 @@ export const model: ProviderPackage.Definition { }) const image = (modelID: string | ModelID) => OpenAIImages.model(media(modelID)) const speech = (modelID: string | ModelID) => OpenAISpeech.model(media(modelID)) + const transcription = (modelID: string | ModelID) => OpenAITranscription.model(media(modelID)) return { id, @@ -117,6 +120,7 @@ export const configure = (input: Config = {}) => { chat, image, speech, + transcription, configure, } } @@ -164,3 +168,4 @@ export const responses = provider.responses export const chat = provider.chat export const image = provider.image export const speech = provider.speech +export const transcription = provider.transcription diff --git a/packages/ai/src/route/media-protocol.ts b/packages/ai/src/route/media-protocol.ts index f68d98c0dae..c898b4115f4 100644 --- a/packages/ai/src/route/media-protocol.ts +++ b/packages/ai/src/route/media-protocol.ts @@ -16,17 +16,37 @@ import { // Bodies // --------------------------------------------------------------------------- -/** JSON `query` is appended to the endpoint URL before the route and caller `http.query` overlays. */ -export type Body = - | { readonly type: "json"; readonly value: Record; readonly query?: Record } - | { readonly type: "multipart"; readonly value: FormData } +/** Array values become repeated parameters (`keyterm=a&keyterm=b`). */ +export type Query = Readonly>> -export const json = (value: Record, query?: Record): Body => ({ +/** `query` is appended to the endpoint URL before the route and caller `http.query` overlays. */ +export type Body = + | { readonly type: "json"; readonly value: Record; readonly query?: Query } + | { readonly type: "multipart"; readonly value: FormData } + | { + readonly type: "binary" + readonly value: Uint8Array + readonly contentType: string + readonly query?: Query + } + +export const json = (value: Record, query?: Query): Body => ({ type: "json", value, query, }) export const multipart = (value: FormData): Body => ({ type: "multipart", value }) +export const binary = (value: Uint8Array, contentType: string, query?: Query): Body => ({ + type: "binary", + value, + contentType, + query, +}) + +export type Send = (path: string, body: Body) => Effect.Effect + +/** Runs after unsupported-field rejection and before `body.from`, for providers that need an upload first. */ +export type Prepare = (request: Request, send: Send) => Effect.Effect // --------------------------------------------------------------------------- // Protocol kinds @@ -91,6 +111,7 @@ export interface Queued { /** Serializable handle. `Generation.token` carries the encoded form so it can be persisted and resumed elsewhere. */ readonly token: Schema.Codec readonly start: { + readonly prepare?: Prepare readonly body: { readonly from: (request: Request) => Effect.Effect } readonly decode: ( response: HttpClientResponse.HttpClientResponse, diff --git a/packages/ai/src/route/media.ts b/packages/ai/src/route/media.ts index d10faa790d1..1b66b723e9d 100644 --- a/packages/ai/src/route/media.ts +++ b/packages/ai/src/route/media.ts @@ -40,6 +40,7 @@ export interface ModelInput { /** One request, one response. */ export interface Route { + readonly kind: "inline" readonly id: string readonly provider: ProviderID readonly protocol: string @@ -48,6 +49,7 @@ export interface Route { /** Submit, then poll through the returned `Generation`. */ export interface QueuedRoute { + readonly kind: "queued" readonly id: string readonly provider: ProviderID readonly protocol: string @@ -62,6 +64,7 @@ export interface QueuedRoute { /** One request whose response parses into events; `generate` runs the same stream and collects it. */ export interface StreamRoute { + readonly kind: "stream" readonly id: string readonly provider: ProviderID readonly protocol: string @@ -95,14 +98,15 @@ export interface StreamInput( input: InlineInput, ): Route => { const transport = makeTransport(input) return { + kind: "inline", id: input.id, provider: transport.provider, protocol: input.protocol.id, @@ -157,7 +161,7 @@ export const queued = ( const start = Effect.fn("MediaRoute.start")(function* (request: Request, execute: Execute) { const submitted = yield* transport.submit( request, - { unsupported: protocol.unsupported, from: protocol.start.body.from }, + { unsupported: protocol.unsupported, prepare: protocol.start.prepare, from: protocol.start.body.from }, execute, ) const started = yield* protocol.start.decode(submitted.response, submitted.context) @@ -185,7 +189,7 @@ export const queued = ( return new Generation(route, encodeToken(token), yield* route.status) }) - return { id: input.id, provider: transport.provider, protocol: protocol.id, start, resume } + return { kind: "queued", id: input.id, provider: transport.provider, protocol: protocol.id, start, resume } } /** Compose a streaming media protocol; `generate` runs the same stream in `generate` mode and folds it with `collect`. */ @@ -236,6 +240,7 @@ export const stream = (input: Composition const provider = ProviderID.make(input.provider) const routeHttp = input.headers === undefined ? undefined : new HttpOptions({ headers: input.headers }) const authorize = Auth.toEffect(input.auth) - const withQuery = (url: URL, query: Record | undefined) => { - for (const [key, value] of Object.entries(query ?? {})) url.searchParams.set(key, value) - return url - } + const baseURL = (path: string) => new URL(`${ProviderShared.trimBaseUrl(input.endpoint.baseURL ?? "")}${path}`) + /** `auth` is only what `Auth` added, never deployment headers. */ + const send = Effect.fn("MediaRoute.send")(function* ( + call: { + readonly method: AuthInput["method"] + readonly url: URL + readonly headers: Headers.Headers + readonly request: AuthInput["request"] + readonly body?: MediaProtocol.Body + }, + execute: Execute, + ) { + const encoded = encode(call.body, call.headers) + const url = call.url.toString() + const headers = yield* authorize({ + request: call.request, + method: call.method, + url, + body: encoded.text, + headers: encoded.headers, + }) + const response = yield* execute( + encoded.apply(HttpClientRequest.make(call.method)(url).pipe(HttpClientRequest.setHeaders(headers))), + ) + return { response, auth: Object.fromEntries(Object.entries(headers).filter(([key]) => !(key in call.headers))) } + }) return { provider, /** Route and model overlays; `start` additionally merges the request's own `http`. */ @@ -266,67 +293,70 @@ const makeTransport = (input: Composition request: Request, protocol: { readonly unsupported?: ReadonlyArray + readonly prepare?: MediaProtocol.Prepare readonly from: (request: Request) => Effect.Effect }, execute: Execute, ) { yield* rejectUnsupported(input.id, provider, request, protocol.unsupported) const http = mergeHttpOptions(routeHttp, request.model.http, request.http) + const headers = Headers.fromInput(http?.headers) + const prepared = + protocol.prepare === undefined + ? request + : yield* protocol.prepare(request, (path, body) => + send({ method: "POST", url: baseURL(path), headers, request, body }, execute).pipe( + Effect.map((sent) => sent.response), + ), + ) // Sanitize after merging so model-level overlays are covered; the model value is restored, not sanitized. - const resolved: Request = { ...sanitizeSurrogates({ ...request, http }), model: request.model } + const resolved: Request = { ...sanitizeSurrogates({ ...prepared, http }), model: request.model } const body = yield* protocol.from(resolved) const url = withQuery( withQuery( Endpoint.render(input.endpoint, { request: resolved, body }), - body.type === "json" ? body.query : undefined, + body.type === "multipart" ? undefined : body.query, ), http?.query, ) - const encoded = body.type === "json" ? ProviderShared.encodeJson(body.value) : "[multipart/form-data]" - const baseHeaders = Headers.fromInput(http?.headers) - const headers = yield* authorize({ - request: resolved, - method: "POST", - url: url.toString(), - body: encoded, - // The HTTP client sets the multipart boundary; a caller-supplied content-type would corrupt it. - headers: body.type === "multipart" ? Headers.remove(baseHeaders, "content-type") : baseHeaders, - }) - const transport = HttpClientRequest.post(url.toString()).pipe( - HttpClientRequest.setHeaders(headers), - body.type === "json" - ? HttpClientRequest.bodyText(encoded, "application/json") - : HttpClientRequest.bodyFormData(body.value), - ) - const response = yield* execute(transport) - return { response, context: { request: resolved, body } } + const sent = yield* send({ method: "POST", url, headers, request: resolved, body }, execute) + return { response: sent.response, context: { request: resolved, body } } }), - /** - * Bodiless follow-up call (status, result, cancel) with the same auth and headers as `submit`. `auth` is only - * what `Auth` added, so protocols can hand download credentials to output assets without deployment headers. - */ - call: Effect.fn("MediaRoute.call")(function* ( - method: AuthInput["method"], - path: string, - http: HttpOptions | undefined, - execute: Execute, - ) { + /** Bodiless follow-up call (status, result, cancel) with the same auth and headers as `submit`. */ + call: (method: AuthInput["method"], path: string, http: HttpOptions | undefined, execute: Execute) => { // Provider-issued absolute URLs (fal `status_url`) are used as-is; everything else resolves against the base. - const url = withQuery( - /^https?:\/\//.test(path) - ? new URL(path) - : new URL(`${ProviderShared.trimBaseUrl(input.endpoint.baseURL ?? "")}${path}`), - http?.query, - ) + const url = withQuery(/^https?:\/\//.test(path) ? new URL(path) : baseURL(path), http?.query) for (const [key, value] of Object.entries(input.endpoint.query ?? {})) url.searchParams.set(key, value) - const base = Headers.fromInput(http?.headers) - const headers = yield* authorize({ request: { http }, method, url: url.toString(), body: "", headers: base }) - const response = yield* execute( - HttpClientRequest.make(method)(url.toString()).pipe(HttpClientRequest.setHeaders(headers)), - ) - const auth = Object.fromEntries(Object.entries(headers).filter(([key]) => !(key in base))) - return { response, auth } - }), + return send({ method, url, headers: Headers.fromInput(http?.headers), request: { http } }, execute) + }, + } +} + +const withQuery = (url: URL, query: MediaProtocol.Query | undefined) => { + for (const [key, value] of Object.entries(query ?? {})) { + url.searchParams.delete(key) + for (const item of typeof value === "string" ? [value] : value) url.searchParams.append(key, item) + } + return url +} + +const encode = (body: MediaProtocol.Body | undefined, headers: Headers.Headers) => { + if (body === undefined) return { text: "", headers, apply: (request: HttpClientRequest.HttpClientRequest) => request } + if (body.type === "json") { + const text = ProviderShared.encodeJson(body.value) + return { text, headers, apply: HttpClientRequest.bodyText(text, "application/json") } + } + if (body.type === "binary") + return { + text: `[${body.contentType}]`, + headers, + apply: HttpClientRequest.bodyUint8Array(body.value, body.contentType), + } + return { + text: "[multipart/form-data]", + // The HTTP client sets the multipart boundary; a caller-supplied content-type would corrupt it. + headers: Headers.remove(headers, "content-type"), + apply: HttpClientRequest.bodyFormData(body.value), } } diff --git a/packages/ai/src/transcription-client.ts b/packages/ai/src/transcription-client.ts new file mode 100644 index 00000000000..786540392c4 --- /dev/null +++ b/packages/ai/src/transcription-client.ts @@ -0,0 +1,122 @@ +import { Context, Effect, Layer, Stream } from "effect" +import { resultEvents, type AwaitOptions, type Generation } from "./generation.js" +import { ProviderShared } from "./protocols/shared.js" +import { RequestExecutor } from "./route/executor.js" +import type { AIError } from "./schema/index.js" +import { + responseEvents, + type TranscriptionEvent, + type TranscriptionModel, + type TranscriptionOptions, + type TranscriptionRequestFor, + type TranscriptionResponse, + type TranscriptionRoute, +} from "./transcription.js" + +export interface Interface { + readonly generate: ( + request: TranscriptionRequestFor, + options?: AwaitOptions, + ) => Effect.Effect + readonly stream: ( + request: TranscriptionRequestFor, + options?: AwaitOptions, + ) => Stream.Stream + readonly start: ( + request: TranscriptionRequestFor, + ) => Effect.Effect, AIError> + readonly resume: ( + model: TranscriptionModel, + token: unknown, + ) => Effect.Effect, AIError> +} + +export class Service extends Context.Service()("@opencode/TranscriptionClient") {} + +export const generate = ( + request: TranscriptionRequestFor, + options?: AwaitOptions, +): Effect.Effect => + Effect.gen(function* () { + const client = yield* Service + return yield* client.generate(request, options) + }) + +export const stream = ( + request: TranscriptionRequestFor, + options?: AwaitOptions, +): Stream.Stream => + Stream.unwrap( + Effect.gen(function* () { + const client = yield* Service + return client.stream(request, options) + }), + ) + +export const start = ( + request: TranscriptionRequestFor, +): Effect.Effect, AIError, Service> => + Effect.gen(function* () { + const client = yield* Service + return yield* client.start(request) + }) + +export const resume = ( + model: TranscriptionModel, + token: unknown, +): Effect.Effect, AIError, Service> => + Effect.gen(function* () { + const client = yield* Service + return yield* client.resume(model, token) + }) + +const notQueued = (route: TranscriptionRoute, operation: string) => + ProviderShared.unsupportedOperation({ + operation: `transcription.${operation}`, + provider: route.provider, + route: route.id, + message: `${route.provider}/${route.id} is not a queued route; use Transcription.generate or Transcription.stream`, + }) + +export const layer: Layer.Layer = Layer.effect( + Service, + Effect.gen(function* () { + const executor = yield* RequestExecutor.Service + const start = (request: TranscriptionRequestFor) => { + const route = request.model.route + if (route.kind !== "queued") return Effect.fail(notQueued(route, "start")) + return route.start(request, executor.execute) + } + return Service.of({ + start, + resume: (model, token) => { + const route = model.route + if (route.kind !== "queued") return Effect.fail(notQueued(route, "resume")) + return route.resume(model, token, executor.execute) + }, + generate: (request, options) => { + const route = request.model.route + if (route.kind !== "queued") return route.generate(request, executor.execute) + return start(request).pipe(Effect.flatMap((generation) => generation.await(options))) + }, + stream: (request, options) => { + const route = request.model.route + if (route.kind === "stream") return route.stream(request, executor.execute) + if (route.kind === "queued") + return Stream.unwrap( + start(request).pipe(Effect.map((generation) => resultEvents(generation, responseEvents, options))), + ) + return Stream.fromIterableEffect(Effect.map(route.generate(request, executor.execute), responseEvents)) + }, + }) + }), +) + +export const TranscriptionClient = { + Service, + layer, + generate, + stream, + start, + resume, +} as const diff --git a/packages/ai/src/transcription.ts b/packages/ai/src/transcription.ts new file mode 100644 index 00000000000..43ac82417f4 --- /dev/null +++ b/packages/ai/src/transcription.ts @@ -0,0 +1,294 @@ +import { Effect, Schema, Stream } from "effect" +import { Generation, ProgressEvent, QueuedEvent, type AwaitOptions } from "./generation.js" +import { Media } from "./media.js" +import { MediaModel, composeRoute, tryRequest } from "./media-model.js" +import { MediaRoute } from "./route/media.js" +import type { MediaProtocol } from "./route/media-protocol.js" +import { AIError, HttpOptions, MediaUsage, ProviderMetadata } from "./schema/index.js" +import { TranscriptionClient, Service } from "./transcription-client.js" + +// --------------------------------------------------------------------------- +// Model +// --------------------------------------------------------------------------- + +export type TranscriptionOptions = Record + +export type TranscriptionRoute = + | MediaRoute.Route, TranscriptionResponse> + | MediaRoute.StreamRoute, TranscriptionEvent, TranscriptionResponse> + | MediaRoute.QueuedRoute, TranscriptionResponse> + +export class TranscriptionModel extends MediaModel< + TranscriptionRoute, + Options +> { + declare protected readonly _TranscriptionModel: void + + static make( + input: MediaModel.Input>, + ) { + return new TranscriptionModel(input) + } + + /** The number of type arguments selects the kind: ``, ``, or ``. */ + static fromRoute( + route: TranscriptionModel.InlineRouteInput, + input: MediaRoute.ModelInput, + ): TranscriptionModel + static fromRoute( + route: TranscriptionModel.StreamRouteInput, + input: MediaRoute.ModelInput, + ): TranscriptionModel + static fromRoute( + route: TranscriptionModel.QueuedRouteInput, + input: MediaRoute.ModelInput, + ): TranscriptionModel + static fromRoute( + route: TranscriptionModel.RouteInput, + input: MediaRoute.ModelInput, + ) { + const composed: TranscriptionRoute = isStreamInput(route) + ? composeRoute((composition) => MediaRoute.stream({ ...composition, collect: collectResponse }), route, input) + : isQueuedInput(route) + ? composeRoute(MediaRoute.queued, route, input) + : composeRoute(MediaRoute.inline, route, input) + return new TranscriptionModel({ + id: input.id, + provider: route.provider, + http: input.http, + route: composed, + }) + } +} + +export namespace TranscriptionModel { + export type InlineRouteInput = MediaModel.RouteInput< + TranscriptionRequestFor, + MediaProtocol.Inline, TranscriptionResponse> + > + + export type StreamRouteInput< + Options extends TranscriptionOptions = TranscriptionOptions, + Frame = unknown, + State = unknown, + > = MediaModel.RouteInput< + MediaProtocol.Addressed>, + MediaProtocol.Streamed, TranscriptionEvent, Frame, State> + > + + export type QueuedRouteInput< + Options extends TranscriptionOptions = TranscriptionOptions, + Token = unknown, + > = MediaModel.RouteInput< + TranscriptionRequestFor, + MediaProtocol.Queued, TranscriptionResponse, Token> + > + + export type RouteInput< + Options extends TranscriptionOptions = TranscriptionOptions, + Frame = unknown, + State = unknown, + Token = unknown, + > = InlineRouteInput | StreamRouteInput | QueuedRouteInput +} + +const isStreamInput = ( + route: TranscriptionModel.RouteInput, +): route is TranscriptionModel.StreamRouteInput => route.protocol.kind === "stream" + +const isQueuedInput = ( + route: TranscriptionModel.RouteInput, +): route is TranscriptionModel.QueuedRouteInput => route.protocol.kind === "queued" + +export const TranscriptionModelSchema = Schema.declare( + (value): value is TranscriptionModel => value instanceof TranscriptionModel, + { expected: "Transcription.Model" }, +) + +// --------------------------------------------------------------------------- +// Request +// --------------------------------------------------------------------------- + +export const TranscriptionTimestamps = Schema.Literals(["none", "segment", "word"]) +export type TranscriptionTimestamps = Schema.Schema.Type + +export class TranscriptionRequest extends Schema.Class("Transcription.Request")({ + model: TranscriptionModelSchema, + audio: Media.AssetSchema, + language: Schema.optional(Schema.String), + prompt: Schema.optional(Schema.String), + /** Routes that cannot produce the requested granularity fail typed; routes may return more than asked. */ + timestamps: Schema.optional(TranscriptionTimestamps), + diarize: Schema.optional(Schema.Boolean), + speakers: Schema.optional(Schema.Int), + providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), + http: Schema.optional(HttpOptions), +}) { + declare protected readonly _TranscriptionRequest: void +} + +export type TranscriptionRequestFor = Omit< + TranscriptionRequest, + "model" | "providerOptions" +> & { + readonly model: TranscriptionModel + readonly providerOptions?: Options +} + +export type TranscriptionModelOptions = Model extends TranscriptionModel ? Options : never + +export type TranscriptionRequestInput = Omit< + ConstructorParameters[0], + "model" | "providerOptions" | "http" +> & { + readonly model: Model + readonly providerOptions?: NoInfer> + readonly http?: HttpOptions.Input +} + +// --------------------------------------------------------------------------- +// Response and events +// --------------------------------------------------------------------------- + +/** Speaker labels are provider-native (`A`, `0`, `spk:0`, or a known speaker name). */ +export const TranscriptionSegment = Schema.Struct({ + text: Schema.String, + startSeconds: Schema.Number, + endSeconds: Schema.Number, + speaker: Schema.optional(Schema.String), +}).annotate({ identifier: "Transcription.Segment" }) +export type TranscriptionSegment = Schema.Schema.Type + +export const TranscriptionWord = Schema.Struct({ + text: Schema.String, + startSeconds: Schema.Number, + endSeconds: Schema.Number, + speaker: Schema.optional(Schema.String), + confidence: Schema.optional(Schema.Number), +}).annotate({ identifier: "Transcription.Word" }) +export type TranscriptionWord = Schema.Schema.Type + +const transcriptFields = { + text: Schema.String, + segments: Schema.optional(Schema.Array(TranscriptionSegment)), + words: Schema.optional(Schema.Array(TranscriptionWord)), + /** Provider-native language as detected or echoed (`en`, `english`, `en_us`), lowercased but not normalized. */ + language: Schema.optional(Schema.String), + durationSeconds: Schema.optional(Schema.Number), + usage: Schema.optional(MediaUsage), + notices: Schema.optional(Schema.Array(Media.Notice)), + providerMetadata: Schema.optional(ProviderMetadata), +} + +export class TranscriptionResponse extends Schema.Class("Transcription.Response")( + transcriptFields, +) {} + +export const TranscriptionTextDeltaEvent = Schema.Struct({ + type: Schema.tag("text-delta"), + delta: Schema.String, +}).annotate({ identifier: "Transcription.Event.TextDelta" }) + +export const TranscriptionSegmentEvent = Schema.Struct({ + type: Schema.tag("segment"), + segment: TranscriptionSegment, +}).annotate({ identifier: "Transcription.Event.Segment" }) + +export const TranscriptionFinishEvent = Schema.Struct({ + type: Schema.tag("finish"), + ...transcriptFields, +}).annotate({ identifier: "Transcription.Event.Finish" }) + +const transcriptionEventTagged = Schema.Union([ + QueuedEvent, + ProgressEvent, + TranscriptionTextDeltaEvent, + TranscriptionSegmentEvent, + TranscriptionFinishEvent, +]).pipe(Schema.toTaggedUnion("type")) +export const TranscriptionEvent = Object.assign(transcriptionEventTagged, { + is: { + generationQueued: transcriptionEventTagged.guards["generation-queued"], + generationProgress: transcriptionEventTagged.guards["generation-progress"], + textDelta: transcriptionEventTagged.guards["text-delta"], + segment: transcriptionEventTagged.guards.segment, + finish: transcriptionEventTagged.guards.finish, + }, +}) +export type TranscriptionEvent = Schema.Schema.Type + +export const responseEvents = (response: TranscriptionResponse): ReadonlyArray => [ + TranscriptionFinishEvent.make({ ...response }), +] + +const collectResponse = (events: ReadonlyArray): Effect.Effect => { + const finish = events.find(TranscriptionEvent.is.finish) + // Every transcription protocol's `finish` emits the terminal event or fails, so a completed stream always has one. + if (finish === undefined) return Effect.die(new Error("The transcription stream completed without a finish event")) + const { type: _type, ...transcript } = finish + return Effect.succeed(new TranscriptionResponse(transcript)) +} + +// --------------------------------------------------------------------------- +// Request-shaped call API +// --------------------------------------------------------------------------- + +export function request( + input: TranscriptionRequestInput, +): TranscriptionRequestFor> +export function request(input: TranscriptionRequest): TranscriptionRequest +export function request(input: TranscriptionRequest | TranscriptionRequestInput) { + if (input instanceof TranscriptionRequest) return input + return new TranscriptionRequest({ + ...input, + http: input.http === undefined ? undefined : HttpOptions.make(input.http), + }) +} + +const requestEffect = (input: TranscriptionRequest | TranscriptionRequestInput) => tryRequest(() => request(input)) + +export function generate( + input: TranscriptionRequestInput, + options?: AwaitOptions, +): Effect.Effect +export function generate( + input: TranscriptionRequest, + options?: AwaitOptions, +): Effect.Effect +export function generate(input: TranscriptionRequest | TranscriptionRequestInput, options?: AwaitOptions) { + return requestEffect(input).pipe(Effect.flatMap((request) => TranscriptionClient.generate(request, options))) +} + +export function stream( + input: TranscriptionRequestInput, + options?: AwaitOptions, +): Stream.Stream +export function stream( + input: TranscriptionRequest, + options?: AwaitOptions, +): Stream.Stream +export function stream(input: TranscriptionRequest | TranscriptionRequestInput, options?: AwaitOptions) { + return Stream.unwrap(requestEffect(input).pipe(Effect.map((request) => TranscriptionClient.stream(request, options)))) +} + +/** Inline and streaming routes fail with `UnsupportedOperation`. */ +export function start( + input: TranscriptionRequestInput, +): Effect.Effect, AIError, Service> +export function start(input: TranscriptionRequest): Effect.Effect, AIError, Service> +export function start(input: TranscriptionRequest | TranscriptionRequestInput) { + return requestEffect(input).pipe(Effect.flatMap((request) => TranscriptionClient.start(request))) +} + +export const resume = ( + model: TranscriptionModel, + token: unknown, +): Effect.Effect, AIError, Service> => TranscriptionClient.resume(model, token) + +export const Transcription = { + request, + generate, + stream, + start, + resume, +} as const diff --git a/packages/ai/src/utils/media-type.ts b/packages/ai/src/utils/media-type.ts index 016d2179275..fb9d2c0a72e 100644 --- a/packages/ai/src/utils/media-type.ts +++ b/packages/ai/src/utils/media-type.ts @@ -38,6 +38,7 @@ const EXTENSIONS: Readonly> = { mp4: "video/mp4", webm: "video/webm", mp3: "audio/mpeg", + m4a: "audio/mp4", wav: "audio/wav", ogg: "audio/ogg", flac: "audio/flac", @@ -49,3 +50,18 @@ const EXTENSIONS: Readonly> = { export const extensionMediaType = (path: string): string | undefined => EXTENSIONS[path.slice(path.lastIndexOf(".") + 1).toLowerCase()] + +const EXTENSION_ALIASES: Readonly> = { + "audio/mp3": "mp3", + "audio/m4a": "m4a", + "audio/x-m4a": "m4a", + "audio/webm": "webm", + "audio/wave": "wav", + "audio/x-wav": "wav", + "audio/x-flac": "flac", +} + +export const mediaTypeExtension = (mediaType: string): string | undefined => { + const type = mediaType.split(";", 1)[0].trim().toLowerCase() + return EXTENSION_ALIASES[type] ?? Object.entries(EXTENSIONS).find(([, known]) => known === type)?.[0] +} diff --git a/packages/ai/src/video-client.ts b/packages/ai/src/video-client.ts index e63ebfa6e02..35dc276d4f0 100644 --- a/packages/ai/src/video-client.ts +++ b/packages/ai/src/video-client.ts @@ -1,9 +1,8 @@ import { Context, Effect, Layer, Stream } from "effect" -import type { AwaitOptions, Generation } from "./generation.js" +import { resultEvents, type AwaitOptions, type Generation } from "./generation.js" import { RequestExecutor } from "./route/executor.js" import type { AIError } from "./schema/index.js" import { - isObservation, responseEvents, type VideoEvent, type VideoModel, @@ -69,15 +68,6 @@ export const stream = ( }), ) -/** Status observations until the first terminal one, then the fetched result expanded into events. */ -const events = (generation: Generation, options: AwaitOptions | undefined) => - generation - .events(options) - .pipe( - Stream.filter(isObservation), - Stream.concat(Stream.fromIterableEffect(Effect.map(generation.result(), responseEvents))), - ) - export const layer: Layer.Layer = Layer.effect( Service, Effect.gen(function* () { @@ -89,7 +79,9 @@ export const layer: Layer.Layer = Layer resume: (model, token) => model.route.resume(model, token, executor.execute), generate: (request, options) => start(request).pipe(Effect.flatMap((generation) => generation.await(options))), stream: (request, options) => - Stream.unwrap(start(request).pipe(Effect.map((generation) => events(generation, options)))), + Stream.unwrap( + start(request).pipe(Effect.map((generation) => resultEvents(generation, responseEvents, options))), + ), }) }), ) diff --git a/packages/ai/src/video.ts b/packages/ai/src/video.ts index f3ae17670c4..9f87a082ff4 100644 --- a/packages/ai/src/video.ts +++ b/packages/ai/src/video.ts @@ -1,5 +1,5 @@ import { Effect, Schema, Stream } from "effect" -import { Generation, type AwaitOptions, type Event as GenerationEvent } from "./generation.js" +import { Generation, ProgressEvent, QueuedEvent, type AwaitOptions } from "./generation.js" import { Media } from "./media.js" import { MediaModel, composeRoute, tryRequest } from "./media-model.js" import { MediaRoute } from "./route/media.js" @@ -123,18 +123,6 @@ export class VideoResponse extends Schema.Class("Video.Response") } } -export const VideoQueuedEvent = Schema.Struct({ - type: Schema.tag("generation-queued"), - id: Schema.String, - position: Schema.optional(Schema.Number), -}).annotate({ identifier: "Video.Event.Queued" }) - -export const VideoProgressEvent = Schema.Struct({ - type: Schema.tag("generation-progress"), - id: Schema.String, - progress: Schema.optional(Schema.Number), -}).annotate({ identifier: "Video.Event.Progress" }) - export const VideoOutputEvent = Schema.Struct({ type: Schema.tag("video"), index: Schema.Number, @@ -148,7 +136,7 @@ export const VideoFinishEvent = Schema.Struct({ providerMetadata: Schema.optional(ProviderMetadata), }).annotate({ identifier: "Video.Event.Finish" }) -const videoEventTagged = Schema.Union([VideoQueuedEvent, VideoProgressEvent, VideoOutputEvent, VideoFinishEvent]).pipe( +const videoEventTagged = Schema.Union([QueuedEvent, ProgressEvent, VideoOutputEvent, VideoFinishEvent]).pipe( Schema.toTaggedUnion("type"), ) export const VideoEvent = Object.assign(videoEventTagged, { @@ -171,10 +159,6 @@ export const responseEvents = (response: VideoResponse): ReadonlyArray => - event.type !== "generation-finished" - // --------------------------------------------------------------------------- // Request-shaped call API // --------------------------------------------------------------------------- diff --git a/packages/ai/test/exports.test.ts b/packages/ai/test/exports.test.ts index 21280fcb7f8..e241df07857 100644 --- a/packages/ai/test/exports.test.ts +++ b/packages/ai/test/exports.test.ts @@ -17,6 +17,7 @@ import { import { Route, Protocol, WebSocketTransport } from "@opencode/ai/route" import { Provider as ProviderSubpath } from "@opencode/ai/provider" import { + AssemblyAI, Baseten, Cartesia, CloudflareAIGateway, @@ -122,6 +123,10 @@ describe("public exports", () => { expect(ElevenLabs.configure({ apiKey: "fixture" }).speech("eleven_flash_v2_5").route.id).toBe("elevenlabs-speech") expect(Cartesia.configure({ apiKey: "fixture" }).speech("sonic-3").route.id).toBe("cartesia-speech") expect(Deepgram.configure({ apiKey: "fixture" }).speech("aura-2-thalia-en").route.id).toBe("deepgram-speech") + expect(OpenAI.configure({ apiKey: "fixture" }).transcription("gpt-transcribe").route.kind).toBe("stream") + expect(Google.configure({ apiKey: "fixture" }).transcription("gemini-3.5-transcribe").route.kind).toBe("stream") + expect(Deepgram.configure({ apiKey: "fixture" }).transcription("nova-3").route.kind).toBe("inline") + expect(AssemblyAI.configure({ apiKey: "fixture" }).transcription("universal-3-5-pro").route.kind).toBe("queued") }) test("protocol barrels expose supported low-level routes", () => { diff --git a/packages/ai/test/fixtures/audio/dialog.mp3 b/packages/ai/test/fixtures/audio/dialog.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..12ea29ed8cc8833de109816e42e6d9a1aeb821f5 GIT binary patch literal 14211 zcmb8$Ra9GD^dRuy4j~Cz+zJ#)fno(3EO?8%L(w9|rMOFRcXugnMN4samr|Sx#Wh2| znU`7XKM#|a+}xFwo8R3#=kBxb3E~Xa4CfGj;^f z!2(dxhZCQ>oBzTF003x2PmibP|9fKzfaXt)Nb>+-^=RFT6399M=qNm>z;7WLz4CR( zA!zS_fq95CiGEU|DSkox9dzt6b`)Y<4Ag{biCHzW3z2%!yPGwQNv#h3rpHx7ai(|& zlWxOeLBsBF4V?DlqoD4!?xbF86-EY4_FwM&KbKiresuWNYNk#W>J-Y=y-swKltvi+ zY1w?(xCj_VJb`%dwrHfzXYc{TI2X|IO`RmO;x_Mn8h95)%lRj?kNQKD-N+ZD!u{g&S$tF->bziL*A^sfG1vzT=j= zV8(e<=$*oPxeyX?`TZcLnGGN7f+)P$lc;>6;N70LA9MH9~p&qpeyRI%lv zJ0L+7L|+)B_?ANfO!?zfVO6U65{$M2xSS}8%U=cOamG3EXe*s~acT83;*Z6_q05l* z&acApr)6b6a;h4{(T2rI4e>M`ij#T8(R~D4AJt7T!(@s|v=nM~;`~bB%BiJ(Vq=s@ z&;XbX#0OY`<4Zlq3wj3>We&uZT;em$!1zoky;kKlsarG=8T;(J!vH=*Kq`(Rr|B28 zC5eA6OK~d{GZk1UQ}$X}`QMV2-UoMctret>7B`$Hzx-oEgqIhtR=M{ZLS16L?_7-p z%|n1Se1IQt?3?rYs?vvbywDGCCDK0kD&GaFt$#0m&fCtpHjUqQ_$$K`e0_33YjkU? zwNXe(@5JLHlbi8U^mx-@u`cy;{?>2v0daNraC6dZCU-NC74EB=EXd$+V$y(@fdp-m zvJnd6t`4Agm5}uPW!|&O&K-CwpgSFNwGD!bXZ&tvP+%X69Z%$H5q~1dq^pwW* zS_6w5At_)uZqb-#A)C&i!VSR+J2LXQ`gcR;IM$Q!QcyH}U}wz9M^kGR(IxkEAXxto zN**3T96fTBm~X^&JU8rWTr6rdVT6^%4}J*5zf1=#!c2LWJSh0y1OuUmw&&^agUC-W&N^X{320WL0&pAOJ0HJ4G zE~(Dt=S5}r=2zkUrEZprR~4@J(#MXYj5lFzo03iTJ8HN|e|XR;uIph&f?ht$-^=eB z4sN1K#B>{Brpbq?#sq}C8PViP8ql2DydObjD(1#FYXu{siV?$n>jK{~T-3?@B6mC$ zm8*(k9y|UGmfX`7zw|(nBnIMZ1IN-GCK*^283_C(*@6O0LM1+i8Y4l}6yp3a6a!fR zk72<`6S*;Vv0fB^EqY*Sc~=98d`WipIEEy<=Y51uIUidR5Op^&FB*eY`TnBCJNeN_ zD+4#Z=!8={-DyH+kqfug9f3Gh5BhowUaSxo*LY_x{D=^m_lrVDf*|B~xN#`4%7Cy( zG=MVco52HFK15FzUO%i3anFqo`-0)v{gDiT2(N23B%;6y_TNMx5J8?oKh1yilyN9B z;hjFcM%)O;=ZKJ!e_NA|?b&{M9f&dIUszq?u&y*;3sFac@W_NVT2azrfa%5^=tX*t zqP$y&cD-CG94sQAR-;dWVKJi|fog1z9rCt7T~VwARfYU(46Wc{2+gZU&9T#%9~ArRMx$t6{f|*uhuVpa)bxB(n3fO3HZh-?eg1zo6aX=qB#~>)v_9f`%piXR7*<`LZczP#zS1X}nTEvyy9+I`VH=&zWC ztnx5+ZcI@Ya0?c5;3e|c%|wb8%D|wcAK_geUm!vB5aoe@oY?`F@{pcEA2O%tebyxE z&xx??W?={#u9!IHQgGoI%_sDWB$cz$@eP}-lgMX?$4LHwiZjKL6nzt}Ec8fIOw7O@ z5=^01Y^VA7KvGl4Ywm;rHR{1?UnV^_Bgh&O#158S@W);tcT9g4-D4QC`%M-dknCTC zjwRh{3NU?>l})44KO_3POLf)a=>!har<}(WP+2 zbG1hC+kn&Uiv87 z;b^`4+&=WoqgFz!VtrLLl8@GDIwu7QQULRy7I%$~I3fy6!~eZ`XXdvZWQH&M)C;JO^l!re*g+PQ1t0%+p+YE;tyiATf{~DTGNR*5Zlq`m zXV2fi0<=3ma9T=1r#seu@9LH!LFN#Zw^{6?_~zaMak35X#=eCsgnX9YtOnM{#fZOt zyGkV2B++~FgWlDXbedx2jrGE4nAL|r6a>MuzZks)X{!AJwn}@%HGdvFqG+1c3D0Mr zkM0Rls+w;Nhc#gb|AKyD!`Cp9bPj~F*qrWH6NA9d>6sHZqir_}LGQ2mCmi;^QwpctDgy(? z^$quyps&x|Ko}SpL19Bidq|K!#B;A*z9%7lRnZ(ha`tsP;_B?IcDDn8SV-1GAZF61 zyx@M*;aJTJ1FEKgeZrMB?cLJr$(qu@S2MNjmzoc9+^ccllqKgz@T|MD2^t+mPYzv&|%u3I`DP~u(L>6G` z6fPB!mVv^;6VGNVjdSKKn8ENxD!56?PSrs)q0WIBZ{2sn`!U%gEG*T+7W8T>`3k!z z(p!f~>t7L@A_xgerZwaG9xXUath_SQ@@0FOPd|`iDMOzDEk8P)RQAKfd6l-f0X|<( zYw^ej)AFuVvgc;U^B0RAa^d?lAEn&A=CfAX+gp9GQH*t+(4J#`#VwMc*2k<90lE1{ zO8UD8lg45HSNb@CAD) zaB}u>=hyNmY(N>KSw4fB+UmD@h*?!dGdbsP}Y2(d#HB-c6==)paJ=%ZoP~nbW z_?7wJD3=IM4?L!UN}capW$yc3%qQFxH*I|ZP~9kH)sEH2un# zLA!H<&L?L+IyR3TTYDf5Ve8~-p_}q<2f6~XagmMoKl`sd{K5q*5v}}zOp+?;M+SL9bizm0kkveLXd4PC)eW-MxMOPh__{Ts#^u|rT6BSBxm zX-6Axx!}5`w-wbmnA~~;4yBK)e0EK&=;XMlH14GyNe2;d2~UmVw74TzITnC7YT49&Hl;J$dzSXYT~h6-KL_lRJ%&0!IZ|$O=1+f?%yk{mDYE?PPMLW1VN5iBcW zCZQ?r8`EMa?1I0xF;IK!KGbyNXeaG*O@IpD02wh)|#C-{|sN7^Eei(=r-Bn?SFa*-(QuEFDHGQ$Z?pNQ4s%#1Z{(dS+vP# zF_Udm4A|c;=wE*t9b_CTA8Yd-{B7tP!7EL990hA|xwEm!2)fp=;h5mY8|kD9jO<_U z+PPk>twg17uw@D6^7)`5Ev-yXrd^KCGkg}sA%A1rCu75sGGb9>CyWH0fLM88_$!Kl zZ!aXE4KxWhYR6uDO9iQ#^?O+})zavg{n8NFF0Mz9@C^~JJ5fZ!hTpO>stOCN^fu1+ zCx$}#IK&EL$WgrL)R@%c51tK1kFHi!^me{EJiw`M3A6kLOZH^wC4}PcU=r|Q zo?>Q^4T;lZ`n^cRkT4EsanuTpYmEGBSf`6ecHi3ngXMr)fIIFVgN+aqpo*9AR12{` zWhS85W&bX7_==2}ffXd{4;qwF(aw{K4ktt036qJ(*>cW$nIl2}Agr1&5j< zXUT7g%P3N1`#}PTSwxV8sXVBKvwD@Y!x-xY%dvsT&(^l$l_uM2Ef&&e)dv-QmUN)O z=PJ<+<^mKNO^+|_QAJ$_RfdBF-u$FfC@;AsrfAuluu4+Gh;MHj!J#g-YJ|3 z1yT|OISQI7!~`*rWxf{OJq;O&FXH%^{8IYI0l&7gs+0~OZrB_UrXv}q1uHygOvb~rzg7>0O|vC zB(|kn5D=)ga!{D8&2&7)uBaV?C@q9hii8XgdE(TV@OT7YX2t=Y;T|Fo_w8cwQtA9A zF)Rv&luBgX2*jx+5X8;~LYEF^p<_~4WlR$40|SZ2i6@_iwi{HcZJaPB{Q}f^MR6r* zUapr=sK)>FK!O0^B}yL};{gDE(a+}3t$>$oY2+IvUp26LAES<`eCIJ3>+Z z{+}&$6aaGTFJllc{+q%_<)`-&m;uV_BpBkxR>A~murhYsY^(6- zQ-5r0^Z9p%#lEqKTLePuvEucK$Xoz4AfE7DK3(~@pBbZqFj`}u!bN(YAB8hW5H<0& zFch#e2GCv!DIUs{TyZ_Cs2h|0aT=Z2DpujpNva6;=}I7_vrB4<0V~0I#w5Bigw4(<^JSW#qBn5WC1wgz>)B2?>Id1e6G3 zFNZQs?jRDdi^O}rk>jJiFWLCVaY!r2tV-&UBW@30b0r78S5jgeP5l&85pXy}sH|8q zvK7`l8x3sAhX_JdJzq!)tUE`v9?x+Zmv9a3v-VtYDLgL^Qt2ut`t|Y{3F0S;6PCrD zfuqY9Was9drdZ5VHaf{MH+$!2Hn*8R-lVhkzu{e=-L zt1GndMZ~b)YXir7)2{+=EP&8}^(X+~2L+}b6g#uEhFpvta9=wtkoP>GYm`}#g6T2dHee2;GtgAfG+ z1N_$ihXO!^nF4l^F*g$ipPMV$zkGaHyU}S-D0N%eO9CK4Voa#MUdl6y*(Z^JX6m#` zMb=+s@;F^{EPK=Obax?D|IGP-@@PGdLC^9+Iv|#FjU}+k83!Tqb;P)(B+qb1;z?NJ zhMJe8%*0HLDl5#ldyHpr04oSkaUxblhNdD4=@~PWMp752B$%9Npa+Fm zU-1_{l>B1OE<$Y%#a`01@~fJEtJl1#yXxfch4Wv#?5Vp%xngLqNEpF1I_%p?GC`)~ zJS7Y!Yh*vH7IfKU9KM>J$xswYZQK|u3E#{fLP>;8pL!>Lu)iZGTA|lLl@6nhe96&E zLT$p>@}iOdvO2pWv)gvX1rwcvj*W)C;O?ID$K-z?lJd7*E`7@3abz}I2lRqfiJ_UR z2W2Ip;#Z-Bxr_!wv+DZ&teJY+Tc*qb!Cm??GCi#w`2}{P<=*B}vZicjC7~y-4ZRpX zUs1_&WRAIJ-25KXs{Mt|{XI{FjUlWjO~p@_2Z??{(l&kS?RMk0WA<*7_WuF#^=TNH5ebM8{#DN@A zV=_ff+T=ggwm)b@l0E8re&QXfXB?KD%PQ9`uKO8aHhDPrn-5RTmQU3~Ze-_bA)HzdCY5s3uTzP|JXPLM)o4SU{+hWREB#0H99y6U|+`p2j!9|2d*{fEZD~C8*qEjZZ z28|g#PH(KYEtGJQQ8m0^nCa4d#g011^kuZR!T93)L{45qf8AmFM-CX&MP> zsuwR^WnQP9{<*(!49_W21paC5VJd#=ZNvIsLFD+m4CO-k9MiU0Ii%FdK0d>o4yO`- zXG^iJRaDvef+`hpF>kyHOsXYherJ&-`%|iHzWvv$C$s1V7zY;(HZfLeE{7?5~%gZYndEUy5bPD2v zS2EdN)45Nj&Est>zu&{_T}2XJn|)-2_#o^Vir8*P_I+u7VPH&|xXa`~ ze@G2}9M@4qQD9a}L}pp+x$*m9om%m=zlvCMV7w_-y)QR(=FU4S8W;(z2{oW^D&nqH z1`)DF1WO%FV#GpN?+~Vud@wol|El^o;MV#ba^p~`u;2vkwDbD%eD_jYJp(;8D>u!R z)en*48jeyY^N~W@`$}tU{8Z~yTzo@m(WZhTFiWY#4;G~9)<3_{tW4x5$Mz&P>>dBePOw{g+m0<&| zB1S>TZTG1PYc)xdN1A0=s9m))GD4c*0Nf*8T{xi$^z29YKiM3-PI|#S;lID&0YRuZ zZKUxR_#c2#AX!x#Cw5GT(vj_YYg#neH6!J&Bk4vMwCK>_%Q>^%6rO&av@Yx6zv2<* ze)rH6^9favLwCH`&W(lseG?L71}^JEP`wPrrIr+q2PFX0M}ntg%APw4N|A%2%S~1K z`grS$?_m&OsXcvzD(YSj2SLGg8~NhFZ%B<>e6GX880i8 zyoyM1t zS^24~^gkRnZuCF>RweqAhy)oFA=7Dqqz;Ww?(_`1`OWfO`*1Pinra{Z#}cJiS_Y6Zjk8;DtTVNW-W(QOqy6uS5kos~AeFS`G}vdj1Zvfp_aY%8Vs zOt}B3skOdBDuvUC!qPB+Rwc^_PdN&$WFYP{P2}0HbNp*?XkMJ$($U~DER4#A1|~S$w%ecDDdfOcim5SrL^ba{>szn*K6k= zGI}QSA_Y_cO2xU6BKJK`S$h?^qs7LEG}F}5Lsx}4g->T;U->DgJ6oJBa-TXL?VCLt zH=)#5Y++NoBY!6(kf6V$FB3#SD{yR?+H`9!Sp0ezv z?-SWP$VqILR`Y z$Ln+Gx`+fs|GTiBJxip70=7f5X;IA37zgqnS5Eh-dT7j^i&MI$eLb(!8imESh)qio zzI~&}wR&*$xUw;zK96Z^kzl-Q1;oi=_~b>%91OJDDo4?3edR&n92z%An8?+$@?9Sj z(|;E*&=+R%0~w($@KA~fV5y%k;3qBGMBQhGv;OD|u~`b@ZKjc?sQec*g{vXDlMD9i zeIwK{F+H-&16T#rOeV8OFq@@;FdJfv$9~w?>QdSE{dA6wpZXl6FAg<+yqolx|MicQ zx=l7LZm$&u8KGlx>@GpTa;S=vLYYNNFmLHk?`~Wh_lM7O4S&F=Z*u;GnBZ*O<@9tT zJcTLpxX`gNO4WIn4Jtl0g?g<2Iyu{Wlim3c`+n)Dr2O0TE2-|cE597o8+&UAzR{zp z^}h%DHHb`DAVGT&a&{@uDki`$iBPxFyf==^L)vpOqMt$09_p|l4yfSLuHH`D8mZCi!L)PxBKrDI#j8`@f}Z#yO4MMqV}TVyoO88nJg+YvWG4OU#YL} zj&0$5_sr?;<&+h1#3#eOPSps6gz^BYaZf;>LNcr%0-v_AFVM;jwmQoue%Q!SMOjaW z1m!WY?X|Tm83UUBN(Z{^N`sj@2cDBGXv|&{H_5g%>YE9kiZyxdJNT?-ZPwB{d`9(< z%dX>uhgW!7m8tEi^w$(r+jeT{@#wi{j*3HqgXNh}!?ZEj{OJx{3Ti7~IE++4PmmxR zM$nI{y>BC(tB3~*^yr}gb4oLzSOu;Gl~7~7jn-b%wRMJd$&y~8$pZJ~CE|~f(W8d*Y3V)*Qi18P(FxfZ z@-d{sUKGbT-VnzHnvOW$lIvedyaqQ3cCUn@SClB4^^P>6vppD|JXS1@^?5^?C$Vw_!?K-Zrb zUZELs;zD={=z^Cb%V3hcx188{_cWFgS?wcf?n(eMo;3Q`5=<(ENDvUhma_vC4<&y7 zN~kT<5HZYA<2~U;s+&o-r@tMa+3Bk+E7#{A2_T2TTd(Xsl@ndgcV^V<+{Hh;aaHV~ z7%l~BwckHHT8>qFx`&0kYbVGF5=h{2@otO}2DMOF%ovm$vmlQ{2|AS_qJhe=i;9Yb z)$WW&>O+N$f)q?sZk8mF=ZU z=ui~m{dz1Y>I8Hw+`ZLC@8G)<1UPUuzLZ(NLK&y-x4=2yK!17tm2L;z4sx@Zq-+$|9WZc44)iv}!# zch<2FcTRTwXosrvuD9Ju5DnD*Tq2uPmCu5v=igcvn52=Q*I+rrO^5;}`i0oYME|HP z#tWz1x1}ITo(nvUq#F}s!0dDeU72`5ek)Ii6O6#U(&loWZ_sy0B91 zeKUJi(srdHJDX$3=hFQ5pxZIZx^KJesIlIU1#!6s)w~9_qRPx1V+VwmjN3}?F87g? zP7_}U9Jl#Fj4%(T@Gqm_Jy!Lm?n=9=jaVe;1B5L>U!)J7B;i*oVDRCjLYH&Brlw+_ zhp#{~pgGXETy#*Xh4v%dT^aJpMl}54_bvFA9iI*zv+M;snn2h$wF&uh>knh^YyN1S ze%-8!Zg^?rL)|(F=5FG+TlK!OpIbrvcfZSxINs1$R2>Ldvm-4yyqIRTq*aN}-JG*d zjisCgt!lCC$K3e&`uaIl$6h-3w#Lh4*d!Pi>c=AX*AW8lw=N~budT^oLvI3JG8{~U zT$AeQDr${HpW^d0=qxw<_l)>&A{oV)xS~&x0R<#w6jq?%@W`admcTQxQ}Zl?1MEqy z*AA)JyC9dQo@qNTP^YtWEc1xUlQmO0XT{6;a60Bcyi}>Zbx5_o6o@RlG)F+KjPf) znXz{=ujTpj-(6ScS@FRMkP&)8`x-T(I~In5_;>qp5Yvb=3_n9x zK@m=L8@lr5nOF9$p?$VL`FFN47+^Yt$mJu0Ni9zz{Cnxh>q0Yr)QRuWlxE+EYaza9 zuora~RiddU8#^WA>+~Q&gd}ZZRVZoN02ROuDNa7e+sdrv9VdR>Qqm7I>}tC6aVx#2 zu1icnMJ6IEmhMl#DzNDtOh8Z-*2|3OOB+QurR=;x&&6Ryg!d7Gz8z4oq~ zcAd?7*udBH6UYeRg7g0U(+$KVC5%`9qN|%=m-(DZyKMx;+1kw__nETjy7kJDs;_+F;HN8e=hDdRj}tmEAhceN*p4OwWgD%cAd&%xJFX19~W-!Zm7| z=%Q#7?ai)85Iscj@$r5*)FPVRU1m~nV62F|{8jFo+|WE_3Q|JR(3m2o8}b+Z%~gCo z)5{!Eh^5&P)ErQ2(rdsPj$H?JBPm`4If}^y%fVT`AWqeA0F!{l>@mITtfOdY8b^Es zBmgMZTZsg5fXkZmyS(6vAAf|Q)v;|16^2X=;uY_H<}|8(OsqS+ynU@4R7*N1PvS1` z|F_JF0)1CqhQ5zwuD&>!@{7pxU21ZU<~QRyNjCPVVa7=qUkOiCb^}V%jjuncVh$hm6MnkS*qjSV}kGBtQR=Gy-$TRKu2#P5~ zv0acL3GmcqHq8ut=zJ6LSmyTk-Rkadkz*1rWxD*i{&=ZhRoZBr)tF!w>OjD_0hg5n z=U6;pLcMM5$9-*0{AQXy8h*{fAzKFW3KMpUii_Q|-`WAvz=nd}e4?i-ULVcczh6(8 z{sdA2dukTZ^al6LQJ`40wR}O-HpbLu#qb!!H>YX|3_8h@flqL}V7xPp37S*QF4yZU z8cx(>(?#4K8*fiZVPg^j@P#nB+RZ$&5f7?HCy(I`t8^x9lEWuN;!VbR{C3zM_b;9n zc)^PSwJ%bj4ss!jE-&4%@*11&S4 z8L@1*>(xMpO2(MR(|4_zzy8kIU0vW>p}B>TzsW6bhi=J|pRKM>mR>~wR?b?G57R}0 z>>v>tt!v)sDf*su2Sgk|Ifa0Nc>M8;6bIr!t4JIiFIzGx4*hVQN;C1^k2sF;Io(em zjyUi5@l99jf*LeWs0&xPCdW6Qi~^j>N8Ezh?XmqeXkrULo>|y;XDwSX^czR~x75cC zf<4ek;|+>S@H@uzxNwkZN!o>o@G)qDl$8z2Xy^&B)%=}k5s1ssaVng!@o2&*4BTQ| zfM^B^nY#8SXTZ0ui0uZ_Pv*VAItt-D%~{X+qlj7=`eyJtZW_AELuB zpW_+qg~ChT4?KJQCEVn@zfQ#xPaS=!K`F+zvoVLx9xE{#^>{;OP2M2QSG;09-z%Sc zkJ0NtS1;G`?cQ1SIhqSyRr?gY>HOozC}w8Z9jl-wS^K=@X@Q^T=gdnjRAXd>A{i4V zTxd4%Jrk|-PHiwTR$-2^bUkv`xHT+<+`b%-{qary8C5X?bor?^SMHO8RIhV%{M_#z z^}99aIeatglvGzzHzxJ#?4N^j!kDJeC-PfKdfcLV@&Yt4aOd&hfznU|BuJB?Qlf05 z2VDkm zM*GPS9Fn{aw?yoh0BgXLQ$qe_ZM&~s`u)7~6C z44ZZJ;actA20k2nFU%8g@CQwm1@ux(>_kt8#@dWyRxVX#<;|X7}Sn&pM$65aN#o#ZJYkYfCLeeqqcv4icouK z`AO&2M8Q0);#EqT;{em*ed;00!|7Qj>2KMY^8!ad&TM(jOUtGPLrLHSRe%QN*oKtU zJA@QOG6HvVcJ=9q2c zgL!`wV9146JlAFb%twqSxX2d+V0|#4@KOsm=PSM%!it|r&@%`M{|B1oxafFj!|2;C zFf@z@eI6)L6a~KcjHg4HU%IEKPJt(N7=cr#om3d`?_sKDBR0fxWplgOTu)o7Ar}oilPjR z!(}$^hq7BAL@4IRTK6ySzkSP7P!UPEtB{<*`#P^H_{6CkQ{7*|z{y+Eq EFRI2f>i_@% literal 0 HcmV?d00001 diff --git a/packages/ai/test/fixtures/audio/hello.mp3 b/packages/ai/test/fixtures/audio/hello.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..94bb4fc281d52d53b1563e4f91abe87a73756387 GIT binary patch literal 6165 zcmbu@1zXh3*9Y+3rI)3b?rvN{I+pGZX#wdHK|s1wqy1;srHJ00tnUVA9bxUjkULZwB&3y-%> zIhGT%oM)Q{W&Sa@DO|x|2j`q%;#f4?o}H=0gGQh^S-{Jq^!ju|G($^%{dImFx8H5< znM`Fvz=omQy~fI_H53!@4os z%70i`87ALXhDdC)=fk^4`B`cexnCQ;r4iIoSq?|15hoX=aMb3ON7r+qWzBA@rWXVz zjTpM{mrRhPMnt$~0vM1r1-=mix`qqlDXTL!NI-PbW}zC}1o@$*TdcL{;8k#mhK_24je4W ziU~jJRg5EGi(}`;fINu1m{i1OqCqYES}FE_j#ycjsO`6^gVt3`-oMZuje8hW!9`Bc z{h-o&;5&UgqoC$ue1r5GhH+zmuVe5f{XTWsnm|-)jSmCU7lf=qxWL=o9S=R8L_3Q) zXc6BrpC^-DfI9~C9RDwsY4#WzRK?h6t*A6>`}L$l|LMFcf&6B-XJKR9IpM5a7CdP9 zs=Nr{r7z0tCx|Zu1O>yN>+(0#ETQXOzOz0k{5l`UN?_A+u;OC!d#XV}GE z#ZRJ>jNIwcT_rUgRtgAD#yJv)Es{mNo@>KdW`AsV8^>yym*&?!MfOJPpNXUC`SRyO zhPzt|LZ6yoKucsuBT4A04#@0fnOO}@2jl7KVZTU_2{nx-~kxV4;KDTJe5gJB`ASY&6z)bWG#?gLgF+_4txiGfY5uu z(~r7QNWy@AQ7TTxnOOkIKks%M| zzdg7A7i}fh`{ALvRE1XBLzEpFp5HadV`SAZw}iq&5$yx{m-_R!LLqnvt*wz*%Et} zr;)D)F`!K{A8{n?lL|KC%RIp-hk8TsaMULX>CMXLR1)EKNLd+Mmn92QE-P~A&UON& zTU1OT9eSUpg8!!~q7N|aWspufrxNh9^F2X+W`R6YB(UC)0{-+L^re8XLPbhmJu+uN zFq2?FXY>|1vXC_xL&2BpF(3L5wUQyuyaI17=5xby@}j=&ku16Rz6yn28tCYi)3K}< z7QX##+;$oq6-D?mT4-+9O3Y-6I9a@5Jgvdm#unjjoOw@-A2}^C)A@1dM{5?Fp|82iJk)2HIaG)Q>#PPN|C^GJM3eEGJ(KuYx!FuOC z45*32$TtNPlu)svwtTx|=M7(&$A`DsD3nt-8M!3g_zqZnT)mx0DXrtaCwtQF+BOd6 zpV1f#d*{`3ed&@?=~!XW+c}1vg=eIRu6yKbeE5PF?MGYHl9I-4^rP3*K2e1Fg$Hkx2e5vszG`vHw;LD z`Eu(G_T0>#VV6pRxRjO0*58^!R7hJ+d!?x)^1Zj)&?uxuG9ZFdPnFFtK(ZrmH3yyG zG)dw2IwbgrMW$M=xyL`S#s zX-HmN&HUnacT$pXCrvKm8}AFPh9H}q+E@kYr`RsyB430dmp9@LK>5;jOC0&a%{h!m zgLZo#t|4#*F8;9yh(0;y7=k=lyUuZx->bp`DOV8?W^_~54cZvVJmBwsS`;(v*!3y{?&W^`nP((Iad>$?ZrA~5NZ|CDy$kcjG+sgl=GIgUH`0xqChJ4gD|aws{tpa@grX#~N16$0 zO|)S%eCZua<72TA=2z5kZbGFYJK6-F8ToHlF#2 z{2P`Nd6x9ru;}!z{Vo0s#5O4%Wb1D+|1yDxOu&31sbv*U7uy+v0nt-LQH|Q>#j(s# z;dVO0CW@CY~CwV+G|C~r#lk*YBFd8=`q7yKEiJPc4zSV1IwY^W@`i2#sWnuEc$bdA}L*sT? zQ8OSq8<;34lCWu}dxVP= zXyd{<0B&$4NrhUzAck!@iRnefI-oHQVEgDNgsaiI7?37Kmve)Tk{ZV|i{kq%NR+nY z3n1DO9A(AlvH10sSBh!lUzg`|s!NxSDx#`V?~7jhF6Q484-Yy4{;MX_yW8;xkTCxF za=U^Hc*<1d>15%bKSl^?ofB4nHxsc`j!6wL24qHIWxHdBRF`L0UQI$Cr>=A}WEadx z#)%UD7R=Z2(eZ@FtMce43>$LcrCI&<4tergD#s@S|k z_o5??0a)+Bim>&#fw)jHxf4n+_jg~5XcC2 z;nn8)3*V@~sATvI-vOp>t8?Y4gp8v}aGd=*`jGcjXp7-&uY%-@D{@`y9~+rzCL@K;O~WKO>HL*%@k zJg3(1OQ_IKz7UIV{LnmS+!j=}<%LKU;4D=;UDGe|FW+Z3U!LpUZv_!n1b|?7rG2$Z z#hygDyRlTx6q`gdCq<^AWf%}64QuJU9C1?lZMNO-F=%!JDc_sxrkO4Vp|DLJ<~Yv% zt*>=}i~hc;g-kV#tZJA3dEITY*uLM!ZjdSiSr#@!Z2;s4v^X>3PXBW|%!Dmoo4((0 z=SUeLnfmmebiYFKpE;3A42XtGYh%Nfa9q0Rj2~gekYdP&#wBe|>Wfxn_5c2Avm`EG z+@Adw=Fs=jRDXQB(U?%;?DF+=9lYW`P|;r4|4)RdJA0iO9U{(`9!-%i`MZrNrOb^C zZzw9q!=N@p#K9{y2m_*p2Q?1%Yw|#Zczra)D)>#LpP2PDr=Nxpknp-Xv{)$Dr_IO# z5;-ojlO@#_+J;p9Q^W08%4o& zSPy_kc^z(^QPJB7;I1l*fkn~!th?wR=%>SLUD$HecQh0LPSUhwx2btYIn0zBB(P&a zqtk6eJro!a1@(u_dT^RKh<$Q8(=vvlLnNsY_PAtLU(%ULZHXN|CK*l)&(h?lk2sjy z{(R+!U9Kn9e{nwH6xhc()yZ@_GT_E4_^4LyYU5HY+vnS2#C2xc*+Rn5dP`)nAB6mPhN6wUQ!Z5=!JX$`lZQCfTB>V$tWz{@(5LI0JD&?u8!;e zbccAbnoi?#+L4W>jMh6g% z5m5wfrQfZ@W^3%GEL5g?HRayXH01cFzFT%EyZ@6@1%#j8!!rzva~V_N^iVN@pGS(WU zgI}eCBJY=%`Ki99Y#vK!WJ2YP8+oJB@$pF_1zd5{&CVZm+~~25suC#G$R_}II!t_A z)*4-{%>4k6U#tkzos*oC;05lYV~I#%bR<;(Iq=2jissiYj#`zs=)WW=p$Vug#+xyb zqu={wC~8Zr?l;A^sJ+sD)^_V1B>Xt~(FST~XyD4EMeuo978lPJm-f5rC1cOtDz?L& zN_t*b;AS1TGCfXAr-jHw4k?Rkk?kG%hxK2vo$-_2TYZPbR%bFZ#efLmWL%#lmXQ$B zuc@>dtzmNq08BwLJo7zoG1bqTfv+=qva+(X?2;uz4FX*&L1Y%ez@2ND@`b@R1O5nHVkAnSL;?e%gcIa6*FT$30f{ku zj3JXwb+r+#UED_>-E#N~NLhSgTX4Dh5f!~ah2NPle#dx4st$&*e&o8`)LZwLnR=o) z4U0Ih%}^^ZD6rGWS4PQF+tWal#a>TX0D+e zC-UV4Z3BSdkU+jxLJOye=)n8onHr<-tbw=3y1dfuyzc@bVv?oOgwp$X!cgOI!rNc%z zot_lOI9%g{~n)bpp+_ef*gm?hy8Vfc6pgC^XOXW-j314 z;|+dVoVUsobi%%ctYSuWc8|@%+!q-FA8b6{Y%P>dJukX;aLGxyvnITJi51JO(UWBGbo{p}8*& z43Mn#w2zMRYRt7%fJWhLwuJKtlJ}G|W%1QABeV`2A3oje?iMq#pr~SU` zMXfJ3hd$TNy(u#qxWt?gJKWIMPRF=QGm~_VG%-5=h{#_y{ndBW)P7Gg2LwPzVq{a049UbBEVVvftfpD!5KaX7FZNSph{Sb4sj; zP>~M?*`_4DGcW F{SPym) => RequestExecutor.layer.pipe( @@ -44,6 +45,27 @@ const executor = (seen: Array) => { headers: { "content-type": "text/event-stream" } }, ) : input.respond(Uint8Array.from([1, 2, 3]), { headers: { "content-type": "audio/pcm" } }) + if (web.url.endsWith("/audio/transcriptions")) + return input.text.includes('name="stream"') + ? input.respond( + sseEvents( + { type: "transcript.text.delta", delta: "Hello" }, + { type: "transcript.text.delta", delta: " there." }, + { type: "transcript.text.done", text: "Hello there." }, + ), + { headers: { "content-type": "text/event-stream" } }, + ) + : input.respond(JSON.stringify({ text: "Hello there." }), { + headers: { "content-type": "application/json" }, + }) + if (web.url.endsWith("/v2/transcript")) + return input.respond(JSON.stringify({ id: "tr_1", status: "queued" }), { + headers: { "content-type": "application/json" }, + }) + if (web.url.endsWith("/v2/transcript/tr_1")) + return input.respond(JSON.stringify({ id: "tr_1", status: "completed", text: "Hello there." }), { + headers: { "content-type": "application/json" }, + }) if (web.url.endsWith("/text_to_video")) return input.respond(JSON.stringify({ id: "task_1" }), { headers: { "content-type": "application/json" } }) if (web.url.endsWith("/tasks/task_1")) { @@ -134,6 +156,32 @@ describe("AI promise client", () => { await ai.dispose() }) + test("generates, streams, and starts transcriptions over the same runtime", async () => { + const ai = AI.make({ layer: executor([]) }) + const audio = Media.url("https://audio.test/hello.mp3") + const bytes = Media.bytes(Uint8Array.from([0x49, 0x44, 0x33]), "audio/mpeg") + + expect( + (await ai.transcription.generate({ model: openai.transcription("gpt-4o-mini-transcribe"), audio: bytes })).text, + ).toBe("Hello there.") + + const deltas: Array = [] + for await (const event of ai.transcription.stream({ + model: openai.transcription("gpt-4o-mini-transcribe"), + audio: bytes, + })) + if (TranscriptionEvent.is.textDelta(event)) deltas.push(event.delta) + expect(deltas).toEqual(["Hello", " there."]) + + const model = AssemblyAI.configure({ apiKey: "test", baseURL: "https://assemblyai.test" }).transcription( + "universal-2", + ) + const generation = await ai.transcription.start({ model, audio }) + expect(generation.token).toEqual({ transcriptID: "tr_1" }) + expect((await generation.await({ poll: { interval: 10 } })).text).toBe("Hello there.") + await ai.dispose() + }) + test("generates and streams speech over the same runtime", async () => { const ai = AI.make({ layer: executor([]) }) const model = openai.speech("gpt-4o-mini-tts") diff --git a/packages/ai/test/provider/assemblyai-transcription.recorded.test.ts b/packages/ai/test/provider/assemblyai-transcription.recorded.test.ts new file mode 100644 index 00000000000..bfbc5767486 --- /dev/null +++ b/packages/ai/test/provider/assemblyai-transcription.recorded.test.ts @@ -0,0 +1,39 @@ +import { describe, expect } from "bun:test" +import { Effect } from "effect" +import { Transcription } from "../../src/index.js" +import { AssemblyAI } from "../../src/providers.js" +import { recordedTests } from "../recorded-test.js" +import { TRANSCRIPT, audio, audioRecording } from "./transcription-recording.js" +import { queuedPoll } from "./queued-recording.js" + +// Recorded against the EU region: the US endpoint was unreachable from the recording network. Same API and shapes. +const model = AssemblyAI.configure({ + apiKey: process.env.ASSEMBLYAI_API_KEY ?? "fixture", + baseURL: "https://api.eu.assemblyai.com", +}).transcription("universal-3-5-pro") + +const recorded = recordedTests({ + prefix: "assemblyai-transcription", + provider: "assemblyai", + protocol: "assemblyai-transcription", + requires: ["ASSEMBLYAI_API_KEY"], + options: audioRecording, +}) + +describe("AssemblyAI Transcription recorded", () => { + recorded.effect( + "uploads, submits, and polls a transcript", + () => + Effect.gen(function* () { + const generation = yield* Transcription.start({ model, audio: yield* audio, diarize: true }) + const response = yield* generation.await({ poll: queuedPoll }) + + expect(response.text).toMatch(TRANSCRIPT) + expect(response.words?.[0]).toMatchObject({ text: "Hello", speaker: "A" }) + expect(response.words?.every((word) => word.startSeconds < 5 && word.endSeconds < 5)).toBe(true) + expect(response.segments?.[0]?.speaker).toBe("A") + expect(response.usage).toEqual({ type: "seconds", seconds: expect.any(Number) }) + }), + { timeout: 15 * 60 * 1000 }, + ) +}) diff --git a/packages/ai/test/provider/deepgram-transcription.recorded.test.ts b/packages/ai/test/provider/deepgram-transcription.recorded.test.ts new file mode 100644 index 00000000000..080432b7a8d --- /dev/null +++ b/packages/ai/test/provider/deepgram-transcription.recorded.test.ts @@ -0,0 +1,47 @@ +import { describe, expect } from "bun:test" +import { Effect, Stream } from "effect" +import { Transcription } from "../../src/index.js" +import { Deepgram } from "../../src/providers.js" +import { recordedTests } from "../recorded-test.js" +import { TRANSCRIPT, audio, audioRecording, dialog } from "./transcription-recording.js" + +const model = Deepgram.configure({ apiKey: process.env.DEEPGRAM_API_KEY ?? "fixture" }).transcription("nova-3") + +const recorded = recordedTests({ + prefix: "deepgram-transcription", + provider: "deepgram", + protocol: "deepgram-transcription", + requires: ["DEEPGRAM_API_KEY"], + options: audioRecording, +}) + +describe("Deepgram Transcription recorded", () => { + recorded.effect("transcribes raw audio with speakers and word timestamps", () => + Effect.gen(function* () { + const request = Transcription.request({ model, audio: yield* audio, diarize: true, timestamps: "word" }) + const response = yield* Transcription.generate(request) + + expect(response.text).toMatch(TRANSCRIPT) + expect(response.words?.map((word) => word.text)).toEqual(["Hello", "from", "OpenCode."]) + expect(response.words?.every((word) => word.speaker === "0" && word.confidence !== undefined)).toBe(true) + expect(response.segments).toEqual([ + { text: response.text, startSeconds: expect.any(Number), endSeconds: expect.any(Number), speaker: "0" }, + ]) + expect(response.language).toBe("en") + expect(response.usage).toEqual({ type: "seconds", seconds: response.durationSeconds }) + + const events = Array.from(yield* Stream.runCollect(Transcription.stream(request))) + expect(events.map((event) => event.type)).toEqual(["finish"]) + }), + ) + + recorded.effect("splits utterances at speaker changes", () => + Effect.gen(function* () { + const response = yield* Transcription.generate({ model, audio: yield* dialog, diarize: true }) + + expect(response.segments?.map((segment) => segment.speaker)).toEqual(["0", "1"]) + expect(response.segments?.[0].text).toMatch(/release ship\?$/) + expect(response.segments?.[1].text).toMatch(/shipped this morning\.$/) + }), + ) +}) diff --git a/packages/ai/test/provider/fal-video.recorded.test.ts b/packages/ai/test/provider/fal-video.recorded.test.ts index 27a70aa738a..e6d42770580 100644 --- a/packages/ai/test/provider/fal-video.recorded.test.ts +++ b/packages/ai/test/provider/fal-video.recorded.test.ts @@ -3,7 +3,7 @@ import { Effect } from "effect" import { Video } from "../../src/index.js" import { Fal } from "../../src/providers.js" import { recordedTests } from "../recorded-test.js" -import { videoPoll } from "./video-recording.js" +import { queuedPoll } from "./queued-recording.js" const model = Fal.configure({ apiKey: process.env.FAL_KEY ?? "fixture", @@ -29,7 +29,7 @@ describe("fal Video recorded", () => { resolution: "720p", providerOptions: { duration: "4s" }, }, - { poll: videoPoll }, + { poll: queuedPoll }, ) expect(response.videos).toHaveLength(1) diff --git a/packages/ai/test/provider/google-transcription.recorded.test.ts b/packages/ai/test/provider/google-transcription.recorded.test.ts new file mode 100644 index 00000000000..c92cfdc52ad --- /dev/null +++ b/packages/ai/test/provider/google-transcription.recorded.test.ts @@ -0,0 +1,54 @@ +import { describe, expect } from "bun:test" +import { Effect, Stream } from "effect" +import { Transcription, TranscriptionEvent } from "../../src/index.js" +import { Google } from "../../src/providers.js" +import { recordedTests } from "../recorded-test.js" +import { TRANSCRIPT, audio, audioRecording, dialog } from "./transcription-recording.js" + +const model = Google.configure({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? "fixture" }).transcription( + "gemini-3.5-transcribe", +) + +const recorded = recordedTests({ + prefix: "google-transcription", + provider: "google", + protocol: "google-transcription", + requires: ["GOOGLE_GENERATIVE_AI_API_KEY"], + options: audioRecording, +}) + +describe("Google Transcription recorded", () => { + recorded.effect("transcribes with speakers and word timestamps", () => + Effect.gen(function* () { + const response = yield* Transcription.generate({ model, audio: yield* audio, diarize: true, language: "en-US" }) + + expect(response.text).toMatch(TRANSCRIPT) + expect(response.words?.length).toBeGreaterThan(2) + expect(response.words?.every((word) => word.endSeconds > word.startSeconds)).toBe(true) + expect(response.segments).toEqual([ + { + text: response.text, + startSeconds: response.words?.[0]?.startSeconds, + endSeconds: response.words?.at(-1)?.endSeconds, + speaker: expect.any(String), + }, + ]) + expect(response.usage?.type).toBe("tokens") + }), + ) + + recorded.effect("streams speaker turns", () => + Effect.gen(function* () { + const events = Array.from( + yield* Stream.runCollect(Transcription.stream({ model, audio: yield* dialog, diarize: true })), + ) + const finish = events.at(-1) + if (finish === undefined || !TranscriptionEvent.is.finish(finish)) throw new Error("Expected a finish event") + + expect(finish.text).toMatch(/release ship\? Yes, it shipped this morning\.$/) + const speakers = events.filter(TranscriptionEvent.is.segment).map((event) => event.segment.speaker) + expect(speakers).toHaveLength(2) + expect(new Set(speakers).size).toBe(2) + }), + ) +}) diff --git a/packages/ai/test/provider/google-video.recorded.test.ts b/packages/ai/test/provider/google-video.recorded.test.ts index db23091c2f7..b8daf170fb0 100644 --- a/packages/ai/test/provider/google-video.recorded.test.ts +++ b/packages/ai/test/provider/google-video.recorded.test.ts @@ -3,7 +3,7 @@ import { Effect } from "effect" import { Video } from "../../src/index.js" import { Google } from "../../src/providers.js" import { recordedTests } from "../recorded-test.js" -import { videoPoll } from "./video-recording.js" +import { queuedPoll } from "./queued-recording.js" const model = Google.configure({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? "fixture", @@ -28,7 +28,7 @@ describe("Google Veo recorded", () => { aspectRatio: "16:9", durationSeconds: 4, }, - { poll: videoPoll }, + { poll: queuedPoll }, ) expect(response.videos).toHaveLength(1) diff --git a/packages/ai/test/provider/openai-transcription.recorded.test.ts b/packages/ai/test/provider/openai-transcription.recorded.test.ts new file mode 100644 index 00000000000..e8ce460bf79 --- /dev/null +++ b/packages/ai/test/provider/openai-transcription.recorded.test.ts @@ -0,0 +1,86 @@ +import { describe, expect } from "bun:test" +import { Effect, Stream } from "effect" +import { Transcription, TranscriptionEvent } from "../../src/index.js" +import { OpenAI } from "../../src/providers.js" +import { recordedTests } from "../recorded-test.js" +import { TRANSCRIPT, audio, audioRecording } from "./transcription-recording.js" + +const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY ?? "fixture" }) + +const recorded = recordedTests({ + prefix: "openai-transcription", + provider: "openai", + protocol: "openai-transcription", + requires: ["OPENAI_API_KEY"], + options: audioRecording, +}) + +describe("OpenAI Transcription recorded", () => { + recorded.effect("transcribes with a language hint", () => + Effect.gen(function* () { + const response = yield* Transcription.generate({ + model: openai.transcription("gpt-transcribe"), + audio: yield* audio, + language: "en", + }) + + expect(response.text).toMatch(TRANSCRIPT) + expect(response.language).toBe("en") + expect(response.usage?.type).toBeDefined() + }), + ) + + recorded.effect("streams text deltas", () => + Effect.gen(function* () { + const events = Array.from( + yield* Stream.runCollect( + Transcription.stream({ model: openai.transcription("gpt-4o-mini-transcribe"), audio: yield* audio }), + ), + ) + const finish = events.at(-1) + if (finish === undefined || !TranscriptionEvent.is.finish(finish)) throw new Error("Expected a finish event") + + const deltas = events.filter(TranscriptionEvent.is.textDelta).map((event) => event.delta) + expect(deltas.length).toBeGreaterThan(1) + expect(deltas.join("")).toBe(finish.text) + expect(finish.text).toMatch(TRANSCRIPT) + expect(finish.usage).toMatchObject({ type: "tokens", input: expect.any(Number), output: expect.any(Number) }) + }), + ) + + recorded.effect("diarizes speakers into segments", () => + Effect.gen(function* () { + const response = yield* Transcription.generate({ + model: openai.transcription("gpt-4o-transcribe-diarize"), + audio: yield* audio, + diarize: true, + }) + + expect(response.text).toMatch(TRANSCRIPT) + expect(response.segments).toEqual([ + { + text: expect.stringMatching(TRANSCRIPT), + startSeconds: expect.any(Number), + endSeconds: expect.any(Number), + speaker: "A", + }, + ]) + expect(response.durationSeconds).toBeGreaterThan(1) + }), + ) + + recorded.effect("returns whisper word timestamps", () => + Effect.gen(function* () { + const response = yield* Transcription.generate({ + model: openai.transcription("whisper-1"), + audio: yield* audio, + timestamps: "word", + }) + + expect(response.words?.map((word) => word.text.toLowerCase())).toEqual(["hello", "from", "opencode"]) + expect(response.words?.every((word) => word.endSeconds > word.startSeconds)).toBe(true) + expect(response.language).toBe("english") + expect(response.usage).toEqual({ type: "seconds", seconds: expect.any(Number) }) + }), + ) +}) diff --git a/packages/ai/test/provider/video-recording.ts b/packages/ai/test/provider/queued-recording.ts similarity index 63% rename from packages/ai/test/provider/video-recording.ts rename to packages/ai/test/provider/queued-recording.ts index bbb95897e84..257ca06a67b 100644 --- a/packages/ai/test/provider/video-recording.ts +++ b/packages/ai/test/provider/queued-recording.ts @@ -1,10 +1,10 @@ import type { Poll } from "../../src/index.js" /** - * Video generations take minutes live, so record with a realistic interval. Replay consumes the recorded polls in + * Queued generations take minutes live, so record with a realistic interval. Replay consumes the recorded polls in * cassette order regardless of timing, so it runs them back-to-back instead of sleeping between each one. */ -export const videoPoll: Poll = { +export const queuedPoll: Poll = { interval: process.env.RECORD === "true" ? "10 seconds" : "10 millis", timeout: "15 minutes", } diff --git a/packages/ai/test/provider/runway-video.recorded.test.ts b/packages/ai/test/provider/runway-video.recorded.test.ts index 672119efc40..0bec3c7b66f 100644 --- a/packages/ai/test/provider/runway-video.recorded.test.ts +++ b/packages/ai/test/provider/runway-video.recorded.test.ts @@ -3,7 +3,7 @@ import { Effect } from "effect" import { Video } from "../../src/index.js" import { Runway } from "../../src/providers.js" import { recordedTests } from "../recorded-test.js" -import { videoPoll } from "./video-recording.js" +import { queuedPoll } from "./queued-recording.js" const model = Runway.configure({ apiKey: process.env.RUNWAYML_API_SECRET ?? "fixture", @@ -28,7 +28,7 @@ describe("Runway recorded", () => { aspectRatio: "1280:720", durationSeconds: 2, }, - { poll: videoPoll }, + { poll: queuedPoll }, ) expect(response.videos).toHaveLength(1) diff --git a/packages/ai/test/provider/transcription-recording.ts b/packages/ai/test/provider/transcription-recording.ts new file mode 100644 index 00000000000..d01b4aa0e22 --- /dev/null +++ b/packages/ai/test/provider/transcription-recording.ts @@ -0,0 +1,41 @@ +import type { HttpRecorder } from "@opencode/http-recorder" +import { Effect } from "effect" +import { Media } from "../../src/index.js" + +const fixture = (name: string) => + Effect.promise(() => Bun.file(new URL(`../fixtures/audio/${name}`, import.meta.url)).bytes()).pipe( + Effect.map((bytes) => Media.bytes(bytes, "audio/mpeg")), + ) + +/** "Hello from OpenCode." spoken by Deepgram Aura (`aura-2-thalia-en`, 32 kbps MP3). */ +export const audio = fixture("hello.mp3") + +/** "Did the release ship?" (`aura-2-thalia-en`), then "Yes, it shipped this morning." (`aura-2-orion-en`). */ +export const dialog = fixture("dialog.mp3") + +/** Providers disagree on "OpenCode" versus "Open Code" and on the final punctuation. */ +export const TRANSCRIPT = /^hello,? from open ?code[.!]?$/i + +/** + * Request bodies are snapshotted as text, so the audio would be stored as mangled UTF-8. Replace it with a marker and + * pin the random multipart boundary, which keeps the remaining form fields and query checked on replay. + */ +const redactAudio = (body: string) => { + const boundary = /^--(\S+)\r\n/.exec(body)?.[1] + if (boundary === undefined) return body.includes("\uFFFD") ? "[audio]" : body + return body + .replaceAll(boundary, "BOUNDARY") + .replace(/(filename="[^"]*"\r\nContent-Type: [^\r]*\r\n\r\n)[\s\S]*?(\r\n--BOUNDARY)/, "$1[audio]$2") +} + +const contentType = (snapshot: HttpRecorder.RequestSnapshot) => + snapshot.headers["content-type"]?.replace(/boundary=.*/, "") + +export const audioRecording: HttpRecorder.RecorderOptions = { + redact: { body: redactAudio }, + match: (incoming, recorded) => + incoming.method === recorded.method && + incoming.url === recorded.url && + contentType(incoming) === contentType(recorded) && + incoming.body === recorded.body, +} diff --git a/packages/ai/test/provider/xai-video.recorded.test.ts b/packages/ai/test/provider/xai-video.recorded.test.ts index b7dfd3791f2..96b25de013b 100644 --- a/packages/ai/test/provider/xai-video.recorded.test.ts +++ b/packages/ai/test/provider/xai-video.recorded.test.ts @@ -3,7 +3,7 @@ import { Effect, Stream } from "effect" import { Video } from "../../src/index.js" import { XAI } from "../../src/providers.js" import { recordedTests } from "../recorded-test.js" -import { videoPoll } from "./video-recording.js" +import { queuedPoll } from "./queued-recording.js" const model = XAI.configure({ apiKey: process.env.XAI_API_KEY ?? "fixture", @@ -28,7 +28,7 @@ describe("xAI Video recorded", () => { resolution: "480p", durationSeconds: 2, }) - const observed = yield* generation.events({ poll: videoPoll }).pipe(Stream.runCollect) + const observed = yield* generation.events({ poll: queuedPoll }).pipe(Stream.runCollect) const progress = Array.from(observed).flatMap((event) => event.type === "generation-progress" && event.progress !== undefined ? [event.progress] : [], ) diff --git a/packages/ai/test/recorded-test.ts b/packages/ai/test/recorded-test.ts index 7998c03c5dd..2c507f5a0dd 100644 --- a/packages/ai/test/recorded-test.ts +++ b/packages/ai/test/recorded-test.ts @@ -8,11 +8,13 @@ import { LLMClient, RequestExecutor } from "../src/route.js" import { ImageClient } from "../src/image-client.js" import { VideoClient } from "../src/video-client.js" import { SpeechClient } from "../src/speech-client.js" +import { TranscriptionClient } from "../src/transcription-client.js" import { EvaluationClient } from "../src/experimental/evaluation-client.js" import type { Service as EvaluationClientService } from "../src/experimental/evaluation-client.js" import type { Service as ImageClientService } from "../src/image-client.js" import type { Service as VideoClientService } from "../src/video-client.js" import type { Service as SpeechClientService } from "../src/speech-client.js" +import type { Service as TranscriptionClientService } from "../src/transcription-client.js" import type { Service as LLMClientService } from "../src/route/client.js" import type { Service as RequestExecutorService } from "../src/route/executor.js" import { @@ -30,6 +32,7 @@ type RecordedEnv = | ImageClientService | VideoClientService | SpeechClientService + | TranscriptionClientService | EvaluationClientService | Socket.WebSocketConstructor @@ -107,6 +110,7 @@ export const recordedTests = (options: RecordedTestsOptions) => ImageClient.layer.pipe(Layer.provide(requestExecutor)), VideoClient.layer.pipe(Layer.provide(requestExecutor)), SpeechClient.layer.pipe(Layer.provide(requestExecutor)), + TranscriptionClient.layer.pipe(Layer.provide(requestExecutor)), EvaluationClient.layer.pipe(Layer.provide(requestExecutor)), webSocket, ) diff --git a/packages/ai/test/transcription.test.ts b/packages/ai/test/transcription.test.ts new file mode 100644 index 00000000000..aa469a141b2 --- /dev/null +++ b/packages/ai/test/transcription.test.ts @@ -0,0 +1,174 @@ +import { describe, expect } from "bun:test" +import { Effect, Fiber, Layer, Stream } from "effect" +import * as TestClock from "effect/testing/TestClock" +import { HttpClientRequest } from "effect/unstable/http" +import { Media, Transcription, TranscriptionClient } from "../src/index.js" +import { AssemblyAI, Deepgram, Google, OpenAI } from "../src/providers.js" +import { it } from "./lib/effect.js" +import { dynamicResponse } from "./lib/http.js" + +const layer = (handler: Parameters[0]) => + TranscriptionClient.layer.pipe(Layer.provideMerge(dynamicResponse(handler))) + +const audio = Media.bytes(Uint8Array.from([0x49, 0x44, 0x33, 1, 2, 3]), "audio/mpeg") +const openai = OpenAI.configure({ apiKey: "test", baseURL: "https://openai.test/v1" }) +const deepgram = Deepgram.configure({ apiKey: "test", baseURL: "https://deepgram.test" }).transcription("nova-3") +const google = Google.configure({ apiKey: "test", baseURL: "https://google.test/v1beta" }).transcription( + "gemini-3.5-transcribe", +) +const assemblyai = AssemblyAI.configure({ apiKey: "aai-key", baseURL: "https://assemblyai.test" }).transcription( + "universal-3-5-pro", +) + +describe("Transcription", () => { + it.effect("rejects what a route cannot honor before sending anything", () => + Effect.gen(function* () { + const errors = yield* Effect.all( + [ + Stream.runCollect(Transcription.stream({ model: openai.transcription("whisper-1"), audio })), + Transcription.generate({ model: openai.transcription("gpt-4o-mini-transcribe"), audio, diarize: true }), + Transcription.generate({ model: openai.transcription("gpt-4o-mini-transcribe"), audio, timestamps: "word" }), + Transcription.generate({ model: openai.transcription("gpt-4o-transcribe-diarize"), audio, prompt: "Names" }), + Transcription.generate({ model: deepgram, audio, prompt: "OpenCode" }), + Transcription.generate({ model: google, audio, speakers: 2 }), + Transcription.start({ model: deepgram, audio }), + Transcription.generate({ model: deepgram, audio, http: { body: { callback: "https://hook.test" } } }), + Transcription.generate({ + model: openai.transcription("gpt-transcribe"), + audio: Media.url("https://a.test/x.mp3"), + }), + Transcription.start({ + model: assemblyai, + audio: Media.ref("file_1", { provider: "openai", mediaType: "audio/mpeg" }), + }), + Transcription.generate({ + model: openai.transcription("gpt-transcribe"), + audio: Media.bytes(Uint8Array.from([1, 2, 3]), "audio/x-unknown"), + }), + Transcription.generate({ + model: Google.configure({ apiKey: "test" }).transcription("gemini-3.6-flash"), + audio, + }), + ].map((effect) => Effect.flip(effect)), + ) + expect(errors.map((error) => [error.reason._tag, "operation" in error.reason && error.reason.operation])).toEqual( + [ + ["UnsupportedOperation", "media.stream"], + ["UnsupportedOperation", "media.diarize"], + ["UnsupportedOperation", "media.timestamps"], + ["UnsupportedOperation", "media.prompt"], + ["UnsupportedOperation", "media.prompt"], + ["UnsupportedOperation", "media.speakers"], + ["UnsupportedOperation", "transcription.start"], + ["InvalidRequest", false], + ["InvalidRequest", false], + ["InvalidRequest", false], + ["InvalidRequest", false], + ["UnsupportedOperation", "transcription.model"], + ], + ) + }).pipe(Effect.provide(layer(() => Effect.die("an unsupported request reached the network")))), + ) + + it.effect( + "uploads inline audio to AssemblyAI, resumes polling from a persisted token, and surfaces failed transcripts", + () => + Effect.gen(function* () { + const calls: Array<{ + readonly method: string + readonly url: string + readonly auth: string | null + readonly body: string + }> = [] + const json = (value: unknown) => ({ + body: JSON.stringify(value), + init: { headers: { "content-type": "application/json" } }, + }) + const polls = () => calls.filter((call) => call.url.endsWith("/tr_1")).length + const failed = JSON.stringify({ id: "tr_2", status: "error", error: "Audio file could not be decoded" }) + const program = Effect.gen(function* () { + const started = yield* Transcription.start({ model: assemblyai, audio, diarize: true, speakers: 2 }) + expect(started.status).toBe("queued") + const resumed = yield* Transcription.resume(assemblyai, JSON.parse(JSON.stringify(started.token))) + const response = yield* resumed.await({ poll: { interval: "1 second" } }) + const failure = yield* Transcription.resume(assemblyai, { transcriptID: "tr_2" }).pipe( + Effect.flatMap((generation) => generation.await()), + Effect.flip, + ) + return { response, failure } + }).pipe( + Effect.provide( + layer((input) => + Effect.gen(function* () { + const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie) + calls.push({ + method: web.method, + url: web.url, + auth: web.headers.get("authorization"), + body: input.text, + }) + if (web.url.endsWith("/tr_2")) + return input.respond(failed, { headers: { "content-type": "application/json" } }) + const reply = web.url.endsWith("/v2/upload") + ? json({ upload_url: "https://cdn.assemblyai.test/upload/1" }) + : web.method === "POST" + ? json({ id: "tr_1", status: "queued" }) + : polls() < 3 + ? json({ id: "tr_1", status: "processing" }) + : json({ + id: "tr_1", + status: "completed", + text: "Hello there.", + words: [ + { text: "Hello", start: 250, end: 700, confidence: 0.9, speaker: "A" }, + { text: "there.", start: 700, end: 1250, confidence: 0.8, speaker: "B" }, + ], + utterances: [ + { text: "Hello", start: 250, end: 700, speaker: "A" }, + { text: "there.", start: 700, end: 1250, speaker: "B" }, + ], + language_code: "en_us", + audio_duration: 2, + }) + return input.respond(reply.body, reply.init) + }), + ), + ), + ) + + const fiber = yield* Effect.forkChild(program) + yield* TestClock.adjust("5 seconds") + const { response, failure } = yield* Fiber.join(fiber) + + expect(calls.slice(0, 2).map((call) => [call.method, call.url, call.auth])).toEqual([ + ["POST", "https://assemblyai.test/v2/upload", "aai-key"], + ["POST", "https://assemblyai.test/v2/transcript", "aai-key"], + ]) + expect(JSON.parse(calls[1].body)).toEqual({ + audio_url: "https://cdn.assemblyai.test/upload/1", + speech_models: ["universal-3-5-pro"], + language_detection: true, + speaker_labels: true, + speakers_expected: 2, + }) + expect(calls[2].url).toBe("https://assemblyai.test/v2/transcript/tr_1") + expect(response.segments).toEqual([ + { text: "Hello", startSeconds: 0.25, endSeconds: 0.7, speaker: "A" }, + { text: "there.", startSeconds: 0.7, endSeconds: 1.25, speaker: "B" }, + ]) + expect(response.words?.[1]).toEqual({ + text: "there.", + startSeconds: 0.7, + endSeconds: 1.25, + speaker: "B", + confidence: 0.8, + }) + expect(response).toMatchObject({ + text: "Hello there.", + language: "en_us", + usage: { type: "seconds", seconds: 2 }, + }) + expect(failure.reason).toMatchObject({ _tag: "ProviderInternal", body: failed }) + }), + ) +}) diff --git a/packages/ai/test/transcription.types.ts b/packages/ai/test/transcription.types.ts new file mode 100644 index 00000000000..e14ca86624e --- /dev/null +++ b/packages/ai/test/transcription.types.ts @@ -0,0 +1,34 @@ +import type { Stream } from "effect" +import { Media, Transcription, type TranscriptionEvent } from "../src/index.js" +import { AssemblyAI, Deepgram, OpenAI } from "../src/providers.js" + +type StreamItem = T extends Stream.Stream ? A : never +type Equal = [A, B] extends [B, A] ? true : false +type Assert = T + +const audio = Media.url("https://example.com/call.mp3") +const deepgram = Deepgram.configure({ apiKey: "test" }).transcription("nova-3") + +Transcription.generate({ + model: deepgram, + audio, + timestamps: "word", + providerOptions: { keyterm: ["OpenCode"], diarize_model: "future-model", futureOption: true }, +}) +Transcription.start({ model: AssemblyAI.configure({ apiKey: "test" }).transcription("universal-3-5-pro"), audio }) + +// @ts-expect-error Known provider options are inferred from the selected model. +Transcription.generate({ model: deepgram, audio, providerOptions: { keyterm: "OpenCode" } }) +// @ts-expect-error Timestamp granularity is a closed set. +Transcription.generate({ model: deepgram, audio, timestamps: "character" }) +// @ts-expect-error Audio is required. +Transcription.request({ model: deepgram }) +// @ts-expect-error Speech models cannot be used for transcription requests. +Transcription.request({ model: OpenAI.configure({ apiKey: "test" }).speech("gpt-4o-mini-tts"), audio }) + +const streamed = Transcription.stream({ + model: OpenAI.configure({ apiKey: "test" }).transcription("gpt-transcribe"), + audio, +}) +type StreamEvent = Assert, TranscriptionEvent>> +void (true satisfies StreamEvent)