From 92d2b1700ffa5d852e7fb7ad2543fd5eebff1fa4 Mon Sep 17 00:00:00 2001 From: Shoubhit Dash Date: Fri, 25 Sep 2026 03:15:45 +0530 Subject: [PATCH] refactor(ai): one media client shape and route types erased over options (#51226) --- packages/ai/AGENTS.md | 2 +- packages/ai/docs/media-design.md | 6 +- packages/ai/src/image-client.ts | 101 ++++-------------------- packages/ai/src/image.ts | 79 ++++-------------- packages/ai/src/media-client.ts | 77 ++++++++++++++++++ packages/ai/src/media-model.ts | 72 ++++++++++------- packages/ai/src/promise.ts | 53 +++++-------- packages/ai/src/route/media.ts | 61 +------------- packages/ai/src/speech-client.ts | 66 ++++++---------- packages/ai/src/speech.ts | 72 ++++++++--------- packages/ai/src/transcription-client.ts | 93 ++-------------------- packages/ai/src/transcription.ts | 97 +++++------------------ packages/ai/src/video-client.ts | 100 ++++------------------- packages/ai/src/video.ts | 78 +++++++++--------- packages/ai/test/exports.test.ts | 13 +++ packages/ai/test/image.types.ts | 6 +- packages/ai/test/promise.test.ts | 12 ++- packages/ai/test/speech.types.ts | 4 +- packages/ai/test/transcription.types.ts | 12 ++- packages/ai/test/video.types.ts | 6 +- 20 files changed, 356 insertions(+), 654 deletions(-) create mode 100644 packages/ai/src/media-client.ts diff --git a/packages/ai/AGENTS.md b/packages/ai/AGENTS.md index 80a77993ab1..c8a18fe0414 100644 --- a/packages/ai/AGENTS.md +++ b/packages/ai/AGENTS.md @@ -102,7 +102,7 @@ Media does not fit the SSE-frames-to-event-state-machine LLM route. `MediaRoute. `MediaProtocol.stream` is the incremental kind every speech route uses, with the same discipline as LLM protocols. `MediaRoute.stream` submits the caller's request as `MediaProtocol.Addressed` (`{ ...request, mode }`, `mode: "generate" | "stream"`), so one provider stays one protocol: `body.from`, the endpoint path, and `frames` read `request.mode` to pick the body, path, and framing. `frames(bytes, context)` returns frames — `Framing.sse`, `Framing.lines`, `Framing.document` (a single-document response shaped like a streamed record), or the raw `bytes` for chunked audio. `initial()` is fresh per-response parser state; `step` folds each frame into it and emits modality events; `finish(state, context)` runs once after the last frame with the request, body, and observed `http` (header-only usage lives there) and emits exactly one terminal event or fails with `route.incomplete()`. Keep parser state to real accumulators and derive anything the request or body determines in `finish`. `generate` runs the same stream and folds it with the modality's `collect`. Request-derived URL parameters go on the body's `query` (array values repeat the parameter), applied before route and caller `http.query`. Decode frames with `route.decodeFrame` and raise stream-time failures with `route.frameError` (the frame stays on `reason.body`); protocols never thread HTTP context, because the route fills `reason.http` on stream errors that lack it. Speech protocols share `protocols/utils/speech-stream.ts` for deltas, timestamps, voice ids, PCM and container descriptions, and the terminal asset. -Transcription uses all three kinds (OpenAI and Gemini stream, Deepgram is inline, AssemblyAI is queued): every route carries its `kind` and `TranscriptionClient` dispatches on it. `ImageRoute` is the same union; both clients dispatch through `MediaRoute.dispatch` and models compose through `composeAnyRoute`, and fal queue protocols come from `protocols/utils/fal-queue.ts`. Bodies are `json`, `multipart`, or `binary` (a raw upload), and a queued protocol that must upload media before submitting implements `start.prepare` (`MediaProtocol.Prepare`; AssemblyAI `/v2/upload`). +Every modality route is the inline | stream | queued union (transcription uses all three: OpenAI and Gemini stream, Deepgram is inline, AssemblyAI is queued), every client is `MediaClient.make(Service, { modality, responseEvents })` (`src/media-client.ts`), which dispatches on the route's `kind`, and every model composes through `composeRoute`. fal queue protocols come from `protocols/utils/fal-queue.ts`, bodies are `json`, `multipart`, or `binary` (a raw upload), and a queued protocol that must upload media before submitting implements `start.prepare` (`MediaProtocol.Prepare`; AssemblyAI `/v2/upload`). ### URL Construction diff --git a/packages/ai/docs/media-design.md b/packages/ai/docs/media-design.md index 689c0607222..ceca1a589b7 100644 --- a/packages/ai/docs/media-design.md +++ b/packages/ai/docs/media-design.md @@ -140,7 +140,7 @@ portability matrix. Editing is not a separate function; `images`/`mask` on the request select the edit path in the route (OpenAI `/images/edits`, Gemini multimodal parts, xAI `/images/edits`). Routes that cannot honor `mask` fail with `Unsupported`. -`ImageRoute` is the inline | stream | queued union, dispatched on `route.kind`. `Image.stream` on a streaming route emits `image-partial` previews before each `image`; on a queued route it emits `generation-queued` / `generation-progress` observations, then the result's `image` and `finish` events. +`ImageRoute` is the inline | stream | queued union, dispatched on `route.kind`, like every modality route. `Image.stream` on a streaming route emits `image-partial` previews before each `image`; on a queued route it emits `generation-queued` / `generation-progress` observations, then the result's `image` and `finish` events. #### Video @@ -215,7 +215,7 @@ const request = Speech.request({ }) const response = yield* Speech.generate(request) // SpeechResponse: audio: Media.Asset, timestamps?, usage?, providerMetadata? -yield* Speech.stream(request) // Stream: audio-delta { chunk } | timestamps { items } | finish { audio, usage? } +yield* Speech.stream(request) // Stream: generation-queued | generation-progress | audio-delta { chunk } | timestamps { items } | finish { audio, usage? } ``` Execution is `MediaProtocol.stream` for every provider: one request whose body is framed and folded by a `step` @@ -420,7 +420,7 @@ Existing facades gain per-modality selectors; the modality routes each facade pr New facades follow the existing one-file-per-provider rule. The facade selector is the public path for media models; modality-specific package entrypoints (for example `@opencode/ai/providers/openai/images`) are deferred until Core has a modality-aware model resolver. -`ImageModel` gives typed `providerOptions` per model; `VideoModel`, `SpeechModel`, and `TranscriptionModel` follow the same generic. They share an internal `MediaModel` base class (ids, route, `http` overlays) that is not part of the public exports; `Generation` and the promise client work with the concrete modality models. +`ImageModel` gives typed `providerOptions` per model; `VideoModel`, `SpeechModel`, and `TranscriptionModel` follow the same generic. As with `LanguageModel`, the route type does not carry `Options`, so `ImageModel` is an `ImageModel` and client methods take plain `ImageRequestFor`. They share an internal `MediaModel` base class (ids, route, `http` overlays) that is not part of the public exports; `Generation` and the promise client work with the concrete modality models. ### Routes and protocols diff --git a/packages/ai/src/image-client.ts b/packages/ai/src/image-client.ts index 6b8453254d2..070c0129709 100644 --- a/packages/ai/src/image-client.ts +++ b/packages/ai/src/image-client.ts @@ -1,99 +1,30 @@ -import { Context, Effect, Layer, Stream } from "effect" -import type { AwaitOptions, Generation } from "./generation.js" -import { RequestExecutor } from "./route/executor.js" -import { MediaRoute } from "./route/media.js" -import type { AIError } from "./schema/index.js" +import { Context } from "effect" +import { MediaClient } from "./media-client.js" import { - responseEvents, + ImageOutputEvent, + ImageFinishEvent, type ImageEvent, - type ImageModel, - type ImageOptions, type ImageRequestFor, type ImageResponse, } from "./image.js" -export interface Interface { - readonly generate: ( - request: ImageRequestFor, - options?: AwaitOptions, - ) => Effect.Effect - readonly stream: ( - request: ImageRequestFor, - options?: AwaitOptions, - ) => Stream.Stream - readonly start: ( - request: ImageRequestFor, - ) => Effect.Effect, AIError> - readonly resume: ( - model: ImageModel, - token: unknown, - ) => Effect.Effect, AIError> -} +export type Interface = MediaClient.Interface export class ImageClientService extends Context.Service()("@opencode/ImageClient") {} export const Service = ImageClientService export type Service = ImageClientService -export const generate = ( - request: ImageRequestFor, - options?: AwaitOptions, -): Effect.Effect => - Effect.gen(function* () { - const client = yield* Service - return yield* client.generate(request, options) - }) - -export const stream = ( - request: ImageRequestFor, - options?: AwaitOptions, -): Stream.Stream => - Stream.unwrap( - Effect.gen(function* () { - const client = yield* Service - return client.stream(request, options) - }), - ) - -export const start = ( - request: ImageRequestFor, -): Effect.Effect, AIError, Service> => - Effect.gen(function* () { - const client = yield* Service - return yield* client.start(request) - }) - -export const resume = ( - model: ImageModel, - token: unknown, -): Effect.Effect, AIError, Service> => - Effect.gen(function* () { - const client = yield* Service - return yield* client.resume(model, token) - }) - -export const layer: Layer.Layer = Layer.effect( - Service, - Effect.gen(function* () { - const executor = yield* RequestExecutor.Service - const dispatch = MediaRoute.dispatch({ - modality: "image", - execute: executor.execute, - responseEvents, - }) - return Service.of({ - start: (request) => dispatch.start(request.model.route, request), - resume: (model, token) => dispatch.resume(model.route, model, token), - generate: (request, options) => dispatch.generate(request.model.route, request, options), - stream: (request, options) => dispatch.stream(request.model.route, request, options), - }) - }), -) - export const ImageClient = { Service, - layer, - generate, - stream, - start, - resume, + ...MediaClient.make(Service, { + modality: "image", + responseEvents: (response: ImageResponse) => [ + ...response.images.map((image, index) => ImageOutputEvent.make({ index, image })), + ImageFinishEvent.make({ + usage: response.usage, + notices: response.notices, + providerMetadata: response.providerMetadata, + }), + ], + }), } as const diff --git a/packages/ai/src/image.ts b/packages/ai/src/image.ts index 45f0127779e..b25efee811f 100644 --- a/packages/ai/src/image.ts +++ b/packages/ai/src/image.ts @@ -1,9 +1,8 @@ import { Effect, Schema, Stream } from "effect" import { Generation, ProgressEvent, QueuedEvent, type AwaitOptions } from "./generation.js" import { Media } from "./media.js" -import { MediaModel, composeAnyRoute, tryRequest } from "./media-model.js" +import { MediaModel, composeRoute, tryRequest } from "./media-model.js" import { MediaRoute } from "./route/media.js" -import type { MediaProtocol } from "./route/media-protocol.js" import { AIError, HttpOptions, MediaUsage, ProviderMetadata, type OpenString } from "./schema/index.js" import { ImageClient, Service } from "./image-client.js" @@ -11,75 +10,39 @@ import { ImageClient, Service } from "./image-client.js" // Model // --------------------------------------------------------------------------- -export type ImageOptions = Record +export type ImageOptions = MediaModel.Options -export type ImageRoute = MediaRoute.AnyRoute< - ImageRequestFor, - ImageEvent, - ImageResponse -> +export type ImageRoute = MediaRoute.AnyRoute -export class ImageModel extends MediaModel, Options> { +export class ImageModel extends MediaModel { declare protected readonly _ImageModel: void - static make(input: MediaModel.Input>) { - return new ImageModel(input) - } - /** The number of type arguments selects the kind: ``, ``, or ``. */ static fromRoute( - route: ImageModel.InlineRouteInput, + route: MediaModel.InlineRouteInput, ImageResponse>, input: MediaRoute.ModelInput, ): ImageModel static fromRoute( - route: ImageModel.StreamRouteInput, + route: MediaModel.StreamRouteInput, ImageEvent, Frame, State>, input: MediaRoute.ModelInput, ): ImageModel static fromRoute( - route: ImageModel.QueuedRouteInput, + route: MediaModel.QueuedRouteInput, ImageResponse, Token>, input: MediaRoute.ModelInput, ): ImageModel static fromRoute( - route: ImageModel.RouteInput, + route: MediaModel.AnyRouteInput, ImageEvent, ImageResponse, Frame, State, Token>, input: MediaRoute.ModelInput, ) { return new ImageModel({ id: input.id, provider: route.protocol.provider, http: input.http, - route: composeAnyRoute(route, input, collectResponse), + route: composeRoute(route, input, collectResponse) as ImageRoute, }) } } -export namespace ImageModel { - export type InlineRouteInput = MediaModel.RouteInput< - ImageRequestFor, - MediaProtocol.Inline, ImageResponse> - > - - export type StreamRouteInput< - Options extends ImageOptions = ImageOptions, - Frame = unknown, - State = unknown, - > = MediaModel.RouteInput< - MediaProtocol.Addressed>, - MediaProtocol.Streamed, ImageEvent, Frame, State> - > - - export type QueuedRouteInput = MediaModel.RouteInput< - ImageRequestFor, - MediaProtocol.Queued, ImageResponse, Token> - > - - export type RouteInput< - Options extends ImageOptions = ImageOptions, - Frame = unknown, - State = unknown, - Token = unknown, - > = MediaModel.AnyRouteInput, ImageEvent, ImageResponse, Frame, State, Token> -} - export const ImageModelSchema = Schema.declare((value): value is ImageModel => value instanceof ImageModel, { expected: "Image.Model", }) @@ -190,15 +153,6 @@ export const ImageEvent = Object.assign(imageEventTagged, { }) export type ImageEvent = Schema.Schema.Type -export const responseEvents = (response: ImageResponse): ReadonlyArray => [ - ...response.images.map((image, index) => ImageOutputEvent.make({ index, image })), - ImageFinishEvent.make({ - usage: response.usage, - notices: response.notices, - providerMetadata: response.providerMetadata, - }), -] - const collectResponse = (events: ReadonlyArray): Effect.Effect => { const finish = events.find(ImageEvent.is.finish) // Every image protocol's `finish` emits the terminal event or fails, so a completed stream always has one. @@ -232,36 +186,31 @@ export function request(input: ImageRequest | ImageRequestInput) { const requestEffect = (input: ImageRequest | ImageRequestInput) => tryRequest(() => request(input)) export function generate( - input: ImageRequestInput, + input: ImageRequest | ImageRequestInput, options?: AwaitOptions, ): Effect.Effect -export function generate(input: ImageRequest, options?: AwaitOptions): Effect.Effect export function generate(input: ImageRequest | ImageRequestInput, options?: AwaitOptions) { return requestEffect(input).pipe(Effect.flatMap((request) => ImageClient.generate(request, options))) } export function stream( - input: ImageRequestInput, + input: ImageRequest | ImageRequestInput, options?: AwaitOptions, ): Stream.Stream -export function stream(input: ImageRequest, options?: AwaitOptions): Stream.Stream export function stream(input: ImageRequest | ImageRequestInput, options?: AwaitOptions) { return Stream.unwrap(requestEffect(input).pipe(Effect.map((request) => ImageClient.stream(request, options)))) } /** Inline and streaming routes fail with `UnsupportedOperation`. */ export function start( - input: ImageRequestInput, + input: ImageRequest | ImageRequestInput, ): Effect.Effect, AIError, Service> -export function start(input: ImageRequest): Effect.Effect, AIError, Service> export function start(input: ImageRequest | ImageRequestInput) { return requestEffect(input).pipe(Effect.flatMap((request) => ImageClient.start(request))) } -export const resume = ( - model: ImageModel, - token: unknown, -): Effect.Effect, AIError, Service> => ImageClient.resume(model, token) +export const resume = (model: ImageModel, token: unknown): Effect.Effect, AIError, Service> => + ImageClient.resume(model, token) export const Image = { request, diff --git a/packages/ai/src/media-client.ts b/packages/ai/src/media-client.ts new file mode 100644 index 00000000000..01867a2f62f --- /dev/null +++ b/packages/ai/src/media-client.ts @@ -0,0 +1,77 @@ +import { type Context, Effect, Layer, Stream } from "effect" +import { resultEvents, type AwaitOptions, type Generation, type Observation } from "./generation.js" +import { RequestExecutor } from "./route/executor.js" +import type { MediaRoute } from "./route/media.js" +import { AIError, UnsupportedOperationError } from "./schema/index.js" + +/** A media request whose model carries the route that executes it. */ +export interface RoutedRequest extends MediaRoute.MediaRequest { + readonly model: MediaRoute.MediaRequest["model"] & { readonly route: MediaRoute.AnyRoute } +} + +/** `start` and `resume` fail with `UnsupportedOperation` on inline and stream routes. */ +export interface Interface, Event, Response> { + readonly generate: (request: Req, options?: AwaitOptions) => Effect.Effect + readonly stream: (request: Req, options?: AwaitOptions) => Stream.Stream + readonly start: (request: Req) => Effect.Effect, AIError> + readonly resume: (model: Req["model"], token: unknown) => Effect.Effect, AIError> +} + +/** One modality's layer and service accessors, dispatching each request on its route's `kind`. */ +export const make = , Event, Response>( + service: Context.Service>, + input: { + readonly modality: string + /** A completed response expanded into the streaming event shape. */ + readonly responseEvents: (response: Response) => ReadonlyArray + }, +) => ({ + layer: Layer.effect( + service, + Effect.gen(function* () { + const executor = yield* RequestExecutor.Service + const notQueued = (route: MediaRoute.AnyRoute, operation: string) => + new AIError({ + reason: new UnsupportedOperationError({ + operation: `${input.modality}.${operation}`, + provider: route.provider, + route: route.id, + message: `${route.provider}/${route.id} is not a queued route; use generate or stream`, + }), + }) + const start = (request: Req) => { + const route = request.model.route + if (route.kind !== "queued") return Effect.fail(notQueued(route, "start")) + return route.start(request, executor.execute) + } + return service.of({ + start, + resume: (model, token) => { + if (model.route.kind !== "queued") return Effect.fail(notQueued(model.route, "resume")) + return model.route.resume(model, token, executor.execute) + }, + generate: (request, options) => { + const route = request.model.route + if (route.kind !== "queued") return route.generate(request, executor.execute) + return start(request).pipe(Effect.flatMap((generation) => generation.await(options))) + }, + stream: (request, options) => { + const route = request.model.route + if (route.kind === "stream") return route.stream(request, executor.execute) + if (route.kind === "queued") + return Stream.unwrap( + start(request).pipe(Effect.map((generation) => resultEvents(generation, input.responseEvents, options))), + ) + return Stream.fromIterableEffect(Effect.map(route.generate(request, executor.execute), input.responseEvents)) + }, + }) + }), + ), + generate: (request: Req, options?: AwaitOptions) => service.use((client) => client.generate(request, options)), + stream: (request: Req, options?: AwaitOptions) => + Stream.unwrap(service.useSync((client) => client.stream(request, options))), + start: (request: Req) => service.use((client) => client.start(request)), + resume: (model: Req["model"], token: unknown) => service.use((client) => client.resume(model, token)), +}) + +export * as MediaClient from "./media-client.js" diff --git a/packages/ai/src/media-model.ts b/packages/ai/src/media-model.ts index 1acb43ac4bc..6e453ab7ad3 100644 --- a/packages/ai/src/media-model.ts +++ b/packages/ai/src/media-model.ts @@ -6,11 +6,13 @@ import { AIError, HttpOptions, InvalidRequestError, ModelID, ProviderID } from " /** * What every media model carries: ids, the configured route, and deployment `http` overlays. Modality classes - * (`ImageModel`, `VideoModel`, `SpeechModel`) extend it with their route type and a nominal marker so one cannot stand - * in for the other in requests. + * (`ImageModel`, `VideoModel`, `SpeechModel`, `TranscriptionModel`) extend it with their route type and a nominal + * marker so one cannot stand in for the other in requests. */ export class MediaModel { - declare protected readonly _Options: (options: Options) => Options + // As with `LanguageModel`, the route type is erased over `Options`; `fromRoute` and the constructor trust that the + // route accepts every request this model's `Options` admit. + declare protected readonly _Options: Options readonly id: ModelID readonly provider: ProviderID readonly route: Route @@ -25,6 +27,8 @@ export class MediaModel { } export namespace MediaModel { + export type Options = Record + export interface Input { readonly id: string | ModelID readonly provider: string | ProviderID @@ -41,48 +45,56 @@ export namespace MediaModel { readonly headers?: Record } + export type InlineRouteInput = RouteInput< + Request, + MediaProtocol.Inline + > + + export type StreamRouteInput = RouteInput< + MediaProtocol.Addressed, + MediaProtocol.Streamed + > + + export type QueuedRouteInput = RouteInput< + Request, + MediaProtocol.Queued + > + export type AnyRouteInput = - | RouteInput> - | RouteInput, MediaProtocol.Streamed> - | RouteInput> + | InlineRouteInput + | StreamRouteInput + | QueuedRouteInput } /** Compose a protocol route input with one deployment through `MediaRoute.inline`, `queued`, or `stream`. */ -export const composeRoute = ( - compose: (input: MediaRoute.Composition & { readonly protocol: Protocol }) => Route, - route: MediaModel.RouteInput, - input: MediaRoute.ModelInput, -): Route => - compose({ - protocol: route.protocol, - endpoint: Endpoint.path(route.path, { baseURL: input.baseURL ?? route.baseURL }), - auth: input.auth, - headers: - route.headers === undefined && input.headers === undefined ? undefined : { ...route.headers, ...input.headers }, - }) - -export const composeAnyRoute = ( +export const composeRoute = ( route: MediaModel.AnyRouteInput, input: MediaRoute.ModelInput, collect: (events: ReadonlyArray) => Effect.Effect, ): MediaRoute.AnyRoute => { - if (isStreamInput(route)) - return composeRoute((composition) => MediaRoute.stream({ ...composition, collect }), route, input) - if (isQueuedInput(route)) return composeRoute(MediaRoute.queued, route, input) - return composeRoute(MediaRoute.inline, route, input) + if (isStreamInput(route)) return MediaRoute.stream({ ...composition(route, input), collect }) + if (isQueuedInput(route)) return MediaRoute.queued(composition(route, input)) + return MediaRoute.inline(composition(route, input)) } +const composition = ( + route: MediaModel.RouteInput, + input: MediaRoute.ModelInput, +): MediaRoute.Composition & { readonly protocol: Protocol } => ({ + protocol: route.protocol, + endpoint: Endpoint.path(route.path, { baseURL: input.baseURL ?? route.baseURL }), + auth: input.auth, + headers: + route.headers === undefined && input.headers === undefined ? undefined : { ...route.headers, ...input.headers }, +}) + const isStreamInput = ( route: MediaModel.AnyRouteInput, -): route is MediaModel.RouteInput< - MediaProtocol.Addressed, - MediaProtocol.Streamed -> => route.protocol.kind === "stream" +): route is MediaModel.StreamRouteInput => route.protocol.kind === "stream" const isQueuedInput = ( route: MediaModel.AnyRouteInput, -): route is MediaModel.RouteInput> => - route.protocol.kind === "queued" +): route is MediaModel.QueuedRouteInput => route.protocol.kind === "queued" /** Lift a synchronous Schema-class constructor into a typed `InvalidRequest` failure. */ export const tryRequest = (make: () => A): Effect.Effect => diff --git a/packages/ai/src/promise.ts b/packages/ai/src/promise.ts index fed0474fafa..3d6ac256244 100644 --- a/packages/ai/src/promise.ts +++ b/packages/ai/src/promise.ts @@ -1,23 +1,22 @@ import { Effect, Layer, ManagedRuntime, Stream } from "effect" import { AIClient } from "./ai-client.js" import type { AwaitOptions, Event, Generation, Snapshot } from "./generation.js" -import { Image, ImageModel, ImageRequest, type ImageOptions, type ImageRequestInput } from "./image.js" +import { Image, type ImageModel, type ImageRequest, type ImageRequestInput } from "./image.js" import { LLM } from "./index.js" import { Media } from "./media.js" import { tryRequest } from "./media-model.js" import { RequestExecutor } from "./route/executor.js" import { AIError, InvalidRequestError, LanguageModel, LLMRequest } from "./schema/index.js" import type { RequestInput } from "./llm.js" -import { Speech, SpeechModel, SpeechRequest, type SpeechRequestInput } from "./speech.js" +import { Speech, type SpeechModel, type SpeechRequest, type SpeechRequestInput } from "./speech.js" import { Transcription, - TranscriptionModel, - TranscriptionRequest, - type TranscriptionOptions, + type TranscriptionModel, + type TranscriptionRequest, type TranscriptionRequestInput, } from "./transcription.js" import { fileMediaType } from "./utils/media-type.js" -import { Video, VideoModel, VideoRequest, type VideoOptions, type VideoRequestInput } from "./video.js" +import { Video, type VideoModel, type VideoRequest, type VideoRequestInput } from "./video.js" /** * Promise-first entrypoint for scripts and non-Effect callers. One `ManagedRuntime` hosts the LLM, image, video, speech, @@ -93,17 +92,8 @@ export const make = (options: Options = {}) => { cancel: (options) => run(generation.cancel(), options), }) - // The typed `generate`/`stream` overloads take a concrete input or a request, not the union; normalize once here. const llmRequest = (input: RequestInput | LLMRequest) => input instanceof LLMRequest ? Effect.succeed(input) : tryRequest(() => LLM.request(input)) - const imageRequest = (input: ImageRequestInput | ImageRequest) => - input instanceof ImageRequest ? input : Image.request(input) - const videoRequest = (input: VideoRequestInput | VideoRequest) => - input instanceof VideoRequest ? input : Video.request(input) - const speechRequest = (input: SpeechRequestInput | SpeechRequest) => - input instanceof SpeechRequest ? input : Speech.request(input) - const transcriptionRequest = (input: TranscriptionRequestInput | TranscriptionRequest) => - input instanceof TranscriptionRequest ? input : Transcription.request(input) return { run, @@ -155,61 +145,58 @@ export const make = (options: Options = {}) => { generate: ( input: ImageRequestInput | ImageRequest, options?: AwaitOptions & RunOptions, - ) => run(Image.generate(imageRequest(input), { poll: options?.poll }), options), + ) => run(Image.generate(input, { poll: options?.poll }), options), stream: ( input: ImageRequestInput | ImageRequest, options?: AwaitOptions & RunOptions, - ) => iterate(Image.stream(imageRequest(input), { poll: options?.poll }), options), + ) => iterate(Image.stream(input, { poll: options?.poll }), options), start: (input: ImageRequestInput | ImageRequest, options?: RunOptions) => - run(Image.start(imageRequest(input)), options).then(handle), - resume: (model: ImageModel, token: unknown, options?: RunOptions) => + run(Image.start(input), options).then(handle), + resume: (model: ImageModel, token: unknown, options?: RunOptions) => run(Image.resume(model, token), options).then(handle), }, video: { request: Video.request, start: (input: VideoRequestInput | VideoRequest, options?: RunOptions) => - run(Video.start(videoRequest(input)), options).then(handle), + run(Video.start(input), options).then(handle), generate: ( input: VideoRequestInput | VideoRequest, options?: AwaitOptions & RunOptions, - ) => run(Video.generate(videoRequest(input), { poll: options?.poll }), options), - resume: (model: VideoModel, token: unknown, options?: RunOptions) => + ) => run(Video.generate(input, { poll: options?.poll }), options), + resume: (model: VideoModel, token: unknown, options?: RunOptions) => run(Video.resume(model, token), options).then(handle), stream: ( input: VideoRequestInput | VideoRequest, options?: AwaitOptions & RunOptions, - ) => iterate(Video.stream(videoRequest(input), { poll: options?.poll }), options), + ) => iterate(Video.stream(input, { poll: options?.poll }), options), }, speech: { request: Speech.request, generate: ( input: SpeechRequestInput | SpeechRequest, options?: RunOptions, - ) => run(Speech.generate(speechRequest(input)), options), + ) => run(Speech.generate(input), options), stream: ( input: SpeechRequestInput | SpeechRequest, options?: RunOptions, - ) => iterate(Speech.stream(speechRequest(input)), options), + ) => iterate(Speech.stream(input), options), }, transcription: { request: Transcription.request, generate: ( input: TranscriptionRequestInput | TranscriptionRequest, options?: AwaitOptions & RunOptions, - ) => run(Transcription.generate(transcriptionRequest(input), { poll: options?.poll }), options), + ) => run(Transcription.generate(input, { poll: options?.poll }), options), stream: ( input: TranscriptionRequestInput | TranscriptionRequest, options?: AwaitOptions & RunOptions, - ) => iterate(Transcription.stream(transcriptionRequest(input), { poll: options?.poll }), options), + ) => iterate(Transcription.stream(input, { poll: options?.poll }), options), start: ( input: TranscriptionRequestInput | TranscriptionRequest, options?: RunOptions, - ) => run(Transcription.start(transcriptionRequest(input)), options).then(handle), - resume: ( - model: TranscriptionModel, - token: unknown, - options?: RunOptions, - ) => run(Transcription.resume(model, token), options).then(handle), + ) => run(Transcription.start(input), options).then(handle), + resume: (model: TranscriptionModel, token: unknown, options?: RunOptions) => + run(Transcription.resume(model, token), options).then(handle), }, dispose: () => runtime.dispose(), } diff --git a/packages/ai/src/route/media.ts b/packages/ai/src/route/media.ts index cdcef21d3c9..af62f0441ae 100644 --- a/packages/ai/src/route/media.ts +++ b/packages/ai/src/route/media.ts @@ -5,7 +5,7 @@ import { Endpoint } from "./endpoint.js" import { RequestExecutorService, type Interface } from "./executor-service.js" import { RequestExecutor } from "./executor.js" import { MediaProtocol } from "./media-protocol.js" -import { Generation, resultEvents, type AwaitOptions, type Observation } from "../generation.js" +import { Generation } from "../generation.js" import type { Media } from "../media.js" import { AIError, @@ -52,7 +52,7 @@ export const deployment = ( // --------------------------------------------------------------------------- /** One request, one response. */ -export interface Route { +export interface InlineRoute { readonly kind: "inline" readonly id: string readonly provider: ProviderID @@ -86,7 +86,7 @@ export interface StreamRoute { } export type AnyRoute = - | Route + | InlineRoute | StreamRoute | QueuedRoute @@ -119,7 +119,7 @@ export interface StreamInput( input: InlineInput, -): Route => { +): InlineRoute => { const transport = makeTransport(input) return { kind: "inline", @@ -267,59 +267,6 @@ export const stream = (input: { - readonly modality: string - readonly execute: Execute - readonly responseEvents: (response: Response) => ReadonlyArray -}) => { - const notQueued = (route: { readonly provider: ProviderID; readonly id: string }, operation: string) => - new AIError({ - reason: new UnsupportedOperationError({ - operation: `${input.modality}.${operation}`, - provider: route.provider, - route: route.id, - message: `${route.provider}/${route.id} is not a queued route; use generate or stream`, - }), - }) - const start = (route: AnyRoute, request: Request) => { - if (route.kind !== "queued") return Effect.fail(notQueued(route, "start")) - return route.start(request, input.execute) - } - return { - start, - resume: ( - route: AnyRoute, - model: MediaRequest["model"], - token: unknown, - ) => { - if (route.kind !== "queued") return Effect.fail(notQueued(route, "resume")) - return route.resume(model, token, input.execute) - }, - generate: ( - route: AnyRoute, - request: Request, - options?: AwaitOptions, - ) => { - if (route.kind !== "queued") return route.generate(request, input.execute) - return start(route, request).pipe(Effect.flatMap((generation) => generation.await(options))) - }, - stream: ( - route: AnyRoute, - request: Request, - options?: AwaitOptions, - ): Stream.Stream => { - if (route.kind === "stream") return route.stream(request, input.execute) - if (route.kind === "queued") - return Stream.unwrap( - start(route, request).pipe( - Effect.map((generation) => resultEvents(generation, input.responseEvents, options)), - ), - ) - return Stream.fromIterableEffect(Effect.map(route.generate(request, input.execute), input.responseEvents)) - }, - } -} - // --------------------------------------------------------------------------- // Transport plumbing shared by every kind // --------------------------------------------------------------------------- diff --git a/packages/ai/src/speech-client.ts b/packages/ai/src/speech-client.ts index fef03abed71..fc9f309e548 100644 --- a/packages/ai/src/speech-client.ts +++ b/packages/ai/src/speech-client.ts @@ -1,53 +1,31 @@ -import { Context, Effect, Layer, Stream } from "effect" -import { RequestExecutor } from "./route/executor.js" -import type { AIError } from "./schema/index.js" -import type { SpeechEvent, SpeechOptions, SpeechRequestFor, SpeechResponse } from "./speech.js" +import { Context } from "effect" +import { MediaClient } from "./media-client.js" +import { + SpeechTimestampsEvent, + SpeechFinishEvent, + type SpeechEvent, + type SpeechRequestFor, + type SpeechResponse, +} from "./speech.js" -export interface Interface { - readonly generate: ( - request: SpeechRequestFor, - ) => Effect.Effect - readonly stream: ( - request: SpeechRequestFor, - ) => Stream.Stream -} +export type Interface = MediaClient.Interface export class SpeechClientService extends Context.Service()("@opencode/SpeechClient") {} export const Service = SpeechClientService export type Service = SpeechClientService -export const generate = ( - request: SpeechRequestFor, -): Effect.Effect => - Effect.gen(function* () { - const client = yield* Service - return yield* client.generate(request) - }) - -export const stream = ( - request: SpeechRequestFor, -): Stream.Stream => - Stream.unwrap( - Effect.gen(function* () { - const client = yield* Service - return client.stream(request) - }), - ) - -export const layer: Layer.Layer = Layer.effect( - Service, - Effect.gen(function* () { - const executor = yield* RequestExecutor.Service - return Service.of({ - generate: (request) => request.model.route.generate(request, executor.execute), - stream: (request) => request.model.route.stream(request, executor.execute), - }) - }), -) - export const SpeechClient = { Service, - layer, - generate, - stream, + ...MediaClient.make(Service, { + modality: "speech", + responseEvents: (response: SpeechResponse) => [ + ...(response.timestamps === undefined ? [] : [SpeechTimestampsEvent.make({ items: response.timestamps })]), + SpeechFinishEvent.make({ + audio: response.audio, + usage: response.usage, + notices: response.notices, + providerMetadata: response.providerMetadata, + }), + ], + }), } as const diff --git a/packages/ai/src/speech.ts b/packages/ai/src/speech.ts index dbb2b0586eb..9c07febe72c 100644 --- a/packages/ai/src/speech.ts +++ b/packages/ai/src/speech.ts @@ -1,8 +1,8 @@ import { Effect, Schema, Stream } from "effect" +import { ProgressEvent, QueuedEvent } from "./generation.js" import { Media } from "./media.js" import { MediaModel, composeRoute, tryRequest } from "./media-model.js" import { MediaRoute } from "./route/media.js" -import type { MediaProtocol } from "./route/media-protocol.js" import { AIError, HttpOptions, MediaUsage, ProviderMetadata, type OpenString } from "./schema/index.js" import { SpeechClient, Service } from "./speech-client.js" @@ -10,53 +10,39 @@ import { SpeechClient, Service } from "./speech-client.js" // Model // --------------------------------------------------------------------------- -export type SpeechOptions = Record +export type SpeechOptions = MediaModel.Options -export type SpeechRoute = MediaRoute.StreamRoute< - SpeechRequestFor, - SpeechEvent, - SpeechResponse -> +export type SpeechRoute = MediaRoute.AnyRoute -export class SpeechModel extends MediaModel< - SpeechRoute, - Options -> { +export class SpeechModel extends MediaModel { declare protected readonly _SpeechModel: void - static make(input: MediaModel.Input>) { - return new SpeechModel(input) - } - - /** Compose a streaming speech protocol with its canonical path into a model for one deployment. */ - static fromRoute( - route: SpeechModel.RouteInput, + /** The number of type arguments selects the kind: ``, ``, or ``. */ + static fromRoute( + route: MediaModel.InlineRouteInput, SpeechResponse>, + input: MediaRoute.ModelInput, + ): SpeechModel + static fromRoute( + route: MediaModel.StreamRouteInput, SpeechEvent, Frame, State>, + input: MediaRoute.ModelInput, + ): SpeechModel + static fromRoute( + route: MediaModel.QueuedRouteInput, SpeechResponse, Token>, + input: MediaRoute.ModelInput, + ): SpeechModel + static fromRoute( + route: MediaModel.AnyRouteInput, SpeechEvent, SpeechResponse, Frame, State, Token>, input: MediaRoute.ModelInput, ) { return new SpeechModel({ id: input.id, provider: route.protocol.provider, http: input.http, - route: composeRoute( - (composition) => MediaRoute.stream({ ...composition, collect: collectResponse }), - route, - input, - ), + route: composeRoute(route, input, collectResponse) as SpeechRoute, }) } } -export namespace SpeechModel { - export type RouteInput< - Options extends SpeechOptions = SpeechOptions, - Frame = unknown, - State = unknown, - > = MediaModel.RouteInput< - MediaProtocol.Addressed>, - MediaProtocol.Streamed, SpeechEvent, Frame, State> - > -} - export const SpeechModelSchema = Schema.declare((value): value is SpeechModel => value instanceof SpeechModel, { expected: "Speech.Model", }) @@ -148,11 +134,17 @@ export const SpeechFinishEvent = Schema.Struct({ providerMetadata: Schema.optional(ProviderMetadata), }).annotate({ identifier: "Speech.Event.Finish" }) -const speechEventTagged = Schema.Union([SpeechAudioDeltaEvent, SpeechTimestampsEvent, SpeechFinishEvent]).pipe( - Schema.toTaggedUnion("type"), -) +const speechEventTagged = Schema.Union([ + QueuedEvent, + ProgressEvent, + SpeechAudioDeltaEvent, + SpeechTimestampsEvent, + SpeechFinishEvent, +]).pipe(Schema.toTaggedUnion("type")) export const SpeechEvent = Object.assign(speechEventTagged, { is: { + generationQueued: speechEventTagged.guards["generation-queued"], + generationProgress: speechEventTagged.guards["generation-progress"], audioDelta: speechEventTagged.guards["audio-delta"], timestamps: speechEventTagged.guards.timestamps, finish: speechEventTagged.guards.finish, @@ -195,17 +187,15 @@ export function request(input: SpeechRequest | SpeechRequestInput) { const requestEffect = (input: SpeechRequest | SpeechRequestInput) => tryRequest(() => request(input)) export function generate( - input: SpeechRequestInput, + input: SpeechRequest | SpeechRequestInput, ): Effect.Effect -export function generate(input: SpeechRequest): Effect.Effect export function generate(input: SpeechRequest | SpeechRequestInput) { return requestEffect(input).pipe(Effect.flatMap((request) => SpeechClient.generate(request))) } export function stream( - input: SpeechRequestInput, + input: SpeechRequest | SpeechRequestInput, ): Stream.Stream -export function stream(input: SpeechRequest): Stream.Stream export function stream(input: SpeechRequest | SpeechRequestInput) { return Stream.unwrap(requestEffect(input).pipe(Effect.map((request) => SpeechClient.stream(request)))) } diff --git a/packages/ai/src/transcription-client.ts b/packages/ai/src/transcription-client.ts index dbcfc638c40..8c3a3b66270 100644 --- a/packages/ai/src/transcription-client.ts +++ b/packages/ai/src/transcription-client.ts @@ -1,34 +1,13 @@ -import { Context, Effect, Layer, Stream } from "effect" -import type { AwaitOptions, Generation } from "./generation.js" -import { RequestExecutor } from "./route/executor.js" -import { MediaRoute } from "./route/media.js" -import type { AIError } from "./schema/index.js" +import { Context } from "effect" +import { MediaClient } from "./media-client.js" import { - responseEvents, + TranscriptionFinishEvent, type TranscriptionEvent, - type TranscriptionModel, - type TranscriptionOptions, type TranscriptionRequestFor, type TranscriptionResponse, } from "./transcription.js" -export interface Interface { - readonly generate: ( - request: TranscriptionRequestFor, - options?: AwaitOptions, - ) => Effect.Effect - readonly stream: ( - request: TranscriptionRequestFor, - options?: AwaitOptions, - ) => Stream.Stream - readonly start: ( - request: TranscriptionRequestFor, - ) => Effect.Effect, AIError> - readonly resume: ( - model: TranscriptionModel, - token: unknown, - ) => Effect.Effect, AIError> -} +export type Interface = MediaClient.Interface export class TranscriptionClientService extends Context.Service()( "@opencode/TranscriptionClient", @@ -36,66 +15,10 @@ export class TranscriptionClientService extends Context.Service( - request: TranscriptionRequestFor, - options?: AwaitOptions, -): Effect.Effect => - Effect.gen(function* () { - const client = yield* Service - return yield* client.generate(request, options) - }) - -export const stream = ( - request: TranscriptionRequestFor, - options?: AwaitOptions, -): Stream.Stream => - Stream.unwrap( - Effect.gen(function* () { - const client = yield* Service - return client.stream(request, options) - }), - ) - -export const start = ( - request: TranscriptionRequestFor, -): Effect.Effect, AIError, Service> => - Effect.gen(function* () { - const client = yield* Service - return yield* client.start(request) - }) - -export const resume = ( - model: TranscriptionModel, - token: unknown, -): Effect.Effect, AIError, Service> => - Effect.gen(function* () { - const client = yield* Service - return yield* client.resume(model, token) - }) - -export const layer: Layer.Layer = Layer.effect( - Service, - Effect.gen(function* () { - const executor = yield* RequestExecutor.Service - const dispatch = MediaRoute.dispatch({ - modality: "transcription", - execute: executor.execute, - responseEvents, - }) - return Service.of({ - start: (request) => dispatch.start(request.model.route, request), - resume: (model, token) => dispatch.resume(model.route, model, token), - generate: (request, options) => dispatch.generate(request.model.route, request, options), - stream: (request, options) => dispatch.stream(request.model.route, request, options), - }) - }), -) - export const TranscriptionClient = { Service, - layer, - generate, - stream, - start, - resume, + ...MediaClient.make(Service, { + modality: "transcription", + responseEvents: (response: TranscriptionResponse) => [TranscriptionFinishEvent.make({ ...response })], + }), } as const diff --git a/packages/ai/src/transcription.ts b/packages/ai/src/transcription.ts index ed8b22b8b55..9a5e01b1be5 100644 --- a/packages/ai/src/transcription.ts +++ b/packages/ai/src/transcription.ts @@ -1,9 +1,8 @@ import { Effect, Schema, Stream } from "effect" import { Generation, ProgressEvent, QueuedEvent, type AwaitOptions } from "./generation.js" import { Media } from "./media.js" -import { MediaModel, composeAnyRoute, tryRequest } from "./media-model.js" +import { MediaModel, composeRoute, tryRequest } from "./media-model.js" import { MediaRoute } from "./route/media.js" -import type { MediaProtocol } from "./route/media-protocol.js" import { AIError, HttpOptions, MediaUsage, ProviderMetadata } from "./schema/index.js" import { TranscriptionClient, Service } from "./transcription-client.js" @@ -11,90 +10,49 @@ import { TranscriptionClient, Service } from "./transcription-client.js" // Model // --------------------------------------------------------------------------- -export type TranscriptionOptions = Record +export type TranscriptionOptions = MediaModel.Options -export type TranscriptionRoute = MediaRoute.AnyRoute< - TranscriptionRequestFor, - TranscriptionEvent, - TranscriptionResponse -> +export type TranscriptionRoute = MediaRoute.AnyRoute export class TranscriptionModel extends MediaModel< - TranscriptionRoute, + TranscriptionRoute, Options > { declare protected readonly _TranscriptionModel: void - static make( - input: MediaModel.Input>, - ) { - return new TranscriptionModel(input) - } - /** The number of type arguments selects the kind: ``, ``, or ``. */ static fromRoute( - route: TranscriptionModel.InlineRouteInput, + route: MediaModel.InlineRouteInput, TranscriptionResponse>, input: MediaRoute.ModelInput, ): TranscriptionModel static fromRoute( - route: TranscriptionModel.StreamRouteInput, + route: MediaModel.StreamRouteInput, TranscriptionEvent, Frame, State>, input: MediaRoute.ModelInput, ): TranscriptionModel static fromRoute( - route: TranscriptionModel.QueuedRouteInput, + route: MediaModel.QueuedRouteInput, TranscriptionResponse, Token>, input: MediaRoute.ModelInput, ): TranscriptionModel static fromRoute( - route: TranscriptionModel.RouteInput, + route: MediaModel.AnyRouteInput< + TranscriptionRequestFor, + TranscriptionEvent, + TranscriptionResponse, + Frame, + State, + Token + >, input: MediaRoute.ModelInput, ) { return new TranscriptionModel({ id: input.id, provider: route.protocol.provider, http: input.http, - route: composeAnyRoute(route, input, collectResponse), + route: composeRoute(route, input, collectResponse) as TranscriptionRoute, }) } } -export namespace TranscriptionModel { - export type InlineRouteInput = MediaModel.RouteInput< - TranscriptionRequestFor, - MediaProtocol.Inline, TranscriptionResponse> - > - - export type StreamRouteInput< - Options extends TranscriptionOptions = TranscriptionOptions, - Frame = unknown, - State = unknown, - > = MediaModel.RouteInput< - MediaProtocol.Addressed>, - MediaProtocol.Streamed, TranscriptionEvent, Frame, State> - > - - export type QueuedRouteInput< - Options extends TranscriptionOptions = TranscriptionOptions, - Token = unknown, - > = MediaModel.RouteInput< - TranscriptionRequestFor, - MediaProtocol.Queued, TranscriptionResponse, Token> - > - - export type RouteInput< - Options extends TranscriptionOptions = TranscriptionOptions, - Frame = unknown, - State = unknown, - Token = unknown, - > = MediaModel.AnyRouteInput< - TranscriptionRequestFor, - TranscriptionEvent, - TranscriptionResponse, - Frame, - State, - Token - > -} - export const TranscriptionModelSchema = Schema.declare( (value): value is TranscriptionModel => value instanceof TranscriptionModel, { expected: "Transcription.Model" }, @@ -212,10 +170,6 @@ export const TranscriptionEvent = Object.assign(transcriptionEventTagged, { }) export type TranscriptionEvent = Schema.Schema.Type -export const responseEvents = (response: TranscriptionResponse): ReadonlyArray => [ - TranscriptionFinishEvent.make({ ...response }), -] - const collectResponse = (events: ReadonlyArray): Effect.Effect => { const finish = events.find(TranscriptionEvent.is.finish) // Every transcription protocol's `finish` emits the terminal event or fails, so a completed stream always has one. @@ -243,11 +197,7 @@ export function request(input: TranscriptionRequest | TranscriptionRequestInput) const requestEffect = (input: TranscriptionRequest | TranscriptionRequestInput) => tryRequest(() => request(input)) export function generate( - input: TranscriptionRequestInput, - options?: AwaitOptions, -): Effect.Effect -export function generate( - input: TranscriptionRequest, + input: TranscriptionRequest | TranscriptionRequestInput, options?: AwaitOptions, ): Effect.Effect export function generate(input: TranscriptionRequest | TranscriptionRequestInput, options?: AwaitOptions) { @@ -255,11 +205,7 @@ export function generate(input: TranscriptionRequest | TranscriptionRequestInput } export function stream( - input: TranscriptionRequestInput, - options?: AwaitOptions, -): Stream.Stream -export function stream( - input: TranscriptionRequest, + input: TranscriptionRequest | TranscriptionRequestInput, options?: AwaitOptions, ): Stream.Stream export function stream(input: TranscriptionRequest | TranscriptionRequestInput, options?: AwaitOptions) { @@ -268,15 +214,14 @@ export function stream(input: TranscriptionRequest | TranscriptionRequestInput, /** Inline and streaming routes fail with `UnsupportedOperation`. */ export function start( - input: TranscriptionRequestInput, + input: TranscriptionRequest | TranscriptionRequestInput, ): Effect.Effect, AIError, Service> -export function start(input: TranscriptionRequest): Effect.Effect, AIError, Service> export function start(input: TranscriptionRequest | TranscriptionRequestInput) { return requestEffect(input).pipe(Effect.flatMap((request) => TranscriptionClient.start(request))) } -export const resume = ( - model: TranscriptionModel, +export const resume = ( + model: TranscriptionModel, token: unknown, ): Effect.Effect, AIError, Service> => TranscriptionClient.resume(model, token) diff --git a/packages/ai/src/video-client.ts b/packages/ai/src/video-client.ts index 5872c94d09f..248680e635c 100644 --- a/packages/ai/src/video-client.ts +++ b/packages/ai/src/video-client.ts @@ -1,98 +1,30 @@ -import { Context, Effect, Layer, Stream } from "effect" -import { resultEvents, type AwaitOptions, type Generation } from "./generation.js" -import { RequestExecutor } from "./route/executor.js" -import type { AIError } from "./schema/index.js" +import { Context } from "effect" +import { MediaClient } from "./media-client.js" import { - responseEvents, + VideoOutputEvent, + VideoFinishEvent, type VideoEvent, - type VideoModel, - type VideoOptions, type VideoRequestFor, type VideoResponse, } from "./video.js" -export interface Interface { - readonly start: ( - request: VideoRequestFor, - ) => Effect.Effect, AIError> - readonly resume: ( - model: VideoModel, - token: unknown, - ) => Effect.Effect, AIError> - readonly generate: ( - request: VideoRequestFor, - options?: AwaitOptions, - ) => Effect.Effect - readonly stream: ( - request: VideoRequestFor, - options?: AwaitOptions, - ) => Stream.Stream -} +export type Interface = MediaClient.Interface export class VideoClientService extends Context.Service()("@opencode/VideoClient") {} export const Service = VideoClientService export type Service = VideoClientService -export const start = ( - request: VideoRequestFor, -): Effect.Effect, AIError, Service> => - Effect.gen(function* () { - const client = yield* Service - return yield* client.start(request) - }) - -export const resume = ( - model: VideoModel, - token: unknown, -): Effect.Effect, AIError, Service> => - Effect.gen(function* () { - const client = yield* Service - return yield* client.resume(model, token) - }) - -export const generate = ( - request: VideoRequestFor, - options?: AwaitOptions, -): Effect.Effect => - Effect.gen(function* () { - const client = yield* Service - return yield* client.generate(request, options) - }) - -export const stream = ( - request: VideoRequestFor, - options?: AwaitOptions, -): Stream.Stream => - Stream.unwrap( - Effect.gen(function* () { - const client = yield* Service - return client.stream(request, options) - }), - ) - -export const layer: Layer.Layer = Layer.effect( - Service, - Effect.gen(function* () { - const executor = yield* RequestExecutor.Service - const start = (request: VideoRequestFor) => - request.model.route.start(request, executor.execute) - return Service.of({ - start, - resume: (model, token) => model.route.resume(model, token, executor.execute), - generate: (request, options) => start(request).pipe(Effect.flatMap((generation) => generation.await(options))), - stream: (request, options) => - Stream.unwrap( - start(request).pipe(Effect.map((generation) => resultEvents(generation, responseEvents, options))), - ), - }) - }), -) - export const VideoClient = { Service, - layer, - start, - resume, - generate, - stream, + ...MediaClient.make(Service, { + modality: "video", + responseEvents: (response: VideoResponse) => [ + ...response.videos.map((video, index) => VideoOutputEvent.make({ index, video })), + VideoFinishEvent.make({ + usage: response.usage, + notices: response.notices, + providerMetadata: response.providerMetadata, + }), + ], + }), } as const diff --git a/packages/ai/src/video.ts b/packages/ai/src/video.ts index afa7ae59ac8..c97ac08f43f 100644 --- a/packages/ai/src/video.ts +++ b/packages/ai/src/video.ts @@ -3,7 +3,6 @@ import { Generation, ProgressEvent, QueuedEvent, type AwaitOptions } from "./gen import { Media } from "./media.js" import { MediaModel, composeRoute, tryRequest } from "./media-model.js" import { MediaRoute } from "./route/media.js" -import type { MediaProtocol } from "./route/media-protocol.js" import { AIError, HttpOptions, MediaUsage, ProviderMetadata, type OpenString } from "./schema/index.js" import { VideoClient, Service } from "./video-client.js" @@ -11,41 +10,39 @@ import { VideoClient, Service } from "./video-client.js" // Model // --------------------------------------------------------------------------- -export type VideoOptions = Record +export type VideoOptions = MediaModel.Options -export type VideoRoute = MediaRoute.QueuedRoute< - VideoRequestFor, - VideoResponse -> +export type VideoRoute = MediaRoute.AnyRoute -export class VideoModel extends MediaModel, Options> { +export class VideoModel extends MediaModel { declare protected readonly _VideoModel: void - static make(input: MediaModel.Input>) { - return new VideoModel(input) - } - - /** Compose a queued video protocol with its canonical start path into a model for one deployment. */ - static fromRoute( - route: VideoModel.RouteInput, + /** The number of type arguments selects the kind: ``, ``, or ``. */ + static fromRoute( + route: MediaModel.InlineRouteInput, VideoResponse>, + input: MediaRoute.ModelInput, + ): VideoModel + static fromRoute( + route: MediaModel.StreamRouteInput, VideoEvent, Frame, State>, + input: MediaRoute.ModelInput, + ): VideoModel + static fromRoute( + route: MediaModel.QueuedRouteInput, VideoResponse, Token>, + input: MediaRoute.ModelInput, + ): VideoModel + static fromRoute( + route: MediaModel.AnyRouteInput, VideoEvent, VideoResponse, Frame, State, Token>, input: MediaRoute.ModelInput, ) { return new VideoModel({ id: input.id, provider: route.protocol.provider, http: input.http, - route: composeRoute(MediaRoute.queued, route, input), + route: composeRoute(route, input, collectResponse) as VideoRoute, }) } } -export namespace VideoModel { - export type RouteInput = MediaModel.RouteInput< - VideoRequestFor, - MediaProtocol.Queued, VideoResponse, Token> - > -} - export const VideoModelSchema = Schema.declare((value): value is VideoModel => value instanceof VideoModel, { expected: "Video.Model", }) @@ -149,15 +146,19 @@ export const VideoEvent = Object.assign(videoEventTagged, { }) export type VideoEvent = Schema.Schema.Type -/** A completed response expanded into the streaming event shape. */ -export const responseEvents = (response: VideoResponse): ReadonlyArray => [ - ...response.videos.map((video, index) => VideoOutputEvent.make({ index, video })), - VideoFinishEvent.make({ - usage: response.usage, - notices: response.notices, - providerMetadata: response.providerMetadata, - }), -] +const collectResponse = (events: ReadonlyArray): Effect.Effect => { + const finish = events.find(VideoEvent.is.finish) + // A streaming video protocol's `finish` emits the terminal event or fails, so a completed stream always has one. + if (finish === undefined) return Effect.die(new Error("The video stream completed without a finish event")) + return Effect.succeed( + new VideoResponse({ + videos: events.filter(VideoEvent.is.video).map((event) => event.video), + usage: finish.usage, + notices: finish.notices, + providerMetadata: finish.providerMetadata, + }), + ) +} // --------------------------------------------------------------------------- // Request-shaped call API @@ -178,33 +179,28 @@ export function request(input: VideoRequest | VideoRequestInput) { const requestEffect = (input: VideoRequest | VideoRequestInput) => tryRequest(() => request(input)) export function start( - input: VideoRequestInput, + input: VideoRequest | VideoRequestInput, ): Effect.Effect, AIError, Service> -export function start(input: VideoRequest): Effect.Effect, AIError, Service> export function start(input: VideoRequest | VideoRequestInput) { return requestEffect(input).pipe(Effect.flatMap((request) => VideoClient.start(request))) } export function generate( - input: VideoRequestInput, + input: VideoRequest | VideoRequestInput, options?: AwaitOptions, ): Effect.Effect -export function generate(input: VideoRequest, options?: AwaitOptions): Effect.Effect export function generate(input: VideoRequest | VideoRequestInput, options?: AwaitOptions) { return requestEffect(input).pipe(Effect.flatMap((request) => VideoClient.generate(request, options))) } /** Rebuild a generation handle from a persisted `Generation.token`, refreshing its status once. */ -export const resume = ( - model: VideoModel, - token: unknown, -): Effect.Effect, AIError, Service> => VideoClient.resume(model, token) +export const resume = (model: VideoModel, token: unknown): Effect.Effect, AIError, Service> => + VideoClient.resume(model, token) export function stream( - input: VideoRequestInput, + input: VideoRequest | VideoRequestInput, options?: AwaitOptions, ): Stream.Stream -export function stream(input: VideoRequest, options?: AwaitOptions): Stream.Stream export function stream(input: VideoRequest | VideoRequestInput, options?: AwaitOptions) { return Stream.unwrap(requestEffect(input).pipe(Effect.map((request) => VideoClient.stream(request, options)))) } diff --git a/packages/ai/test/exports.test.ts b/packages/ai/test/exports.test.ts index 3b4f640d55e..2748b933ad9 100644 --- a/packages/ai/test/exports.test.ts +++ b/packages/ai/test/exports.test.ts @@ -54,6 +54,19 @@ import { TestLLM } from "@opencode/ai/testing" import { Evaluation, EvaluationClient } from "@opencode/ai/experimental" describe("public exports", () => { + test("modality, provider, and protocol entrypoints load first in a fresh process", async () => { + const results = await Promise.all( + ["image", "video", "speech", "transcription", "providers", "protocols"].map(async (entry) => { + const child = Bun.spawn( + [process.execPath, "-e", `await import(${JSON.stringify(`${import.meta.dir}/../src/${entry}.ts`)})`], + { stderr: "pipe" }, + ) + return { entry, exitCode: await child.exited, stderr: await new Response(child.stderr).text() } + }), + ) + expect(results.filter((result) => result.exitCode !== 0)).toEqual([]) + }) + test("root exposes app-facing runtime APIs", () => { expect(LLM.request).toBeFunction() expect(LLMClient.Service).toBeFunction() diff --git a/packages/ai/test/image.types.ts b/packages/ai/test/image.types.ts index 380b583acba..0a2406cec50 100644 --- a/packages/ai/test/image.types.ts +++ b/packages/ai/test/image.types.ts @@ -7,7 +7,6 @@ import { type ImageModelOptions, type ImageOptions, type ImageRequestFor, - type ImageRoute, } from "../src/index.js" import type { Service } from "../src/image-client.js" import { Anthropic, BlackForestLabs, Google, OpenAI, Stability, XAI, ZAI } from "../src/providers.js" @@ -21,8 +20,7 @@ type GoogleLikeOptions = { readonly thinkingLevel?: "LOW" | "HIGH" } & Record -declare const route: ImageRoute -const google = ImageModel.make({ id: "gemini-image", provider: "google", route }) +declare const google: ImageModel // @ts-expect-error Extracted model options retain known provider fields. const invalidGoogleOptions: ImageModelOptions = { imageSize: "8K" } void invalidGoogleOptions @@ -152,6 +150,8 @@ Image.generate({ model: zai, prompt: "A lighthouse", providerOptions: { quality: Image.generate({ model: zai, prompt: "A lighthouse", providerOptions: { userID: 1 } }) declare const generic: ImageModel +const widenImage = (model: ImageModel): ImageModel => model +void widenImage Image.generate({ model: generic, prompt: "A lighthouse", providerOptions: { arbitrary: true } }) const explicitAsset: Media.Asset = Media.url("https://example.com/image.png") void explicitAsset diff --git a/packages/ai/test/promise.test.ts b/packages/ai/test/promise.test.ts index 1f2fb714f7e..f93bf784fa1 100644 --- a/packages/ai/test/promise.test.ts +++ b/packages/ai/test/promise.test.ts @@ -127,6 +127,10 @@ describe("AI promise client", () => { expect(image.image).toBeInstanceOf(Media.Asset) expect(image.image.mediaType).toBe("image/png") expect(await ai.run(image.image.bytes())).toEqual(Uint8Array.from([1, 2, 3])) + const requested = await ai.image.generate( + ai.image.request({ model: openai.image("gpt-image-2"), prompt: "A lighthouse" }), + ) + expect(requested.image.mediaType).toBe("image/png") const deltas: Array = [] for await (const event of ai.llm.stream(request)) { @@ -149,6 +153,7 @@ describe("AI promise client", () => { "https://openai.test/v1/chat/completions", "https://openai.test/v1/chat/completions", "https://openai.test/v1/images/generations", + "https://openai.test/v1/images/generations", "https://openai.test/v1/chat/completions", "https://openai.test/v1/chat/completions", "https://openai.test/v1/images/generations", @@ -277,10 +282,15 @@ describe("AI promise client", () => { expect(failure).toBeInstanceOf(AIError) expect(failure instanceof AIError && failure.reason.http?.status).toBe(404) - const invalid = await ai.llm + const invalidLLM = await ai.llm // @ts-expect-error Invalid input must reject with AIError instead of throwing synchronously. .generate({ model: openai.responses("gpt-5"), messages: [{ role: "bogus" }] }) .catch((error: unknown) => error) + expect(invalidLLM instanceof AIError && invalidLLM.reason._tag).toBe("InvalidRequest") + + const invalid = await ai.image + .generate({ model: openai.image("gpt-image-2"), prompt: "A lighthouse", n: 1.5 }) + .catch((error: unknown) => error) expect(invalid instanceof AIError && invalid.reason._tag).toBe("InvalidRequest") const controller = new AbortController() diff --git a/packages/ai/test/speech.types.ts b/packages/ai/test/speech.types.ts index b9de6b42e87..4f5d40c5117 100644 --- a/packages/ai/test/speech.types.ts +++ b/packages/ai/test/speech.types.ts @@ -1,5 +1,5 @@ import type { Stream } from "effect" -import { Speech, type SpeechEvent } from "../src/index.js" +import { Speech, SpeechModel, type SpeechEvent, type SpeechOptions } from "../src/index.js" import { ElevenLabs, OpenAI, Runway } from "../src/providers.js" type StreamItem = T extends Stream.Stream ? A : never @@ -7,6 +7,8 @@ type Equal = [A, B] extends [B, A] ? true : false type Assert = T const elevenlabs = ElevenLabs.configure({ apiKey: "test" }).speech("eleven_flash_v2_5") +const widenSpeech = (model: SpeechModel): SpeechModel => model +void widenSpeech Speech.generate({ model: elevenlabs, diff --git a/packages/ai/test/transcription.types.ts b/packages/ai/test/transcription.types.ts index e14ca86624e..4a81b6f4aac 100644 --- a/packages/ai/test/transcription.types.ts +++ b/packages/ai/test/transcription.types.ts @@ -1,5 +1,11 @@ import type { Stream } from "effect" -import { Media, Transcription, type TranscriptionEvent } from "../src/index.js" +import { + Media, + Transcription, + TranscriptionModel, + type TranscriptionEvent, + type TranscriptionOptions, +} from "../src/index.js" import { AssemblyAI, Deepgram, OpenAI } from "../src/providers.js" type StreamItem = T extends Stream.Stream ? A : never @@ -8,6 +14,10 @@ type Assert = T const audio = Media.url("https://example.com/call.mp3") const deepgram = Deepgram.configure({ apiKey: "test" }).transcription("nova-3") +const widenTranscription = ( + model: TranscriptionModel, +): TranscriptionModel => model +void widenTranscription Transcription.generate({ model: deepgram, diff --git a/packages/ai/test/video.types.ts b/packages/ai/test/video.types.ts index 57f66a46aca..f1bbd4fe19e 100644 --- a/packages/ai/test/video.types.ts +++ b/packages/ai/test/video.types.ts @@ -9,7 +9,6 @@ import { type VideoModelOptions, type VideoOptions, type VideoRequestFor, - type VideoRoute, } from "../src/index.js" import type { Service } from "../src/video-client.js" import { Anthropic, Fal, Google, OpenAI, Runway, XAI } from "../src/providers.js" @@ -23,8 +22,7 @@ type VeoLikeOptions = { readonly personGeneration?: "allow_all" | "allow_adult" } & Record -declare const route: VideoRoute -const veo = VideoModel.make({ id: "veo", provider: "google", route }) +declare const veo: VideoModel // @ts-expect-error Extracted model options retain known provider fields. const invalidVeoOptions: VideoModelOptions = { personGeneration: "everyone" } void invalidVeoOptions @@ -99,6 +97,8 @@ Video.generate({ model: google, prompt: "A kitten", durationSeconds: "8s" }) Video.generate({ model: google, prompt: "A kitten", options: { personGeneration: "allow_all" } }) declare const generic: VideoModel +const widenVideo = (model: VideoModel): VideoModel => model +void widenVideo Video.generate({ model: generic, prompt: "A kitten", providerOptions: { arbitrary: true } }) const request = Video.request({ model: veo, prompt: "A kitten", providerOptions: { personGeneration: "allow_all" } })