From 9810d98bc2db41fe9fb59e069fb96bd2e8a93f79 Mon Sep 17 00:00:00 2001 From: Shoubhit Dash Date: Thu, 24 Sep 2026 18:34:14 +0530 Subject: [PATCH] feat(ai): polish media DX with promise file helpers and one client layer (#50978) --- packages/ai/AGENTS.md | 4 +- packages/ai/README.md | 285 +++++++++++------- packages/ai/docs/media-design.md | 147 +++++---- packages/ai/example/tutorial.ts | 27 +- packages/ai/src/ai-client.ts | 24 ++ .../ai/src/experimental/evaluation-client.ts | 2 +- packages/ai/src/image-client.ts | 4 +- packages/ai/src/index.ts | 3 +- packages/ai/src/media.ts | 4 +- packages/ai/src/promise.ts | 94 ++++-- packages/ai/src/protocols/openai-speech.ts | 6 +- packages/ai/src/protocols/shared.ts | 3 +- packages/ai/src/route/auth.ts | 33 +- packages/ai/src/route/client.ts | 4 +- packages/ai/src/route/executor-service.ts | 6 +- packages/ai/src/route/executor.ts | 16 + packages/ai/src/route/index.ts | 2 +- packages/ai/src/speech-client.ts | 4 +- packages/ai/src/transcription-client.ts | 6 +- packages/ai/src/utils/media-type.ts | 5 +- packages/ai/src/video-client.ts | 4 +- packages/ai/test/auth.test.ts | 9 + packages/ai/test/executor.test.ts | 24 +- packages/ai/test/exports.test.ts | 28 +- packages/ai/test/promise.test.ts | 39 +++ packages/ai/test/video.test.ts | 1 + 26 files changed, 522 insertions(+), 262 deletions(-) create mode 100644 packages/ai/src/ai-client.ts diff --git a/packages/ai/AGENTS.md b/packages/ai/AGENTS.md index c30cc3bd311..144e8b0c801 100644 --- a/packages/ai/AGENTS.md +++ b/packages/ai/AGENTS.md @@ -55,7 +55,7 @@ const request = LLM.request({ prompt: "Say hello.", }) -const response = yield * LLMClient.generate(request) +const response = yield * LLMClient.generate(request) // inside Effect.gen ``` `LLM.request(...)` builds an `LLMRequest`. `LLMClient.generate(...)` reads the executable route carried by `request.model.route`, builds the provider-native body, asks the route's transport for a real `HttpClientRequest.HttpClientRequest`, sends it through `RequestExecutor.Service`, parses the provider stream into common `LLMEvent`s, and finally returns an `LLMResponse`. @@ -98,7 +98,7 @@ When a provider supports multiple physical transports, selection remains executi Media does not fit the SSE-frames-to-event-state-machine LLM route. `MediaRoute.make(...)` (`src/route/media.ts`) composes a `MediaProtocol` kind with `Endpoint` and `Auth` and owns the transport plumbing: `http` option merging, URL/query rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol. `MediaProtocol.inline` (`src/route/media-protocol.ts`) is `body.from(request)` plus `response.decode(response, context)`; use `MediaProtocol.decodeJson` / `text` / `bytes` so decode failures retain the raw body and HTTP context. `Generation` (`src/generation.ts`) is the provider-neutral handle for a queued generation over a `GenerationRoute` (`status`, `result`, `cancel`, `pollHint`). Image protocol files follow the same section order as LLM protocols and declare unsupported common fields once through `MediaInput.rejectUnsupported`. -`MediaProtocol.queued` is the submit-then-poll kind every video route uses: `start` (body + decode into `{ token, snapshot }`), `status`, `result`, and optional `cancel`, each addressed by a route-owned `token` whose `Schema.Codec` makes it serializable. `MediaRoute.inline` and `MediaRoute.queued` compose the two kinds with `Endpoint` and `Auth`; the queued route decodes the token once at the boundary (`start` output or `resume` input) and closes over it in a token-free `GenerationRoute` (`status`/`result`/`cancel` are plain Effects), so `Generation` never sees the token's shape and only carries the encoded JSON for persistence. Polls reuse the route's auth and deployment headers plus the request's `http` overlay after `start`, and resolve relative paths against the route base URL (provider-issued absolute URLs such as fal's `status_url` pass through). `result` is always its own GET even when the provider returns output inside the status document, so `Generation.await` behaves the same after `start` and after `resume`. `PollContext.auth` carries only what `Auth` added so protocols can hand download credentials to output assets as transient `Media.Asset.headers` (Veo) — never part of `source` or JSON. Status strings map through a per-protocol `STATUS` table via `MediaProtocol.status`; terminal generations without output fail through `output.ended` / `output.contentPolicy` with the provider document on `reason.body`. `Generation.AwaitOptions` (`{ poll?: Poll }`) is the one options type for `await`, `events`, `Video.generate`, and `Video.stream`. +`MediaProtocol.queued` is the submit-then-poll kind every video route uses: `start` (body + decode into `{ token, snapshot }`), `status`, `result`, and optional `cancel`, each addressed by a route-owned `token` whose `Schema.Codec` makes it serializable. `MediaRoute.inline` and `MediaRoute.queued` compose the two kinds with `Endpoint` and `Auth`; the queued route decodes the token once at the boundary (`start` output or `resume` input) and closes over it in a token-free `GenerationRoute` (`status`/`result`/`cancel` are plain Effects), so `Generation` never sees the token's shape and only carries the encoded JSON for persistence. Polls reuse the route's auth and deployment headers plus the request's `http` overlay after `start`, and resolve relative paths against the route base URL (provider-issued absolute URLs such as fal's `status_url` pass through). `result` is always its own GET even when the provider returns output inside the status document, so `Generation.await` behaves the same after `start` and after `resume`. `PollContext.auth` carries only what `Auth` added so protocols can hand download credentials to output assets as transient `Media.Asset.headers` (Veo) — never part of `source` or JSON. Status strings map through a per-protocol `STATUS` table via `MediaProtocol.status`; terminal generations without output fail through `output.ended` / `output.contentPolicy` with the provider document on `reason.body`. `GenerationAwaitOptions` (`AwaitOptions` in `src/generation.ts`, `{ poll?: Poll }`) is the one options type for `await`, `events`, `Video.generate`, and `Video.stream`. `MediaProtocol.stream` is the incremental kind every speech route uses, with the same discipline as LLM protocols. `MediaRoute.stream` submits the caller's request as `MediaProtocol.Addressed` (`{ ...request, mode }`, `mode: "generate" | "stream"`), so one provider stays one protocol: `body.from`, the endpoint path, and `frames` read `request.mode` to pick the body, path, and framing. `frames(bytes, context)` returns frames — `Framing.sse`, `Framing.lines`, `Framing.document` (a single-document response shaped like a streamed record), or the raw `bytes` for chunked audio. `initial()` is fresh per-response parser state; `step` folds each frame into it and emits modality events; `finish(state, context)` runs once after the last frame with the request, body, and observed `http` (header-only usage lives there) and emits exactly one terminal event or fails with `MediaProtocol.incomplete`. Keep parser state to real accumulators and derive anything the request or body determines in `finish`. `generate` runs the same stream and folds it with the modality's `collect`. Request-derived URL parameters go on the body's `query` (array values repeat the parameter), applied before route and caller `http.query`. Decode frames with `MediaProtocol.decodeFrame` and raise stream-time failures with `MediaProtocol.frameError` (the frame stays on `reason.body`); protocols never thread HTTP context, because the route fills `reason.http` on stream errors that lack it. Speech protocols share `protocols/utils/speech-stream.ts` for deltas, timestamps, voice ids, PCM and container descriptions, and the terminal asset. diff --git a/packages/ai/README.md b/packages/ai/README.md index 5f24fb79768..bc079002ef4 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -1,11 +1,10 @@ # @opencode/ai -Schema-first language model and image-generation APIs built with Effect. +Schema-first APIs for text, images, video, speech, and transcription, built with Effect. ```ts -import { Effect, Layer } from "effect" -import { LLM, LLMClient } from "@opencode/ai" -import { RequestExecutor } from "@opencode/ai/route" +import { Effect } from "effect" +import { AIClient, LLM } from "@opencode/ai" import { OpenAI } from "@opencode/ai/providers" const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY }) @@ -18,23 +17,23 @@ const request = LLM.request({ }) const program = Effect.gen(function* () { - const response = yield* LLMClient.generate(request) + const response = yield* LLM.generate(request) console.log(response.text) }) -const llmLayer = LLMClient.layer.pipe(Layer.provide(RequestExecutor.fetchLayer)) - -await Effect.runPromise(program.pipe(Effect.provide(llmLayer))) +// Every modality client plus the HTTP request executor; `AIClient.layerWith(executor)` swaps the executor. +await Effect.runPromise(program.pipe(Effect.provide(AIClient.layer))) ``` -Run `LLMClient.stream(request)` instead of `generate` when you want incremental `LLMEvent`s. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses, Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment. +Run `LLM.stream(request)` instead of `generate` when you want incremental `LLMEvent`s. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses, +Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment. -The same configured facade names image models. `Image.request` resolves the provider's image route from the ref and -returns `Media.Asset`s with lazily decoded bytes: +The same configured facade names image, video, speech, and transcription models. `Image.generate` resolves the +provider's image route from the model and returns `Media.Asset`s with lazily decoded bytes: ```ts import { NodeFileSystem } from "@effect/platform-node" -import { Image, ImageClient, Media } from "@opencode/ai" +import { Image, Media } from "@opencode/ai" const image = Effect.gen(function* () { const response = yield* Image.generate({ @@ -46,13 +45,28 @@ const image = Effect.gen(function* () { yield* Media.write(response.image, "./garden.png") }) -// `asset.bytes()` / `Media.write` also need the executor, so merge it into the environment instead of hiding it. -const imageLayer = ImageClient.layer.pipe(Layer.provideMerge(RequestExecutor.fetchLayer)) - -await Effect.runPromise(image.pipe(Effect.provide(imageLayer), Effect.provide(NodeFileSystem.layer))) +// `Media.file` / `Media.write` use the Effect `FileSystem` service; provide your platform's layer. +await Effect.runPromise(image.pipe(Effect.provide(AIClient.layer), Effect.provide(NodeFileSystem.layer))) ``` -Prefer promises? `@opencode/ai/promise` exposes the same LLM and image APIs over one managed runtime: +Advanced: each client also has its own `layer`, which requires `RequestExecutor.Service`. Compose client layers with +`Layer.provideMerge`, not `Layer.provide`: `asset.bytes()`, `Media.write`, and Gemini's `media` output parts need the +executor too, and hiding it fails type-checking with `RequestExecutorService` left in the requirements. + +To share a policy such as logging across every client, wrap the executor once with `RequestExecutor.middleware`: + +```ts +import { RequestExecutor } from "@opencode/ai/route" + +const logged = RequestExecutor.middleware((request, next) => + Effect.log(`${request.method} ${request.url}`).pipe(Effect.andThen(next(request))), +) + +const everything = AIClient.layerWith(logged) // or AI.make({ layer: logged }) +``` + +Prefer promises? `@opencode/ai/promise` exposes the same LLM and media APIs over one managed runtime, plus asset +helpers; `ai.file` and `ai.write` load `node:fs/promises` on first use, so no Effect `FileSystem` is needed: ```ts import { AI } from "@opencode/ai/promise" @@ -60,6 +74,7 @@ import { AI } from "@opencode/ai/promise" const ai = AI.make() const text = await ai.llm.generate({ model: openai.responses("gpt-4o-mini"), prompt: "Say hello." }) const generated = await ai.image.generate({ model: openai.image("gpt-image-2"), prompt: "A lighthouse" }) +await ai.write(generated.image, "./lighthouse.png") // also ai.file(path), ai.bytes(asset), ai.base64(asset), ai.materialize(asset) for await (const event of ai.llm.stream({ model: openai.responses("gpt-4o-mini"), prompt: "Stream hello." })) { // LLMEvent } @@ -322,10 +337,9 @@ and `moonshot/responses`; each exports `model(modelID, settings)`. MiniMax defaults to its Messages API and reads `MINIMAX_API_KEY` when `apiKey` is omitted: ```ts -import { Effect, Layer } from "effect" -import { LLM, LLMClient } from "@opencode/ai" +import { Effect } from "effect" +import { AIClient, LLM } from "@opencode/ai" import { MiniMax } from "@opencode/ai/providers" -import { RequestExecutor } from "@opencode/ai/route" const minimax = MiniMax.configure({ apiKey: process.env.MINIMAX_API_KEY }) const request = LLM.request({ @@ -335,8 +349,7 @@ const request = LLM.request({ generation: { maxTokens: 1536 }, }) -const layer = LLMClient.layer.pipe(Layer.provide(RequestExecutor.fetchLayer)) -const response = await Effect.runPromise(LLMClient.generate(request).pipe(Effect.provide(layer))) +const response = await Effect.runPromise(LLM.generate(request).pipe(Effect.provide(AIClient.layer))) console.log(response.text) ``` @@ -405,14 +418,14 @@ Use `Image.generate` for one-off generation or editing: import { Image, Media } from "@opencode/ai" const generation = Image.generate({ - model: meta("muse-image-1.0"), + model: meta.image("muse-image-1.0"), prompt: "A flat black square on a white background.", n: 1, providerOptions: { reasoningStrength: "low" }, }) const edit = Image.generate({ - model: meta("muse-image-1.0"), + model: meta.image("muse-image-1.0"), prompt: "Make the square purple.", images: [Media.bytes(imageBytes, "image/webp")], format: "png", @@ -459,6 +472,25 @@ const program = Effect.gen(function* () { }) ``` +Common fields are portable in shape, not in support. Unsupported fields fail with a typed `AIError` before any network +call rather than being dropped, so check this table before swapping only the `model`: + +| Provider | `n` | `size` | `aspectRatio` | `seed` | `format` | `images` | `mask` | +| --------------------- | --- | --------- | ------------- | ------ | -------- | ------------------------- | ------------------- | +| OpenAI | ✓¹ | ✓ | ✗ | ✗ | ✓ | ✓ | ✓ | +| Google (Gemini) | 1 | ✗ | ✓ | ✓ | ✗ | ✓ (no public URLs) | ✗ | +| xAI | ✓ | ✗ | ✓ | ✗ | ✗ | ✓ | ✗ | +| Z.ai | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | +| Meta | ✓ | ✓ (hint) | ✗ | ✗ | ✓ | ✓ | ✗ | +| Black Forest Labs | 1 | per model | per model | ✓ | ✓ | per model (1–8) | `flux-pro-1.0-fill` | +| fal | ✓ | per model | per model | ✓ | ✓ | 1 (several on `/edit`) | ✓ | +| Replicate | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ (use `providerOptions`) | ✗ | +| Stability `image` | 1 | ✗ | ✓ | ✓ | ✓ | 1 (not on `core`) | ✗ | +| Stability `upscale()` | ✗ | ✗ | ✗ | ✓ | ✓ | exactly 1 (required) | ✗ | + +✓ lowers natively; ✗ fails whenever the field is set (including `n: 1`); `1` means `n > 1` fails. ¹ `Image.stream` on OpenAI generates one image. fal +rejects `size` and `aspectRatio` together; which one a fal or BFL model takes depends on the model. + `Media.Asset` is the one asset type shared by image requests, image responses, LLM messages, and tool results. `asset.source` is the serializable `Media.Source` (`bytes`, `base64`, `url`, or `ref`); `asset.bytes()`, `asset.base64()`, and `asset.dataUrl()` decode or download lazily and cache; `asset.materialize()` pulls a `url` @@ -468,9 +500,8 @@ asset into owned bytes before the provider URL expires. Construct assets with `M Pass ordered image inputs to the same method for editing, composition, or image-conditioned generation: ```ts -const response = - yield * - Image.generate({ +const composed = Effect.gen(function* () { + const response = yield* Image.generate({ model, prompt: "Combine these product photos into one studio scene", images: [ @@ -481,23 +512,25 @@ const response = providerOptions, http, }) + return response.images +}) ``` `Media.ref(provider, id)` represents provider file handles such as OpenAI file IDs or Gemini Files URIs; routes -only forward refs that belong to their own provider. Raw strings are not accepted as image inputs, avoiding -ambiguity between base64, URLs, and provider IDs. Empty or omitted `images` uses text-to-image generation; a -non-empty array selects the provider's edit behavior without enforcing provider image-count limits locally. OpenAI +only forward refs that belong to their own provider (OpenAI, xAI, and Gemini images accept them). No shipped route +returns a ref yet, and `asset.bytes()` / `materialize()` on a ref fail by design. Raw strings are not accepted as +image inputs, avoiding ambiguity between base64, URLs, and provider IDs. Empty or omitted `images` uses text-to-image generation; a +non-empty array selects the provider's edit behavior (see the table above for routes that limit the count). OpenAI uses multipart for byte/data-URL edits and its JSON reference body for URL or file-ID edits. The common `mask` field selects inpainting; routes that cannot honor it fail with `UnsupportedOperation`: ```ts -yield * - Image.generate({ - model: openai.image("gpt-image-2"), - prompt, - images: [Media.bytes(sourceBytes, "image/png")], - mask: Media.bytes(maskBytes, "image/png"), - }) +const inpainted = Image.generate({ + model: openai.image("gpt-image-2"), + prompt, + images: [Media.bytes(sourceBytes, "image/png")], + mask: Media.bytes(maskBytes, "image/png"), +}) ``` On multipart requests, `http.body` can override option fields but not structural `model`, `prompt`, `image[]`, @@ -508,31 +541,31 @@ not accept image inputs. These cases fail with a typed `AIError` before network Provider-native image options belong to each request. Raw `http.body` fields have final precedence over them: ```ts -yield * - Image.generate({ - model: openai.image("gpt-image-2"), - prompt, - providerOptions: { quality: "medium" }, - http, - }) +const medium = Image.generate({ + model: openai.image("gpt-image-2"), + prompt, + providerOptions: { quality: "medium" }, + http, +}) ``` xAI image models use the same request API with xAI-native controls: ```ts -yield * - Image.generate({ - model: XAI.configure({ apiKey })("any-model-id"), - prompt, - n: 2, - aspectRatio: "16:9", - providerOptions: { - resolution: "1k", - responseFormat: "b64_json", - future_option: true, - }, - http, - }) +import { XAI } from "@opencode/ai/providers" + +const xai = Image.generate({ + model: XAI.configure({ apiKey }).image("any-model-id"), + prompt, + n: 2, + aspectRatio: "16:9", + providerOptions: { + resolution: "1k", + responseFormat: "b64_json", + future_option: true, + }, + http, +}) ``` Google's current Gemini image models use the same direct API: @@ -542,7 +575,7 @@ import { Google } from "@opencode/ai/providers" const googleProgram = Effect.gen(function* () { const response = yield* Image.generate({ - model: Google.configure({ apiKey })("any-model-id"), + model: Google.configure({ apiKey }).image("any-model-id"), prompt: "A robot tending a rooftop garden", aspectRatio: "16:9", seed: 42, @@ -567,17 +600,18 @@ their mapped aliases, and `http.body` is the final deep overlay. The selected mo Z.ai image models infer open Z.ai-native options from the selected model: ```ts -yield * - Image.generate({ - model: ZAI.configure({ apiKey })("any-model-id"), - prompt, - providerOptions: { - quality: "hd", - userID: "user-123", - future_option: true, - }, - http, - }) +import { ZAI } from "@opencode/ai/providers" + +const zai = Image.generate({ + model: ZAI.configure({ apiKey }).image("any-model-id"), + prompt, + providerOptions: { + quality: "hd", + userID: "user-123", + future_option: true, + }, + http, +}) ``` Z.ai does not include trustworthy MIME metadata for output URLs, so generated images use @@ -591,12 +625,14 @@ and emits `image-partial` events before each final `image`; `Image.generate` kee `dall-e-*` models do not stream and fail typed: ```ts -yield * - Image.stream({ - model: openai.image("gpt-image-2"), - prompt: "A lighthouse at dusk", - providerOptions: { partialImages: 2 }, - }).pipe(Stream.runForEach((event) => (ImageEvent.is.imagePartial(event) ? showPreview(event.image) : Effect.void))) +import { Stream } from "effect" +import { ImageEvent } from "@opencode/ai" + +const previews = Image.stream({ + model: openai.image("gpt-image-2"), + prompt: "A lighthouse at dusk", + providerOptions: { partialImages: 2 }, +}).pipe(Stream.runForEach((event) => (ImageEvent.is.imagePartial(event) ? showPreview(event.image) : Effect.void))) ``` The provider may send fewer previews than requested when the final image is ready first. @@ -612,13 +648,20 @@ import { BlackForestLabs, Stability } from "@opencode/ai/providers" const bfl = BlackForestLabs.configure({ apiKey: process.env.BFL_API_KEY }) -const generation = yield * Image.start({ model: bfl.image("flux-2-pro"), prompt, size: "1024x768" }) -persist(generation.token) +const submit = Effect.gen(function* () { + const generation = yield* Image.start({ model: bfl.image("flux-2-pro"), prompt, size: "1024x768" }) + persist({ provider: "black-forest-labs", modelID: "flux-2-pro", token: generation.token }) +}) -const resumed = yield * Image.resume(bfl.image("flux-2-pro"), loadToken()) -const response = yield * resumed.await({ poll: { interval: "2 seconds" } }) +const finish = Effect.gen(function* () { + const saved = load() + const resumed = yield* Image.resume(bfl.image(saved.modelID), saved.token) + return yield* resumed.await({ poll: { interval: "2 seconds" } }) +}) ``` +The token carries no route identity, so persist the provider and model ID alongside it: `resume` needs the model. + - **Black Forest Labs** — results are downloaded before returning, because `result.sample` expires in 10 minutes. - **Replicate** — inputs are model-defined, so only `prompt` lowers: sizing, count, seed, format, and files go in `providerOptions` under the model's names, with files as `Media.Asset` (data URLs up to 256 KB, larger by URL). @@ -628,12 +671,13 @@ const response = yield * resumed.await({ poll: { interval: "2 seconds" } }) ```ts const stability = Stability.configure({ apiKey: process.env.STABILITY_API_KEY }) -const upscaled = - yield * - Image.generate( - { model: stability.upscale(), prompt: "A lighthouse", images: [yield * Media.file("./small.png")] }, +const upscaled = Effect.gen(function* () { + const small = yield* Media.file("./small.png") + return yield* Image.generate( + { model: stability.upscale(), prompt: "A lighthouse", images: [small] }, { poll: { interval: "5 seconds" } }, ) +}) ``` Imagen is not available: Google shut it down on the Gemini API, and Vertex discontinued the Imagen 4 models on @@ -666,8 +710,8 @@ Common fields (`frames`, `references`, `video`, `durationSeconds`, `aspectRatio` under `providerOptions`, inferred from the selected model. ```ts -import { Video, VideoClient } from "@opencode/ai" -import { Google } from "@opencode/ai/providers" +import { Video } from "@opencode/ai" +import { Google, Runway } from "@opencode/ai/providers" const google = Google.configure({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY }) @@ -696,6 +740,7 @@ const controlled = Effect.gen(function* () { generation.id // provider operation / task / request id generation.status // "queued" | "running" | "completed" | "failed" | "cancelled" | "expired" generation.token // route-owned JSON: `{ operation }`, `{ requestID }`, `{ taskID }`, or fal's follow-up URLs + // The token carries no route identity: persist the provider and model ID alongside it, since `resume` needs the model. const saved = JSON.stringify(generation.token) const resumed = yield* Video.resume(google.video("veo-3.1-generate-preview"), JSON.parse(saved)) @@ -706,8 +751,8 @@ const controlled = Effect.gen(function* () { const events = Video.stream({ model: Runway.configure({ apiKey }).video("gen4.5"), prompt }, { poll }) ``` -`VideoClient.layer` needs `RequestExecutor.Service`, and status polls, result fetches, cancels, and asset downloads -all run through the same executor with the route's auth. `Generation.await` and `Generation.events` fail with a +Status polls, result fetches, cancels, and asset downloads all run through the same request executor with the route's +auth. `Generation.await` and `Generation.events` fail with a `Timeout` reason when `poll.timeout` (default 10 minutes) elapses. Failed, cancelled, and expired generations fail typed with the provider's terminal document on `reason.body`; moderation outcomes (Veo `raiMediaFilteredReasons`, xAI `respect_moderation`, Runway `SAFETY.*` codes) surface as `notices` when @@ -726,15 +771,18 @@ Provider notes: - **Runway** expects pixel ratios in `aspectRatio` for most models (`"1280:720"`), pins `X-Runway-Version`, reports `usage: { type: "credits" }`, and its output URLs expire after 24–48 hours. -The promise client exposes the same surface: `ai.video.start(...)` resolves to a handle with `await`, `refresh`, -`cancel`, and `token`; `ai.video.generate`, `ai.video.resume(model, token)`, and `ai.video.stream` mirror the Effect -API. +The promise client exposes the same surface: `ai.video.start(...)` resolves to a handle with `await`, `events`, +`result`, `refresh`, `cancel`, and `token`; `ai.video.generate`, `ai.video.resume(model, token)`, and +`ai.video.stream` mirror the Effect API. The handle's `status` and `progress` are a snapshot from when it was +created; `refresh()` resolves to a new handle. ```ts import { ai } from "@opencode/ai/promise" const generation = await ai.video.start({ model, prompt }) -const video = await generation.await({ poll: { interval: 10_000 }, signal }) +for await (const event of generation.events({ poll: { interval: 10_000 } })) console.log(event.type) +const video = await generation.result({ signal }) +await ai.write(video.video, "./kite.mp4") ``` ## Speech generation @@ -786,7 +834,6 @@ ElevenLabs and Cartesia. `{ id }` selects an OpenAI custom voice (`{ id: "voice_ plain string elsewhere. There is no cross-provider voice catalog. `format` is the container-level word (`mp3`, `wav`, `pcm`, `opus`, `aac`, `flac`); sample rates and bitrates live under `providerOptions`, and a value the route cannot produce fails as `UnsupportedOperation`. Streams buffer every chunk so `finish` can carry the whole clip. -`SpeechClient.layer` needs `RequestExecutor.Service`. Provider notes: @@ -814,7 +861,7 @@ The promise client mirrors the Effect API; `ai.speech.stream` is an `AsyncIterab import { ai } from "@opencode/ai/promise" const response = await ai.speech.generate({ model, text: "Hello from OpenCode.", voice: "coral" }) -await Bun.write("hello.mp3", await ai.run(response.audio.bytes())) +await ai.write(response.audio, "hello.mp3") for await (const event of ai.speech.stream({ model, text: "Hello from OpenCode.", voice: "coral" })) { if (event.type === "audio-delta") player.write(event.chunk) @@ -831,6 +878,7 @@ facades. Common fields (`language`, `prompt`, `timestamps: "none" | "segment" | natively or fail with a typed `AIError` before any network call; a route may return more than asked. ```ts +import { Console, Effect, Stream } from "effect" import { Media, Transcription, TranscriptionEvent } from "@opencode/ai" import { AssemblyAI, Deepgram, OpenAI } from "@opencode/ai/providers" @@ -857,7 +905,7 @@ const program = Effect.gen(function* () { Stream.runDrain, ) - // Queued: persist the token, resume from another process, and await. + // Queued: persist the token with the provider and model ID (the token alone cannot pick the model), resume, and await. const model = AssemblyAI.configure({ apiKey }).transcription("universal-3-5-pro") const generation = yield* Transcription.start({ model, audio }) const resumed = yield* Transcription.resume(model, JSON.parse(JSON.stringify(generation.token))) @@ -866,7 +914,7 @@ const program = Effect.gen(function* () { ``` Inline routes emit only `finish` from `stream` (no faked deltas); queued routes emit `generation-queued` / -`generation-progress` before it. `TranscriptionClient.layer` needs `RequestExecutor.Service`. +`generation-progress` before it. Provider notes: @@ -878,6 +926,7 @@ Provider notes: The promise client mirrors the Effect API: ```ts +const audio = await ai.file("./call.mp3") const text = (await ai.transcription.generate({ model, audio })).text for await (const event of ai.transcription.stream({ model, audio })) if (event.type === "text-delta") write(event.delta) const generation = await ai.transcription.start({ model: assemblyai, audio }) @@ -897,7 +946,8 @@ const transcript = await generation.await({ poll: { interval: 3_000 } }) - **`Generation`** — provider-neutral handle for an in-flight media generation (`await`, `refresh`, `cancel`, `events`) used by queued media routes. - **`Speech.request` / `Speech.generate` / `Speech.stream`** — text-to-speech through a provider-neutral request; `SpeechClient` is its Effect service and layer. - **`Transcription.request` / `generate` / `stream` / `start` / `resume`** — speech-to-text over inline, streaming, and queued routes; `TranscriptionClient` is its Effect service and layer. -- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm`, `image`, `video`, `speech`, and `transcription` as Promise / `AsyncIterable` APIs. +- **`AIClient.layer` / `AIClient.layerWith(executor)`** — every modality client plus the request executor in one layer. +- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm`, `image`, `video`, `speech`, and `transcription` as Promise / `AsyncIterable` APIs, plus `file`, `write`, `bytes`, `base64`, and `materialize` for assets. ## Testing @@ -960,11 +1010,13 @@ This is different from prompt caching, server-side history storage, or truncatio Prefer this operation, where supported, when the application owns compaction policy and durable context updates. ```ts -const result = yield * LLMClient.compact(request) -const next = LLMRequest.update(request, { - messages: result.replacement, +const compacted = Effect.gen(function* () { + const result = yield* LLMClient.compact(request) + const next = LLMRequest.update(request, { + messages: result.replacement, + }) + return yield* LLMClient.generate(next) }) -const response = yield * LLMClient.generate(next) ``` `replacement` replaces the complete input window. Do not append it to the original transcript or extract only the encrypted item: the provider may retain additional messages in its output. Retained user and assistant messages remain ordinary messages with typed text, media, or reasoning parts, in their original order. Provider-specific message IDs, status, and phase use `providerMetadata`, not a raw output array hidden in an assistant message. Unsupported returned item types fail explicitly. @@ -980,16 +1032,16 @@ The input must still fit the model's context window. Explicit compaction is not OpenAI Responses also exposes a separate, explicitly selected mechanism: ```ts -const result = - yield * - LLMClient.compact(request, { +const checkpoint = Effect.gen(function* () { + const result = yield* LLMClient.compact(request, { mechanism: "trigger", webSocket, // Optional: without it, the request uses HTTP/SSE. }) -result.checkpoint // Successful encrypted CompactionPart. -result.responseID -result.usage + result.checkpoint // Successful encrypted CompactionPart. + result.responseID + result.usage +}) ``` This appends a native `compaction_trigger` control item to the full input and sends a normal Responses request, with tools and instructions retained, `stream: true`, `store: false`, and parallel tool calls enabled. It removes normal-answer text/output-format controls, forced tool choices, output-token/tool-call limits, and automatic `context_management`. Body overlays cannot replace `input` or supply `previous_response_id`/`conversation`; the complete canonical history is required for safe stateless replay. Request metadata, auth, headers, query parameters, service tier, and supported prompt-cache settings are preserved. @@ -1003,9 +1055,11 @@ The supplied WebSocket executor can reuse a compatible append baseline for the c Trigger support is separate from endpoint support. Only the OpenAI Responses route advertises it; Azure, xAI, Chat, and compatible Responses routes do not inherit it. Untyped calls still fail before sending: missing route capabilities return `UnsupportedOperation`, while unknown mechanism names and invalid inputs return `InvalidRequest`. Dynamic callers must narrow for the selected mechanism: ```ts -if (LLMClient.canCompact(request, { mechanism: "trigger" })) { - const result = yield * LLMClient.compact(request, { mechanism: "trigger" }) -} +const narrowed = Effect.gen(function* () { + if (LLMClient.canCompact(request, { mechanism: "trigger" })) { + const result = yield* LLMClient.compact(request, { mechanism: "trigger" }) + } +}) ``` This capability describes protocol implementation, **not universal availability on OpenAI API deployments**. The host application owns subscription/deployment eligibility, OAuth, endpoint selection, and deployment-specific headers. Local protocol/socket tests do not establish live provider support. @@ -1014,9 +1068,10 @@ This capability describes protocol implementation, **not universal availability `providerOptions.contextManagement` lets the provider decide when to compact during an ordinary `generate` or `stream` call. This is an advanced option for callers that own persistence and recovery: persist the complete assistant message, including its checkpoint, before continuing. Enabling the option does not provide durable checkpoint storage, interruption recovery, or model-switch policy. Keep the prior context until a successful checkpoint has been persisted. -Inside an `Effect.gen`, enable OpenAI compaction with typed provider options: +Enable OpenAI compaction with typed provider options: ```ts +import { Effect } from "effect" import { LLM, LLMClient, LLMRequest, Message } from "@opencode/ai" import { OpenAI } from "@opencode/ai/providers" @@ -1027,9 +1082,11 @@ const request = LLM.request({ contextManagement: [{ type: "compaction", compactThreshold: 200_000 }], }, }) -const response = yield * LLMClient.generate(request) -const next = LLMRequest.update(request, { - messages: [...request.messages, response.message, Message.user("Continue")], +const continued = Effect.gen(function* () { + const response = yield* LLMClient.generate(request) + return LLMRequest.update(request, { + messages: [...request.messages, response.message, Message.user("Continue")], + }) }) ``` @@ -1257,7 +1314,7 @@ Compose a route with `Route.make({ protocol, endpoint, auth, framing, ... })`. T ## Effect -This package is built on Effect. Public methods return `Effect` or `Stream`; provide `LLMClient.layer` for LLM dispatch and `ImageClient.layer` for image dispatch, then import the provider/protocol modules for the routes you use. The example at `example/tutorial.ts` is a runnable walkthrough. +This package is built on Effect. Public methods return `Effect` or `Stream`; provide `AIClient.layer` (or `AIClient.layerWith(executor)`) for every modality, then import the provider/protocol modules for the routes you use. The example at `example/tutorial.ts` is a runnable walkthrough. ## See also diff --git a/packages/ai/docs/media-design.md b/packages/ai/docs/media-design.md index b1522f5179d..d0b032623e6 100644 --- a/packages/ai/docs/media-design.md +++ b/packages/ai/docs/media-design.md @@ -1,6 +1,6 @@ # Media generation in `@opencode/ai` — public API direction -Status: phases 1–3 implemented (Speech and Transcription); phases 4–5 proposal. +Status: phases 1–4 implemented (through Image queued routes and partial images); phase 5 proposal. ## Goal @@ -54,7 +54,7 @@ Speech.request({ model: openai.speech("gpt-4o-mini-tts"), text }) Transcription.request({ model: openai.transcription("gpt-4o-transcribe"), audio }) ``` -The request namespace and the selector share one word (`Image.request` + `.image(...)`). That redundancy is accepted: a callable facade returning a lazily resolved ref would be a second way to construct the same model, and the type machinery to infer `providerOptions` through it is not worth one word. Where a provider has two routes for one modality, the selectors stay explicit (`openai.chat`, `stability.image` inline vs `stability.upscale()` queued), and one default per modality per provider is part of the facade definition (OpenAI image → Images API, Google image → Gemini-native; Imagen is shut down, so there is no `google.imagen`). Provider package entrypoints keep `model(modelID, settings)` per modality-specific path, e.g. `@opencode/ai/providers/openai/responses`. +The request namespace and the selector share one word (`Image.request` + `.image(...)`). That redundancy is accepted: a callable facade returning a lazily resolved ref would be a second way to construct the same model, and the type machinery to infer `providerOptions` through it is not worth one word. Where a provider has two routes for one modality, the selectors stay explicit (`openai.chat`, `stability.image` inline vs `stability.upscale()` queued), and one default per modality per provider is part of the facade definition (OpenAI image → Images API, Google image → Gemini-native; Imagen is shut down, so there is no `google.imagen`). The facade selector (`openai.image(id)`) is the public path for media models. Modality-specific package entrypoints (`model(modelID, settings)` beside today's LLM paths such as `@opencode/ai/providers/openai/responses`) are deferred until Core has a modality-aware model resolver; Core's resolver accepts only `LanguageModel` today. ### `Media` — the asset type @@ -86,10 +86,18 @@ class Media.Asset { Media.bytes(data, mediaType?) Media.base64(data, mediaType?) Media.url(url, options?) Media.ref(provider, id) -Media.file(path) // Bun/Node: reads + sniffs; Effect FileSystem variant for layers -Media.write(asset, path) // convenience, uses FileSystem +Media.file(path) // Effect: reads + sniffs +Media.write(asset, path) // Effect ``` +`Media.file` and `Media.write` stay Effect-only: bring your platform's `FileSystem` layer. The Promise client owns the +runtime path: `ai.file(path)` and `ai.write(asset, path)` read and write through `node:fs/promises` (loaded on first +use) with the same media-type sniffing and `InvalidRequest` failures, and `ai.bytes`, `ai.base64`, and +`ai.materialize` run the asset methods in its runtime. + +A `ref` source is accepted as input only by routes whose provider issues file handles. No shipped route produces one +yet, so `bytes()` and `materialize()` on a ref fail by design until a producer exists. + Raw-PCM outputs (Gemini TTS, Cartesia raw, Deepgram WS) carry `info.encoding/sampleRate/channels` because there is no container header. ### Modality namespaces @@ -104,28 +112,32 @@ import { OpenAI, Google, ElevenLabs, Fal } from "@opencode/ai/providers" #### Image ```ts -const request = Image.request({ - model: openai.image("gpt-image-2"), - prompt: "A robot tending a rooftop garden", - images: [Media.file("./ref.png")], // references / edit sources - mask: Media.file("./mask.png"), - n: 2, - size: "1536x1024", // or aspectRatio: "3:2" - seed: 7, - format: "webp", - providerOptions: { quality: "high", background: "transparent" }, // typed per model +Effect.gen(function* () { + const request = Image.request({ + model: openai.image("gpt-image-2"), + prompt: "A robot tending a rooftop garden", + images: [yield* Media.file("./ref.png")], // references / edit sources + mask: yield* Media.file("./mask.png"), + n: 2, + size: "1536x1024", // OpenAI sizes by pixels; Gemini/xAI take aspectRatio instead + format: "webp", + providerOptions: { quality: "high", background: "transparent" }, // typed per model + }) + + const response = yield* Image.generate(request) // ImageResponse + response.image // Media.Asset (first) + response.images // Media.Asset[] + response.usage // Usage union (see below) + response.notices // moderation / partial-result notices + + Image.stream(request) // Stream + // ImageEvent: generation-queued | generation-progress | image-partial { index, image } | image { index, image } | finish { usage } }) - -const response = yield* Image.generate(request) // ImageResponse -response.image // Media.Asset (first) -response.images // Media.Asset[] -response.usage // Usage union (see below) -response.notices // moderation / partial-result notices - -yield* Image.stream(request) // Stream -// ImageEvent: generation-queued | generation-progress | image-partial { index, image } | image { index, image } | finish { usage } ``` +`size` and `aspectRatio` are not interchangeable; each route rejects fields it cannot lower — see the README's Image +portability matrix. + Editing is not a separate function; `images`/`mask` on the request select the edit path in the route (OpenAI `/images/edits`, Gemini multimodal parts, xAI `/images/edits`). Routes that cannot honor `mask` fail with `Unsupported`. `ImageRoute` is the inline | stream | queued union, dispatched on `route.kind`. `Image.stream` on a streaming route emits `image-partial` previews before each `image`; on a queued route it emits `generation-queued` / `generation-progress` observations, then the result's `image` and `finish` events. @@ -135,40 +147,43 @@ Editing is not a separate function; `images`/`mask` on the request select the ed Shipped in phase 2 (`src/video.ts`, `src/video-client.ts`, protocols `google-video`, `xai-video`, `fal-video`, `runway-video`). ```ts -const request = Video.request({ - model: google.video("veo-3.1-generate-preview"), - prompt: "Panning wide shot of a calico kitten sleeping in the sunshine", - frames: { first: Media.file("./start.png"), last: Media.file("./end.png") }, - references: [Media.file("./style.png")], - video: Media.bytes(previous, "video/mp4"), // edit / extend source - durationSeconds: 8, - aspectRatio: "16:9", - resolution: "1080p", - audio: true, - n: 1, - seed: 7, - negativePrompt: "text, watermark", // common, not provider-native - providerOptions: { personGeneration: "allow_adult" }, +Effect.gen(function* () { + const request = Video.request({ + model: google.video("veo-3.1-generate-preview"), + prompt: "Panning wide shot of a calico kitten sleeping in the sunshine", + frames: { first: yield* Media.file("./start.png"), last: yield* Media.file("./end.png") }, + references: [yield* Media.file("./style.png")], + video: Media.bytes(previous, "video/mp4"), // edit / extend source + durationSeconds: 8, + aspectRatio: "16:9", + resolution: "1080p", + audio: true, + n: 1, + seed: 7, + negativePrompt: "text, watermark", // common, not provider-native + providerOptions: { personGeneration: "allow_adult" }, + }) + + // Simple: wait for it. + const response = yield* Video.generate(request, { poll: { interval: "10 seconds", timeout: "10 minutes" } }) + response.video // Media.Asset: url with expiresAt (+ transient `headers` for Veo downloads) + response.usage // credits on Runway; the other three report none + response.notices // Veo raiMediaFilteredReasons → filtered, xAI respect_moderation → moderated + yield* response.video.materialize() // pull bytes before the URL expires + + // Explicit generation control. + const generation = yield* Video.start(request) // Generation + generation.id; generation.status; generation.progress; generation.position; generation.token + yield* generation.await({ poll }) // VideoResponse + yield* generation.cancel() // fal PUT cancel_url, Runway DELETE /tasks/{id}; no-op for Veo and xAI + + // Resume from another process. The token is validated against the route's codec and refreshed once. It carries no + // route identity, so persist the provider and model ID alongside it: `resume` needs the model. + const resumed = yield* Video.resume(model, JSON.parse(saved)) + + // Progress as a stream. + Video.stream(request, { poll }) // Stream: generation-queued { id, position } | generation-progress { id, progress } | video { index, video } | finish { usage, notices } }) - -// Simple: wait for it. -const response = yield* Video.generate(request, { poll: { interval: "10 seconds", timeout: "10 minutes" } }) -response.video // Media.Asset: url with expiresAt (+ transient `headers` for Veo downloads) -response.usage // credits on Runway; the other three report none -response.notices // Veo raiMediaFilteredReasons → filtered, xAI respect_moderation → moderated -yield* response.video.materialize() // pull bytes before the URL expires - -// Explicit generation control. -const generation = yield* Video.start(request) // Generation -generation.id; generation.status; generation.progress; generation.position; generation.token -yield* generation.await({ poll }) // VideoResponse -yield* generation.cancel() // fal PUT cancel_url, Runway DELETE /tasks/{id}; no-op for Veo and xAI - -// Resume from another process. The token is validated against the route's codec and refreshed once. -const resumed = yield* Video.resume(model, JSON.parse(saved)) - -// Progress as a stream. -yield* Video.stream(request, { poll }) // Stream: generation-queued { id, position } | generation-progress { id, progress } | video { index, video } | finish { usage, notices } ``` Tokens are route-owned JSON: Veo `{ operation }`, xAI `{ requestID }`, Runway `{ taskID }`, fal @@ -330,12 +345,12 @@ class Generation { readonly expiresAt?: number refresh(): Effect, AIError> result(): Effect - await(options?: AwaitOptions): Effect + await(options?: GenerationAwaitOptions): Effect cancel(): Effect - events(options?: AwaitOptions): Stream // fails with Timeout past poll.timeout, checked per observation + events(options?: GenerationAwaitOptions): Stream // fails with Timeout past poll.timeout, checked per observation } -AwaitOptions = { poll?: Poll } +GenerationAwaitOptions = { poll?: Poll } Poll = { interval?: Duration; timeout?: Duration; schedule?: Schedule } // route may override from provider hints (`openai-poll-after-ms`) ``` @@ -365,15 +380,17 @@ const ai = AI.make() // ManagedRuntime over Reque // AI.make({ layer }) to inject a custom executor / recorder / middleware const image = await ai.image.generate({ model, prompt }) -await image.image.bytes() +await ai.bytes(image.image) // also ai.base64, ai.materialize, ai.write(asset, path) +const reference = await ai.file("./ref.png") for await (const event of ai.speech.stream({ model, text, voice })) { … } -const generation = await ai.video.start({ model, prompt }) +const generation = await ai.video.start({ model, prompt }) // snapshot handle; refresh() returns a new one +for await (const event of generation.events({ poll: { interval: 10_000 } })) { … } const video = await generation.await({ poll: { interval: 10_000 }, signal }) -const resumed = ai.video.resume(model, JSON.parse(saved)) +const resumed = await ai.video.resume(model, JSON.parse(saved)) // persist provider + model ID with the token -const text = await ai.llm.generate({ model, prompt }) // closes today's gap: LLM has no promise API either +const text = await ai.llm.generate({ model, prompt }) for await (const event of ai.llm.stream(request)) { … } await ai.dispose() @@ -401,9 +418,9 @@ Existing facades gain per-modality selectors; the modality routes each facade pr | `Runway` | | | ✓ | | | | | `Luma`, `Kling`, `MiniMax` | | per provider | | | | | -New facades follow the existing one-file-per-provider rule. Package entrypoints are modality-specific, such as `@opencode/ai/providers/openai/images`, and return the concrete model. +New facades follow the existing one-file-per-provider rule. The facade selector is the public path for media models; modality-specific package entrypoints (for example `@opencode/ai/providers/openai/images`) are deferred until Core has a modality-aware model resolver. -`ImageModel` already gives typed `providerOptions` per model; `VideoModel`, `SpeechModel`, `TranscriptionModel` follow the same generic. A shared `MediaModel` union is what `Generation` and the promise client key on. +`ImageModel` gives typed `providerOptions` per model; `VideoModel`, `SpeechModel`, and `TranscriptionModel` follow the same generic. They share an internal `MediaModel` base class (ids, route, `http` overlays) that is not part of the public exports; `Generation` and the promise client work with the concrete modality models. ### Routes and protocols diff --git a/packages/ai/example/tutorial.ts b/packages/ai/example/tutorial.ts index 985260ecd5f..b97fbb212df 100644 --- a/packages/ai/example/tutorial.ts +++ b/packages/ai/example/tutorial.ts @@ -1,18 +1,7 @@ -import { Config, Effect, Formatter, Layer, Schema, Stream } from "effect" +import { Config, Effect, Formatter, Schema, Stream } from "effect" import { NodeFileSystem } from "@effect/platform-node" -import { - Image, - ImageClient, - LLM, - LLMClient, - LLMRequest, - Media, - Message, - ProviderID, - Tool, - ToolRuntime, -} from "@opencode/ai" -import { Route, Auth, Endpoint, Framing, Protocol, RequestExecutor } from "@opencode/ai/route" +import { AIClient, Image, LLM, LLMRequest, Media, Message, ProviderID, Tool, ToolRuntime } from "@opencode/ai" +import { Route, Auth, Endpoint, Framing, Protocol } from "@opencode/ai/route" import { OpenAI } from "@opencode/ai/providers" /** @@ -243,12 +232,10 @@ const generateImage = Effect.gen(function* () { yield* Media.write(response.image, "tutorial-image.jpg").pipe(Effect.provide(NodeFileSystem.layer)) }) -// Provide the LLM runtime and the HTTP request executor once. Keep one path -// enabled at a time so the tutorial can demonstrate generate, stream, or +// Provide every modality client and the HTTP request executor once with +// `AIClient.layer` (`AIClient.layerWith(executor)` swaps the executor). Keep one +// path enabled at a time so the tutorial can demonstrate generate, stream, or // tool-loop behavior without spending tokens on every example. -const requestExecutorLayer = RequestExecutor.fetchLayer -const llmClientLayer = LLMClient.layer.pipe(Layer.provide(requestExecutorLayer)) -const imageClientLayer = ImageClient.layer.pipe(Layer.provide(requestExecutorLayer)) const program = Effect.gen(function* () { // yield* generateOnce @@ -257,6 +244,6 @@ const program = Effect.gen(function* () { // yield* generateDynamicObject.pipe(Effect.andThen((response) => Effect.sync(() => console.log(response.object)))) // yield* generateImage yield* streamWithTools -}).pipe(Effect.provide(Layer.mergeAll(requestExecutorLayer, llmClientLayer, imageClientLayer))) +}).pipe(Effect.provide(AIClient.layer)) Effect.runPromise(program) diff --git a/packages/ai/src/ai-client.ts b/packages/ai/src/ai-client.ts new file mode 100644 index 00000000000..268e32b4a6b --- /dev/null +++ b/packages/ai/src/ai-client.ts @@ -0,0 +1,24 @@ +import { Layer } from "effect" +import { ImageClient } from "./image-client.js" +import { LLMClient } from "./route/client.js" +import { RequestExecutor } from "./route/executor.js" +import { SpeechClient } from "./speech-client.js" +import { TranscriptionClient } from "./transcription-client.js" +import { VideoClient } from "./video-client.js" + +/** Every modality client over `executor`, which stays in the output so `asset.bytes()` and `Media.write` resolve. */ +export const layerWith = (executor: Layer.Layer) => + Layer.mergeAll( + LLMClient.layer, + ImageClient.layer, + VideoClient.layer, + SpeechClient.layer, + TranscriptionClient.layer, + ).pipe(Layer.provideMerge(executor)) + +/** Every modality client plus the executor over `RequestExecutor.fetchLayer`: the one layer most programs need. */ +export const layer = layerWith(RequestExecutor.fetchLayer) + +export type Services = Layer.Success + +export * as AIClient from "./ai-client.js" diff --git a/packages/ai/src/experimental/evaluation-client.ts b/packages/ai/src/experimental/evaluation-client.ts index 8fdd2ceb410..511cf4ec26c 100644 --- a/packages/ai/src/experimental/evaluation-client.ts +++ b/packages/ai/src/experimental/evaluation-client.ts @@ -86,7 +86,7 @@ export const layer: Layer.Layer = Layer }) }), ) -export const fetchLayer = layer.pipe(Layer.provide(RequestExecutor.fetchLayer)) +export const fetchLayer = layer.pipe(Layer.provideMerge(RequestExecutor.fetchLayer)) export const EvaluationClient = { Service, diff --git a/packages/ai/src/image-client.ts b/packages/ai/src/image-client.ts index b155f5159e9..6b8453254d2 100644 --- a/packages/ai/src/image-client.ts +++ b/packages/ai/src/image-client.ts @@ -30,7 +30,9 @@ export interface Interface { ) => Effect.Effect, AIError> } -export class Service extends Context.Service()("@opencode/ImageClient") {} +export class ImageClientService extends Context.Service()("@opencode/ImageClient") {} +export const Service = ImageClientService +export type Service = ImageClientService export const generate = ( request: ImageRequestFor, diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 594ae5d6d58..f43b90d90d8 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -1,3 +1,4 @@ +export { AIClient } from "./ai-client.js" export { LLMClient } from "./route/client.js" export { ImageClient } from "./image-client.js" export { Auth } from "./route/auth.js" @@ -8,7 +9,7 @@ export type { RouteLanguageModelInput, RouteRoutedLanguageModelInput, Interface as LLMClientShape, - Service as LLMClientService, + LLMClientService, } from "./route/client.js" export * from "./schema/index.js" export { diff --git a/packages/ai/src/media.ts b/packages/ai/src/media.ts index 6f231e25b40..fca61a3cc2a 100644 --- a/packages/ai/src/media.ts +++ b/packages/ai/src/media.ts @@ -6,7 +6,7 @@ import { ProviderID } from "./schema/ids.js" import { AIError, HttpContext, InvalidProviderOutputError, InvalidRequestError } from "./schema/errors.js" import { ProviderMetadata } from "./schema/options.js" import { Service } from "./route/executor-service.js" -import { detectMediaType, extensionMediaType } from "./utils/media-type.js" +import { detectMediaType, fileMediaType } from "./utils/media-type.js" export { detectMediaType } from "./utils/media-type.js" @@ -309,7 +309,7 @@ export const file = (path: string, options?: AssetOptions): Effect.Effect invalid(`Failed to read media file ${path}`, cause))) - return bytes(data, detectMediaType(data) ?? extensionMediaType(path), options) + return bytes(data, fileMediaType(data, path), options) }) /** Materialize an asset and write its bytes through `FileSystem`. */ diff --git a/packages/ai/src/promise.ts b/packages/ai/src/promise.ts index b2a775b5acf..fed0474fafa 100644 --- a/packages/ai/src/promise.ts +++ b/packages/ai/src/promise.ts @@ -1,14 +1,14 @@ import { Effect, Layer, ManagedRuntime, Stream } from "effect" -import type { AwaitOptions, Generation, Snapshot } from "./generation.js" +import { AIClient } from "./ai-client.js" +import type { AwaitOptions, Event, Generation, Snapshot } from "./generation.js" import { Image, ImageModel, ImageRequest, type ImageOptions, type ImageRequestInput } from "./image.js" -import { ImageClient } from "./image-client.js" import { LLM } from "./index.js" -import { LLMClient } from "./route/client.js" +import { Media } from "./media.js" +import { tryRequest } from "./media-model.js" import { RequestExecutor } from "./route/executor.js" -import { LanguageModel, LLMRequest } from "./schema/index.js" +import { AIError, InvalidRequestError, LanguageModel, LLMRequest } from "./schema/index.js" import type { RequestInput } from "./llm.js" import { Speech, SpeechModel, SpeechRequest, type SpeechRequestInput } from "./speech.js" -import { SpeechClient } from "./speech-client.js" import { Transcription, TranscriptionModel, @@ -16,17 +16,16 @@ import { type TranscriptionOptions, type TranscriptionRequestInput, } from "./transcription.js" -import { TranscriptionClient } from "./transcription-client.js" +import { fileMediaType } from "./utils/media-type.js" import { Video, VideoModel, VideoRequest, type VideoOptions, type VideoRequestInput } from "./video.js" -import { VideoClient } from "./video-client.js" /** * Promise-first entrypoint for scripts and non-Effect callers. One `ManagedRuntime` hosts the LLM, image, video, speech, * and transcription clients over a request executor; every method runs the corresponding Effect API and rethrows - * `AIError` unchanged. + * `AIError` unchanged. `file` and `write` load `node:fs/promises` on first use, so importing this module does not. */ export interface Options { - /** Executor layer; defaults to `RequestExecutor.fetchLayer`. Inject a recorder or middleware here. */ + /** Executor layer; defaults to `RequestExecutor.fetchLayer`. Inject a recorder or `RequestExecutor.middleware(fn)` here. */ readonly layer?: Layer.Layer } @@ -34,19 +33,20 @@ export interface RunOptions { readonly signal?: AbortSignal } -export type Services = - | Layer.Success - | Layer.Success - | Layer.Success - | Layer.Success - | Layer.Success - | RequestExecutor.Service +export type Services = AIClient.Services -/** Promise view of a `Generation`: its snapshot plus `await`, `refresh`, and `cancel` returning promises. */ +/** + * Promise view of a `Generation`. Its fields are a snapshot taken when the handle was created; `refresh()` resolves to a + * new handle rather than updating this one. + */ export type GenerationHandle = Snapshot & { /** Serializable JSON; pass it back to `resume` from another process. */ readonly token: unknown readonly await: (options?: AwaitOptions & RunOptions) => Promise + /** Status observations until the first terminal one, polling like `await`; abort ends iteration without throwing. */ + readonly events: (options?: AwaitOptions & RunOptions) => AsyncIterable + /** The result without polling; fails when the generation has not completed. */ + readonly result: (options?: RunOptions) => Promise readonly refresh: (options?: RunOptions) => Promise> readonly cancel: (options?: RunOptions) => Promise } @@ -65,17 +65,9 @@ const abortEffect = (signal: AbortSignal | undefined) => }) export const make = (options: Options = {}) => { - const runtime = ManagedRuntime.make( - Layer.mergeAll( - LLMClient.layer, - ImageClient.layer, - VideoClient.layer, - SpeechClient.layer, - TranscriptionClient.layer, - ).pipe(Layer.provideMerge(options.layer ?? RequestExecutor.fetchLayer)), - ) + const runtime = ManagedRuntime.make(AIClient.layerWith(options.layer ?? RequestExecutor.fetchLayer)) - /** Run any package Effect (for example `asset.bytes()`) inside this runtime. */ + /** Run any package Effect (for example `LLMClient.compact(...)`) inside this runtime. */ const run = (effect: Effect.Effect, options?: RunOptions) => runtime.runPromise(effect, { signal: options?.signal }) @@ -95,12 +87,15 @@ export const make = (options: Options = {}) => { ...generation.snapshot, token: generation.token, await: (options) => run(generation.await({ poll: options?.poll }), options), + events: (options) => iterate(generation.events({ poll: options?.poll }), options), + result: (options) => run(generation.result(), options), refresh: (options) => run(generation.refresh(), options).then(handle), cancel: (options) => run(generation.cancel(), options), }) // The typed `generate`/`stream` overloads take a concrete input or a request, not the union; normalize once here. - const llmRequest = (input: RequestInput | LLMRequest) => (input instanceof LLMRequest ? input : LLM.request(input)) + const llmRequest = (input: RequestInput | LLMRequest) => + input instanceof LLMRequest ? Effect.succeed(input) : tryRequest(() => LLM.request(input)) const imageRequest = (input: ImageRequestInput | ImageRequest) => input instanceof ImageRequest ? input : Image.request(input) const videoRequest = (input: VideoRequestInput | VideoRequest) => @@ -112,12 +107,48 @@ export const make = (options: Options = {}) => { return { run, + /** Decoded asset bytes, downloading `url` sources through the executor. */ + bytes: (asset: Media.Asset, options?: RunOptions) => run(asset.bytes(), options), + base64: (asset: Media.Asset, options?: RunOptions) => run(asset.base64(), options), + /** Pull a `url` asset into owned bytes before the provider URL expires. */ + materialize: (asset: Media.Asset, options?: RunOptions) => run(asset.materialize(), options), + /** Read a file into an asset like `Media.file`: sniffed media type, then the extension's. */ + file: async (path: string, options?: Media.AssetOptions & RunOptions) => { + const { readFile } = await import("node:fs/promises") + return run( + Effect.tryPromise({ + try: (signal) => readFile(path, { signal }), + catch: (cause) => fileError(`Failed to read media file ${path}`, cause), + }).pipe( + Effect.map((buffer) => { + const data = new Uint8Array(buffer) + return Media.bytes(data, fileMediaType(data, path), options) + }), + ), + options, + ) + }, + /** Write an asset's bytes like `Media.write`, downloading `url` sources through the executor. */ + write: async (asset: Media.Asset, path: string, options?: RunOptions) => { + const { writeFile } = await import("node:fs/promises") + return run( + asset.bytes().pipe( + Effect.flatMap((data) => + Effect.tryPromise({ + try: (signal) => writeFile(path, data, { signal }), + catch: (cause) => fileError(`Failed to write media file ${path}`, cause), + }), + ), + ), + options, + ) + }, llm: { request: LLM.request, generate: (input: RequestInput | LLMRequest, options?: RunOptions) => - run(LLM.generate(llmRequest(input)), options), + run(Effect.flatMap(llmRequest(input), LLM.generate), options), stream: (input: RequestInput | LLMRequest, options?: RunOptions) => - iterate(LLM.stream(llmRequest(input)), options), + iterate(Stream.unwrap(Effect.map(llmRequest(input), LLM.stream)), options), }, image: { request: Image.request, @@ -186,6 +217,9 @@ export const make = (options: Options = {}) => { export type Client = ReturnType +const fileError = (message: string, cause: unknown) => + new AIError({ reason: new InvalidRequestError({ message, cause }) }) + /** Default client over `RequestExecutor.fetchLayer` for scripts; the runtime builds its layer on first use. */ export const ai = make() diff --git a/packages/ai/src/protocols/openai-speech.ts b/packages/ai/src/protocols/openai-speech.ts index e48633b035b..53784a494c0 100644 --- a/packages/ai/src/protocols/openai-speech.ts +++ b/packages/ai/src/protocols/openai-speech.ts @@ -18,7 +18,11 @@ const PCM_SAMPLE_RATE = 24000 // 1. Public model input // --------------------------------------------------------------------------- -export type OpenAISpeechOptions = Record +/** `voice`, `instructions`, `speed`, and `format` are common request fields; other native body fields pass through. */ +export type OpenAISpeechOptions = { + /** Defaults to `"sse"` in `stream` mode on models that support it; the merged value selects the response framing. */ + readonly stream_format?: "sse" | "audio" +} & Record export type Request = SpeechRequestFor diff --git a/packages/ai/src/protocols/shared.ts b/packages/ai/src/protocols/shared.ts index d69c1483bb7..fc94a464cea 100644 --- a/packages/ai/src/protocols/shared.ts +++ b/packages/ai/src/protocols/shared.ts @@ -211,7 +211,8 @@ export const mediaReference = ( if (provider !== undefined && asset.source.type === "ref" && asset.source.provider === provider) return Effect.succeed({ type: "ref", value: asset.source.id }) const accepted = provider === undefined ? "" : `, and ${provider} references` - return Effect.fail(invalidRequest(`${label} accepts inline bytes, data URLs, http(s) URLs${accepted}`)) + const got = asset.source.type === "ref" ? `; got ${asset.source.provider}:${asset.source.id}` : "" + return Effect.fail(invalidRequest(`${label} accepts inline bytes, data URLs, http(s) URLs${accepted}${got}`)) } /** diff --git a/packages/ai/src/route/auth.ts b/packages/ai/src/route/auth.ts index d48be6172cc..5494a86ebdc 100644 --- a/packages/ai/src/route/auth.ts +++ b/packages/ai/src/route/auth.ts @@ -1,12 +1,15 @@ -import { Config, Effect, Redacted } from "effect" +import { Config, Effect, Option, Redacted } from "effect" import { Headers } from "effect/unstable/http" -import { AuthenticationError, InvalidRequestError, AIError, type HttpOptions } from "../schema/index.js" +import { AuthenticationError, AIError, type HttpOptions } from "../schema/index.js" export class MissingCredentialError extends Error { readonly _tag = "MissingCredentialError" - constructor(readonly source: string) { - super(`Missing auth credential: ${source}`) + constructor( + readonly source: string, + message = `Missing auth credential: ${source}`, + ) { + super(message) } } @@ -89,7 +92,14 @@ export const optional = (secret: Secret | undefined, source = "optional value") ? credential(Effect.fail(new MissingCredentialError(source))) : credentialFromSecret(secret, source) -export const config = (name: string) => credentialFromSecret(Config.redacted(name), name) +export const config = (name: string) => + credential( + Effect.gen(function* () { + const secret = yield* Config.option(Config.redacted(name)) + if (Option.isSome(secret) && Redacted.value(secret.value) !== "") return secret.value + return yield* Effect.fail(new MissingCredentialError(name, `${name} is not set`)) + }), + ) export const effect = (load: Effect.Effect) => credential(load) @@ -145,15 +155,10 @@ export function scheme(name: string, source?: Secret | Credential) { } const toAIError = (error: AuthError): AIError => { - if (error instanceof MissingCredentialError || error instanceof Config.ConfigError) { - return new AIError({ - reason: - error instanceof MissingCredentialError - ? new AuthenticationError({ message: error.message, cause: error }) - : new InvalidRequestError({ message: `Failed to resolve auth config: ${error.message}`, cause: error }), - }) - } - return error + if (error instanceof AIError) return error + const message = + error instanceof MissingCredentialError ? error.message : `Failed to resolve auth config: ${error.message}` + return new AIError({ reason: new AuthenticationError({ message, cause: error }) }) } export const toEffect = diff --git a/packages/ai/src/route/client.ts b/packages/ai/src/route/client.ts index b7592d0f6eb..49425b41591 100644 --- a/packages/ai/src/route/client.ts +++ b/packages/ai/src/route/client.ts @@ -261,7 +261,9 @@ const unsupportedCompaction = (request: LLMRequest, mechanism: string | undefine }) } -export class Service extends Context.Service()("@opencode/LLMClient") {} +export class LLMClientService extends Context.Service()("@opencode/LLMClient") {} +export const Service = LLMClientService +export type Service = LLMClientService const resolveRequestOptions = (request: LLMRequest) => { const messages = normalizeToolHistory(request.messages) diff --git a/packages/ai/src/route/executor-service.ts b/packages/ai/src/route/executor-service.ts index f341177bcfa..5cd0c3fd24c 100644 --- a/packages/ai/src/route/executor-service.ts +++ b/packages/ai/src/route/executor-service.ts @@ -19,4 +19,8 @@ export type HttpMiddleware = ( handler: HttpHandler, ) => Effect.Effect -export class Service extends Context.Service()("@opencode/AI/RequestExecutor") {} +export class RequestExecutorService extends Context.Service()( + "@opencode/AI/RequestExecutor", +) {} +export const Service = RequestExecutorService +export type Service = RequestExecutorService diff --git a/packages/ai/src/route/executor.ts b/packages/ai/src/route/executor.ts index 815f0bee7f9..059d0f7acc1 100644 --- a/packages/ai/src/route/executor.ts +++ b/packages/ai/src/route/executor.ts @@ -255,4 +255,20 @@ export const layer: Layer.Layer = Layer.e export const fetchLayer = layer.pipe(Layer.provide(FetchHttpClient.layer)) +/** Run `fn` on every request: it sees the raw response before status classification, inside middleware already on `executor`, and outside per-call middleware. */ +export const middleware = (fn: HttpMiddleware, executor: Layer.Layer = fetchLayer): Layer.Layer => + Layer.effect( + Service, + Effect.gen(function* () { + const inner = yield* Service + return Service.of({ + execute: (request, next) => + inner.execute( + request, + next === undefined ? fn : (input, handler) => fn(input, (forwarded) => next(forwarded, handler)), + ), + }) + }), + ).pipe(Layer.provide(executor)) + export * as RequestExecutor from "./executor.js" diff --git a/packages/ai/src/route/index.ts b/packages/ai/src/route/index.ts index e7a62c6b7e2..3e840afdc99 100644 --- a/packages/ai/src/route/index.ts +++ b/packages/ai/src/route/index.ts @@ -7,7 +7,7 @@ export type { RouteDefaultsInput, AnyRoute, Interface as LLMClientShape, - Service as LLMClientService, + LLMClientService, StreamOptions, CompactMethod, CompactionOperations, diff --git a/packages/ai/src/speech-client.ts b/packages/ai/src/speech-client.ts index e13e30a2f31..fef03abed71 100644 --- a/packages/ai/src/speech-client.ts +++ b/packages/ai/src/speech-client.ts @@ -12,7 +12,9 @@ export interface Interface { ) => Stream.Stream } -export class Service extends Context.Service()("@opencode/SpeechClient") {} +export class SpeechClientService extends Context.Service()("@opencode/SpeechClient") {} +export const Service = SpeechClientService +export type Service = SpeechClientService export const generate = ( request: SpeechRequestFor, diff --git a/packages/ai/src/transcription-client.ts b/packages/ai/src/transcription-client.ts index 37d6099e88d..dbcfc638c40 100644 --- a/packages/ai/src/transcription-client.ts +++ b/packages/ai/src/transcription-client.ts @@ -30,7 +30,11 @@ export interface Interface { ) => Effect.Effect, AIError> } -export class Service extends Context.Service()("@opencode/TranscriptionClient") {} +export class TranscriptionClientService extends Context.Service()( + "@opencode/TranscriptionClient", +) {} +export const Service = TranscriptionClientService +export type Service = TranscriptionClientService export const generate = ( request: TranscriptionRequestFor, diff --git a/packages/ai/src/utils/media-type.ts b/packages/ai/src/utils/media-type.ts index fb9d2c0a72e..114e2c6efdf 100644 --- a/packages/ai/src/utils/media-type.ts +++ b/packages/ai/src/utils/media-type.ts @@ -48,9 +48,12 @@ const EXTENSIONS: Readonly> = { csv: "text/csv", } -export const extensionMediaType = (path: string): string | undefined => +const extensionMediaType = (path: string): string | undefined => EXTENSIONS[path.slice(path.lastIndexOf(".") + 1).toLowerCase()] +/** Media type of a file's contents: sniffed magic bytes, then the path's extension. */ +export const fileMediaType = (bytes: Uint8Array, path: string) => detectMediaType(bytes) ?? extensionMediaType(path) + const EXTENSION_ALIASES: Readonly> = { "audio/mp3": "mp3", "audio/m4a": "m4a", diff --git a/packages/ai/src/video-client.ts b/packages/ai/src/video-client.ts index 35dc276d4f0..5872c94d09f 100644 --- a/packages/ai/src/video-client.ts +++ b/packages/ai/src/video-client.ts @@ -29,7 +29,9 @@ export interface Interface { ) => Stream.Stream } -export class Service extends Context.Service()("@opencode/VideoClient") {} +export class VideoClientService extends Context.Service()("@opencode/VideoClient") {} +export const Service = VideoClientService +export type Service = VideoClientService export const start = ( request: VideoRequestFor, diff --git a/packages/ai/test/auth.test.ts b/packages/ai/test/auth.test.ts index 5732ab798dd..cabf3013bbd 100644 --- a/packages/ai/test/auth.test.ts +++ b/packages/ai/test/auth.test.ts @@ -92,6 +92,15 @@ describe("Auth", () => { }), ) + it.effect("reports a missing config credential as Authentication naming the variable", () => + Effect.gen(function* () { + const error = yield* Auth.toEffect(Auth.config("OPENAI_API_KEY").bearer())(input).pipe(withEnv({}), Effect.flip) + + expect(error.reason._tag).toBe("Authentication") + expect(error.message).toContain("OPENAI_API_KEY is not set") + }), + ) + it.effect("can intentionally leave auth untouched", () => Effect.gen(function* () { const headers = yield* Auth.none.apply(input) diff --git a/packages/ai/test/executor.test.ts b/packages/ai/test/executor.test.ts index 9d0532d52d5..e83d2142c5b 100644 --- a/packages/ai/test/executor.test.ts +++ b/packages/ai/test/executor.test.ts @@ -1,11 +1,11 @@ import { describe, expect } from "bun:test" -import { Deferred, Effect, Fiber, Ref, Stream } from "effect" +import { Deferred, Effect, Fiber, Layer, Ref, Stream } from "effect" import { Headers, HttpClientError, HttpClientRequest } from "effect/unstable/http" import { LLM, AIError, HttpContext, InvalidProviderOutputError, TransportError } from "../src/index.js" import { LLMClient, RequestExecutor, WebSocketTransport, type WebSocketChannelExecutor } from "../src/route.js" import { route } from "../src/protocols/openai-chat.js" import { configure } from "../src/providers/openai.js" -import { dynamicResponse, fixedResponse, systemError } from "./lib/http.js" +import { dynamicResponse, fixedResponse, handlerLayer, systemError } from "./lib/http.js" import { deltaChunk } from "./lib/openai-chunks.js" import { sseEvents, sseRaw } from "./lib/sse.js" import { it } from "./lib/effect.js" @@ -195,6 +195,26 @@ describe("RequestExecutor", () => { ), ) + it.effect("runs shared middleware outside per-call middleware", () => { + const calls: Array = [] + const record = (name: string) => Effect.sync(() => calls.push(name)) + const base = RequestExecutor.layer.pipe( + Layer.provide(handlerLayer((input) => record("handler").pipe(Effect.as(input.respond("ok"))))), + ) + return Effect.gen(function* () { + const executor = yield* RequestExecutor.Service + yield* executor.execute(request, (input, next) => record("per-call").pipe(Effect.andThen(next(input)))) + expect(calls).toEqual(["outer", "per-call", "handler"]) + calls.length = 0 + yield* executor.execute(request) + expect(calls).toEqual(["outer", "handler"]) + }).pipe( + Effect.provide( + RequestExecutor.middleware((input, next) => record("outer").pipe(Effect.andThen(next(input))), base), + ), + ) + }) + it.effect("classifies context overflow responses", () => Effect.gen(function* () { const executor = yield* RequestExecutor.Service diff --git a/packages/ai/test/exports.test.ts b/packages/ai/test/exports.test.ts index e241df07857..3b4f640d55e 100644 --- a/packages/ai/test/exports.test.ts +++ b/packages/ai/test/exports.test.ts @@ -1,8 +1,11 @@ import { describe, expect, test } from "bun:test" +import { Effect, Layer } from "effect" import { + AIClient, AIError, Generation, Image, + ImageClient, LanguageModel, LLM, LLMClient, @@ -11,10 +14,11 @@ import { Speech, SpeechClient, SpeechEvent, + TranscriptionClient, Video, VideoClient, } from "@opencode/ai" -import { Route, Protocol, WebSocketTransport } from "@opencode/ai/route" +import { Route, Protocol, RequestExecutor, WebSocketTransport } from "@opencode/ai/route" import { Provider as ProviderSubpath } from "@opencode/ai/provider" import { AssemblyAI, @@ -76,6 +80,28 @@ describe("public exports", () => { expect(EvaluationClient.fetchLayer).toBeDefined() }) + test("AIClient.layerWith shares one executor across every client", async () => { + let built = 0 + const counting = Layer.effect( + RequestExecutor.Service, + Effect.sync(() => { + built++ + return RequestExecutor.Service.of({ execute: () => Effect.die("unexpected request") }) + }), + ) + await Effect.runPromise( + Effect.gen(function* () { + yield* LLMClient.Service + yield* ImageClient.Service + yield* VideoClient.Service + yield* SpeechClient.Service + yield* TranscriptionClient.Service + yield* RequestExecutor.Service + }).pipe(Effect.provide(AIClient.layerWith(counting))), + ) + expect(built).toBe(1) + }) + test("route barrel exposes route-authoring APIs", () => { expect(Route.make).toBeFunction() expect(Protocol.make).toBeFunction() diff --git a/packages/ai/test/promise.test.ts b/packages/ai/test/promise.test.ts index 85331a43fe1..1f2a32bfa29 100644 --- a/packages/ai/test/promise.test.ts +++ b/packages/ai/test/promise.test.ts @@ -1,4 +1,7 @@ import { describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" import { Effect, Layer } from "effect" import { HttpClientRequest } from "effect/unstable/http" import { AIError, LLMEvent, Media, SpeechEvent, TranscriptionEvent } from "../src/index.js" @@ -182,6 +185,36 @@ describe("AI promise client", () => { await ai.dispose() }) + test("observes a started generation's events and fetches its result", async () => { + const ai = AI.make({ layer: executor([]) }) + const model = Runway.configure({ apiKey: "test", baseURL: "https://runway.test/v1" }).video("gen4.5") + const generation = await ai.video.start({ model, prompt: "A kite" }) + + const events: Array = [] + for await (const event of generation.events({ poll: { interval: 10 } })) events.push(event.type) + expect(events).toEqual(["generation-progress", "generation-finished"]) + expect(generation.status).toBe("queued") + expect((await generation.result()).video.source).toMatchObject({ url: "https://runway.test/out.mp4" }) + await ai.dispose() + }) + + test("reads, writes, and decodes assets without leaving promises", async () => { + const ai = AI.make({ layer: executor([]) }) + const dir = await mkdtemp(join(tmpdir(), "ai-promise-")) + try { + const png = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3]) + await Bun.write(join(dir, "source.bin"), png) + + const asset = await ai.file(join(dir, "source.bin")) + expect(asset.mediaType).toBe("image/png") + await ai.write(asset, join(dir, "copy.png")) + expect(await ai.bytes(await ai.file(join(dir, "copy.png")))).toEqual(png) + } finally { + await rm(dir, { recursive: true }) + await ai.dispose() + } + }) + test("generates, streams, and starts transcriptions over the same runtime", async () => { const ai = AI.make({ layer: executor([]) }) const audio = Media.url("https://audio.test/hello.mp3") @@ -233,6 +266,12 @@ describe("AI promise client", () => { expect(failure).toBeInstanceOf(AIError) expect(failure instanceof AIError && failure.reason.http?.status).toBe(404) + const invalid = await ai.llm + // @ts-expect-error Invalid input must reject with AIError, not throw synchronously. + .generate({ model: openai.responses("gpt-5"), messages: [{ role: "bogus" }] }) + .catch((error: unknown) => error) + expect(invalid instanceof AIError && invalid.reason._tag).toBe("InvalidRequest") + const controller = new AbortController() controller.abort() const aborted = await ai.llm diff --git a/packages/ai/test/video.test.ts b/packages/ai/test/video.test.ts index 49e63c75219..aecf45d65c4 100644 --- a/packages/ai/test/video.test.ts +++ b/packages/ai/test/video.test.ts @@ -533,6 +533,7 @@ describe("Video / fal", () => { "InvalidRequest", ]) expect(errors[1].message).toContain("end_image_url") + expect(errors[4].message).toContain("; got fal:handle") }).pipe(Effect.provide(layer(() => Effect.die("unsupported input reached the network")))), ) })