mirror of
https://gitee.com/zhangsken/opencode.git
synced 2026-10-16 10:30:01 +08:00
feat(ai): polish media DX with promise file helpers and one client layer (#50978)
This commit is contained in:
Shoubhit Dash
@@ -55,7 +55,7 @@ const request = LLM.request({
|
||||
prompt: "Say hello.",
|
||||
})
|
||||
|
||||
const response = yield * LLMClient.generate(request)
|
||||
const response = yield * LLMClient.generate(request) // inside Effect.gen
|
||||
```
|
||||
|
||||
`LLM.request(...)` builds an `LLMRequest`. `LLMClient.generate(...)` reads the executable route carried by `request.model.route`, builds the provider-native body, asks the route's transport for a real `HttpClientRequest.HttpClientRequest`, sends it through `RequestExecutor.Service`, parses the provider stream into common `LLMEvent`s, and finally returns an `LLMResponse`.
|
||||
@@ -98,7 +98,7 @@ When a provider supports multiple physical transports, selection remains executi
|
||||
|
||||
Media does not fit the SSE-frames-to-event-state-machine LLM route. `MediaRoute.make(...)` (`src/route/media.ts`) composes a `MediaProtocol` kind with `Endpoint` and `Auth` and owns the transport plumbing: `http` option merging, URL/query rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol. `MediaProtocol.inline` (`src/route/media-protocol.ts`) is `body.from(request)` plus `response.decode(response, context)`; use `MediaProtocol.decodeJson` / `text` / `bytes` so decode failures retain the raw body and HTTP context. `Generation` (`src/generation.ts`) is the provider-neutral handle for a queued generation over a `GenerationRoute` (`status`, `result`, `cancel`, `pollHint`). Image protocol files follow the same section order as LLM protocols and declare unsupported common fields once through `MediaInput.rejectUnsupported`.
|
||||
|
||||
`MediaProtocol.queued` is the submit-then-poll kind every video route uses: `start` (body + decode into `{ token, snapshot }`), `status`, `result`, and optional `cancel`, each addressed by a route-owned `token` whose `Schema.Codec` makes it serializable. `MediaRoute.inline` and `MediaRoute.queued` compose the two kinds with `Endpoint` and `Auth`; the queued route decodes the token once at the boundary (`start` output or `resume` input) and closes over it in a token-free `GenerationRoute` (`status`/`result`/`cancel` are plain Effects), so `Generation` never sees the token's shape and only carries the encoded JSON for persistence. Polls reuse the route's auth and deployment headers plus the request's `http` overlay after `start`, and resolve relative paths against the route base URL (provider-issued absolute URLs such as fal's `status_url` pass through). `result` is always its own GET even when the provider returns output inside the status document, so `Generation.await` behaves the same after `start` and after `resume`. `PollContext.auth` carries only what `Auth` added so protocols can hand download credentials to output assets as transient `Media.Asset.headers` (Veo) — never part of `source` or JSON. Status strings map through a per-protocol `STATUS` table via `MediaProtocol.status`; terminal generations without output fail through `output.ended` / `output.contentPolicy` with the provider document on `reason.body`. `Generation.AwaitOptions` (`{ poll?: Poll }`) is the one options type for `await`, `events`, `Video.generate`, and `Video.stream`.
|
||||
`MediaProtocol.queued` is the submit-then-poll kind every video route uses: `start` (body + decode into `{ token, snapshot }`), `status`, `result`, and optional `cancel`, each addressed by a route-owned `token` whose `Schema.Codec` makes it serializable. `MediaRoute.inline` and `MediaRoute.queued` compose the two kinds with `Endpoint` and `Auth`; the queued route decodes the token once at the boundary (`start` output or `resume` input) and closes over it in a token-free `GenerationRoute` (`status`/`result`/`cancel` are plain Effects), so `Generation` never sees the token's shape and only carries the encoded JSON for persistence. Polls reuse the route's auth and deployment headers plus the request's `http` overlay after `start`, and resolve relative paths against the route base URL (provider-issued absolute URLs such as fal's `status_url` pass through). `result` is always its own GET even when the provider returns output inside the status document, so `Generation.await` behaves the same after `start` and after `resume`. `PollContext.auth` carries only what `Auth` added so protocols can hand download credentials to output assets as transient `Media.Asset.headers` (Veo) — never part of `source` or JSON. Status strings map through a per-protocol `STATUS` table via `MediaProtocol.status`; terminal generations without output fail through `output.ended` / `output.contentPolicy` with the provider document on `reason.body`. `GenerationAwaitOptions` (`AwaitOptions` in `src/generation.ts`, `{ poll?: Poll }`) is the one options type for `await`, `events`, `Video.generate`, and `Video.stream`.
|
||||
|
||||
`MediaProtocol.stream` is the incremental kind every speech route uses, with the same discipline as LLM protocols. `MediaRoute.stream` submits the caller's request as `MediaProtocol.Addressed<Request>` (`{ ...request, mode }`, `mode: "generate" | "stream"`), so one provider stays one protocol: `body.from`, the endpoint path, and `frames` read `request.mode` to pick the body, path, and framing. `frames(bytes, context)` returns frames — `Framing.sse`, `Framing.lines`, `Framing.document` (a single-document response shaped like a streamed record), or the raw `bytes` for chunked audio. `initial()` is fresh per-response parser state; `step` folds each frame into it and emits modality events; `finish(state, context)` runs once after the last frame with the request, body, and observed `http` (header-only usage lives there) and emits exactly one terminal event or fails with `MediaProtocol.incomplete`. Keep parser state to real accumulators and derive anything the request or body determines in `finish`. `generate` runs the same stream and folds it with the modality's `collect`. Request-derived URL parameters go on the body's `query` (array values repeat the parameter), applied before route and caller `http.query`. Decode frames with `MediaProtocol.decodeFrame` and raise stream-time failures with `MediaProtocol.frameError` (the frame stays on `reason.body`); protocols never thread HTTP context, because the route fills `reason.http` on stream errors that lack it. Speech protocols share `protocols/utils/speech-stream.ts` for deltas, timestamps, voice ids, PCM and container descriptions, and the terminal asset.
|
||||
|
||||
|
||||
+171
-114
@@ -1,11 +1,10 @@
|
||||
# @opencode/ai
|
||||
|
||||
Schema-first language model and image-generation APIs built with Effect.
|
||||
Schema-first APIs for text, images, video, speech, and transcription, built with Effect.
|
||||
|
||||
```ts
|
||||
import { Effect, Layer } from "effect"
|
||||
import { LLM, LLMClient } from "@opencode/ai"
|
||||
import { RequestExecutor } from "@opencode/ai/route"
|
||||
import { Effect } from "effect"
|
||||
import { AIClient, LLM } from "@opencode/ai"
|
||||
import { OpenAI } from "@opencode/ai/providers"
|
||||
|
||||
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY })
|
||||
@@ -18,23 +17,23 @@ const request = LLM.request({
|
||||
})
|
||||
|
||||
const program = Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(request)
|
||||
const response = yield* LLM.generate(request)
|
||||
console.log(response.text)
|
||||
})
|
||||
|
||||
const llmLayer = LLMClient.layer.pipe(Layer.provide(RequestExecutor.fetchLayer))
|
||||
|
||||
await Effect.runPromise(program.pipe(Effect.provide(llmLayer)))
|
||||
// Every modality client plus the HTTP request executor; `AIClient.layerWith(executor)` swaps the executor.
|
||||
await Effect.runPromise(program.pipe(Effect.provide(AIClient.layer)))
|
||||
```
|
||||
|
||||
Run `LLMClient.stream(request)` instead of `generate` when you want incremental `LLMEvent`s. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses, Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment.
|
||||
Run `LLM.stream(request)` instead of `generate` when you want incremental `LLMEvent`s. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses,
|
||||
Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment.
|
||||
|
||||
The same configured facade names image models. `Image.request` resolves the provider's image route from the ref and
|
||||
returns `Media.Asset`s with lazily decoded bytes:
|
||||
The same configured facade names image, video, speech, and transcription models. `Image.generate` resolves the
|
||||
provider's image route from the model and returns `Media.Asset`s with lazily decoded bytes:
|
||||
|
||||
```ts
|
||||
import { NodeFileSystem } from "@effect/platform-node"
|
||||
import { Image, ImageClient, Media } from "@opencode/ai"
|
||||
import { Image, Media } from "@opencode/ai"
|
||||
|
||||
const image = Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
@@ -46,13 +45,28 @@ const image = Effect.gen(function* () {
|
||||
yield* Media.write(response.image, "./garden.png")
|
||||
})
|
||||
|
||||
// `asset.bytes()` / `Media.write` also need the executor, so merge it into the environment instead of hiding it.
|
||||
const imageLayer = ImageClient.layer.pipe(Layer.provideMerge(RequestExecutor.fetchLayer))
|
||||
|
||||
await Effect.runPromise(image.pipe(Effect.provide(imageLayer), Effect.provide(NodeFileSystem.layer)))
|
||||
// `Media.file` / `Media.write` use the Effect `FileSystem` service; provide your platform's layer.
|
||||
await Effect.runPromise(image.pipe(Effect.provide(AIClient.layer), Effect.provide(NodeFileSystem.layer)))
|
||||
```
|
||||
|
||||
Prefer promises? `@opencode/ai/promise` exposes the same LLM and image APIs over one managed runtime:
|
||||
Advanced: each client also has its own `layer`, which requires `RequestExecutor.Service`. Compose client layers with
|
||||
`Layer.provideMerge`, not `Layer.provide`: `asset.bytes()`, `Media.write`, and Gemini's `media` output parts need the
|
||||
executor too, and hiding it fails type-checking with `RequestExecutorService` left in the requirements.
|
||||
|
||||
To share a policy such as logging across every client, wrap the executor once with `RequestExecutor.middleware`:
|
||||
|
||||
```ts
|
||||
import { RequestExecutor } from "@opencode/ai/route"
|
||||
|
||||
const logged = RequestExecutor.middleware((request, next) =>
|
||||
Effect.log(`${request.method} ${request.url}`).pipe(Effect.andThen(next(request))),
|
||||
)
|
||||
|
||||
const everything = AIClient.layerWith(logged) // or AI.make({ layer: logged })
|
||||
```
|
||||
|
||||
Prefer promises? `@opencode/ai/promise` exposes the same LLM and media APIs over one managed runtime, plus asset
|
||||
helpers; `ai.file` and `ai.write` load `node:fs/promises` on first use, so no Effect `FileSystem` is needed:
|
||||
|
||||
```ts
|
||||
import { AI } from "@opencode/ai/promise"
|
||||
@@ -60,6 +74,7 @@ import { AI } from "@opencode/ai/promise"
|
||||
const ai = AI.make()
|
||||
const text = await ai.llm.generate({ model: openai.responses("gpt-4o-mini"), prompt: "Say hello." })
|
||||
const generated = await ai.image.generate({ model: openai.image("gpt-image-2"), prompt: "A lighthouse" })
|
||||
await ai.write(generated.image, "./lighthouse.png") // also ai.file(path), ai.bytes(asset), ai.base64(asset), ai.materialize(asset)
|
||||
for await (const event of ai.llm.stream({ model: openai.responses("gpt-4o-mini"), prompt: "Stream hello." })) {
|
||||
// LLMEvent
|
||||
}
|
||||
@@ -322,10 +337,9 @@ and `moonshot/responses`; each exports `model(modelID, settings)`.
|
||||
MiniMax defaults to its Messages API and reads `MINIMAX_API_KEY` when `apiKey` is omitted:
|
||||
|
||||
```ts
|
||||
import { Effect, Layer } from "effect"
|
||||
import { LLM, LLMClient } from "@opencode/ai"
|
||||
import { Effect } from "effect"
|
||||
import { AIClient, LLM } from "@opencode/ai"
|
||||
import { MiniMax } from "@opencode/ai/providers"
|
||||
import { RequestExecutor } from "@opencode/ai/route"
|
||||
|
||||
const minimax = MiniMax.configure({ apiKey: process.env.MINIMAX_API_KEY })
|
||||
const request = LLM.request({
|
||||
@@ -335,8 +349,7 @@ const request = LLM.request({
|
||||
generation: { maxTokens: 1536 },
|
||||
})
|
||||
|
||||
const layer = LLMClient.layer.pipe(Layer.provide(RequestExecutor.fetchLayer))
|
||||
const response = await Effect.runPromise(LLMClient.generate(request).pipe(Effect.provide(layer)))
|
||||
const response = await Effect.runPromise(LLM.generate(request).pipe(Effect.provide(AIClient.layer)))
|
||||
console.log(response.text)
|
||||
```
|
||||
|
||||
@@ -405,14 +418,14 @@ Use `Image.generate` for one-off generation or editing:
|
||||
import { Image, Media } from "@opencode/ai"
|
||||
|
||||
const generation = Image.generate({
|
||||
model: meta("muse-image-1.0"),
|
||||
model: meta.image("muse-image-1.0"),
|
||||
prompt: "A flat black square on a white background.",
|
||||
n: 1,
|
||||
providerOptions: { reasoningStrength: "low" },
|
||||
})
|
||||
|
||||
const edit = Image.generate({
|
||||
model: meta("muse-image-1.0"),
|
||||
model: meta.image("muse-image-1.0"),
|
||||
prompt: "Make the square purple.",
|
||||
images: [Media.bytes(imageBytes, "image/webp")],
|
||||
format: "png",
|
||||
@@ -459,6 +472,25 @@ const program = Effect.gen(function* () {
|
||||
})
|
||||
```
|
||||
|
||||
Common fields are portable in shape, not in support. Unsupported fields fail with a typed `AIError` before any network
|
||||
call rather than being dropped, so check this table before swapping only the `model`:
|
||||
|
||||
| Provider | `n` | `size` | `aspectRatio` | `seed` | `format` | `images` | `mask` |
|
||||
| --------------------- | --- | --------- | ------------- | ------ | -------- | ------------------------- | ------------------- |
|
||||
| OpenAI | ✓¹ | ✓ | ✗ | ✗ | ✓ | ✓ | ✓ |
|
||||
| Google (Gemini) | 1 | ✗ | ✓ | ✓ | ✗ | ✓ (no public URLs) | ✗ |
|
||||
| xAI | ✓ | ✗ | ✓ | ✗ | ✗ | ✓ | ✗ |
|
||||
| Z.ai | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ |
|
||||
| Meta | ✓ | ✓ (hint) | ✗ | ✗ | ✓ | ✓ | ✗ |
|
||||
| Black Forest Labs | 1 | per model | per model | ✓ | ✓ | per model (1–8) | `flux-pro-1.0-fill` |
|
||||
| fal | ✓ | per model | per model | ✓ | ✓ | 1 (several on `/edit`) | ✓ |
|
||||
| Replicate | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ (use `providerOptions`) | ✗ |
|
||||
| Stability `image` | 1 | ✗ | ✓ | ✓ | ✓ | 1 (not on `core`) | ✗ |
|
||||
| Stability `upscale()` | ✗ | ✗ | ✗ | ✓ | ✓ | exactly 1 (required) | ✗ |
|
||||
|
||||
✓ lowers natively; ✗ fails whenever the field is set (including `n: 1`); `1` means `n > 1` fails. ¹ `Image.stream` on OpenAI generates one image. fal
|
||||
rejects `size` and `aspectRatio` together; which one a fal or BFL model takes depends on the model.
|
||||
|
||||
`Media.Asset` is the one asset type shared by image requests, image responses, LLM messages, and tool results.
|
||||
`asset.source` is the serializable `Media.Source` (`bytes`, `base64`, `url`, or `ref`); `asset.bytes()`,
|
||||
`asset.base64()`, and `asset.dataUrl()` decode or download lazily and cache; `asset.materialize()` pulls a `url`
|
||||
@@ -468,9 +500,8 @@ asset into owned bytes before the provider URL expires. Construct assets with `M
|
||||
Pass ordered image inputs to the same method for editing, composition, or image-conditioned generation:
|
||||
|
||||
```ts
|
||||
const response =
|
||||
yield *
|
||||
Image.generate({
|
||||
const composed = Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model,
|
||||
prompt: "Combine these product photos into one studio scene",
|
||||
images: [
|
||||
@@ -481,23 +512,25 @@ const response =
|
||||
providerOptions,
|
||||
http,
|
||||
})
|
||||
return response.images
|
||||
})
|
||||
```
|
||||
|
||||
`Media.ref(provider, id)` represents provider file handles such as OpenAI file IDs or Gemini Files URIs; routes
|
||||
only forward refs that belong to their own provider. Raw strings are not accepted as image inputs, avoiding
|
||||
ambiguity between base64, URLs, and provider IDs. Empty or omitted `images` uses text-to-image generation; a
|
||||
non-empty array selects the provider's edit behavior without enforcing provider image-count limits locally. OpenAI
|
||||
only forward refs that belong to their own provider (OpenAI, xAI, and Gemini images accept them). No shipped route
|
||||
returns a ref yet, and `asset.bytes()` / `materialize()` on a ref fail by design. Raw strings are not accepted as
|
||||
image inputs, avoiding ambiguity between base64, URLs, and provider IDs. Empty or omitted `images` uses text-to-image generation; a
|
||||
non-empty array selects the provider's edit behavior (see the table above for routes that limit the count). OpenAI
|
||||
uses multipart for byte/data-URL edits and its JSON reference body for URL or file-ID edits. The common `mask`
|
||||
field selects inpainting; routes that cannot honor it fail with `UnsupportedOperation`:
|
||||
|
||||
```ts
|
||||
yield *
|
||||
Image.generate({
|
||||
model: openai.image("gpt-image-2"),
|
||||
prompt,
|
||||
images: [Media.bytes(sourceBytes, "image/png")],
|
||||
mask: Media.bytes(maskBytes, "image/png"),
|
||||
})
|
||||
const inpainted = Image.generate({
|
||||
model: openai.image("gpt-image-2"),
|
||||
prompt,
|
||||
images: [Media.bytes(sourceBytes, "image/png")],
|
||||
mask: Media.bytes(maskBytes, "image/png"),
|
||||
})
|
||||
```
|
||||
|
||||
On multipart requests, `http.body` can override option fields but not structural `model`, `prompt`, `image[]`,
|
||||
@@ -508,31 +541,31 @@ not accept image inputs. These cases fail with a typed `AIError` before network
|
||||
Provider-native image options belong to each request. Raw `http.body` fields have final precedence over them:
|
||||
|
||||
```ts
|
||||
yield *
|
||||
Image.generate({
|
||||
model: openai.image("gpt-image-2"),
|
||||
prompt,
|
||||
providerOptions: { quality: "medium" },
|
||||
http,
|
||||
})
|
||||
const medium = Image.generate({
|
||||
model: openai.image("gpt-image-2"),
|
||||
prompt,
|
||||
providerOptions: { quality: "medium" },
|
||||
http,
|
||||
})
|
||||
```
|
||||
|
||||
xAI image models use the same request API with xAI-native controls:
|
||||
|
||||
```ts
|
||||
yield *
|
||||
Image.generate({
|
||||
model: XAI.configure({ apiKey })("any-model-id"),
|
||||
prompt,
|
||||
n: 2,
|
||||
aspectRatio: "16:9",
|
||||
providerOptions: {
|
||||
resolution: "1k",
|
||||
responseFormat: "b64_json",
|
||||
future_option: true,
|
||||
},
|
||||
http,
|
||||
})
|
||||
import { XAI } from "@opencode/ai/providers"
|
||||
|
||||
const xai = Image.generate({
|
||||
model: XAI.configure({ apiKey }).image("any-model-id"),
|
||||
prompt,
|
||||
n: 2,
|
||||
aspectRatio: "16:9",
|
||||
providerOptions: {
|
||||
resolution: "1k",
|
||||
responseFormat: "b64_json",
|
||||
future_option: true,
|
||||
},
|
||||
http,
|
||||
})
|
||||
```
|
||||
|
||||
Google's current Gemini image models use the same direct API:
|
||||
@@ -542,7 +575,7 @@ import { Google } from "@opencode/ai/providers"
|
||||
|
||||
const googleProgram = Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model: Google.configure({ apiKey })("any-model-id"),
|
||||
model: Google.configure({ apiKey }).image("any-model-id"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
aspectRatio: "16:9",
|
||||
seed: 42,
|
||||
@@ -567,17 +600,18 @@ their mapped aliases, and `http.body` is the final deep overlay. The selected mo
|
||||
Z.ai image models infer open Z.ai-native options from the selected model:
|
||||
|
||||
```ts
|
||||
yield *
|
||||
Image.generate({
|
||||
model: ZAI.configure({ apiKey })("any-model-id"),
|
||||
prompt,
|
||||
providerOptions: {
|
||||
quality: "hd",
|
||||
userID: "user-123",
|
||||
future_option: true,
|
||||
},
|
||||
http,
|
||||
})
|
||||
import { ZAI } from "@opencode/ai/providers"
|
||||
|
||||
const zai = Image.generate({
|
||||
model: ZAI.configure({ apiKey }).image("any-model-id"),
|
||||
prompt,
|
||||
providerOptions: {
|
||||
quality: "hd",
|
||||
userID: "user-123",
|
||||
future_option: true,
|
||||
},
|
||||
http,
|
||||
})
|
||||
```
|
||||
|
||||
Z.ai does not include trustworthy MIME metadata for output URLs, so generated images use
|
||||
@@ -591,12 +625,14 @@ and emits `image-partial` events before each final `image`; `Image.generate` kee
|
||||
`dall-e-*` models do not stream and fail typed:
|
||||
|
||||
```ts
|
||||
yield *
|
||||
Image.stream({
|
||||
model: openai.image("gpt-image-2"),
|
||||
prompt: "A lighthouse at dusk",
|
||||
providerOptions: { partialImages: 2 },
|
||||
}).pipe(Stream.runForEach((event) => (ImageEvent.is.imagePartial(event) ? showPreview(event.image) : Effect.void)))
|
||||
import { Stream } from "effect"
|
||||
import { ImageEvent } from "@opencode/ai"
|
||||
|
||||
const previews = Image.stream({
|
||||
model: openai.image("gpt-image-2"),
|
||||
prompt: "A lighthouse at dusk",
|
||||
providerOptions: { partialImages: 2 },
|
||||
}).pipe(Stream.runForEach((event) => (ImageEvent.is.imagePartial(event) ? showPreview(event.image) : Effect.void)))
|
||||
```
|
||||
|
||||
The provider may send fewer previews than requested when the final image is ready first.
|
||||
@@ -612,13 +648,20 @@ import { BlackForestLabs, Stability } from "@opencode/ai/providers"
|
||||
|
||||
const bfl = BlackForestLabs.configure({ apiKey: process.env.BFL_API_KEY })
|
||||
|
||||
const generation = yield * Image.start({ model: bfl.image("flux-2-pro"), prompt, size: "1024x768" })
|
||||
persist(generation.token)
|
||||
const submit = Effect.gen(function* () {
|
||||
const generation = yield* Image.start({ model: bfl.image("flux-2-pro"), prompt, size: "1024x768" })
|
||||
persist({ provider: "black-forest-labs", modelID: "flux-2-pro", token: generation.token })
|
||||
})
|
||||
|
||||
const resumed = yield * Image.resume(bfl.image("flux-2-pro"), loadToken())
|
||||
const response = yield * resumed.await({ poll: { interval: "2 seconds" } })
|
||||
const finish = Effect.gen(function* () {
|
||||
const saved = load()
|
||||
const resumed = yield* Image.resume(bfl.image(saved.modelID), saved.token)
|
||||
return yield* resumed.await({ poll: { interval: "2 seconds" } })
|
||||
})
|
||||
```
|
||||
|
||||
The token carries no route identity, so persist the provider and model ID alongside it: `resume` needs the model.
|
||||
|
||||
- **Black Forest Labs** — results are downloaded before returning, because `result.sample` expires in 10 minutes.
|
||||
- **Replicate** — inputs are model-defined, so only `prompt` lowers: sizing, count, seed, format, and files go in
|
||||
`providerOptions` under the model's names, with files as `Media.Asset` (data URLs up to 256 KB, larger by URL).
|
||||
@@ -628,12 +671,13 @@ const response = yield * resumed.await({ poll: { interval: "2 seconds" } })
|
||||
|
||||
```ts
|
||||
const stability = Stability.configure({ apiKey: process.env.STABILITY_API_KEY })
|
||||
const upscaled =
|
||||
yield *
|
||||
Image.generate(
|
||||
{ model: stability.upscale(), prompt: "A lighthouse", images: [yield * Media.file("./small.png")] },
|
||||
const upscaled = Effect.gen(function* () {
|
||||
const small = yield* Media.file("./small.png")
|
||||
return yield* Image.generate(
|
||||
{ model: stability.upscale(), prompt: "A lighthouse", images: [small] },
|
||||
{ poll: { interval: "5 seconds" } },
|
||||
)
|
||||
})
|
||||
```
|
||||
|
||||
Imagen is not available: Google shut it down on the Gemini API, and Vertex discontinued the Imagen 4 models on
|
||||
@@ -666,8 +710,8 @@ Common fields (`frames`, `references`, `video`, `durationSeconds`, `aspectRatio`
|
||||
under `providerOptions`, inferred from the selected model.
|
||||
|
||||
```ts
|
||||
import { Video, VideoClient } from "@opencode/ai"
|
||||
import { Google } from "@opencode/ai/providers"
|
||||
import { Video } from "@opencode/ai"
|
||||
import { Google, Runway } from "@opencode/ai/providers"
|
||||
|
||||
const google = Google.configure({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY })
|
||||
|
||||
@@ -696,6 +740,7 @@ const controlled = Effect.gen(function* () {
|
||||
generation.id // provider operation / task / request id
|
||||
generation.status // "queued" | "running" | "completed" | "failed" | "cancelled" | "expired"
|
||||
generation.token // route-owned JSON: `{ operation }`, `{ requestID }`, `{ taskID }`, or fal's follow-up URLs
|
||||
// The token carries no route identity: persist the provider and model ID alongside it, since `resume` needs the model.
|
||||
const saved = JSON.stringify(generation.token)
|
||||
|
||||
const resumed = yield* Video.resume(google.video("veo-3.1-generate-preview"), JSON.parse(saved))
|
||||
@@ -706,8 +751,8 @@ const controlled = Effect.gen(function* () {
|
||||
const events = Video.stream({ model: Runway.configure({ apiKey }).video("gen4.5"), prompt }, { poll })
|
||||
```
|
||||
|
||||
`VideoClient.layer` needs `RequestExecutor.Service`, and status polls, result fetches, cancels, and asset downloads
|
||||
all run through the same executor with the route's auth. `Generation.await` and `Generation.events` fail with a
|
||||
Status polls, result fetches, cancels, and asset downloads all run through the same request executor with the route's
|
||||
auth. `Generation.await` and `Generation.events` fail with a
|
||||
`Timeout` reason when `poll.timeout` (default 10 minutes) elapses. Failed,
|
||||
cancelled, and expired generations fail typed with the provider's terminal document on `reason.body`; moderation
|
||||
outcomes (Veo `raiMediaFilteredReasons`, xAI `respect_moderation`, Runway `SAFETY.*` codes) surface as `notices` when
|
||||
@@ -726,15 +771,18 @@ Provider notes:
|
||||
- **Runway** expects pixel ratios in `aspectRatio` for most models (`"1280:720"`), pins `X-Runway-Version`, reports
|
||||
`usage: { type: "credits" }`, and its output URLs expire after 24–48 hours.
|
||||
|
||||
The promise client exposes the same surface: `ai.video.start(...)` resolves to a handle with `await`, `refresh`,
|
||||
`cancel`, and `token`; `ai.video.generate`, `ai.video.resume(model, token)`, and `ai.video.stream` mirror the Effect
|
||||
API.
|
||||
The promise client exposes the same surface: `ai.video.start(...)` resolves to a handle with `await`, `events`,
|
||||
`result`, `refresh`, `cancel`, and `token`; `ai.video.generate`, `ai.video.resume(model, token)`, and
|
||||
`ai.video.stream` mirror the Effect API. The handle's `status` and `progress` are a snapshot from when it was
|
||||
created; `refresh()` resolves to a new handle.
|
||||
|
||||
```ts
|
||||
import { ai } from "@opencode/ai/promise"
|
||||
|
||||
const generation = await ai.video.start({ model, prompt })
|
||||
const video = await generation.await({ poll: { interval: 10_000 }, signal })
|
||||
for await (const event of generation.events({ poll: { interval: 10_000 } })) console.log(event.type)
|
||||
const video = await generation.result({ signal })
|
||||
await ai.write(video.video, "./kite.mp4")
|
||||
```
|
||||
|
||||
## Speech generation
|
||||
@@ -786,7 +834,6 @@ ElevenLabs and Cartesia. `{ id }` selects an OpenAI custom voice (`{ id: "voice_
|
||||
plain string elsewhere. There is no cross-provider voice catalog. `format` is the container-level word (`mp3`, `wav`,
|
||||
`pcm`, `opus`, `aac`, `flac`); sample rates and bitrates live under `providerOptions`, and a value the route cannot
|
||||
produce fails as `UnsupportedOperation`. Streams buffer every chunk so `finish` can carry the whole clip.
|
||||
`SpeechClient.layer` needs `RequestExecutor.Service`.
|
||||
|
||||
Provider notes:
|
||||
|
||||
@@ -814,7 +861,7 @@ The promise client mirrors the Effect API; `ai.speech.stream` is an `AsyncIterab
|
||||
import { ai } from "@opencode/ai/promise"
|
||||
|
||||
const response = await ai.speech.generate({ model, text: "Hello from OpenCode.", voice: "coral" })
|
||||
await Bun.write("hello.mp3", await ai.run(response.audio.bytes()))
|
||||
await ai.write(response.audio, "hello.mp3")
|
||||
|
||||
for await (const event of ai.speech.stream({ model, text: "Hello from OpenCode.", voice: "coral" })) {
|
||||
if (event.type === "audio-delta") player.write(event.chunk)
|
||||
@@ -831,6 +878,7 @@ facades. Common fields (`language`, `prompt`, `timestamps: "none" | "segment" |
|
||||
natively or fail with a typed `AIError` before any network call; a route may return more than asked.
|
||||
|
||||
```ts
|
||||
import { Console, Effect, Stream } from "effect"
|
||||
import { Media, Transcription, TranscriptionEvent } from "@opencode/ai"
|
||||
import { AssemblyAI, Deepgram, OpenAI } from "@opencode/ai/providers"
|
||||
|
||||
@@ -857,7 +905,7 @@ const program = Effect.gen(function* () {
|
||||
Stream.runDrain,
|
||||
)
|
||||
|
||||
// Queued: persist the token, resume from another process, and await.
|
||||
// Queued: persist the token with the provider and model ID (the token alone cannot pick the model), resume, and await.
|
||||
const model = AssemblyAI.configure({ apiKey }).transcription("universal-3-5-pro")
|
||||
const generation = yield* Transcription.start({ model, audio })
|
||||
const resumed = yield* Transcription.resume(model, JSON.parse(JSON.stringify(generation.token)))
|
||||
@@ -866,7 +914,7 @@ const program = Effect.gen(function* () {
|
||||
```
|
||||
|
||||
Inline routes emit only `finish` from `stream` (no faked deltas); queued routes emit `generation-queued` /
|
||||
`generation-progress` before it. `TranscriptionClient.layer` needs `RequestExecutor.Service`.
|
||||
`generation-progress` before it.
|
||||
|
||||
Provider notes:
|
||||
|
||||
@@ -878,6 +926,7 @@ Provider notes:
|
||||
The promise client mirrors the Effect API:
|
||||
|
||||
```ts
|
||||
const audio = await ai.file("./call.mp3")
|
||||
const text = (await ai.transcription.generate({ model, audio })).text
|
||||
for await (const event of ai.transcription.stream({ model, audio })) if (event.type === "text-delta") write(event.delta)
|
||||
const generation = await ai.transcription.start({ model: assemblyai, audio })
|
||||
@@ -897,7 +946,8 @@ const transcript = await generation.await({ poll: { interval: 3_000 } })
|
||||
- **`Generation`** — provider-neutral handle for an in-flight media generation (`await`, `refresh`, `cancel`, `events`) used by queued media routes.
|
||||
- **`Speech.request` / `Speech.generate` / `Speech.stream`** — text-to-speech through a provider-neutral request; `SpeechClient` is its Effect service and layer.
|
||||
- **`Transcription.request` / `generate` / `stream` / `start` / `resume`** — speech-to-text over inline, streaming, and queued routes; `TranscriptionClient` is its Effect service and layer.
|
||||
- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm`, `image`, `video`, `speech`, and `transcription` as Promise / `AsyncIterable` APIs.
|
||||
- **`AIClient.layer` / `AIClient.layerWith(executor)`** — every modality client plus the request executor in one layer.
|
||||
- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm`, `image`, `video`, `speech`, and `transcription` as Promise / `AsyncIterable` APIs, plus `file`, `write`, `bytes`, `base64`, and `materialize` for assets.
|
||||
|
||||
## Testing
|
||||
|
||||
@@ -960,11 +1010,13 @@ This is different from prompt caching, server-side history storage, or truncatio
|
||||
Prefer this operation, where supported, when the application owns compaction policy and durable context updates.
|
||||
|
||||
```ts
|
||||
const result = yield * LLMClient.compact(request)
|
||||
const next = LLMRequest.update(request, {
|
||||
messages: result.replacement,
|
||||
const compacted = Effect.gen(function* () {
|
||||
const result = yield* LLMClient.compact(request)
|
||||
const next = LLMRequest.update(request, {
|
||||
messages: result.replacement,
|
||||
})
|
||||
return yield* LLMClient.generate(next)
|
||||
})
|
||||
const response = yield * LLMClient.generate(next)
|
||||
```
|
||||
|
||||
`replacement` replaces the complete input window. Do not append it to the original transcript or extract only the encrypted item: the provider may retain additional messages in its output. Retained user and assistant messages remain ordinary messages with typed text, media, or reasoning parts, in their original order. Provider-specific message IDs, status, and phase use `providerMetadata`, not a raw output array hidden in an assistant message. Unsupported returned item types fail explicitly.
|
||||
@@ -980,16 +1032,16 @@ The input must still fit the model's context window. Explicit compaction is not
|
||||
OpenAI Responses also exposes a separate, explicitly selected mechanism:
|
||||
|
||||
```ts
|
||||
const result =
|
||||
yield *
|
||||
LLMClient.compact(request, {
|
||||
const checkpoint = Effect.gen(function* () {
|
||||
const result = yield* LLMClient.compact(request, {
|
||||
mechanism: "trigger",
|
||||
webSocket, // Optional: without it, the request uses HTTP/SSE.
|
||||
})
|
||||
|
||||
result.checkpoint // Successful encrypted CompactionPart.
|
||||
result.responseID
|
||||
result.usage
|
||||
result.checkpoint // Successful encrypted CompactionPart.
|
||||
result.responseID
|
||||
result.usage
|
||||
})
|
||||
```
|
||||
|
||||
This appends a native `compaction_trigger` control item to the full input and sends a normal Responses request, with tools and instructions retained, `stream: true`, `store: false`, and parallel tool calls enabled. It removes normal-answer text/output-format controls, forced tool choices, output-token/tool-call limits, and automatic `context_management`. Body overlays cannot replace `input` or supply `previous_response_id`/`conversation`; the complete canonical history is required for safe stateless replay. Request metadata, auth, headers, query parameters, service tier, and supported prompt-cache settings are preserved.
|
||||
@@ -1003,9 +1055,11 @@ The supplied WebSocket executor can reuse a compatible append baseline for the c
|
||||
Trigger support is separate from endpoint support. Only the OpenAI Responses route advertises it; Azure, xAI, Chat, and compatible Responses routes do not inherit it. Untyped calls still fail before sending: missing route capabilities return `UnsupportedOperation`, while unknown mechanism names and invalid inputs return `InvalidRequest`. Dynamic callers must narrow for the selected mechanism:
|
||||
|
||||
```ts
|
||||
if (LLMClient.canCompact(request, { mechanism: "trigger" })) {
|
||||
const result = yield * LLMClient.compact(request, { mechanism: "trigger" })
|
||||
}
|
||||
const narrowed = Effect.gen(function* () {
|
||||
if (LLMClient.canCompact(request, { mechanism: "trigger" })) {
|
||||
const result = yield* LLMClient.compact(request, { mechanism: "trigger" })
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
This capability describes protocol implementation, **not universal availability on OpenAI API deployments**. The host application owns subscription/deployment eligibility, OAuth, endpoint selection, and deployment-specific headers. Local protocol/socket tests do not establish live provider support.
|
||||
@@ -1014,9 +1068,10 @@ This capability describes protocol implementation, **not universal availability
|
||||
|
||||
`providerOptions.contextManagement` lets the provider decide when to compact during an ordinary `generate` or `stream` call. This is an advanced option for callers that own persistence and recovery: persist the complete assistant message, including its checkpoint, before continuing. Enabling the option does not provide durable checkpoint storage, interruption recovery, or model-switch policy. Keep the prior context until a successful checkpoint has been persisted.
|
||||
|
||||
Inside an `Effect.gen`, enable OpenAI compaction with typed provider options:
|
||||
Enable OpenAI compaction with typed provider options:
|
||||
|
||||
```ts
|
||||
import { Effect } from "effect"
|
||||
import { LLM, LLMClient, LLMRequest, Message } from "@opencode/ai"
|
||||
import { OpenAI } from "@opencode/ai/providers"
|
||||
|
||||
@@ -1027,9 +1082,11 @@ const request = LLM.request({
|
||||
contextManagement: [{ type: "compaction", compactThreshold: 200_000 }],
|
||||
},
|
||||
})
|
||||
const response = yield * LLMClient.generate(request)
|
||||
const next = LLMRequest.update(request, {
|
||||
messages: [...request.messages, response.message, Message.user("Continue")],
|
||||
const continued = Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(request)
|
||||
return LLMRequest.update(request, {
|
||||
messages: [...request.messages, response.message, Message.user("Continue")],
|
||||
})
|
||||
})
|
||||
```
|
||||
|
||||
@@ -1257,7 +1314,7 @@ Compose a route with `Route.make({ protocol, endpoint, auth, framing, ... })`. T
|
||||
|
||||
## Effect
|
||||
|
||||
This package is built on Effect. Public methods return `Effect` or `Stream`; provide `LLMClient.layer` for LLM dispatch and `ImageClient.layer` for image dispatch, then import the provider/protocol modules for the routes you use. The example at `example/tutorial.ts` is a runnable walkthrough.
|
||||
This package is built on Effect. Public methods return `Effect` or `Stream`; provide `AIClient.layer` (or `AIClient.layerWith(executor)`) for every modality, then import the provider/protocol modules for the routes you use. The example at `example/tutorial.ts` is a runnable walkthrough.
|
||||
|
||||
## See also
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Media generation in `@opencode/ai` — public API direction
|
||||
|
||||
Status: phases 1–3 implemented (Speech and Transcription); phases 4–5 proposal.
|
||||
Status: phases 1–4 implemented (through Image queued routes and partial images); phase 5 proposal.
|
||||
|
||||
## Goal
|
||||
|
||||
@@ -54,7 +54,7 @@ Speech.request({ model: openai.speech("gpt-4o-mini-tts"), text })
|
||||
Transcription.request({ model: openai.transcription("gpt-4o-transcribe"), audio })
|
||||
```
|
||||
|
||||
The request namespace and the selector share one word (`Image.request` + `.image(...)`). That redundancy is accepted: a callable facade returning a lazily resolved ref would be a second way to construct the same model, and the type machinery to infer `providerOptions` through it is not worth one word. Where a provider has two routes for one modality, the selectors stay explicit (`openai.chat`, `stability.image` inline vs `stability.upscale()` queued), and one default per modality per provider is part of the facade definition (OpenAI image → Images API, Google image → Gemini-native; Imagen is shut down, so there is no `google.imagen`). Provider package entrypoints keep `model(modelID, settings)` per modality-specific path, e.g. `@opencode/ai/providers/openai/responses`.
|
||||
The request namespace and the selector share one word (`Image.request` + `.image(...)`). That redundancy is accepted: a callable facade returning a lazily resolved ref would be a second way to construct the same model, and the type machinery to infer `providerOptions` through it is not worth one word. Where a provider has two routes for one modality, the selectors stay explicit (`openai.chat`, `stability.image` inline vs `stability.upscale()` queued), and one default per modality per provider is part of the facade definition (OpenAI image → Images API, Google image → Gemini-native; Imagen is shut down, so there is no `google.imagen`). The facade selector (`openai.image(id)`) is the public path for media models. Modality-specific package entrypoints (`model(modelID, settings)` beside today's LLM paths such as `@opencode/ai/providers/openai/responses`) are deferred until Core has a modality-aware model resolver; Core's resolver accepts only `LanguageModel` today.
|
||||
|
||||
### `Media` — the asset type
|
||||
|
||||
@@ -86,10 +86,18 @@ class Media.Asset {
|
||||
|
||||
Media.bytes(data, mediaType?) Media.base64(data, mediaType?)
|
||||
Media.url(url, options?) Media.ref(provider, id)
|
||||
Media.file(path) // Bun/Node: reads + sniffs; Effect FileSystem variant for layers
|
||||
Media.write(asset, path) // convenience, uses FileSystem
|
||||
Media.file(path) // Effect<Asset, AIError, FileSystem>: reads + sniffs
|
||||
Media.write(asset, path) // Effect<void, AIError, FileSystem | RequestExecutor.Service>
|
||||
```
|
||||
|
||||
`Media.file` and `Media.write` stay Effect-only: bring your platform's `FileSystem` layer. The Promise client owns the
|
||||
runtime path: `ai.file(path)` and `ai.write(asset, path)` read and write through `node:fs/promises` (loaded on first
|
||||
use) with the same media-type sniffing and `InvalidRequest` failures, and `ai.bytes`, `ai.base64`, and
|
||||
`ai.materialize` run the asset methods in its runtime.
|
||||
|
||||
A `ref` source is accepted as input only by routes whose provider issues file handles. No shipped route produces one
|
||||
yet, so `bytes()` and `materialize()` on a ref fail by design until a producer exists.
|
||||
|
||||
Raw-PCM outputs (Gemini TTS, Cartesia raw, Deepgram WS) carry `info.encoding/sampleRate/channels` because there is no container header.
|
||||
|
||||
### Modality namespaces
|
||||
@@ -104,28 +112,32 @@ import { OpenAI, Google, ElevenLabs, Fal } from "@opencode/ai/providers"
|
||||
#### Image
|
||||
|
||||
```ts
|
||||
const request = Image.request({
|
||||
model: openai.image("gpt-image-2"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
images: [Media.file("./ref.png")], // references / edit sources
|
||||
mask: Media.file("./mask.png"),
|
||||
n: 2,
|
||||
size: "1536x1024", // or aspectRatio: "3:2"
|
||||
seed: 7,
|
||||
format: "webp",
|
||||
providerOptions: { quality: "high", background: "transparent" }, // typed per model
|
||||
Effect.gen(function* () {
|
||||
const request = Image.request({
|
||||
model: openai.image("gpt-image-2"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
images: [yield* Media.file("./ref.png")], // references / edit sources
|
||||
mask: yield* Media.file("./mask.png"),
|
||||
n: 2,
|
||||
size: "1536x1024", // OpenAI sizes by pixels; Gemini/xAI take aspectRatio instead
|
||||
format: "webp",
|
||||
providerOptions: { quality: "high", background: "transparent" }, // typed per model
|
||||
})
|
||||
|
||||
const response = yield* Image.generate(request) // ImageResponse
|
||||
response.image // Media.Asset (first)
|
||||
response.images // Media.Asset[]
|
||||
response.usage // Usage union (see below)
|
||||
response.notices // moderation / partial-result notices
|
||||
|
||||
Image.stream(request) // Stream<ImageEvent>
|
||||
// ImageEvent: generation-queued | generation-progress | image-partial { index, image } | image { index, image } | finish { usage }
|
||||
})
|
||||
|
||||
const response = yield* Image.generate(request) // ImageResponse
|
||||
response.image // Media.Asset (first)
|
||||
response.images // Media.Asset[]
|
||||
response.usage // Usage union (see below)
|
||||
response.notices // moderation / partial-result notices
|
||||
|
||||
yield* Image.stream(request) // Stream<ImageEvent>
|
||||
// ImageEvent: generation-queued | generation-progress | image-partial { index, image } | image { index, image } | finish { usage }
|
||||
```
|
||||
|
||||
`size` and `aspectRatio` are not interchangeable; each route rejects fields it cannot lower — see the README's Image
|
||||
portability matrix.
|
||||
|
||||
Editing is not a separate function; `images`/`mask` on the request select the edit path in the route (OpenAI `/images/edits`, Gemini multimodal parts, xAI `/images/edits`). Routes that cannot honor `mask` fail with `Unsupported`.
|
||||
|
||||
`ImageRoute` is the inline | stream | queued union, dispatched on `route.kind`. `Image.stream` on a streaming route emits `image-partial` previews before each `image`; on a queued route it emits `generation-queued` / `generation-progress` observations, then the result's `image` and `finish` events.
|
||||
@@ -135,40 +147,43 @@ Editing is not a separate function; `images`/`mask` on the request select the ed
|
||||
Shipped in phase 2 (`src/video.ts`, `src/video-client.ts`, protocols `google-video`, `xai-video`, `fal-video`, `runway-video`).
|
||||
|
||||
```ts
|
||||
const request = Video.request({
|
||||
model: google.video("veo-3.1-generate-preview"),
|
||||
prompt: "Panning wide shot of a calico kitten sleeping in the sunshine",
|
||||
frames: { first: Media.file("./start.png"), last: Media.file("./end.png") },
|
||||
references: [Media.file("./style.png")],
|
||||
video: Media.bytes(previous, "video/mp4"), // edit / extend source
|
||||
durationSeconds: 8,
|
||||
aspectRatio: "16:9",
|
||||
resolution: "1080p",
|
||||
audio: true,
|
||||
n: 1,
|
||||
seed: 7,
|
||||
negativePrompt: "text, watermark", // common, not provider-native
|
||||
providerOptions: { personGeneration: "allow_adult" },
|
||||
Effect.gen(function* () {
|
||||
const request = Video.request({
|
||||
model: google.video("veo-3.1-generate-preview"),
|
||||
prompt: "Panning wide shot of a calico kitten sleeping in the sunshine",
|
||||
frames: { first: yield* Media.file("./start.png"), last: yield* Media.file("./end.png") },
|
||||
references: [yield* Media.file("./style.png")],
|
||||
video: Media.bytes(previous, "video/mp4"), // edit / extend source
|
||||
durationSeconds: 8,
|
||||
aspectRatio: "16:9",
|
||||
resolution: "1080p",
|
||||
audio: true,
|
||||
n: 1,
|
||||
seed: 7,
|
||||
negativePrompt: "text, watermark", // common, not provider-native
|
||||
providerOptions: { personGeneration: "allow_adult" },
|
||||
})
|
||||
|
||||
// Simple: wait for it.
|
||||
const response = yield* Video.generate(request, { poll: { interval: "10 seconds", timeout: "10 minutes" } })
|
||||
response.video // Media.Asset: url with expiresAt (+ transient `headers` for Veo downloads)
|
||||
response.usage // credits on Runway; the other three report none
|
||||
response.notices // Veo raiMediaFilteredReasons → filtered, xAI respect_moderation → moderated
|
||||
yield* response.video.materialize() // pull bytes before the URL expires
|
||||
|
||||
// Explicit generation control.
|
||||
const generation = yield* Video.start(request) // Generation<VideoResponse>
|
||||
generation.id; generation.status; generation.progress; generation.position; generation.token
|
||||
yield* generation.await({ poll }) // VideoResponse
|
||||
yield* generation.cancel() // fal PUT cancel_url, Runway DELETE /tasks/{id}; no-op for Veo and xAI
|
||||
|
||||
// Resume from another process. The token is validated against the route's codec and refreshed once. It carries no
|
||||
// route identity, so persist the provider and model ID alongside it: `resume` needs the model.
|
||||
const resumed = yield* Video.resume(model, JSON.parse(saved))
|
||||
|
||||
// Progress as a stream.
|
||||
Video.stream(request, { poll }) // Stream<VideoEvent>: generation-queued { id, position } | generation-progress { id, progress } | video { index, video } | finish { usage, notices }
|
||||
})
|
||||
|
||||
// Simple: wait for it.
|
||||
const response = yield* Video.generate(request, { poll: { interval: "10 seconds", timeout: "10 minutes" } })
|
||||
response.video // Media.Asset: url with expiresAt (+ transient `headers` for Veo downloads)
|
||||
response.usage // credits on Runway; the other three report none
|
||||
response.notices // Veo raiMediaFilteredReasons → filtered, xAI respect_moderation → moderated
|
||||
yield* response.video.materialize() // pull bytes before the URL expires
|
||||
|
||||
// Explicit generation control.
|
||||
const generation = yield* Video.start(request) // Generation<VideoResponse>
|
||||
generation.id; generation.status; generation.progress; generation.position; generation.token
|
||||
yield* generation.await({ poll }) // VideoResponse
|
||||
yield* generation.cancel() // fal PUT cancel_url, Runway DELETE /tasks/{id}; no-op for Veo and xAI
|
||||
|
||||
// Resume from another process. The token is validated against the route's codec and refreshed once.
|
||||
const resumed = yield* Video.resume(model, JSON.parse(saved))
|
||||
|
||||
// Progress as a stream.
|
||||
yield* Video.stream(request, { poll }) // Stream<VideoEvent>: generation-queued { id, position } | generation-progress { id, progress } | video { index, video } | finish { usage, notices }
|
||||
```
|
||||
|
||||
Tokens are route-owned JSON: Veo `{ operation }`, xAI `{ requestID }`, Runway `{ taskID }`, fal
|
||||
@@ -330,12 +345,12 @@ class Generation<Response> {
|
||||
readonly expiresAt?: number
|
||||
refresh(): Effect<Generation<Response>, AIError>
|
||||
result(): Effect<Response, AIError>
|
||||
await(options?: AwaitOptions): Effect<Response, AIError>
|
||||
await(options?: GenerationAwaitOptions): Effect<Response, AIError>
|
||||
cancel(): Effect<void, AIError>
|
||||
events(options?: AwaitOptions): Stream<GenerationEvent, AIError> // fails with Timeout past poll.timeout, checked per observation
|
||||
events(options?: GenerationAwaitOptions): Stream<GenerationEvent, AIError> // fails with Timeout past poll.timeout, checked per observation
|
||||
}
|
||||
|
||||
AwaitOptions = { poll?: Poll }
|
||||
GenerationAwaitOptions = { poll?: Poll }
|
||||
Poll = { interval?: Duration; timeout?: Duration; schedule?: Schedule } // route may override from provider hints (`openai-poll-after-ms`)
|
||||
```
|
||||
|
||||
@@ -365,15 +380,17 @@ const ai = AI.make() // ManagedRuntime over Reque
|
||||
// AI.make({ layer }) to inject a custom executor / recorder / middleware
|
||||
|
||||
const image = await ai.image.generate({ model, prompt })
|
||||
await image.image.bytes()
|
||||
await ai.bytes(image.image) // also ai.base64, ai.materialize, ai.write(asset, path)
|
||||
const reference = await ai.file("./ref.png")
|
||||
|
||||
for await (const event of ai.speech.stream({ model, text, voice })) { … }
|
||||
|
||||
const generation = await ai.video.start({ model, prompt })
|
||||
const generation = await ai.video.start({ model, prompt }) // snapshot handle; refresh() returns a new one
|
||||
for await (const event of generation.events({ poll: { interval: 10_000 } })) { … }
|
||||
const video = await generation.await({ poll: { interval: 10_000 }, signal })
|
||||
const resumed = ai.video.resume(model, JSON.parse(saved))
|
||||
const resumed = await ai.video.resume(model, JSON.parse(saved)) // persist provider + model ID with the token
|
||||
|
||||
const text = await ai.llm.generate({ model, prompt }) // closes today's gap: LLM has no promise API either
|
||||
const text = await ai.llm.generate({ model, prompt })
|
||||
for await (const event of ai.llm.stream(request)) { … }
|
||||
|
||||
await ai.dispose()
|
||||
@@ -401,9 +418,9 @@ Existing facades gain per-modality selectors; the modality routes each facade pr
|
||||
| `Runway` | | | ✓ | | | |
|
||||
| `Luma`, `Kling`, `MiniMax` | | per provider | | | | |
|
||||
|
||||
New facades follow the existing one-file-per-provider rule. Package entrypoints are modality-specific, such as `@opencode/ai/providers/openai/images`, and return the concrete model.
|
||||
New facades follow the existing one-file-per-provider rule. The facade selector is the public path for media models; modality-specific package entrypoints (for example `@opencode/ai/providers/openai/images`) are deferred until Core has a modality-aware model resolver.
|
||||
|
||||
`ImageModel<Options>` already gives typed `providerOptions` per model; `VideoModel`, `SpeechModel`, `TranscriptionModel` follow the same generic. A shared `MediaModel` union is what `Generation` and the promise client key on.
|
||||
`ImageModel<Options>` gives typed `providerOptions` per model; `VideoModel`, `SpeechModel`, and `TranscriptionModel` follow the same generic. They share an internal `MediaModel` base class (ids, route, `http` overlays) that is not part of the public exports; `Generation` and the promise client work with the concrete modality models.
|
||||
|
||||
### Routes and protocols
|
||||
|
||||
|
||||
@@ -1,18 +1,7 @@
|
||||
import { Config, Effect, Formatter, Layer, Schema, Stream } from "effect"
|
||||
import { Config, Effect, Formatter, Schema, Stream } from "effect"
|
||||
import { NodeFileSystem } from "@effect/platform-node"
|
||||
import {
|
||||
Image,
|
||||
ImageClient,
|
||||
LLM,
|
||||
LLMClient,
|
||||
LLMRequest,
|
||||
Media,
|
||||
Message,
|
||||
ProviderID,
|
||||
Tool,
|
||||
ToolRuntime,
|
||||
} from "@opencode/ai"
|
||||
import { Route, Auth, Endpoint, Framing, Protocol, RequestExecutor } from "@opencode/ai/route"
|
||||
import { AIClient, Image, LLM, LLMRequest, Media, Message, ProviderID, Tool, ToolRuntime } from "@opencode/ai"
|
||||
import { Route, Auth, Endpoint, Framing, Protocol } from "@opencode/ai/route"
|
||||
import { OpenAI } from "@opencode/ai/providers"
|
||||
|
||||
/**
|
||||
@@ -243,12 +232,10 @@ const generateImage = Effect.gen(function* () {
|
||||
yield* Media.write(response.image, "tutorial-image.jpg").pipe(Effect.provide(NodeFileSystem.layer))
|
||||
})
|
||||
|
||||
// Provide the LLM runtime and the HTTP request executor once. Keep one path
|
||||
// enabled at a time so the tutorial can demonstrate generate, stream, or
|
||||
// Provide every modality client and the HTTP request executor once with
|
||||
// `AIClient.layer` (`AIClient.layerWith(executor)` swaps the executor). Keep one
|
||||
// path enabled at a time so the tutorial can demonstrate generate, stream, or
|
||||
// tool-loop behavior without spending tokens on every example.
|
||||
const requestExecutorLayer = RequestExecutor.fetchLayer
|
||||
const llmClientLayer = LLMClient.layer.pipe(Layer.provide(requestExecutorLayer))
|
||||
const imageClientLayer = ImageClient.layer.pipe(Layer.provide(requestExecutorLayer))
|
||||
|
||||
const program = Effect.gen(function* () {
|
||||
// yield* generateOnce
|
||||
@@ -257,6 +244,6 @@ const program = Effect.gen(function* () {
|
||||
// yield* generateDynamicObject.pipe(Effect.andThen((response) => Effect.sync(() => console.log(response.object))))
|
||||
// yield* generateImage
|
||||
yield* streamWithTools
|
||||
}).pipe(Effect.provide(Layer.mergeAll(requestExecutorLayer, llmClientLayer, imageClientLayer)))
|
||||
}).pipe(Effect.provide(AIClient.layer))
|
||||
|
||||
Effect.runPromise(program)
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import { Layer } from "effect"
|
||||
import { ImageClient } from "./image-client.js"
|
||||
import { LLMClient } from "./route/client.js"
|
||||
import { RequestExecutor } from "./route/executor.js"
|
||||
import { SpeechClient } from "./speech-client.js"
|
||||
import { TranscriptionClient } from "./transcription-client.js"
|
||||
import { VideoClient } from "./video-client.js"
|
||||
|
||||
/** Every modality client over `executor`, which stays in the output so `asset.bytes()` and `Media.write` resolve. */
|
||||
export const layerWith = <E, R>(executor: Layer.Layer<RequestExecutor.Service, E, R>) =>
|
||||
Layer.mergeAll(
|
||||
LLMClient.layer,
|
||||
ImageClient.layer,
|
||||
VideoClient.layer,
|
||||
SpeechClient.layer,
|
||||
TranscriptionClient.layer,
|
||||
).pipe(Layer.provideMerge(executor))
|
||||
|
||||
/** Every modality client plus the executor over `RequestExecutor.fetchLayer`: the one layer most programs need. */
|
||||
export const layer = layerWith(RequestExecutor.fetchLayer)
|
||||
|
||||
export type Services = Layer.Success<typeof layer>
|
||||
|
||||
export * as AIClient from "./ai-client.js"
|
||||
@@ -86,7 +86,7 @@ export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer
|
||||
})
|
||||
}),
|
||||
)
|
||||
export const fetchLayer = layer.pipe(Layer.provide(RequestExecutor.fetchLayer))
|
||||
export const fetchLayer = layer.pipe(Layer.provideMerge(RequestExecutor.fetchLayer))
|
||||
|
||||
export const EvaluationClient = {
|
||||
Service,
|
||||
|
||||
@@ -30,7 +30,9 @@ export interface Interface {
|
||||
) => Effect.Effect<Generation<ImageResponse>, AIError>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/ImageClient") {}
|
||||
export class ImageClientService extends Context.Service<ImageClientService, Interface>()("@opencode/ImageClient") {}
|
||||
export const Service = ImageClientService
|
||||
export type Service = ImageClientService
|
||||
|
||||
export const generate = <Options extends ImageOptions>(
|
||||
request: ImageRequestFor<Options>,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
export { AIClient } from "./ai-client.js"
|
||||
export { LLMClient } from "./route/client.js"
|
||||
export { ImageClient } from "./image-client.js"
|
||||
export { Auth } from "./route/auth.js"
|
||||
@@ -8,7 +9,7 @@ export type {
|
||||
RouteLanguageModelInput,
|
||||
RouteRoutedLanguageModelInput,
|
||||
Interface as LLMClientShape,
|
||||
Service as LLMClientService,
|
||||
LLMClientService,
|
||||
} from "./route/client.js"
|
||||
export * from "./schema/index.js"
|
||||
export {
|
||||
|
||||
@@ -6,7 +6,7 @@ import { ProviderID } from "./schema/ids.js"
|
||||
import { AIError, HttpContext, InvalidProviderOutputError, InvalidRequestError } from "./schema/errors.js"
|
||||
import { ProviderMetadata } from "./schema/options.js"
|
||||
import { Service } from "./route/executor-service.js"
|
||||
import { detectMediaType, extensionMediaType } from "./utils/media-type.js"
|
||||
import { detectMediaType, fileMediaType } from "./utils/media-type.js"
|
||||
|
||||
export { detectMediaType } from "./utils/media-type.js"
|
||||
|
||||
@@ -309,7 +309,7 @@ export const file = (path: string, options?: AssetOptions): Effect.Effect<Asset,
|
||||
const data = yield* fs
|
||||
.readFile(path)
|
||||
.pipe(Effect.mapError((cause) => invalid(`Failed to read media file ${path}`, cause)))
|
||||
return bytes(data, detectMediaType(data) ?? extensionMediaType(path), options)
|
||||
return bytes(data, fileMediaType(data, path), options)
|
||||
})
|
||||
|
||||
/** Materialize an asset and write its bytes through `FileSystem`. */
|
||||
|
||||
+64
-30
@@ -1,14 +1,14 @@
|
||||
import { Effect, Layer, ManagedRuntime, Stream } from "effect"
|
||||
import type { AwaitOptions, Generation, Snapshot } from "./generation.js"
|
||||
import { AIClient } from "./ai-client.js"
|
||||
import type { AwaitOptions, Event, Generation, Snapshot } from "./generation.js"
|
||||
import { Image, ImageModel, ImageRequest, type ImageOptions, type ImageRequestInput } from "./image.js"
|
||||
import { ImageClient } from "./image-client.js"
|
||||
import { LLM } from "./index.js"
|
||||
import { LLMClient } from "./route/client.js"
|
||||
import { Media } from "./media.js"
|
||||
import { tryRequest } from "./media-model.js"
|
||||
import { RequestExecutor } from "./route/executor.js"
|
||||
import { LanguageModel, LLMRequest } from "./schema/index.js"
|
||||
import { AIError, InvalidRequestError, LanguageModel, LLMRequest } from "./schema/index.js"
|
||||
import type { RequestInput } from "./llm.js"
|
||||
import { Speech, SpeechModel, SpeechRequest, type SpeechRequestInput } from "./speech.js"
|
||||
import { SpeechClient } from "./speech-client.js"
|
||||
import {
|
||||
Transcription,
|
||||
TranscriptionModel,
|
||||
@@ -16,17 +16,16 @@ import {
|
||||
type TranscriptionOptions,
|
||||
type TranscriptionRequestInput,
|
||||
} from "./transcription.js"
|
||||
import { TranscriptionClient } from "./transcription-client.js"
|
||||
import { fileMediaType } from "./utils/media-type.js"
|
||||
import { Video, VideoModel, VideoRequest, type VideoOptions, type VideoRequestInput } from "./video.js"
|
||||
import { VideoClient } from "./video-client.js"
|
||||
|
||||
/**
|
||||
* Promise-first entrypoint for scripts and non-Effect callers. One `ManagedRuntime` hosts the LLM, image, video, speech,
|
||||
* and transcription clients over a request executor; every method runs the corresponding Effect API and rethrows
|
||||
* `AIError` unchanged.
|
||||
* `AIError` unchanged. `file` and `write` load `node:fs/promises` on first use, so importing this module does not.
|
||||
*/
|
||||
export interface Options {
|
||||
/** Executor layer; defaults to `RequestExecutor.fetchLayer`. Inject a recorder or middleware here. */
|
||||
/** Executor layer; defaults to `RequestExecutor.fetchLayer`. Inject a recorder or `RequestExecutor.middleware(fn)` here. */
|
||||
readonly layer?: Layer.Layer<RequestExecutor.Service>
|
||||
}
|
||||
|
||||
@@ -34,19 +33,20 @@ export interface RunOptions {
|
||||
readonly signal?: AbortSignal
|
||||
}
|
||||
|
||||
export type Services =
|
||||
| Layer.Success<typeof LLMClient.layer>
|
||||
| Layer.Success<typeof ImageClient.layer>
|
||||
| Layer.Success<typeof VideoClient.layer>
|
||||
| Layer.Success<typeof SpeechClient.layer>
|
||||
| Layer.Success<typeof TranscriptionClient.layer>
|
||||
| RequestExecutor.Service
|
||||
export type Services = AIClient.Services
|
||||
|
||||
/** Promise view of a `Generation`: its snapshot plus `await`, `refresh`, and `cancel` returning promises. */
|
||||
/**
|
||||
* Promise view of a `Generation`. Its fields are a snapshot taken when the handle was created; `refresh()` resolves to a
|
||||
* new handle rather than updating this one.
|
||||
*/
|
||||
export type GenerationHandle<Response> = Snapshot & {
|
||||
/** Serializable JSON; pass it back to `resume` from another process. */
|
||||
readonly token: unknown
|
||||
readonly await: (options?: AwaitOptions & RunOptions) => Promise<Response>
|
||||
/** Status observations until the first terminal one, polling like `await`; abort ends iteration without throwing. */
|
||||
readonly events: (options?: AwaitOptions & RunOptions) => AsyncIterable<Event>
|
||||
/** The result without polling; fails when the generation has not completed. */
|
||||
readonly result: (options?: RunOptions) => Promise<Response>
|
||||
readonly refresh: (options?: RunOptions) => Promise<GenerationHandle<Response>>
|
||||
readonly cancel: (options?: RunOptions) => Promise<void>
|
||||
}
|
||||
@@ -65,17 +65,9 @@ const abortEffect = (signal: AbortSignal | undefined) =>
|
||||
})
|
||||
|
||||
export const make = (options: Options = {}) => {
|
||||
const runtime = ManagedRuntime.make(
|
||||
Layer.mergeAll(
|
||||
LLMClient.layer,
|
||||
ImageClient.layer,
|
||||
VideoClient.layer,
|
||||
SpeechClient.layer,
|
||||
TranscriptionClient.layer,
|
||||
).pipe(Layer.provideMerge(options.layer ?? RequestExecutor.fetchLayer)),
|
||||
)
|
||||
const runtime = ManagedRuntime.make(AIClient.layerWith(options.layer ?? RequestExecutor.fetchLayer))
|
||||
|
||||
/** Run any package Effect (for example `asset.bytes()`) inside this runtime. */
|
||||
/** Run any package Effect (for example `LLMClient.compact(...)`) inside this runtime. */
|
||||
const run = <A, E>(effect: Effect.Effect<A, E, Services>, options?: RunOptions) =>
|
||||
runtime.runPromise(effect, { signal: options?.signal })
|
||||
|
||||
@@ -95,12 +87,15 @@ export const make = (options: Options = {}) => {
|
||||
...generation.snapshot,
|
||||
token: generation.token,
|
||||
await: (options) => run(generation.await({ poll: options?.poll }), options),
|
||||
events: (options) => iterate(generation.events({ poll: options?.poll }), options),
|
||||
result: (options) => run(generation.result(), options),
|
||||
refresh: (options) => run(generation.refresh(), options).then(handle),
|
||||
cancel: (options) => run(generation.cancel(), options),
|
||||
})
|
||||
|
||||
// The typed `generate`/`stream` overloads take a concrete input or a request, not the union; normalize once here.
|
||||
const llmRequest = (input: RequestInput | LLMRequest) => (input instanceof LLMRequest ? input : LLM.request(input))
|
||||
const llmRequest = (input: RequestInput | LLMRequest) =>
|
||||
input instanceof LLMRequest ? Effect.succeed(input) : tryRequest(() => LLM.request(input))
|
||||
const imageRequest = (input: ImageRequestInput | ImageRequest) =>
|
||||
input instanceof ImageRequest ? input : Image.request(input)
|
||||
const videoRequest = (input: VideoRequestInput | VideoRequest) =>
|
||||
@@ -112,12 +107,48 @@ export const make = (options: Options = {}) => {
|
||||
|
||||
return {
|
||||
run,
|
||||
/** Decoded asset bytes, downloading `url` sources through the executor. */
|
||||
bytes: (asset: Media.Asset, options?: RunOptions) => run(asset.bytes(), options),
|
||||
base64: (asset: Media.Asset, options?: RunOptions) => run(asset.base64(), options),
|
||||
/** Pull a `url` asset into owned bytes before the provider URL expires. */
|
||||
materialize: (asset: Media.Asset, options?: RunOptions) => run(asset.materialize(), options),
|
||||
/** Read a file into an asset like `Media.file`: sniffed media type, then the extension's. */
|
||||
file: async (path: string, options?: Media.AssetOptions & RunOptions) => {
|
||||
const { readFile } = await import("node:fs/promises")
|
||||
return run(
|
||||
Effect.tryPromise({
|
||||
try: (signal) => readFile(path, { signal }),
|
||||
catch: (cause) => fileError(`Failed to read media file ${path}`, cause),
|
||||
}).pipe(
|
||||
Effect.map((buffer) => {
|
||||
const data = new Uint8Array(buffer)
|
||||
return Media.bytes(data, fileMediaType(data, path), options)
|
||||
}),
|
||||
),
|
||||
options,
|
||||
)
|
||||
},
|
||||
/** Write an asset's bytes like `Media.write`, downloading `url` sources through the executor. */
|
||||
write: async (asset: Media.Asset, path: string, options?: RunOptions) => {
|
||||
const { writeFile } = await import("node:fs/promises")
|
||||
return run(
|
||||
asset.bytes().pipe(
|
||||
Effect.flatMap((data) =>
|
||||
Effect.tryPromise({
|
||||
try: (signal) => writeFile(path, data, { signal }),
|
||||
catch: (cause) => fileError(`Failed to write media file ${path}`, cause),
|
||||
}),
|
||||
),
|
||||
),
|
||||
options,
|
||||
)
|
||||
},
|
||||
llm: {
|
||||
request: LLM.request,
|
||||
generate: <const Model extends LanguageModel>(input: RequestInput<Model> | LLMRequest, options?: RunOptions) =>
|
||||
run(LLM.generate(llmRequest(input)), options),
|
||||
run(Effect.flatMap(llmRequest(input), LLM.generate), options),
|
||||
stream: <const Model extends LanguageModel>(input: RequestInput<Model> | LLMRequest, options?: RunOptions) =>
|
||||
iterate(LLM.stream(llmRequest(input)), options),
|
||||
iterate(Stream.unwrap(Effect.map(llmRequest(input), LLM.stream)), options),
|
||||
},
|
||||
image: {
|
||||
request: Image.request,
|
||||
@@ -186,6 +217,9 @@ export const make = (options: Options = {}) => {
|
||||
|
||||
export type Client = ReturnType<typeof make>
|
||||
|
||||
const fileError = (message: string, cause: unknown) =>
|
||||
new AIError({ reason: new InvalidRequestError({ message, cause }) })
|
||||
|
||||
/** Default client over `RequestExecutor.fetchLayer` for scripts; the runtime builds its layer on first use. */
|
||||
export const ai = make()
|
||||
|
||||
|
||||
@@ -18,7 +18,11 @@ const PCM_SAMPLE_RATE = 24000
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type OpenAISpeechOptions = Record<string, unknown>
|
||||
/** `voice`, `instructions`, `speed`, and `format` are common request fields; other native body fields pass through. */
|
||||
export type OpenAISpeechOptions = {
|
||||
/** Defaults to `"sse"` in `stream` mode on models that support it; the merged value selects the response framing. */
|
||||
readonly stream_format?: "sse" | "audio"
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = SpeechRequestFor<OpenAISpeechOptions>
|
||||
|
||||
|
||||
@@ -211,7 +211,8 @@ export const mediaReference = (
|
||||
if (provider !== undefined && asset.source.type === "ref" && asset.source.provider === provider)
|
||||
return Effect.succeed({ type: "ref", value: asset.source.id })
|
||||
const accepted = provider === undefined ? "" : `, and ${provider} references`
|
||||
return Effect.fail(invalidRequest(`${label} accepts inline bytes, data URLs, http(s) URLs${accepted}`))
|
||||
const got = asset.source.type === "ref" ? `; got ${asset.source.provider}:${asset.source.id}` : ""
|
||||
return Effect.fail(invalidRequest(`${label} accepts inline bytes, data URLs, http(s) URLs${accepted}${got}`))
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
import { Config, Effect, Redacted } from "effect"
|
||||
import { Config, Effect, Option, Redacted } from "effect"
|
||||
import { Headers } from "effect/unstable/http"
|
||||
import { AuthenticationError, InvalidRequestError, AIError, type HttpOptions } from "../schema/index.js"
|
||||
import { AuthenticationError, AIError, type HttpOptions } from "../schema/index.js"
|
||||
|
||||
export class MissingCredentialError extends Error {
|
||||
readonly _tag = "MissingCredentialError"
|
||||
|
||||
constructor(readonly source: string) {
|
||||
super(`Missing auth credential: ${source}`)
|
||||
constructor(
|
||||
readonly source: string,
|
||||
message = `Missing auth credential: ${source}`,
|
||||
) {
|
||||
super(message)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,7 +92,14 @@ export const optional = (secret: Secret | undefined, source = "optional value")
|
||||
? credential(Effect.fail(new MissingCredentialError(source)))
|
||||
: credentialFromSecret(secret, source)
|
||||
|
||||
export const config = (name: string) => credentialFromSecret(Config.redacted(name), name)
|
||||
export const config = (name: string) =>
|
||||
credential(
|
||||
Effect.gen(function* () {
|
||||
const secret = yield* Config.option(Config.redacted(name))
|
||||
if (Option.isSome(secret) && Redacted.value(secret.value) !== "") return secret.value
|
||||
return yield* Effect.fail(new MissingCredentialError(name, `${name} is not set`))
|
||||
}),
|
||||
)
|
||||
|
||||
export const effect = (load: Effect.Effect<Redacted.Redacted, CredentialError>) => credential(load)
|
||||
|
||||
@@ -145,15 +155,10 @@ export function scheme(name: string, source?: Secret | Credential) {
|
||||
}
|
||||
|
||||
const toAIError = (error: AuthError): AIError => {
|
||||
if (error instanceof MissingCredentialError || error instanceof Config.ConfigError) {
|
||||
return new AIError({
|
||||
reason:
|
||||
error instanceof MissingCredentialError
|
||||
? new AuthenticationError({ message: error.message, cause: error })
|
||||
: new InvalidRequestError({ message: `Failed to resolve auth config: ${error.message}`, cause: error }),
|
||||
})
|
||||
}
|
||||
return error
|
||||
if (error instanceof AIError) return error
|
||||
const message =
|
||||
error instanceof MissingCredentialError ? error.message : `Failed to resolve auth config: ${error.message}`
|
||||
return new AIError({ reason: new AuthenticationError({ message, cause: error }) })
|
||||
}
|
||||
|
||||
export const toEffect =
|
||||
|
||||
@@ -261,7 +261,9 @@ const unsupportedCompaction = (request: LLMRequest, mechanism: string | undefine
|
||||
})
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/LLMClient") {}
|
||||
export class LLMClientService extends Context.Service<LLMClientService, Interface>()("@opencode/LLMClient") {}
|
||||
export const Service = LLMClientService
|
||||
export type Service = LLMClientService
|
||||
|
||||
const resolveRequestOptions = (request: LLMRequest) => {
|
||||
const messages = normalizeToolHistory(request.messages)
|
||||
|
||||
@@ -19,4 +19,8 @@ export type HttpMiddleware = (
|
||||
handler: HttpHandler,
|
||||
) => Effect.Effect<HttpClientResponse.HttpClientResponse, Error>
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/AI/RequestExecutor") {}
|
||||
export class RequestExecutorService extends Context.Service<RequestExecutorService, Interface>()(
|
||||
"@opencode/AI/RequestExecutor",
|
||||
) {}
|
||||
export const Service = RequestExecutorService
|
||||
export type Service = RequestExecutorService
|
||||
|
||||
@@ -255,4 +255,20 @@ export const layer: Layer.Layer<Service, never, HttpClient.HttpClient> = Layer.e
|
||||
|
||||
export const fetchLayer = layer.pipe(Layer.provide(FetchHttpClient.layer))
|
||||
|
||||
/** Run `fn` on every request: it sees the raw response before status classification, inside middleware already on `executor`, and outside per-call middleware. */
|
||||
export const middleware = (fn: HttpMiddleware, executor: Layer.Layer<Service> = fetchLayer): Layer.Layer<Service> =>
|
||||
Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const inner = yield* Service
|
||||
return Service.of({
|
||||
execute: (request, next) =>
|
||||
inner.execute(
|
||||
request,
|
||||
next === undefined ? fn : (input, handler) => fn(input, (forwarded) => next(forwarded, handler)),
|
||||
),
|
||||
})
|
||||
}),
|
||||
).pipe(Layer.provide(executor))
|
||||
|
||||
export * as RequestExecutor from "./executor.js"
|
||||
|
||||
@@ -7,7 +7,7 @@ export type {
|
||||
RouteDefaultsInput,
|
||||
AnyRoute,
|
||||
Interface as LLMClientShape,
|
||||
Service as LLMClientService,
|
||||
LLMClientService,
|
||||
StreamOptions,
|
||||
CompactMethod,
|
||||
CompactionOperations,
|
||||
|
||||
@@ -12,7 +12,9 @@ export interface Interface {
|
||||
) => Stream.Stream<SpeechEvent, AIError>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/SpeechClient") {}
|
||||
export class SpeechClientService extends Context.Service<SpeechClientService, Interface>()("@opencode/SpeechClient") {}
|
||||
export const Service = SpeechClientService
|
||||
export type Service = SpeechClientService
|
||||
|
||||
export const generate = <Options extends SpeechOptions>(
|
||||
request: SpeechRequestFor<Options>,
|
||||
|
||||
@@ -30,7 +30,11 @@ export interface Interface {
|
||||
) => Effect.Effect<Generation<TranscriptionResponse>, AIError>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/TranscriptionClient") {}
|
||||
export class TranscriptionClientService extends Context.Service<TranscriptionClientService, Interface>()(
|
||||
"@opencode/TranscriptionClient",
|
||||
) {}
|
||||
export const Service = TranscriptionClientService
|
||||
export type Service = TranscriptionClientService
|
||||
|
||||
export const generate = <Options extends TranscriptionOptions>(
|
||||
request: TranscriptionRequestFor<Options>,
|
||||
|
||||
@@ -48,9 +48,12 @@ const EXTENSIONS: Readonly<Record<string, string>> = {
|
||||
csv: "text/csv",
|
||||
}
|
||||
|
||||
export const extensionMediaType = (path: string): string | undefined =>
|
||||
const extensionMediaType = (path: string): string | undefined =>
|
||||
EXTENSIONS[path.slice(path.lastIndexOf(".") + 1).toLowerCase()]
|
||||
|
||||
/** Media type of a file's contents: sniffed magic bytes, then the path's extension. */
|
||||
export const fileMediaType = (bytes: Uint8Array, path: string) => detectMediaType(bytes) ?? extensionMediaType(path)
|
||||
|
||||
const EXTENSION_ALIASES: Readonly<Record<string, string>> = {
|
||||
"audio/mp3": "mp3",
|
||||
"audio/m4a": "m4a",
|
||||
|
||||
@@ -29,7 +29,9 @@ export interface Interface {
|
||||
) => Stream.Stream<VideoEvent, AIError>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/VideoClient") {}
|
||||
export class VideoClientService extends Context.Service<VideoClientService, Interface>()("@opencode/VideoClient") {}
|
||||
export const Service = VideoClientService
|
||||
export type Service = VideoClientService
|
||||
|
||||
export const start = <Options extends VideoOptions>(
|
||||
request: VideoRequestFor<Options>,
|
||||
|
||||
@@ -92,6 +92,15 @@ describe("Auth", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("reports a missing config credential as Authentication naming the variable", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* Auth.toEffect(Auth.config("OPENAI_API_KEY").bearer())(input).pipe(withEnv({}), Effect.flip)
|
||||
|
||||
expect(error.reason._tag).toBe("Authentication")
|
||||
expect(error.message).toContain("OPENAI_API_KEY is not set")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("can intentionally leave auth untouched", () =>
|
||||
Effect.gen(function* () {
|
||||
const headers = yield* Auth.none.apply(input)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Deferred, Effect, Fiber, Ref, Stream } from "effect"
|
||||
import { Deferred, Effect, Fiber, Layer, Ref, Stream } from "effect"
|
||||
import { Headers, HttpClientError, HttpClientRequest } from "effect/unstable/http"
|
||||
import { LLM, AIError, HttpContext, InvalidProviderOutputError, TransportError } from "../src/index.js"
|
||||
import { LLMClient, RequestExecutor, WebSocketTransport, type WebSocketChannelExecutor } from "../src/route.js"
|
||||
import { route } from "../src/protocols/openai-chat.js"
|
||||
import { configure } from "../src/providers/openai.js"
|
||||
import { dynamicResponse, fixedResponse, systemError } from "./lib/http.js"
|
||||
import { dynamicResponse, fixedResponse, handlerLayer, systemError } from "./lib/http.js"
|
||||
import { deltaChunk } from "./lib/openai-chunks.js"
|
||||
import { sseEvents, sseRaw } from "./lib/sse.js"
|
||||
import { it } from "./lib/effect.js"
|
||||
@@ -195,6 +195,26 @@ describe("RequestExecutor", () => {
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("runs shared middleware outside per-call middleware", () => {
|
||||
const calls: Array<string> = []
|
||||
const record = (name: string) => Effect.sync(() => calls.push(name))
|
||||
const base = RequestExecutor.layer.pipe(
|
||||
Layer.provide(handlerLayer((input) => record("handler").pipe(Effect.as(input.respond("ok"))))),
|
||||
)
|
||||
return Effect.gen(function* () {
|
||||
const executor = yield* RequestExecutor.Service
|
||||
yield* executor.execute(request, (input, next) => record("per-call").pipe(Effect.andThen(next(input))))
|
||||
expect(calls).toEqual(["outer", "per-call", "handler"])
|
||||
calls.length = 0
|
||||
yield* executor.execute(request)
|
||||
expect(calls).toEqual(["outer", "handler"])
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
RequestExecutor.middleware((input, next) => record("outer").pipe(Effect.andThen(next(input))), base),
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
it.effect("classifies context overflow responses", () =>
|
||||
Effect.gen(function* () {
|
||||
const executor = yield* RequestExecutor.Service
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { Effect, Layer } from "effect"
|
||||
import {
|
||||
AIClient,
|
||||
AIError,
|
||||
Generation,
|
||||
Image,
|
||||
ImageClient,
|
||||
LanguageModel,
|
||||
LLM,
|
||||
LLMClient,
|
||||
@@ -11,10 +14,11 @@ import {
|
||||
Speech,
|
||||
SpeechClient,
|
||||
SpeechEvent,
|
||||
TranscriptionClient,
|
||||
Video,
|
||||
VideoClient,
|
||||
} from "@opencode/ai"
|
||||
import { Route, Protocol, WebSocketTransport } from "@opencode/ai/route"
|
||||
import { Route, Protocol, RequestExecutor, WebSocketTransport } from "@opencode/ai/route"
|
||||
import { Provider as ProviderSubpath } from "@opencode/ai/provider"
|
||||
import {
|
||||
AssemblyAI,
|
||||
@@ -76,6 +80,28 @@ describe("public exports", () => {
|
||||
expect(EvaluationClient.fetchLayer).toBeDefined()
|
||||
})
|
||||
|
||||
test("AIClient.layerWith shares one executor across every client", async () => {
|
||||
let built = 0
|
||||
const counting = Layer.effect(
|
||||
RequestExecutor.Service,
|
||||
Effect.sync(() => {
|
||||
built++
|
||||
return RequestExecutor.Service.of({ execute: () => Effect.die("unexpected request") })
|
||||
}),
|
||||
)
|
||||
await Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
yield* LLMClient.Service
|
||||
yield* ImageClient.Service
|
||||
yield* VideoClient.Service
|
||||
yield* SpeechClient.Service
|
||||
yield* TranscriptionClient.Service
|
||||
yield* RequestExecutor.Service
|
||||
}).pipe(Effect.provide(AIClient.layerWith(counting))),
|
||||
)
|
||||
expect(built).toBe(1)
|
||||
})
|
||||
|
||||
test("route barrel exposes route-authoring APIs", () => {
|
||||
expect(Route.make).toBeFunction()
|
||||
expect(Protocol.make).toBeFunction()
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { mkdtemp, rm } from "node:fs/promises"
|
||||
import { tmpdir } from "node:os"
|
||||
import { join } from "node:path"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { AIError, LLMEvent, Media, SpeechEvent, TranscriptionEvent } from "../src/index.js"
|
||||
@@ -182,6 +185,36 @@ describe("AI promise client", () => {
|
||||
await ai.dispose()
|
||||
})
|
||||
|
||||
test("observes a started generation's events and fetches its result", async () => {
|
||||
const ai = AI.make({ layer: executor([]) })
|
||||
const model = Runway.configure({ apiKey: "test", baseURL: "https://runway.test/v1" }).video("gen4.5")
|
||||
const generation = await ai.video.start({ model, prompt: "A kite" })
|
||||
|
||||
const events: Array<string> = []
|
||||
for await (const event of generation.events({ poll: { interval: 10 } })) events.push(event.type)
|
||||
expect(events).toEqual(["generation-progress", "generation-finished"])
|
||||
expect(generation.status).toBe("queued")
|
||||
expect((await generation.result()).video.source).toMatchObject({ url: "https://runway.test/out.mp4" })
|
||||
await ai.dispose()
|
||||
})
|
||||
|
||||
test("reads, writes, and decodes assets without leaving promises", async () => {
|
||||
const ai = AI.make({ layer: executor([]) })
|
||||
const dir = await mkdtemp(join(tmpdir(), "ai-promise-"))
|
||||
try {
|
||||
const png = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3])
|
||||
await Bun.write(join(dir, "source.bin"), png)
|
||||
|
||||
const asset = await ai.file(join(dir, "source.bin"))
|
||||
expect(asset.mediaType).toBe("image/png")
|
||||
await ai.write(asset, join(dir, "copy.png"))
|
||||
expect(await ai.bytes(await ai.file(join(dir, "copy.png")))).toEqual(png)
|
||||
} finally {
|
||||
await rm(dir, { recursive: true })
|
||||
await ai.dispose()
|
||||
}
|
||||
})
|
||||
|
||||
test("generates, streams, and starts transcriptions over the same runtime", async () => {
|
||||
const ai = AI.make({ layer: executor([]) })
|
||||
const audio = Media.url("https://audio.test/hello.mp3")
|
||||
@@ -233,6 +266,12 @@ describe("AI promise client", () => {
|
||||
expect(failure).toBeInstanceOf(AIError)
|
||||
expect(failure instanceof AIError && failure.reason.http?.status).toBe(404)
|
||||
|
||||
const invalid = await ai.llm
|
||||
// @ts-expect-error Invalid input must reject with AIError, not throw synchronously.
|
||||
.generate({ model: openai.responses("gpt-5"), messages: [{ role: "bogus" }] })
|
||||
.catch((error: unknown) => error)
|
||||
expect(invalid instanceof AIError && invalid.reason._tag).toBe("InvalidRequest")
|
||||
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
const aborted = await ai.llm
|
||||
|
||||
@@ -533,6 +533,7 @@ describe("Video / fal", () => {
|
||||
"InvalidRequest",
|
||||
])
|
||||
expect(errors[1].message).toContain("end_image_url")
|
||||
expect(errors[4].message).toContain("; got fal:handle")
|
||||
}).pipe(Effect.provide(layer(() => Effect.die("unsupported input reached the network")))),
|
||||
)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user