From bad6834a3ea2e8e229f1de8610917e2ce0b46c04 Mon Sep 17 00:00:00 2001 From: Shoubhit Dash Date: Fri, 25 Sep 2026 22:23:14 +0530 Subject: [PATCH] fix(ai): size fal Kontext by aspect ratio and decode sync_mode data URIs (#51371) --- packages/ai/README.md | 24 ++++---- packages/ai/src/protocols/fal-images.ts | 34 +++++++----- packages/ai/test/image.test.ts | 74 +++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 26 deletions(-) diff --git a/packages/ai/README.md b/packages/ai/README.md index 88a25717310..e8554bd9685 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -475,18 +475,18 @@ const program = Effect.gen(function* () { Common fields are portable in shape, not in support. Unsupported fields fail with a typed `AIError` before any network call rather than being dropped, so check this table before swapping only the `model`: -| Provider | `n` | `size` | `aspectRatio` | `seed` | `format` | `images` | `mask` | -| --------------------- | --- | --------- | ------------- | ------ | -------- | ------------------------- | ------------------- | -| OpenAI | ✓¹ | ✓ | ✗ | ✗ | ✓ | ✓ | ✓ | -| Google (Gemini) | 1 | ✗ | ✓ | ✓ | ✗ | ✓ (no public URLs) | ✗ | -| xAI | ✓ | ✗ | ✓ | ✗ | ✗ | ✓ | ✗ | -| Z.ai | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | -| Meta | ✓ | ✓ (hint) | ✗ | ✗ | ✓ | ✓ | ✗ | -| Black Forest Labs | 1 | per model | per model | ✓ | ✓ | per model (1–8) | `flux-pro-1.0-fill` | -| fal | ✓ | per model | per model | ✓ | ✓ | 1 (several on `/edit`) | ✓ | -| Replicate | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ (use `providerOptions`) | ✗ | -| Stability `image` | 1 | ✗ | ✓ | ✓ | ✓ | 1 (not on `core`) | ✗ | -| Stability `upscale()` | ✗ | ✗ | ✗ | ✓ | ✓ | exactly 1 (required) | ✗ | +| Provider | `n` | `size` | `aspectRatio` | `seed` | `format` | `images` | `mask` | +| --------------------- | --- | --------- | ------------- | ------ | -------- | -------------------------------- | ------------------- | +| OpenAI | ✓¹ | ✓ | ✗ | ✗ | ✓ | ✓ | ✓ | +| Google (Gemini) | 1 | ✗ | ✓ | ✓ | ✗ | ✓ (no public URLs) | ✗ | +| xAI | ✓ | ✗ | ✓ | ✗ | ✗ | ✓ | ✗ | +| Z.ai | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | +| Meta | ✓ | ✓ (hint) | ✗ | ✗ | ✓ | ✓ | ✗ | +| Black Forest Labs | 1 | per model | per model | ✓ | ✓ | per model (1–8) | `flux-pro-1.0-fill` | +| fal | ✓ | per model | per model | ✓ | ✓ | 1 (several on `/edit`, `/multi`) | ✓ | +| Replicate | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ (use `providerOptions`) | ✗ | +| Stability `image` | 1 | ✗ | ✓ | ✓ | ✓ | 1 (not on `core`) | ✗ | +| Stability `upscale()` | ✗ | ✗ | ✗ | ✓ | ✓ | exactly 1 (required) | ✗ | ✓ lowers natively; ✗ fails whenever the field is set (including `n: 1`); `1` means `n > 1` fails. ¹ `Image.stream` on OpenAI generates one image. fal rejects `size` and `aspectRatio` together; which one a fal or BFL model takes depends on the model. diff --git a/packages/ai/src/protocols/fal-images.ts b/packages/ai/src/protocols/fal-images.ts index 00e3d05d584..d3de229dc4e 100644 --- a/packages/ai/src/protocols/fal-images.ts +++ b/packages/ai/src/protocols/fal-images.ts @@ -49,7 +49,7 @@ const QueueResult = Schema.StructWithRest( // --------------------------------------------------------------------------- const sizing = (model: string) => { - if (/^fal-ai\/(nano-banana|flux-pro\/v1\.1-ultra)/.test(model)) return "aspect_ratio" + if (/^fal-ai\/(nano-banana|flux-pro\/(v1\.1-ultra|kontext))/.test(model)) return "aspect_ratio" if (model.startsWith("fal-ai/flux")) return "image_size" return undefined } @@ -63,20 +63,24 @@ const validate = (request: Request) => { return Effect.fail(route.unsupported("media.size", `${id} sizes by aspectRatio`)) if (request.aspectRatio !== undefined && field === "image_size") return Effect.fail(route.unsupported("media.aspectRatio", `${id} sizes by size (image_size)`)) - if ((request.images?.length ?? 0) > 1 && !isEdit(id)) + if ((request.images?.length ?? 0) > 1 && !takesImageList(id)) return Effect.fail( - route.unsupported("media.images", `${id} takes one image_url; use an /edit endpoint for several images`), + route.unsupported( + "media.images", + `${id} takes one image_url; use an /edit or /multi endpoint for several images`, + ), ) return Effect.void } -// `/edit` endpoints take an `image_urls` list; image-to-image, fill, and Ultra take one `image_url` (beside `mask_url`). -const isEdit = (model: string) => model.endsWith("/edit") +// `/edit` and `/multi` (Kontext) endpoints take an `image_urls` list; image-to-image, fill, and Ultra take one +// `image_url` (beside `mask_url`). +const takesImageList = (model: string) => model.endsWith("/edit") || model.endsWith("/multi") const fromRequest = Effect.fn("FalImages.fromRequest")(function* (request: Request) { yield* validate(request) const images = yield* Effect.forEach(request.images ?? [], (image) => FalQueue.mediaUrl(image, route.name)) - const edit = isEdit(request.model.id) + const list = takesImageList(request.model.id) return MediaProtocol.json( mergeJsonRecords( { @@ -86,8 +90,8 @@ const fromRequest = Effect.fn("FalImages.fromRequest")(function* (request: Reque image_size: request.size === undefined ? undefined : MediaInput.dimensions(request.size), aspect_ratio: request.aspectRatio, output_format: request.format, - image_urls: edit && images.length > 0 ? images : undefined, - image_url: edit ? undefined : images[0], + image_urls: list && images.length > 0 ? images : undefined, + image_url: list ? undefined : images[0], mask_url: request.mask === undefined ? undefined : yield* FalQueue.mediaUrl(request.mask, route.name), }, request.providerOptions, @@ -112,12 +116,14 @@ const decodeResult = Effect.fn("FalImages.decodeResult")(function* ( // With the safety checker on, flagged images come back blacked out rather than omitted. const flagged = (has_nsfw_concepts ?? []).flatMap((value, index) => (value ? [index] : [])) return new ImageResponse({ - images: images.map((image) => - Media.url(image.url, { - mediaType: image.content_type ?? undefined, - info: { width: image.width ?? undefined, height: image.height ?? undefined }, - }), - ), + images: images.map((image) => { + const info = { width: image.width ?? undefined, height: image.height ?? undefined } + // `sync_mode: true` returns data URIs instead of hosted URLs. + return ( + Media.parseDataUrl(image.url, { info }) ?? + Media.url(image.url, { mediaType: image.content_type ?? undefined, info }) + ) + }), notices: flagged.length === 0 ? undefined diff --git a/packages/ai/test/image.test.ts b/packages/ai/test/image.test.ts index c9b79b62e50..9701835be80 100644 --- a/packages/ai/test/image.test.ts +++ b/packages/ai/test/image.test.ts @@ -761,6 +761,80 @@ describe("Image", () => { }).pipe(Effect.provide(layer(() => Effect.die("an unsupported request reached the network")))), ) + const falToken = { + requestID: "r1", + statusURL: "https://queue.fal.test/fal-ai/flux/requests/r1/status", + responseURL: "https://queue.fal.test/fal-ai/flux/requests/r1", + cancelURL: "https://queue.fal.test/fal-ai/flux/requests/r1/cancel", + } + const falSubmitted = { + request_id: falToken.requestID, + status_url: falToken.statusURL, + response_url: falToken.responseURL, + cancel_url: falToken.cancelURL, + } + const bodies: Array = [] + it.effect("sizes fal Kontext by aspect ratio and sends several images to /multi", () => + Effect.gen(function* () { + const fal = Fal.configure({ apiKey: "test", baseURL: "https://queue.fal.test" }) + const images = [Media.url("https://example.test/a.png"), Media.url("https://example.test/b.png")] + const rejected = yield* Image.start({ + model: fal.image("fal-ai/flux-pro/kontext"), + prompt: "A lighthouse", + size: "512x512", + }).pipe(Effect.flip) + yield* Image.start({ + model: fal.image("fal-ai/flux-pro/kontext"), + prompt: "A lighthouse", + images: images.slice(0, 1), + aspectRatio: "16:9", + }) + yield* Image.start({ model: fal.image("fal-ai/flux-pro/kontext/max/multi"), prompt: "A lighthouse", images }) + + expect(rejected.reason).toMatchObject({ _tag: "UnsupportedOperation", operation: "media.size" }) + expect(bodies).toEqual([ + { prompt: "A lighthouse", aspect_ratio: "16:9", image_url: "https://example.test/a.png" }, + { prompt: "A lighthouse", image_urls: ["https://example.test/a.png", "https://example.test/b.png"] }, + ]) + }).pipe( + Effect.provide( + layer((input) => { + bodies.push(JSON.parse(input.text)) + return Effect.succeed(json(input, falSubmitted)) + }), + ), + ), + ) + + it.effect("decodes fal sync_mode data URIs as inline images", () => + Effect.gen(function* () { + const generation = yield* Image.resume(Fal.configure({ apiKey: "test" }).image("fal-ai/flux/schnell"), falToken) + const response = yield* generation.await() + + expect(response.images.map((image) => image.source)).toEqual([ + { type: "base64", data: "AQID", mediaType: "image/png" }, + { type: "url", url: "https://v3.fal.media/out.jpg", mediaType: "image/jpeg" }, + ]) + expect(response.image.info).toEqual({ width: 512, height: 512 }) + expect(yield* response.image.bytes()).toEqual(Uint8Array.from([1, 2, 3])) + }).pipe( + Effect.provide( + layer((input) => + Effect.succeed( + input.request.url === falToken.statusURL + ? json(input, { status: "COMPLETED" }) + : json(input, { + images: [ + { url: "data:image/png;base64,AQID", width: 512, height: 512, content_type: "image/png" }, + { url: "https://v3.fal.media/out.jpg", width: 512, height: 512, content_type: "image/jpeg" }, + ], + }), + ), + ), + ), + ), + ) + const moderated = { id: "req_1", status: "Content Moderated" } const prediction = { id: "p_1",