fix(ai): size fal Kontext by aspect ratio and decode sync_mode data URIs (#51371)

packages/ai/README.md

@@ -475,18 +475,18 @@ const program = Effect.gen(function* () {
 Common fields are portable in shape, not in support. Unsupported fields fail with a typed `AIError` before any network
 call rather than being dropped, so check this table before swapping only the `model`:
 
-| Provider              | `n` | `size`    | `aspectRatio` | `seed` | `format` | `images`                  | `mask`              |
-| --------------------- | --- | --------- | ------------- | ------ | -------- | ------------------------- | ------------------- |
-| OpenAI                | ✓¹  | ✓         | ✗             | ✗      | ✓        | ✓                         | ✓                   |
-| Google (Gemini)       | 1   | ✗         | ✓             | ✓      | ✗        | ✓ (no public URLs)        | ✗                   |
-| xAI                   | ✓   | ✗         | ✓             | ✗      | ✗        | ✓                         | ✗                   |
-| Z.ai                  | ✗   | ✓         | ✗             | ✗      | ✗        | ✗                         | ✗                   |
-| Meta                  | ✓   | ✓ (hint)  | ✗             | ✗      | ✓        | ✓                         | ✗                   |
-| Black Forest Labs     | 1   | per model | per model     | ✓      | ✓        | per model (1–8)           | `flux-pro-1.0-fill` |
-| fal                   | ✓   | per model | per model     | ✓      | ✓        | 1 (several on `/edit`)    | ✓                   |
-| Replicate             | ✗   | ✗         | ✗             | ✗      | ✗        | ✗ (use `providerOptions`) | ✗                   |
-| Stability `image`     | 1   | ✗         | ✓             | ✓      | ✓        | 1 (not on `core`)         | ✗                   |
-| Stability `upscale()` | ✗   | ✗         | ✗             | ✓      | ✓        | exactly 1 (required)      | ✗                   |
+| Provider              | `n` | `size`    | `aspectRatio` | `seed` | `format` | `images`                         | `mask`              |
+| --------------------- | --- | --------- | ------------- | ------ | -------- | -------------------------------- | ------------------- |
+| OpenAI                | ✓¹  | ✓         | ✗             | ✗      | ✓        | ✓                                | ✓                   |
+| Google (Gemini)       | 1   | ✗         | ✓             | ✓      | ✗        | ✓ (no public URLs)               | ✗                   |
+| xAI                   | ✓   | ✗         | ✓             | ✗      | ✗        | ✓                                | ✗                   |
+| Z.ai                  | ✗   | ✓         | ✗             | ✗      | ✗        | ✗                                | ✗                   |
+| Meta                  | ✓   | ✓ (hint)  | ✗             | ✗      | ✓        | ✓                                | ✗                   |
+| Black Forest Labs     | 1   | per model | per model     | ✓      | ✓        | per model (1–8)                  | `flux-pro-1.0-fill` |
+| fal                   | ✓   | per model | per model     | ✓      | ✓        | 1 (several on `/edit`, `/multi`) | ✓                   |
+| Replicate             | ✗   | ✗         | ✗             | ✗      | ✗        | ✗ (use `providerOptions`)        | ✗                   |
+| Stability `image`     | 1   | ✗         | ✓             | ✓      | ✓        | 1 (not on `core`)                | ✗                   |
+| Stability `upscale()` | ✗   | ✗         | ✗             | ✓      | ✓        | exactly 1 (required)             | ✗                   |
 
 ✓ lowers natively; ✗ fails whenever the field is set (including `n: 1`); `1` means `n > 1` fails. ¹ `Image.stream` on OpenAI generates one image. fal
 rejects `size` and `aspectRatio` together; which one a fal or BFL model takes depends on the model.

packages/ai/src/protocols/fal-images.ts

@@ -49,7 +49,7 @@ const QueueResult = Schema.StructWithRest(
 // ---------------------------------------------------------------------------
 
 const sizing = (model: string) => {
-  if (/^fal-ai\/(nano-banana|flux-pro\/v1\.1-ultra)/.test(model)) return "aspect_ratio"
+  if (/^fal-ai\/(nano-banana|flux-pro\/(v1\.1-ultra|kontext))/.test(model)) return "aspect_ratio"
   if (model.startsWith("fal-ai/flux")) return "image_size"
   return undefined
 }
@@ -63,20 +63,24 @@ const validate = (request: Request) => {
     return Effect.fail(route.unsupported("media.size", `${id} sizes by aspectRatio`))
   if (request.aspectRatio !== undefined && field === "image_size")
     return Effect.fail(route.unsupported("media.aspectRatio", `${id} sizes by size (image_size)`))
-  if ((request.images?.length ?? 0) > 1 && !isEdit(id))
+  if ((request.images?.length ?? 0) > 1 && !takesImageList(id))
     return Effect.fail(
-      route.unsupported("media.images", `${id} takes one image_url; use an /edit endpoint for several images`),
+      route.unsupported(
+        "media.images",
+        `${id} takes one image_url; use an /edit or /multi endpoint for several images`,
+      ),
     )
   return Effect.void
 }
 
-// `/edit` endpoints take an `image_urls` list; image-to-image, fill, and Ultra take one `image_url` (beside `mask_url`).
-const isEdit = (model: string) => model.endsWith("/edit")
+// `/edit` and `/multi` (Kontext) endpoints take an `image_urls` list; image-to-image, fill, and Ultra take one
+// `image_url` (beside `mask_url`).
+const takesImageList = (model: string) => model.endsWith("/edit") || model.endsWith("/multi")
 
 const fromRequest = Effect.fn("FalImages.fromRequest")(function* (request: Request) {
   yield* validate(request)
   const images = yield* Effect.forEach(request.images ?? [], (image) => FalQueue.mediaUrl(image, route.name))
-  const edit = isEdit(request.model.id)
+  const list = takesImageList(request.model.id)
   return MediaProtocol.json(
     mergeJsonRecords(
       {
@@ -86,8 +90,8 @@ const fromRequest = Effect.fn("FalImages.fromRequest")(function* (request: Reque
         image_size: request.size === undefined ? undefined : MediaInput.dimensions(request.size),
         aspect_ratio: request.aspectRatio,
         output_format: request.format,
-        image_urls: edit && images.length > 0 ? images : undefined,
-        image_url: edit ? undefined : images[0],
+        image_urls: list && images.length > 0 ? images : undefined,
+        image_url: list ? undefined : images[0],
         mask_url: request.mask === undefined ? undefined : yield* FalQueue.mediaUrl(request.mask, route.name),
       },
       request.providerOptions,
@@ -112,12 +116,14 @@ const decodeResult = Effect.fn("FalImages.decodeResult")(function* (
   // With the safety checker on, flagged images come back blacked out rather than omitted.
   const flagged = (has_nsfw_concepts ?? []).flatMap((value, index) => (value ? [index] : []))
   return new ImageResponse({
-    images: images.map((image) =>
-      Media.url(image.url, {
-        mediaType: image.content_type ?? undefined,
-        info: { width: image.width ?? undefined, height: image.height ?? undefined },
-      }),
-    ),
+    images: images.map((image) => {
+      const info = { width: image.width ?? undefined, height: image.height ?? undefined }
+      // `sync_mode: true` returns data URIs instead of hosted URLs.
+      return (
+        Media.parseDataUrl(image.url, { info }) ??
+        Media.url(image.url, { mediaType: image.content_type ?? undefined, info })
+      )
+    }),
     notices:
       flagged.length === 0
         ? undefined

packages/ai/test/image.test.ts

@@ -761,6 +761,80 @@ describe("Image", () => {
     }).pipe(Effect.provide(layer(() => Effect.die("an unsupported request reached the network")))),
   )
 
+  const falToken = {
+    requestID: "r1",
+    statusURL: "https://queue.fal.test/fal-ai/flux/requests/r1/status",
+    responseURL: "https://queue.fal.test/fal-ai/flux/requests/r1",
+    cancelURL: "https://queue.fal.test/fal-ai/flux/requests/r1/cancel",
+  }
+  const falSubmitted = {
+    request_id: falToken.requestID,
+    status_url: falToken.statusURL,
+    response_url: falToken.responseURL,
+    cancel_url: falToken.cancelURL,
+  }
+  const bodies: Array<unknown> = []
+  it.effect("sizes fal Kontext by aspect ratio and sends several images to /multi", () =>
+    Effect.gen(function* () {
+      const fal = Fal.configure({ apiKey: "test", baseURL: "https://queue.fal.test" })
+      const images = [Media.url("https://example.test/a.png"), Media.url("https://example.test/b.png")]
+      const rejected = yield* Image.start({
+        model: fal.image("fal-ai/flux-pro/kontext"),
+        prompt: "A lighthouse",
+        size: "512x512",
+      }).pipe(Effect.flip)
+      yield* Image.start({
+        model: fal.image("fal-ai/flux-pro/kontext"),
+        prompt: "A lighthouse",
+        images: images.slice(0, 1),
+        aspectRatio: "16:9",
+      })
+      yield* Image.start({ model: fal.image("fal-ai/flux-pro/kontext/max/multi"), prompt: "A lighthouse", images })
+
+      expect(rejected.reason).toMatchObject({ _tag: "UnsupportedOperation", operation: "media.size" })
+      expect(bodies).toEqual([
+        { prompt: "A lighthouse", aspect_ratio: "16:9", image_url: "https://example.test/a.png" },
+        { prompt: "A lighthouse", image_urls: ["https://example.test/a.png", "https://example.test/b.png"] },
+      ])
+    }).pipe(
+      Effect.provide(
+        layer((input) => {
+          bodies.push(JSON.parse(input.text))
+          return Effect.succeed(json(input, falSubmitted))
+        }),
+      ),
+    ),
+  )
+
+  it.effect("decodes fal sync_mode data URIs as inline images", () =>
+    Effect.gen(function* () {
+      const generation = yield* Image.resume(Fal.configure({ apiKey: "test" }).image("fal-ai/flux/schnell"), falToken)
+      const response = yield* generation.await()
+
+      expect(response.images.map((image) => image.source)).toEqual([
+        { type: "base64", data: "AQID", mediaType: "image/png" },
+        { type: "url", url: "https://v3.fal.media/out.jpg", mediaType: "image/jpeg" },
+      ])
+      expect(response.image.info).toEqual({ width: 512, height: 512 })
+      expect(yield* response.image.bytes()).toEqual(Uint8Array.from([1, 2, 3]))
+    }).pipe(
+      Effect.provide(
+        layer((input) =>
+          Effect.succeed(
+            input.request.url === falToken.statusURL
+              ? json(input, { status: "COMPLETED" })
+              : json(input, {
+                  images: [
+                    { url: "data:image/png;base64,AQID", width: 512, height: 512, content_type: "image/png" },
+                    { url: "https://v3.fal.media/out.jpg", width: 512, height: 512, content_type: "image/jpeg" },
+                  ],
+                }),
+          ),
+        ),
+      ),
+    ),
+  )
+
   const moderated = { id: "req_1", status: "Content Moderated" }
   const prediction = {
     id: "p_1",