fix(simulation): include images in UI captures (#49868)

packages/protocol/src/simulation.ts

@@ -340,6 +340,17 @@ export namespace Frontend {
   export const Color = Schema.Tuple([Schema.Number, Schema.Number, Schema.Number, Schema.Number])
   export type Color = Schema.Schema.Type<typeof Color>
 
+  export const CapturedImage = Schema.Struct({
+    x: Schema.Number,
+    y: Schema.Number,
+    width: Schema.Number,
+    height: Schema.Number,
+    pixelWidth: Schema.Number,
+    pixelHeight: Schema.Number,
+    rgba: Schema.String.check(Schema.isBase64()),
+  })
+  export interface CapturedImage extends Schema.Schema.Type<typeof CapturedImage> {}
+
   export const CapturedFrame = Schema.Struct({
     cols: Schema.Number,
     rows: Schema.Number,
@@ -357,6 +368,7 @@ export namespace Frontend {
         ),
       }),
     ),
+    images: Schema.optionalKey(Schema.Array(CapturedImage)),
   })
   export interface CapturedFrame extends Schema.Schema.Type<typeof CapturedFrame> {}
 

packages/simulation/src/frontend/actions.ts

@@ -1,4 +1,4 @@
-import type { CliRenderer, Renderable } from "@opentui/core"
+import { ImageRenderable, type CliRenderer, type Renderable } from "@opentui/core"
 import {
   createMockKeys,
   createMockMouse,
@@ -7,7 +7,7 @@ import {
   type MockInput,
   type MockMouse,
 } from "@opentui/core/testing"
-import { Effect, Schema } from "effect"
+import { Effect, Encoding, Schema } from "effect"
 import { SimulationProtocol } from "../protocol"
 import { SimulationRenderer } from "./renderer"
 import { SimulationSemantics } from "./semantics"
@@ -165,9 +165,55 @@ export const capture = Effect.fn("SimulationActions.capture")(function* (harness
         width: span.width,
       })),
     })),
+    images: captureImages(harness.renderer),
   } satisfies SimulationProtocol.Frontend.CapturedFrame
 })
 
+function captureImages(renderer: CliRenderer): SimulationProtocol.Frontend.CapturedImage[] {
+  return all(renderer.root).flatMap((renderable) => {
+    if (!(renderable instanceof ImageRenderable) || !renderable.visible || renderable.isDestroyed || !renderable.image)
+      return []
+    const fitted = renderable.fit === "cover"
+      ? { width: renderable.width, height: renderable.height }
+      : renderable.getFittedSize(renderable.width, renderable.height)
+    if (fitted.width <= 0 || fitted.height <= 0) return []
+    const x = renderable.screenX + Math.floor((renderable.width - fitted.width) / 2)
+    const y = renderable.screenY + Math.floor((renderable.height - fitted.height) / 2)
+    let source = renderable.image
+    let extracted: typeof source | undefined
+    if (renderable.fit === "cover") {
+      const targetAspect = renderable.width / (renderable.height * renderable.cellAspectRatio)
+      const sourceAspect = source.width / source.height
+      const width = sourceAspect > targetAspect ? Math.max(1, Math.round(source.height * targetAspect)) : source.width
+      const height = sourceAspect > targetAspect ? source.height : Math.max(1, Math.round(source.width / targetAspect))
+      extracted = source.extract({
+        left: Math.floor((source.width - width) / 2),
+        top: Math.floor((source.height - height) / 2),
+        width,
+        height,
+      })
+      source = extracted
+    }
+    try {
+      const raw = source.raw("rgba8")
+      const rgba = new Uint8Array(raw.width * raw.height * 4)
+      for (let row = 0; row < raw.height; row++)
+        rgba.set(raw.data.subarray(row * raw.stride, row * raw.stride + raw.width * 4), row * raw.width * 4)
+      return [{
+        x,
+        y,
+        width: fitted.width,
+        height: fitted.height,
+        pixelWidth: raw.width,
+        pixelHeight: raw.height,
+        rgba: Encoding.encodeBase64(rgba),
+      }]
+    } finally {
+      extracted?.dispose()
+    }
+  })
+}
+
 export const execute = Effect.fn("SimulationActions.execute")(function* (harness: Harness, action: Action) {
   switch (action.type) {
     case "ui.type":

packages/simulation/test/actions.test.ts

@@ -1,7 +1,7 @@
 import { expect, test } from "bun:test"
-import { BoxRenderable, TextRenderable } from "@opentui/core"
+import { BoxRenderable, ImageRenderable, NativeImage, TextRenderable } from "@opentui/core"
 import { Effect } from "effect"
-import { createHarness, execute, type Harness, matches, snapshot, state } from "../src/frontend/actions"
+import { capture, createHarness, execute, type Harness, matches, snapshot, state } from "../src/frontend/actions"
 import { SimulationRenderer } from "../src/frontend/renderer"
 import { SimulationSemantics } from "../src/frontend/semantics"
 
@@ -74,6 +74,42 @@ test("headless input mirrors the configured kitty keyboard protocol", async () =
   )
 })
 
+test("captures image pixels and their terminal placement", async () => {
+  await Effect.runPromise(
+    Effect.scoped(
+      Effect.gen(function* () {
+        const renderer = yield* SimulationRenderer.create({ width: 20, height: 10 })
+        const image = NativeImage.fromRgba(
+          Uint8Array.from([255, 0, 0, 255, 0, 0, 255, 255]),
+          2,
+          1,
+        )
+        renderer.root.add(new ImageRenderable(renderer, {
+          source: image,
+          position: "absolute",
+          left: 2,
+          top: 1,
+          width: 2,
+          height: 1,
+          fit: "fill",
+        }))
+
+        const frame = yield* capture(createHarness(renderer))
+
+        expect(frame.images).toEqual([{
+          x: 2,
+          y: 1,
+          width: 2,
+          height: 1,
+          pixelWidth: 2,
+          pixelHeight: 1,
+          rgba: "/wAA/wAA//8=",
+        }])
+      }),
+    ),
+  )
+})
+
 test("clicks a target at relative coordinates through descendant text", async () => {
   await Effect.runPromise(
     Effect.scoped(