diff --git a/packages/protocol/src/simulation.ts b/packages/protocol/src/simulation.ts index e73f478a85a..38c9fde40fa 100644 --- a/packages/protocol/src/simulation.ts +++ b/packages/protocol/src/simulation.ts @@ -340,6 +340,17 @@ export namespace Frontend { export const Color = Schema.Tuple([Schema.Number, Schema.Number, Schema.Number, Schema.Number]) export type Color = Schema.Schema.Type + export const CapturedImage = Schema.Struct({ + x: Schema.Number, + y: Schema.Number, + width: Schema.Number, + height: Schema.Number, + pixelWidth: Schema.Number, + pixelHeight: Schema.Number, + rgba: Schema.String.check(Schema.isBase64()), + }) + export interface CapturedImage extends Schema.Schema.Type {} + export const CapturedFrame = Schema.Struct({ cols: Schema.Number, rows: Schema.Number, @@ -357,6 +368,7 @@ export namespace Frontend { ), }), ), + images: Schema.optionalKey(Schema.Array(CapturedImage)), }) export interface CapturedFrame extends Schema.Schema.Type {} diff --git a/packages/simulation/src/frontend/actions.ts b/packages/simulation/src/frontend/actions.ts index e1530fa5b90..82823519be8 100644 --- a/packages/simulation/src/frontend/actions.ts +++ b/packages/simulation/src/frontend/actions.ts @@ -1,4 +1,4 @@ -import type { CliRenderer, Renderable } from "@opentui/core" +import { ImageRenderable, type CliRenderer, type Renderable } from "@opentui/core" import { createMockKeys, createMockMouse, @@ -7,7 +7,7 @@ import { type MockInput, type MockMouse, } from "@opentui/core/testing" -import { Effect, Schema } from "effect" +import { Effect, Encoding, Schema } from "effect" import { SimulationProtocol } from "../protocol" import { SimulationRenderer } from "./renderer" import { SimulationSemantics } from "./semantics" @@ -165,9 +165,55 @@ export const capture = Effect.fn("SimulationActions.capture")(function* (harness width: span.width, })), })), + images: captureImages(harness.renderer), } satisfies SimulationProtocol.Frontend.CapturedFrame }) +function captureImages(renderer: CliRenderer): SimulationProtocol.Frontend.CapturedImage[] { + return all(renderer.root).flatMap((renderable) => { + if (!(renderable instanceof ImageRenderable) || !renderable.visible || renderable.isDestroyed || !renderable.image) + return [] + const fitted = renderable.fit === "cover" + ? { width: renderable.width, height: renderable.height } + : renderable.getFittedSize(renderable.width, renderable.height) + if (fitted.width <= 0 || fitted.height <= 0) return [] + const x = renderable.screenX + Math.floor((renderable.width - fitted.width) / 2) + const y = renderable.screenY + Math.floor((renderable.height - fitted.height) / 2) + let source = renderable.image + let extracted: typeof source | undefined + if (renderable.fit === "cover") { + const targetAspect = renderable.width / (renderable.height * renderable.cellAspectRatio) + const sourceAspect = source.width / source.height + const width = sourceAspect > targetAspect ? Math.max(1, Math.round(source.height * targetAspect)) : source.width + const height = sourceAspect > targetAspect ? source.height : Math.max(1, Math.round(source.width / targetAspect)) + extracted = source.extract({ + left: Math.floor((source.width - width) / 2), + top: Math.floor((source.height - height) / 2), + width, + height, + }) + source = extracted + } + try { + const raw = source.raw("rgba8") + const rgba = new Uint8Array(raw.width * raw.height * 4) + for (let row = 0; row < raw.height; row++) + rgba.set(raw.data.subarray(row * raw.stride, row * raw.stride + raw.width * 4), row * raw.width * 4) + return [{ + x, + y, + width: fitted.width, + height: fitted.height, + pixelWidth: raw.width, + pixelHeight: raw.height, + rgba: Encoding.encodeBase64(rgba), + }] + } finally { + extracted?.dispose() + } + }) +} + export const execute = Effect.fn("SimulationActions.execute")(function* (harness: Harness, action: Action) { switch (action.type) { case "ui.type": diff --git a/packages/simulation/test/actions.test.ts b/packages/simulation/test/actions.test.ts index cf8e91a6170..ff47a3e6b69 100644 --- a/packages/simulation/test/actions.test.ts +++ b/packages/simulation/test/actions.test.ts @@ -1,7 +1,7 @@ import { expect, test } from "bun:test" -import { BoxRenderable, TextRenderable } from "@opentui/core" +import { BoxRenderable, ImageRenderable, NativeImage, TextRenderable } from "@opentui/core" import { Effect } from "effect" -import { createHarness, execute, type Harness, matches, snapshot, state } from "../src/frontend/actions" +import { capture, createHarness, execute, type Harness, matches, snapshot, state } from "../src/frontend/actions" import { SimulationRenderer } from "../src/frontend/renderer" import { SimulationSemantics } from "../src/frontend/semantics" @@ -74,6 +74,42 @@ test("headless input mirrors the configured kitty keyboard protocol", async () = ) }) +test("captures image pixels and their terminal placement", async () => { + await Effect.runPromise( + Effect.scoped( + Effect.gen(function* () { + const renderer = yield* SimulationRenderer.create({ width: 20, height: 10 }) + const image = NativeImage.fromRgba( + Uint8Array.from([255, 0, 0, 255, 0, 0, 255, 255]), + 2, + 1, + ) + renderer.root.add(new ImageRenderable(renderer, { + source: image, + position: "absolute", + left: 2, + top: 1, + width: 2, + height: 1, + fit: "fill", + })) + + const frame = yield* capture(createHarness(renderer)) + + expect(frame.images).toEqual([{ + x: 2, + y: 1, + width: 2, + height: 1, + pixelWidth: 2, + pixelHeight: 1, + rgba: "/wAA/wAA//8=", + }]) + }), + ), + ) +}) + test("clicks a target at relative coordinates through descendant text", async () => { await Effect.runPromise( Effect.scoped(