companion: fix session and voice races

Keep user interactions tied to the current companion turn so stopping,
typing, or reopening a retained session cannot revive stale speech or lose
input. Limit status information to the main session and the companion's
actual permission boundary.
This commit is contained in:
Simon Klee
2026-09-25 23:41:01 +02:00
parent 4b2dda60d6
commit 80241e9718
11 changed files with 49 additions and 69 deletions
+1 -1
View File
@@ -1535,7 +1535,7 @@ export type VoiceSpeechOutput =
| { readonly type: "mp3" }
| { readonly type: "pcm"; readonly sampleRate: number; readonly channels: number }
}
| { readonly type: "audio"; readonly index: number; readonly data: string }
| { readonly type: "audio"; readonly data: string }
| { readonly type: "done" }
| { readonly type: "error"; readonly message: string }
export type VoiceSpeechOperation<E = never> = (input: VoiceSpeechInput) => Stream.Stream<VoiceSpeechOutput, E>
@@ -1477,7 +1477,7 @@ export type ModelCost = {
export type VoiceSpeechEvent =
| { type: "format"; format: VoiceSpeechFormat }
| { type: "audio"; index: number; data: string }
| { type: "audio"; data: string }
| { type: "done" }
| { type: "error"; message: string }
+5 -5
View File
@@ -17,7 +17,7 @@ export const agent = Agent.ID.make("companion")
const names = ["main_status", "main_read", "main_send", "main_cancel", "main_interrupt"]
// User config rules apply after the companion's own and may allow more, such as `edit` or `git *`.
// User config rules apply after the companion's own, but never beyond these actions.
const actions = new Set(["read", "grep", "glob", "webfetch", "websearch", "shell", "external_directory", ...names])
// One read-only git command with no shell operators. The shell parser also drops redirects that
@@ -72,7 +72,7 @@ export const Plugin = {
const status = Effect.fn("CompanionTools.status")(function* (main: SessionSchema.Info) {
const [active, inbox, permissions] = yield* Effect.all(
[sessions.active, sessions.inbox(main.id).pipe(Effect.orElseSucceed(() => [])), permission.list()],
[sessions.active, sessions.inbox(main.id).pipe(Effect.orElseSucceed(() => [])), permission.forSession(main.id)],
{ concurrency: "unbounded" },
)
const pending = inbox.flatMap((item) =>
@@ -80,9 +80,9 @@ export const Plugin = {
? [`- ${item.id} [${item.delivery}] ${clip(item.payload.text, 200)}`]
: [],
)
const asked = permissions
.filter((request) => request.sessionID === main.id)
.map((request) => `- ${request.action}: ${clip(request.message ?? request.resources.join(", "), 200)}`)
const asked = permissions.map(
(request) => `- ${request.action}: ${clip(request.message ?? request.resources.join(", "), 200)}`,
)
return [
`Main session: ${main.title ?? "Untitled"} (${main.id})`,
`Status: ${active.has(main.id) ? "running" : "idle"}${main.outcome ? ` (last run ${main.outcome})` : ""}`,
+9 -26
View File
@@ -55,31 +55,13 @@ type Facade = {
type Settings = { readonly apiKey?: string; readonly baseURL?: string }
// Speech and transcription models are not in the model catalog, so each provider package maps to its facade directly.
const facades: Record<string, (settings: Settings) => Promise<Facade>> = {
"@opencode/ai/providers/openai": async (settings) => {
const { configure } = await import("@opencode/ai/providers/openai")
return configure(settings)
},
"@opencode/ai/providers/google": async (settings) => {
const { configure } = await import("@opencode/ai/providers/google")
return configure(settings)
},
"@opencode/ai/providers/xai": async (settings) => {
const { configure } = await import("@opencode/ai/providers/xai")
return configure(settings)
},
"@opencode/ai/providers/elevenlabs": async (settings) => {
const { configure } = await import("@opencode/ai/providers/elevenlabs")
return configure(settings)
},
"@opencode/ai/providers/deepgram": async (settings) => {
const { configure } = await import("@opencode/ai/providers/deepgram")
return configure(settings)
},
"@opencode/ai/providers/assemblyai": async (settings) => {
const { configure } = await import("@opencode/ai/providers/assemblyai")
return configure(settings)
},
const facades: Record<string, () => Promise<{ readonly configure: (settings: Settings) => Facade }>> = {
"@opencode/ai/providers/openai": () => import("@opencode/ai/providers/openai"),
"@opencode/ai/providers/google": () => import("@opencode/ai/providers/google"),
"@opencode/ai/providers/xai": () => import("@opencode/ai/providers/xai"),
"@opencode/ai/providers/elevenlabs": () => import("@opencode/ai/providers/elevenlabs"),
"@opencode/ai/providers/deepgram": () => import("@opencode/ai/providers/deepgram"),
"@opencode/ai/providers/assemblyai": () => import("@opencode/ai/providers/assemblyai"),
}
// Gemini speech is PCM only; every other supported provider streams MP3.
@@ -120,7 +102,8 @@ export const layer = Layer.effect(
apiKey: credential?.key ?? stringSetting(provider?.settings?.apiKey),
baseURL: stringSetting(provider?.settings?.baseURL),
}
return { specifier, facade: yield* Effect.promise(() => load(settings)) }
const { configure } = yield* Effect.promise(load)
return { specifier, facade: configure(settings) }
})
const selected = Effect.fn("Voice.selected")(function* <Key extends "transcription" | "speech">(key: Key) {
+1 -2
View File
@@ -1,4 +1,4 @@
import { NonNegativeInt, PositiveInt } from "@opencode/schema/schema"
import { PositiveInt } from "@opencode/schema/schema"
import { Schema } from "effect"
import { HttpApiEndpoint, HttpApiGroup, HttpApiSchema, OpenApi } from "effect/unstable/httpapi"
import { ServiceUnavailableError } from "../errors.js"
@@ -14,7 +14,6 @@ const SpeechEvent = Schema.Union([
Schema.Struct({ type: Schema.Literal("format"), format: SpeechFormat }),
Schema.Struct({
type: Schema.Literal("audio"),
index: NonNegativeInt,
data: Schema.String.annotate({ description: "Base64-encoded audio bytes." }),
}),
Schema.Struct({ type: Schema.Literal("done") }),
+2 -5
View File
@@ -34,11 +34,8 @@ export const VoiceHandler = HttpApiBuilder.group(Api, "server.voice", (handlers)
const voice = yield* Voice.Service
const audio = yield* voice.speak({ text: request.payload.text }).pipe(Effect.mapError(unavailable))
return audio.pipe(
Stream.zipWithIndex,
Stream.map(([chunk, index]) =>
chunk.type === "format"
? chunk
: { type: "audio" as const, index, data: Encoding.encodeBase64(chunk.chunk) },
Stream.map((chunk) =>
chunk.type === "format" ? chunk : { type: "audio" as const, data: Encoding.encodeBase64(chunk.chunk) },
),
Stream.concat(Stream.succeed({ type: "done" as const })),
// Provider failures arrive after the response started, so they travel as a terminal event.
@@ -9,8 +9,6 @@ const TAP_FRAMES = 16384
const TAP_BLOCK = 1024
export type Recording = {
/** Root-mean-square level of the latest captured chunk, from 0 to 1. */
readonly level: () => number
/** Stops capture and returns the utterance as 16 kHz mono WAV, or undefined when nothing was captured. */
readonly stop: () => Promise<Uint8Array | undefined>
readonly cancel: () => void
@@ -21,19 +19,16 @@ export async function record(onLevel: (level: number) => void): Promise<Recordin
if (!audio) throw new Error("Audio is unavailable in this terminal")
const capture = await audio.openCapture({ channels: 1 })
const chunks: Float32Array[] = []
let level = 0
const reader = capture.readable.getReader()
const pump = (async () => {
while (true) {
const next = await reader.read()
if (next.done) return
chunks.push(next.value)
level = rms(next.value)
onLevel(level)
onLevel(rms(next.value))
}
})().catch(() => undefined)
return {
level: () => level,
async stop() {
await finish(capture, pump)
const samples = concat(chunks)
@@ -106,10 +101,11 @@ export async function play(format: VoiceSpeechFormat, body: AsyncIterable<Uint8A
signal,
})
).catch((error) => Promise.reject(cause(error)))
const failure = new Promise<never>((_, reject) => stream.on("error", (error) => reject(cause(error))))
const abort = () => stream.dispose()
signal.addEventListener("abort", abort, { once: true })
await Promise.race([stream.closed, failure]).finally(() => signal.removeEventListener("abort", abort))
// Aborting the signal disposes the stream, which closes it.
await Promise.race([
stream.closed,
new Promise<never>((_, reject) => stream.on("error", (error) => reject(cause(error)))),
])
}
// OpenTUI wraps source failures; the provider's message is more useful than "source failed".
@@ -21,17 +21,21 @@ export default Plugin.define({
const [talking, setTalking] = createSignal<string>()
const [bargeIn, setBargeIn] = createSignal(false)
const companionOf = (sessionID: string) => {
// One lookup per main session, so the panel and the talk key cannot race to create two companions.
const lookup = (sessionID: string) => {
const cached = companions.get(sessionID)
if (cached) return cached
const request = context.client.session.companion({ sessionID }).then(async (info) => {
await Promise.all([context.data.session.sync(info.id), context.data.session.message.sync(info.id)])
return info.id
})
const request = context.client.session.companion({ sessionID }).then((info) => info.id)
request.catch(() => companions.delete(sessionID))
companions.set(sessionID, request)
return request
}
// Session retention evicts the companion's messages with its main session, so every use syncs again.
const companionOf = (sessionID: string) =>
lookup(sessionID).then(async (companionID) => {
await Promise.all([context.data.session.sync(companionID), context.data.session.message.sync(companionID)])
return companionID
})
const send = (sessionID: string, text: string, voice: boolean) => {
const known = new Set(
@@ -124,10 +128,9 @@ export default Plugin.define({
error={companion.error ? errorMessage(companion.error) : undefined}
voice={voice}
onSubmit={(text) => {
const companionID = companion()
if (!companionID) return
voice.stop()
send(companionID, text, false)
// The companion may still be loading when the first message is typed.
void companionOf(input.sessionID).then((companionID) => send(companionID, text, false), toastError)
}}
onStop={() => void stop(companion())}
/>
@@ -235,5 +235,5 @@ function toolLabel(tool: SessionMessageAssistantTool) {
if (tool.name === "main_read") return `· read the main session${pending}`
if (tool.name === "main_interrupt") return `■ interrupted the main session${pending}`
if (tool.name === "main_cancel") return `× cancelled a queued prompt${pending}`
return `· ${tool.name} ${text("path") ?? text("pattern") ?? text("filePath") ?? text("url") ?? ""}${pending}`.trimEnd()
return `· ${tool.name} ${text("command") ?? text("pattern") ?? text("path") ?? text("url") ?? text("query") ?? ""}${pending}`.trimEnd()
}
@@ -23,7 +23,6 @@ export function createVoice(input: {
}) {
const [state, setState] = createStore({ status: "idle" as VoiceStatus, level: 0 })
let recording: Recording | undefined
let transcribing = false
let speech = new AbortController()
let playback = Promise.resolve()
let queued = 0
@@ -115,14 +114,15 @@ export function createVoice(input: {
}
const transcribe = async () => {
const current = recording
const active = recording
recording = undefined
if (!current) return
transcribing = true
if (!active) return
const current = turn
setState({ status: "transcribing", level: 0 })
const audio = await current.stop()
const audio = await active.stop()
const text = audio ? await transcript(audio) : ""
transcribing = false
// Stopping while the utterance transcribes discards it.
if (current !== turn) return
setState("status", "idle")
if (text) input.onTranscript(text, false)
}
@@ -138,7 +138,7 @@ export function createVoice(input: {
played = []
unplayed = []
held = undefined
if (!transcribing) setState({ status: "idle", level: 0 })
setState({ status: "idle", level: 0 })
}
const speak = (text: string) => {
@@ -230,7 +230,7 @@ export function speakable(text: string) {
* natural. When `final`, the remainder is flushed too.
*/
export function sentences(text: string, from: number, final: boolean) {
const boundaries = Array.from(text.slice(from).matchAll(/[.!?…:;]+["')\]]*(?=\s)|\n/g), (match) => {
const boundaries = Array.from(text.slice(from).matchAll(/[.!?…:;]+["')\]]*(?=\s)/g), (match) => {
return from + (match.index ?? 0) + match[0].length
})
const cuts = boundaries.reduce<number[]>((result, boundary) => {
+4 -2
View File
@@ -41,7 +41,7 @@ The companion may run these shell commands:
Git commands that contain `>` or `--output` are denied, because both write files.
Configured permission rules apply after the companion's own rules, so they can allow more, for example `git *` or `edit`. Two hooks cap the companion regardless of configuration:
Configured permission rules apply after the companion's own rules, so they can allow more within the capped actions, for example an external directory or `.env` reads. Rules such as `git *` or `edit` have no effect, because two hooks cap the companion regardless of configuration:
- A tool hook runs a companion shell command only when it is exactly one of the git commands above and contains no `;`, `&`, `|`, `<`, `>`, `$`, backtick, newline, or `--output`. This also covers redirects after `&&` or `||`, which the shell parser leaves out of permission resources.
- A permission hook denies every action except `read`, `grep`, `glob`, `webfetch`, `websearch`, `shell`, `external_directory`, and the `main_*` tools. It also turns every `ask` result into `deny`: the companion never asks for permission, because clients do not show permission prompts for companions.
@@ -58,7 +58,7 @@ The `main_*` tools act only on the calling companion's parent. They never accept
Steers are sent without confirmation. They appear in the main timeline like any prompt, so the user can see what the companion sent. The TUI marks user messages whose `metadata.source` is `"companion"` with a muted `via companion` label, so they do not look like the user typed them.
Before each companion request whose tail is a user message, a context hook inserts a short, unpersisted digest of the main session: status, title, the last user prompt, recent assistant text, and running tools. Most questions then need no tool call, which keeps voice replies fast.
Before each companion request whose tail is a user message, a context hook inserts a short, unpersisted digest of the main session before the newest prompt: the `main_status` report and the six most recent main-session messages in `main_read` form. Most questions then need no tool call, which keeps voice replies fast.
## Voice
@@ -94,6 +94,8 @@ The voice loop is a cascade:
4. As companion text streams in, the client splits it into sentences outside code blocks and speaks them in order.
5. Pressing talk while the companion is thinking or speaking stops playback, interrupts the companion, and starts listening again.
The stop key (`companion.stop`) stops playback, capture, and the companion's reply. Stopping, or sending a typed message, also discards an utterance that is still being transcribed.
By default the microphone is closed while the companion speaks, so speaker output does not feed back into the transcript.
### Barge-in