mirror of
https://gitee.com/zhangsken/opencode.git
synced 2026-10-16 10:30:01 +08:00
companion: fix session and voice races
Keep user interactions tied to the current companion turn so stopping, typing, or reopening a retained session cannot revive stale speech or lose input. Limit status information to the main session and the companion's actual permission boundary.
This commit is contained in:
@@ -1535,7 +1535,7 @@ export type VoiceSpeechOutput =
|
||||
| { readonly type: "mp3" }
|
||||
| { readonly type: "pcm"; readonly sampleRate: number; readonly channels: number }
|
||||
}
|
||||
| { readonly type: "audio"; readonly index: number; readonly data: string }
|
||||
| { readonly type: "audio"; readonly data: string }
|
||||
| { readonly type: "done" }
|
||||
| { readonly type: "error"; readonly message: string }
|
||||
export type VoiceSpeechOperation<E = never> = (input: VoiceSpeechInput) => Stream.Stream<VoiceSpeechOutput, E>
|
||||
|
||||
@@ -1477,7 +1477,7 @@ export type ModelCost = {
|
||||
|
||||
export type VoiceSpeechEvent =
|
||||
| { type: "format"; format: VoiceSpeechFormat }
|
||||
| { type: "audio"; index: number; data: string }
|
||||
| { type: "audio"; data: string }
|
||||
| { type: "done" }
|
||||
| { type: "error"; message: string }
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ export const agent = Agent.ID.make("companion")
|
||||
|
||||
const names = ["main_status", "main_read", "main_send", "main_cancel", "main_interrupt"]
|
||||
|
||||
// User config rules apply after the companion's own and may allow more, such as `edit` or `git *`.
|
||||
// User config rules apply after the companion's own, but never beyond these actions.
|
||||
const actions = new Set(["read", "grep", "glob", "webfetch", "websearch", "shell", "external_directory", ...names])
|
||||
|
||||
// One read-only git command with no shell operators. The shell parser also drops redirects that
|
||||
@@ -72,7 +72,7 @@ export const Plugin = {
|
||||
|
||||
const status = Effect.fn("CompanionTools.status")(function* (main: SessionSchema.Info) {
|
||||
const [active, inbox, permissions] = yield* Effect.all(
|
||||
[sessions.active, sessions.inbox(main.id).pipe(Effect.orElseSucceed(() => [])), permission.list()],
|
||||
[sessions.active, sessions.inbox(main.id).pipe(Effect.orElseSucceed(() => [])), permission.forSession(main.id)],
|
||||
{ concurrency: "unbounded" },
|
||||
)
|
||||
const pending = inbox.flatMap((item) =>
|
||||
@@ -80,9 +80,9 @@ export const Plugin = {
|
||||
? [`- ${item.id} [${item.delivery}] ${clip(item.payload.text, 200)}`]
|
||||
: [],
|
||||
)
|
||||
const asked = permissions
|
||||
.filter((request) => request.sessionID === main.id)
|
||||
.map((request) => `- ${request.action}: ${clip(request.message ?? request.resources.join(", "), 200)}`)
|
||||
const asked = permissions.map(
|
||||
(request) => `- ${request.action}: ${clip(request.message ?? request.resources.join(", "), 200)}`,
|
||||
)
|
||||
return [
|
||||
`Main session: ${main.title ?? "Untitled"} (${main.id})`,
|
||||
`Status: ${active.has(main.id) ? "running" : "idle"}${main.outcome ? ` (last run ${main.outcome})` : ""}`,
|
||||
|
||||
@@ -55,31 +55,13 @@ type Facade = {
|
||||
type Settings = { readonly apiKey?: string; readonly baseURL?: string }
|
||||
|
||||
// Speech and transcription models are not in the model catalog, so each provider package maps to its facade directly.
|
||||
const facades: Record<string, (settings: Settings) => Promise<Facade>> = {
|
||||
"@opencode/ai/providers/openai": async (settings) => {
|
||||
const { configure } = await import("@opencode/ai/providers/openai")
|
||||
return configure(settings)
|
||||
},
|
||||
"@opencode/ai/providers/google": async (settings) => {
|
||||
const { configure } = await import("@opencode/ai/providers/google")
|
||||
return configure(settings)
|
||||
},
|
||||
"@opencode/ai/providers/xai": async (settings) => {
|
||||
const { configure } = await import("@opencode/ai/providers/xai")
|
||||
return configure(settings)
|
||||
},
|
||||
"@opencode/ai/providers/elevenlabs": async (settings) => {
|
||||
const { configure } = await import("@opencode/ai/providers/elevenlabs")
|
||||
return configure(settings)
|
||||
},
|
||||
"@opencode/ai/providers/deepgram": async (settings) => {
|
||||
const { configure } = await import("@opencode/ai/providers/deepgram")
|
||||
return configure(settings)
|
||||
},
|
||||
"@opencode/ai/providers/assemblyai": async (settings) => {
|
||||
const { configure } = await import("@opencode/ai/providers/assemblyai")
|
||||
return configure(settings)
|
||||
},
|
||||
const facades: Record<string, () => Promise<{ readonly configure: (settings: Settings) => Facade }>> = {
|
||||
"@opencode/ai/providers/openai": () => import("@opencode/ai/providers/openai"),
|
||||
"@opencode/ai/providers/google": () => import("@opencode/ai/providers/google"),
|
||||
"@opencode/ai/providers/xai": () => import("@opencode/ai/providers/xai"),
|
||||
"@opencode/ai/providers/elevenlabs": () => import("@opencode/ai/providers/elevenlabs"),
|
||||
"@opencode/ai/providers/deepgram": () => import("@opencode/ai/providers/deepgram"),
|
||||
"@opencode/ai/providers/assemblyai": () => import("@opencode/ai/providers/assemblyai"),
|
||||
}
|
||||
|
||||
// Gemini speech is PCM only; every other supported provider streams MP3.
|
||||
@@ -120,7 +102,8 @@ export const layer = Layer.effect(
|
||||
apiKey: credential?.key ?? stringSetting(provider?.settings?.apiKey),
|
||||
baseURL: stringSetting(provider?.settings?.baseURL),
|
||||
}
|
||||
return { specifier, facade: yield* Effect.promise(() => load(settings)) }
|
||||
const { configure } = yield* Effect.promise(load)
|
||||
return { specifier, facade: configure(settings) }
|
||||
})
|
||||
|
||||
const selected = Effect.fn("Voice.selected")(function* <Key extends "transcription" | "speech">(key: Key) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { NonNegativeInt, PositiveInt } from "@opencode/schema/schema"
|
||||
import { PositiveInt } from "@opencode/schema/schema"
|
||||
import { Schema } from "effect"
|
||||
import { HttpApiEndpoint, HttpApiGroup, HttpApiSchema, OpenApi } from "effect/unstable/httpapi"
|
||||
import { ServiceUnavailableError } from "../errors.js"
|
||||
@@ -14,7 +14,6 @@ const SpeechEvent = Schema.Union([
|
||||
Schema.Struct({ type: Schema.Literal("format"), format: SpeechFormat }),
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("audio"),
|
||||
index: NonNegativeInt,
|
||||
data: Schema.String.annotate({ description: "Base64-encoded audio bytes." }),
|
||||
}),
|
||||
Schema.Struct({ type: Schema.Literal("done") }),
|
||||
|
||||
@@ -34,11 +34,8 @@ export const VoiceHandler = HttpApiBuilder.group(Api, "server.voice", (handlers)
|
||||
const voice = yield* Voice.Service
|
||||
const audio = yield* voice.speak({ text: request.payload.text }).pipe(Effect.mapError(unavailable))
|
||||
return audio.pipe(
|
||||
Stream.zipWithIndex,
|
||||
Stream.map(([chunk, index]) =>
|
||||
chunk.type === "format"
|
||||
? chunk
|
||||
: { type: "audio" as const, index, data: Encoding.encodeBase64(chunk.chunk) },
|
||||
Stream.map((chunk) =>
|
||||
chunk.type === "format" ? chunk : { type: "audio" as const, data: Encoding.encodeBase64(chunk.chunk) },
|
||||
),
|
||||
Stream.concat(Stream.succeed({ type: "done" as const })),
|
||||
// Provider failures arrive after the response started, so they travel as a terminal event.
|
||||
|
||||
@@ -9,8 +9,6 @@ const TAP_FRAMES = 16384
|
||||
const TAP_BLOCK = 1024
|
||||
|
||||
export type Recording = {
|
||||
/** Root-mean-square level of the latest captured chunk, from 0 to 1. */
|
||||
readonly level: () => number
|
||||
/** Stops capture and returns the utterance as 16 kHz mono WAV, or undefined when nothing was captured. */
|
||||
readonly stop: () => Promise<Uint8Array | undefined>
|
||||
readonly cancel: () => void
|
||||
@@ -21,19 +19,16 @@ export async function record(onLevel: (level: number) => void): Promise<Recordin
|
||||
if (!audio) throw new Error("Audio is unavailable in this terminal")
|
||||
const capture = await audio.openCapture({ channels: 1 })
|
||||
const chunks: Float32Array[] = []
|
||||
let level = 0
|
||||
const reader = capture.readable.getReader()
|
||||
const pump = (async () => {
|
||||
while (true) {
|
||||
const next = await reader.read()
|
||||
if (next.done) return
|
||||
chunks.push(next.value)
|
||||
level = rms(next.value)
|
||||
onLevel(level)
|
||||
onLevel(rms(next.value))
|
||||
}
|
||||
})().catch(() => undefined)
|
||||
return {
|
||||
level: () => level,
|
||||
async stop() {
|
||||
await finish(capture, pump)
|
||||
const samples = concat(chunks)
|
||||
@@ -106,10 +101,11 @@ export async function play(format: VoiceSpeechFormat, body: AsyncIterable<Uint8A
|
||||
signal,
|
||||
})
|
||||
).catch((error) => Promise.reject(cause(error)))
|
||||
const failure = new Promise<never>((_, reject) => stream.on("error", (error) => reject(cause(error))))
|
||||
const abort = () => stream.dispose()
|
||||
signal.addEventListener("abort", abort, { once: true })
|
||||
await Promise.race([stream.closed, failure]).finally(() => signal.removeEventListener("abort", abort))
|
||||
// Aborting the signal disposes the stream, which closes it.
|
||||
await Promise.race([
|
||||
stream.closed,
|
||||
new Promise<never>((_, reject) => stream.on("error", (error) => reject(cause(error)))),
|
||||
])
|
||||
}
|
||||
|
||||
// OpenTUI wraps source failures; the provider's message is more useful than "source failed".
|
||||
|
||||
@@ -21,17 +21,21 @@ export default Plugin.define({
|
||||
const [talking, setTalking] = createSignal<string>()
|
||||
const [bargeIn, setBargeIn] = createSignal(false)
|
||||
|
||||
const companionOf = (sessionID: string) => {
|
||||
// One lookup per main session, so the panel and the talk key cannot race to create two companions.
|
||||
const lookup = (sessionID: string) => {
|
||||
const cached = companions.get(sessionID)
|
||||
if (cached) return cached
|
||||
const request = context.client.session.companion({ sessionID }).then(async (info) => {
|
||||
await Promise.all([context.data.session.sync(info.id), context.data.session.message.sync(info.id)])
|
||||
return info.id
|
||||
})
|
||||
const request = context.client.session.companion({ sessionID }).then((info) => info.id)
|
||||
request.catch(() => companions.delete(sessionID))
|
||||
companions.set(sessionID, request)
|
||||
return request
|
||||
}
|
||||
// Session retention evicts the companion's messages with its main session, so every use syncs again.
|
||||
const companionOf = (sessionID: string) =>
|
||||
lookup(sessionID).then(async (companionID) => {
|
||||
await Promise.all([context.data.session.sync(companionID), context.data.session.message.sync(companionID)])
|
||||
return companionID
|
||||
})
|
||||
|
||||
const send = (sessionID: string, text: string, voice: boolean) => {
|
||||
const known = new Set(
|
||||
@@ -124,10 +128,9 @@ export default Plugin.define({
|
||||
error={companion.error ? errorMessage(companion.error) : undefined}
|
||||
voice={voice}
|
||||
onSubmit={(text) => {
|
||||
const companionID = companion()
|
||||
if (!companionID) return
|
||||
voice.stop()
|
||||
send(companionID, text, false)
|
||||
// The companion may still be loading when the first message is typed.
|
||||
void companionOf(input.sessionID).then((companionID) => send(companionID, text, false), toastError)
|
||||
}}
|
||||
onStop={() => void stop(companion())}
|
||||
/>
|
||||
|
||||
@@ -235,5 +235,5 @@ function toolLabel(tool: SessionMessageAssistantTool) {
|
||||
if (tool.name === "main_read") return `· read the main session${pending}`
|
||||
if (tool.name === "main_interrupt") return `■ interrupted the main session${pending}`
|
||||
if (tool.name === "main_cancel") return `× cancelled a queued prompt${pending}`
|
||||
return `· ${tool.name} ${text("path") ?? text("pattern") ?? text("filePath") ?? text("url") ?? ""}${pending}`.trimEnd()
|
||||
return `· ${tool.name} ${text("command") ?? text("pattern") ?? text("path") ?? text("url") ?? text("query") ?? ""}${pending}`.trimEnd()
|
||||
}
|
||||
|
||||
@@ -23,7 +23,6 @@ export function createVoice(input: {
|
||||
}) {
|
||||
const [state, setState] = createStore({ status: "idle" as VoiceStatus, level: 0 })
|
||||
let recording: Recording | undefined
|
||||
let transcribing = false
|
||||
let speech = new AbortController()
|
||||
let playback = Promise.resolve()
|
||||
let queued = 0
|
||||
@@ -115,14 +114,15 @@ export function createVoice(input: {
|
||||
}
|
||||
|
||||
const transcribe = async () => {
|
||||
const current = recording
|
||||
const active = recording
|
||||
recording = undefined
|
||||
if (!current) return
|
||||
transcribing = true
|
||||
if (!active) return
|
||||
const current = turn
|
||||
setState({ status: "transcribing", level: 0 })
|
||||
const audio = await current.stop()
|
||||
const audio = await active.stop()
|
||||
const text = audio ? await transcript(audio) : ""
|
||||
transcribing = false
|
||||
// Stopping while the utterance transcribes discards it.
|
||||
if (current !== turn) return
|
||||
setState("status", "idle")
|
||||
if (text) input.onTranscript(text, false)
|
||||
}
|
||||
@@ -138,7 +138,7 @@ export function createVoice(input: {
|
||||
played = []
|
||||
unplayed = []
|
||||
held = undefined
|
||||
if (!transcribing) setState({ status: "idle", level: 0 })
|
||||
setState({ status: "idle", level: 0 })
|
||||
}
|
||||
|
||||
const speak = (text: string) => {
|
||||
@@ -230,7 +230,7 @@ export function speakable(text: string) {
|
||||
* natural. When `final`, the remainder is flushed too.
|
||||
*/
|
||||
export function sentences(text: string, from: number, final: boolean) {
|
||||
const boundaries = Array.from(text.slice(from).matchAll(/[.!?…:;]+["')\]]*(?=\s)|\n/g), (match) => {
|
||||
const boundaries = Array.from(text.slice(from).matchAll(/[.!?…:;]+["')\]]*(?=\s)/g), (match) => {
|
||||
return from + (match.index ?? 0) + match[0].length
|
||||
})
|
||||
const cuts = boundaries.reduce<number[]>((result, boundary) => {
|
||||
|
||||
@@ -41,7 +41,7 @@ The companion may run these shell commands:
|
||||
|
||||
Git commands that contain `>` or `--output` are denied, because both write files.
|
||||
|
||||
Configured permission rules apply after the companion's own rules, so they can allow more, for example `git *` or `edit`. Two hooks cap the companion regardless of configuration:
|
||||
Configured permission rules apply after the companion's own rules, so they can allow more within the capped actions, for example an external directory or `.env` reads. Rules such as `git *` or `edit` have no effect, because two hooks cap the companion regardless of configuration:
|
||||
|
||||
- A tool hook runs a companion shell command only when it is exactly one of the git commands above and contains no `;`, `&`, `|`, `<`, `>`, `$`, backtick, newline, or `--output`. This also covers redirects after `&&` or `||`, which the shell parser leaves out of permission resources.
|
||||
- A permission hook denies every action except `read`, `grep`, `glob`, `webfetch`, `websearch`, `shell`, `external_directory`, and the `main_*` tools. It also turns every `ask` result into `deny`: the companion never asks for permission, because clients do not show permission prompts for companions.
|
||||
@@ -58,7 +58,7 @@ The `main_*` tools act only on the calling companion's parent. They never accept
|
||||
|
||||
Steers are sent without confirmation. They appear in the main timeline like any prompt, so the user can see what the companion sent. The TUI marks user messages whose `metadata.source` is `"companion"` with a muted `via companion` label, so they do not look like the user typed them.
|
||||
|
||||
Before each companion request whose tail is a user message, a context hook inserts a short, unpersisted digest of the main session: status, title, the last user prompt, recent assistant text, and running tools. Most questions then need no tool call, which keeps voice replies fast.
|
||||
Before each companion request whose tail is a user message, a context hook inserts a short, unpersisted digest of the main session before the newest prompt: the `main_status` report and the six most recent main-session messages in `main_read` form. Most questions then need no tool call, which keeps voice replies fast.
|
||||
|
||||
## Voice
|
||||
|
||||
@@ -94,6 +94,8 @@ The voice loop is a cascade:
|
||||
4. As companion text streams in, the client splits it into sentences outside code blocks and speaks them in order.
|
||||
5. Pressing talk while the companion is thinking or speaking stops playback, interrupts the companion, and starts listening again.
|
||||
|
||||
The stop key (`companion.stop`) stops playback, capture, and the companion's reply. Stopping, or sending a typed message, also discards an utterance that is still being transcribed.
|
||||
|
||||
By default the microphone is closed while the companion speaks, so speaker output does not feed back into the transcript.
|
||||
|
||||
### Barge-in
|
||||
|
||||
Reference in New Issue
Block a user