From 09f347db0a9fe2c2289e4d320d1337dc8ca5d7ec Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 16:32:55 -0700 Subject: [PATCH 01/32] Add bounded production agent adapters --- .github/workflows/agent-isolation.yml | 2 +- agents/adapters/claude.ts | 29 +++ agents/adapters/codex.ts | 29 +++ agents/adapters/supervisor.ts | 277 ++++++++++++++++++++++++++ agents/adapters/types.ts | 15 ++ agents/container/probe.sh | 13 ++ agents/container/profile.ts | 9 +- agents/container/run.ts | 8 + agents/policy.ts | 15 +- test/agent-adapter.test.ts | 34 ++++ test/agent-supervisor.test.ts | 175 ++++++++++++++++ 11 files changed, 601 insertions(+), 5 deletions(-) create mode 100644 agents/adapters/claude.ts create mode 100644 agents/adapters/codex.ts create mode 100644 agents/adapters/supervisor.ts create mode 100644 agents/adapters/types.ts create mode 100644 test/agent-adapter.test.ts create mode 100644 test/agent-supervisor.test.ts diff --git a/.github/workflows/agent-isolation.yml b/.github/workflows/agent-isolation.yml index 395e87e..9e66e35 100644 --- a/.github/workflows/agent-isolation.yml +++ b/.github/workflows/agent-isolation.yml @@ -29,4 +29,4 @@ jobs: - run: npm ci --ignore-scripts - run: npm run typecheck # The Docker suites share one image tag and daemon, so run test files one at a time. - - run: npx vitest run --no-file-parallelism test/agent-contract.test.ts test/agent-clone.test.ts test/agent-container.test.ts test/agent-network.test.ts test/agent-policy.test.ts test/agent-proxy.test.ts + - run: npx vitest run --no-file-parallelism test/agent-contract.test.ts test/agent-clone.test.ts test/agent-container.test.ts test/agent-network.test.ts test/agent-policy.test.ts test/agent-proxy.test.ts test/agent-adapter.test.ts test/agent-supervisor.test.ts diff --git a/agents/adapters/claude.ts b/agents/adapters/claude.ts new file mode 100644 index 0000000..41c7cc3 --- /dev/null +++ b/agents/adapters/claude.ts @@ -0,0 +1,29 @@ +import type { InvocationHandle } from '../contract.ts'; +import { createContainerProfile } from '../container/profile.ts'; +import { createVendorNetwork, removeVendorNetwork } from '../network/network.ts'; +import { createClaudeCommand, createPhasePolicy } from '../policy.ts'; +import { startProfileInvocation } from './supervisor.ts'; +import type { AgentAdapterOptions, AgentAdapterRequest } from './types.ts'; + +export function parseClaudeOutput(raw: Buffer): { text: string; providerFailed: boolean } { + const envelope = JSON.parse(raw.toString('utf8')) as { result?: unknown; is_error?: unknown }; + if (typeof envelope.result !== 'string' || typeof envelope.is_error !== 'boolean') + throw new Error('Claude returned a malformed output envelope.'); + return Object.freeze({ text: envelope.result, providerFailed: envelope.is_error }); +} + +export function startClaudeInvocation(request: AgentAdapterRequest, + oauthToken: string, options: AgentAdapterOptions = {}): InvocationHandle { + if (!oauthToken || oauthToken.includes('\0')) throw new Error('Claude OAuth token is malformed.'); + const policy = createPhasePolicy(request.invocation); + const network = createVendorNetwork(request.invocation, request.imageId); + try { + const profile = createContainerProfile({ ...request, policy, network, + command: createClaudeCommand(policy, request.prompt), claudeToken: oauthToken }); + return startProfileInvocation(profile, { ...options, secrets: { CLAUDE_CODE_OAUTH_TOKEN: oauthToken }, + decode: (_profile, raw) => parseClaudeOutput(raw) }); + } catch (error) { + removeVendorNetwork(network); + throw error; + } +} diff --git a/agents/adapters/codex.ts b/agents/adapters/codex.ts new file mode 100644 index 0000000..deef695 --- /dev/null +++ b/agents/adapters/codex.ts @@ -0,0 +1,29 @@ +import type { InvocationHandle } from '../contract.ts'; +import { createContainerProfile } from '../container/profile.ts'; +import { createVendorNetwork, removeVendorNetwork } from '../network/network.ts'; +import { createCodexCommand, createPhasePolicy } from '../policy.ts'; +import { readBoundedContainerFile, startProfileInvocation } from './supervisor.ts'; +import type { AgentAdapterOptions, AgentAdapterRequest } from './types.ts'; + +export const CODEX_OUTPUT_FILE = '/tmp/codeboost-output/final.txt'; + +export function readCodexOutput(container: string, maximumBytes: number) { + const output = readBoundedContainerFile(container, CODEX_OUTPUT_FILE, maximumBytes); + return Object.freeze({ text: output.toString('utf8'), additionalBytes: output.length }); +} + +export function startCodexInvocation(request: AgentAdapterRequest, + authFile: string, options: AgentAdapterOptions = {}): InvocationHandle { + if (!authFile || authFile.includes('\0')) throw new Error('Codex auth path is malformed.'); + const policy = createPhasePolicy(request.invocation); + const network = createVendorNetwork(request.invocation, request.imageId); + try { + const profile = createContainerProfile({ ...request, policy, network, + command: createCodexCommand(policy, request.prompt), codexAuthFile: authFile, deferredOutput: true }); + return startProfileInvocation(profile, { ...options, + decode: (current, _raw, maximum) => readCodexOutput(current.name, maximum) }); + } catch (error) { + removeVendorNetwork(network); + throw error; + } +} diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts new file mode 100644 index 0000000..d1d84c3 --- /dev/null +++ b/agents/adapters/supervisor.ts @@ -0,0 +1,277 @@ +import { execFileSync, spawn, type ChildProcess } from 'node:child_process'; +import type { InvocationHandle, InvocationResult, StopReason } from '../contract.ts'; +import { assertPhasePolicy } from '../policy.ts'; +import { createValidatedContainer, disposeValidatedContainer, validateContainer } from '../container/run.ts'; +import type { ContainerProfile } from '../container/profile.ts'; + +export const OUTPUT_LIMITS = Object.freeze({ + stdoutBytes: 16 * 1024 * 1024, + stderrBytes: 4 * 1024 * 1024, + combinedBytes: 20 * 1024 * 1024, +}); +const DEFAULT_TIMEOUT_MS = 10 * 60_000; +const DIAGNOSTIC_BYTES = 1024; +const active = new Map(); + +export interface CaptureLimits { + readonly stdoutBytes: number; + readonly stderrBytes: number; + readonly combinedBytes: number; +} +export interface DecodedOutput { + readonly text: string; + /** Bytes captured outside process stdout, such as Codex's final-output file. */ + readonly additionalBytes?: number; + readonly providerFailed?: boolean; +} +export interface SupervisorOptions { + readonly secrets?: Readonly>; + readonly timeoutMs?: number; + readonly limits?: Partial; + readonly decode?: (profile: ContainerProfile, rawStdout: Buffer, maximumBytes: number) => DecodedOutput; +} +export class OutputLimitError extends Error {} + +const dockerEnvironment = () => ({ PATH: process.env.PATH, DOCKER_HOST: process.env.DOCKER_HOST }); +const positiveInteger = (value: number, name: string) => { + if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${name} must be a positive integer.`); +}; +const captureLimits = (override: Partial | undefined): CaptureLimits => { + const limits = Object.freeze({ ...OUTPUT_LIMITS, ...override }); + positiveInteger(limits.stdoutBytes, 'stdoutBytes'); + positiveInteger(limits.stderrBytes, 'stderrBytes'); + positiveInteger(limits.combinedBytes, 'combinedBytes'); + if (limits.stdoutBytes > OUTPUT_LIMITS.stdoutBytes || limits.stderrBytes > OUTPUT_LIMITS.stderrBytes + || limits.combinedBytes > OUTPUT_LIMITS.combinedBytes) + throw new Error('Capture limits cannot exceed the production hard limits.'); + return limits; +}; +const diagnosticFor = (reason: StopReason, detail?: string) => Buffer.from( + `[codeboost: ${reason}${detail ? `: ${detail.replace(/[\r\n]+/g, ' ').slice(0, 512)}` : ''}]\n`); +const withDiagnostic = (stderr: Buffer, stdoutBytes: number, reason: StopReason, limits: CaptureLimits, + detail?: string) => { + const diagnostic = diagnosticFor(reason, detail).subarray(0, DIAGNOSTIC_BYTES); + const maximum = Math.max(0, Math.min(limits.stderrBytes, limits.combinedBytes - stdoutBytes)); + if (maximum <= diagnostic.length) return diagnostic.subarray(0, maximum); + return Buffer.concat([stderr.subarray(0, maximum - diagnostic.length), diagnostic]); +}; +const runControl = (args: readonly string[], timeoutMs = 5_000): ChildProcess => { + const child = spawn('docker', [...args], { env: dockerEnvironment(), stdio: 'ignore' }); + const timer = setTimeout(() => child.kill('SIGKILL'), timeoutMs); + timer.unref(); + child.once('close', () => clearTimeout(timer)); + child.once('error', () => clearTimeout(timer)); + return child; +}; + +/** Read a running container's tmpfs file with a pinned no-follow bounded reader. */ +export function readBoundedContainerFile(container: string, source: string, maximumBytes: number): Buffer { + positiveInteger(maximumBytes, 'maximumBytes'); + if (!source.startsWith('/tmp/codeboost-output/') || source.includes('\0')) + throw new Error('Adapter output must come from the bounded output directory.'); + try { + const reader = [ + "const fs=require('node:fs'),path=process.argv[1],maximum=Number(process.argv[2]);", + 'let fd;try{fd=fs.openSync(path,fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW);', + "const before=fs.fstatSync(fd);if(before.size>maximum)throw new Error('OUTPUT_LIMIT');", + "if(!before.isFile()||before.nlink!==1)throw new Error('UNSAFE_FILE');", + 'const output=Buffer.allocUnsafe(maximum+1);let length=0,count=0;', + 'do{count=fs.readSync(fd,output,length,output.length-length,null);length+=count}', + "while(count>0&&lengthmaximum)throw new Error('OUTPUT_LIMIT');", + 'const after=fs.fstatSync(fd);if(before.dev!==after.dev||before.ino!==after.ino||before.size!==after.size', + "||before.mtimeMs!==after.mtimeMs||before.ctimeMs!==after.ctimeMs)throw new Error('CHANGED_FILE');", + 'process.stdout.write(output.subarray(0,length))}finally{if(fd!==undefined)fs.closeSync(fd)}', + ].join(''); + return execFileSync('docker', ['exec', container, 'node', '-e', reader, source, String(maximumBytes)], { + env: dockerEnvironment(), timeout: 30_000, killSignal: 'SIGKILL', maxBuffer: maximumBytes + 1, + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch (error) { + const diagnostic = error && typeof error === 'object' && 'stderr' in error ? String(error.stderr) : String(error); + if (diagnostic.includes('OUTPUT_LIMIT')) throw new OutputLimitError('Adapter output exceeds its capture limit.'); + throw new Error('Adapter output is not a stable bounded unlinked regular file.'); + } +} + +export function isInvocationActive(attemptId: string): boolean { + return active.has(attemptId); +} + +export function startProfileInvocation(profile: ContainerProfile, options: SupervisorOptions = {}): InvocationHandle { + const invocation = assertPhasePolicy(profile.policy); + if (active.has(invocation.attemptId)) { + disposeValidatedContainer(profile); + throw new Error('An invocation with this attempt ID is still active.'); + } + let limits: CaptureLimits, configuredTimeout: number; + try { + limits = captureLimits(options.limits); + configuredTimeout = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + positiveInteger(configuredTimeout, 'timeoutMs'); + } catch (error) { + disposeValidatedContainer(profile); + throw error; + } + const now = Date.now(), deadline = Math.min(invocation.deadline, now + configuredTimeout); + if (!Number.isSafeInteger(deadline) || deadline <= now) { + disposeValidatedContainer(profile); + throw new Error('Invocation deadline has already expired.'); + } + const remaining = () => { + const value = deadline - Date.now(); + if (value < 1) throw new Error('Invocation deadline has already expired.'); + return value; + }; + try { + createValidatedContainer(profile, remaining(), options.secrets ?? {}); + validateContainer(profile.name, profile, remaining()); + } catch (error) { + try { disposeValidatedContainer(profile); } catch { /* createValidatedContainer already reports unsettled cleanup */ } + throw error; + } + + const stdoutChunks: Buffer[] = [], stderrChunks: Buffer[] = []; + let stdoutBytes = 0, stderrBytes = 0, combinedBytes = 0; + let stopReason: StopReason | undefined, failureDetail: string | undefined, closed = false, terminating = false; + let decodedOutput: DecodedOutput | undefined, protocolToken: string | undefined; + let protocolBuffer = Buffer.alloc(0); + const child = spawn('docker', ['start', '--attach', profile.name], { + env: dockerEnvironment(), stdio: ['ignore', 'pipe', 'pipe'], + }); + const timers = new Set>(); + const later = (callback: () => void, delay: number) => { + const timer = setTimeout(() => { timers.delete(timer); callback(); }, delay); + timer.unref(); timers.add(timer); return timer; + }; + const terminate = () => { + if (terminating || closed) return; + terminating = true; + child.stdout?.resume(); child.stderr?.resume(); + runControl(['stop', '--signal=TERM', '--time=1', profile.name]); + later(() => { if (!closed) runControl(['kill', '--signal=KILL', profile.name]); }, 1_500); + later(() => { + if (!closed) { + runControl(['rm', '--force', profile.name]); + child.kill('SIGKILL'); + } + }, 4_000); + }; + const stop = (reason: StopReason) => { + if (closed || stopReason) return; + stopReason = reason; + terminate(); + }; + const capture = (stream: 'stdout' | 'stderr', value: Buffer | string) => { + if (stopReason || closed) return; + const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value); + const streamBytes = stream === 'stdout' ? stdoutBytes : stderrBytes; + const streamLimit = stream === 'stdout' ? limits.stdoutBytes : limits.stderrBytes; + const available = Math.max(0, Math.min(streamLimit - streamBytes, limits.combinedBytes - combinedBytes)); + if (available > 0) { + const retained = chunk.subarray(0, available); + (stream === 'stdout' ? stdoutChunks : stderrChunks).push(retained); + if (stream === 'stdout') stdoutBytes += retained.length; + else stderrBytes += retained.length; + combinedBytes += retained.length; + } + if (chunk.length > available) stop('output-limit'); + }; + const decodeOutput = () => { + if (!options.decode || decodedOutput || stopReason) return; + try { + const raw = Buffer.concat(stdoutChunks, stdoutBytes); + const decoded = options.decode(profile, raw, + Math.max(1, Math.min(limits.stdoutBytes - stdoutBytes, limits.combinedBytes - combinedBytes))); + const additional = decoded.additionalBytes ?? 0; + if (!Number.isSafeInteger(additional) || additional < 0) throw new Error('Adapter returned an invalid byte count.'); + if (stdoutBytes + additional > limits.stdoutBytes || combinedBytes + additional > limits.combinedBytes) + throw new OutputLimitError('Adapter output exceeds its capture limit.'); + decodedOutput = decoded; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const reason = error instanceof OutputLimitError || /exceeds its capture limit/i.test(message) + ? 'output-limit' : 'capture-failure'; + failureDetail ??= message; + if (closed) stopReason ??= reason; + else stop(reason); + } + }; + const protocolLine = (line: Buffer) => { + const text = line.toString('utf8').trim(); + const started = /^\x1eCODEBOOST_START:([0-9a-f-]{36})\x1e$/.exec(text); + if (started) { + if (protocolToken && protocolToken !== started[1]) return false; + protocolToken = started[1]; + return true; + } + const ready = /^\x1eCODEBOOST_READY:([0-9a-f-]{36}):([0-9]+)\x1e$/.exec(text); + if (!ready || ready[1] !== protocolToken) return false; + decodeOutput(); + if (decodedOutput) runControl(['exec', profile.name, 'touch', `/tmp/codeboost-output/collected-${ready[1]}`]); + return true; + }; + const captureStderr = (value: Buffer | string) => { + if (!profile.deferredOutput) { capture('stderr', value); return; } + const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value); + protocolBuffer = Buffer.concat([protocolBuffer, chunk]); + let newline: number; + while ((newline = protocolBuffer.indexOf(0x0a)) >= 0) { + const line = protocolBuffer.subarray(0, newline + 1); + protocolBuffer = protocolBuffer.subarray(newline + 1); + if (!protocolLine(line)) capture('stderr', line); + } + if (protocolBuffer.length > 1024) { + const flush = protocolBuffer.subarray(0, protocolBuffer.length - 128); + protocolBuffer = protocolBuffer.subarray(protocolBuffer.length - 128); + capture('stderr', flush); + } + }; + child.stdout?.on('data', value => capture('stdout', value)); + child.stderr?.on('data', captureStderr); + child.stdout?.once('error', error => { failureDetail ??= error.message; stop('capture-failure'); }); + child.stderr?.once('error', error => { failureDetail ??= error.message; stop('capture-failure'); }); + child.once('error', error => { failureDetail ??= error.message; stop('capture-failure'); }); + later(() => stop('timeout'), Math.max(1, deadline - Date.now())); + + let resolveSettled!: (result: InvocationResult) => void; + const settled = new Promise(resolve => { resolveSettled = resolve; }); + const handle: InvocationHandle = Object.freeze({ + attemptId: invocation.attemptId, + settled, + cancel: (reason: StopReason) => stop(reason), + }); + active.set(invocation.attemptId, handle); + + child.once('close', (code, signal) => { + for (const timer of timers) clearTimeout(timer); + timers.clear(); + if (protocolBuffer.length) { + if (!protocolLine(protocolBuffer)) capture('stderr', protocolBuffer); + protocolBuffer = Buffer.alloc(0); + } + closed = true; + let finalStdout = Buffer.concat(stdoutChunks, stdoutBytes), finalStderr = Buffer.concat(stderrChunks, stderrBytes); + let exitCode = code, finalSignal = signal; + if (!stopReason && code === 0 && options.decode && !profile.deferredOutput) decodeOutput(); + if (!stopReason && code === 0 && profile.deferredOutput && !decodedOutput) { + stopReason = 'capture-failure'; failureDetail ??= 'Deferred output protocol did not complete.'; + } + if (decodedOutput) { + finalStdout = Buffer.from(decodedOutput.text); + if (decodedOutput.providerFailed) exitCode = exitCode === 0 ? 1 : exitCode; + } + try { + disposeValidatedContainer(profile); + } catch { + stopReason ??= 'capture-failure'; + return; // Ownership remains active because termination/cleanup was not confirmed. + } + if (stopReason) finalStderr = withDiagnostic(finalStderr, finalStdout.length, stopReason, limits, failureDetail); + const result = Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, + exitCode, signal: finalSignal, ...(stopReason ? { stopReason } : {}), + stdout: finalStdout.toString('utf8'), stderr: finalStderr.toString('utf8') }); + active.delete(invocation.attemptId); + resolveSettled(result); + }); + return handle; +} diff --git a/agents/adapters/types.ts b/agents/adapters/types.ts new file mode 100644 index 0000000..ab942c7 --- /dev/null +++ b/agents/adapters/types.ts @@ -0,0 +1,15 @@ +import type { InvocationInput } from '../contract.ts'; +import type { TaskFilesystems } from '../container/storage.ts'; +import type { CaptureLimits } from './supervisor.ts'; + +export interface AgentAdapterRequest { + readonly invocation: InvocationInput; + readonly filesystems: TaskFilesystems; + readonly inputDirectory: string; + readonly imageId: string; + readonly prompt: string; +} +export interface AgentAdapterOptions { + readonly timeoutMs?: number; + readonly limits?: Partial; +} diff --git a/agents/container/probe.sh b/agents/container/probe.sh index 70f9f24..40f2af1 100644 --- a/agents/container/probe.sh +++ b/agents/container/probe.sh @@ -81,4 +81,17 @@ esac [ "$(codex --version)" = 'codex-cli 0.153.4' ] || fail 'unexpected Codex version' [ "$(claude --version | awk '{print $1}')" = '2.1.281' ] || fail 'unexpected Claude version' +install --directory --owner=10001 --group=10001 --mode=0700 /tmp/codeboost-output +if [ "${CODEBOOST_DEFERRED_OUTPUT:-}" = '1' ]; then + token="$(cat /proc/sys/kernel/random/uuid)" + printf '\036CODEBOOST_START:%s\036\n' "$token" >&2 + set +e + "$@" + status="$?" + set -e + printf '\036CODEBOOST_READY:%s:%s\036\n' "$token" "$status" >&2 + acknowledgement="/tmp/codeboost-output/collected-$token" + while [ ! -e "$acknowledgement" ]; do sleep 0.05; done + exit "$status" +fi exec "$@" diff --git a/agents/container/profile.ts b/agents/container/profile.ts index 02c5f26..b9bd742 100644 --- a/agents/container/profile.ts +++ b/agents/container/profile.ts @@ -21,6 +21,7 @@ export interface ContainerProfile { readonly ownershipId: string; readonly network: VendorNetwork; readonly policy: PhasePolicy; + readonly deferredOutput: boolean; } export interface ProfileOptions { readonly invocation: InvocationInput; @@ -32,6 +33,7 @@ export interface ProfileOptions { readonly claudeToken?: string; readonly network: VendorNetwork; readonly policy: PhasePolicy; + readonly deferredOutput?: boolean; } interface FileIdentity { @@ -229,6 +231,10 @@ export function createContainerProfile(options: ProfileOptions): ContainerProfil '--mount', mount({ type: 'volume', source: filesystems.workVolume, target: '/work', readonly: readOnlyWork }), '--mount', mount({ type: 'volume', source: filesystems.metadataVolume, target: '/work/.git', readonly: true }), '--mount', mount({ type: 'bind', source: inputIdentity.inputDirectory, target: '/run/codeboost-input', readonly: true })]; + if (options.deferredOutput) { + if (invocation.vendor !== 'codex') throw new Error('Deferred output is available only for Codex.'); + args.push('--env', 'CODEBOOST_DEFERRED_OUTPUT=1'); + } if (invocation.vendor === 'codex') { args.push('--env', 'CODEX_HOME=/run/codeboost-auth/codex', '--tmpfs', '/run/codeboost-auth/codex:rw,nosuid,nodev,size=4194304,nr_inodes=256,uid=10001,gid=10001,mode=0700', @@ -239,7 +245,8 @@ export function createContainerProfile(options: ProfileOptions): ContainerProfil const profile = Object.freeze({ name, args: Object.freeze(args), expectedImage: options.imageId, phase: invocation.phase, vendor: invocation.vendor, filesystems: capturedFilesystems, inputDirectory: inputIdentity.inputDirectory, codexAuthFile, - command: Object.freeze([...command]), ownershipId, network: options.network, policy: options.policy }); + command: Object.freeze([...command]), ownershipId, network: options.network, policy: options.policy, + deferredOutput: options.deferredOutput === true }); identities.set(profile, Object.freeze({ inputDirectory: inputIdentity.inputDirectory, schema: inputIdentity.schema, auth: authIdentity, cleanupDirectories: Object.freeze([...cleanupDirectories]), filesystems, clone: invocation.clone, diff --git a/agents/container/run.ts b/agents/container/run.ts index e9d7b7f..38cebdd 100644 --- a/agents/container/run.ts +++ b/agents/container/run.ts @@ -88,6 +88,11 @@ const removeContainerOrThrow = (profile: ContainerProfile, createUnsettled = fal disposeContainerProfile(profile); }; +/** Remove a validated invocation container, then its profile-owned staging and network resources. */ +export function disposeValidatedContainer(profile: ContainerProfile): void { + removeContainerOrThrow(profile); +} + type Inspect = { Image: string; Config: { Image: string; User: string; Env: string[]; Entrypoint: string[] | null; Cmd: string[] | null; @@ -239,6 +244,7 @@ export function validateContainer(container: string, profile: ContainerProfile, const allowedEnvironment = new Set(['PATH', 'NODE_VERSION', 'YARN_VERSION', 'HOME', 'CODEBOOST_PHASE', 'CODEBOOST_VENDOR', 'CODEBOOST_WORK_BYTES', 'CODEBOOST_WORK_INODES', 'CODEBOOST_METADATA_BYTES', 'CODEBOOST_METADATA_INODES', 'npm_config_cache', 'XDG_CACHE_HOME', 'HTTPS_PROXY', 'HTTP_PROXY', 'NO_PROXY', + ...(profile.deferredOutput ? ['CODEBOOST_DEFERRED_OUTPUT'] : []), ...(profile.vendor === 'codex' ? ['CODEX_HOME'] : ['CLAUDE_CODE_OAUTH_TOKEN'])]); if (new Set(names).size !== names.length || names.some(name => !allowedEnvironment.has(name))) throw new Error('Container includes an unexpected environment variable.'); @@ -255,6 +261,8 @@ export function validateContainer(container: string, profile: ContainerProfile, || environment.get('HTTP_PROXY') !== profile.network.proxyUrl || environment.get('NO_PROXY') !== 'localhost,127.0.0.1') throw new Error('Container isolation environment changed.'); + if (profile.deferredOutput && environment.get('CODEBOOST_DEFERRED_OUTPUT') !== '1') + throw new Error('Container deferred-output protocol changed.'); if (profile.vendor === 'codex' && (names.includes('CLAUDE_CODE_OAUTH_TOKEN') || environment.get('CODEX_HOME') !== '/run/codeboost-auth/codex')) throw new Error('Credential profiles must not be combined or redirected.'); diff --git a/agents/policy.ts b/agents/policy.ts index 17c6349..59d585d 100644 --- a/agents/policy.ts +++ b/agents/policy.ts @@ -84,12 +84,14 @@ export function createCodexCommand(policy: PhasePolicy, prompt: string): AgentCo if (!prompt || prompt.includes('\0')) throw new Error('Codex prompt must be nonempty and contain no NUL.'); const sandbox = policy.worktree === 'read-write' ? 'workspace-write' : 'read-only'; // `--` ends option parsing, so a prompt beginning with `-` stays prompt data. - return command(policy, [...codexBaseArguments(policy), 'exec', '--sandbox', sandbox, '--skip-git-repo-check', '--', - prompt]); + return command(policy, [...codexBaseArguments(policy), 'exec', '--sandbox', sandbox, '--skip-git-repo-check', + '--output-last-message', '/tmp/codeboost-output/final.txt', '--', prompt]); } export type IsolationProbe = 'noop' | 'phase-worktree' | 'read-only-isolation' | 'persist-write' - | 'persist-read' | 'capacity' | 'metadata' | 'must-not-run' | 'input-marker'; + | 'persist-read' | 'capacity' | 'metadata' | 'must-not-run' | 'input-marker' | 'finite-output' + | 'infinite-stdout' | 'infinite-stderr' | 'infinite-mixed' | 'ignore-term' | 'symlink-output' + | 'oversized-output'; /** Fixed startup probes validate the sandbox itself without granting an agent a process tool. */ export function createIsolationProbeCommand(policy: PhasePolicy, probe: IsolationProbe): AgentCommand { @@ -112,6 +114,13 @@ export function createIsolationProbeCommand(policy: PhasePolicy, probe: Isolatio 'must-not-run': 'touch /tmp/command-ran', 'input-marker': 'set -eu; grep -q codeboost-schema-marker /run/codeboost-input/schema.json; ' + 'test ! -e /run/codeboost-input/extra.json', + 'finite-output': 'printf stdout-marker; printf stderr-marker >&2', + 'infinite-stdout': "while :; do head -c 4096 /dev/zero | tr '\\0' x; done", + 'infinite-stderr': "while :; do head -c 4096 /dev/zero | tr '\\0' x >&2; done", + 'infinite-mixed': "while :; do head -c 4096 /dev/zero | tr '\\0' x; head -c 4096 /dev/zero | tr '\\0' y >&2; done", + 'ignore-term': "trap '' TERM; while :; do sleep 1; done", + 'symlink-output': 'ln -s /etc/passwd /tmp/codeboost-output/final.txt', + 'oversized-output': 'head -c 131072 /dev/zero > /tmp/codeboost-output/final.txt', }; return command(policy, probe === 'noop' ? ['true'] : ['sh', '-c', scripts[probe]]); } diff --git a/test/agent-adapter.test.ts b/test/agent-adapter.test.ts new file mode 100644 index 0000000..eddc772 --- /dev/null +++ b/test/agent-adapter.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest'; +import { parseClaudeOutput } from '../agents/adapters/claude.ts'; +import { CODEX_OUTPUT_FILE } from '../agents/adapters/codex.ts'; +import { OUTPUT_LIMITS } from '../agents/adapters/supervisor.ts'; +import { captureInvocation } from '../agents/contract.ts'; +import { createCodexCommand, createPhasePolicy } from '../agents/policy.ts'; + +describe('production agent adapters', () => { + it('parses recorded Claude success and failure envelopes', () => { + expect(parseClaudeOutput(Buffer.from('{"result":"planned","is_error":false}'))) + .toEqual({ text: 'planned', providerFailed: false }); + expect(parseClaudeOutput(Buffer.from('{"result":"login required","is_error":true}'))) + .toEqual({ text: 'login required', providerFailed: true }); + expect(() => parseClaudeOutput(Buffer.from('{"result":3,"is_error":false}'))).toThrow('malformed'); + expect(() => parseClaudeOutput(Buffer.from('not json'))).toThrow(); + }); + + it('routes Codex final output to the bounded scratch directory', () => { + const invocation = captureInvocation({ + clone: { id: 'clone', taskId: 'task', directory: '/tmp/task', head: 'a'.repeat(40) }, + phase: 'planning', vendor: 'codex', approvedArgv: [], deadline: 2_000, attemptId: 'adapter-command', + context: { snapshotId: 's', planId: 'p', planRevision: 1, assignmentId: 'a', referencedCodeHash: 'c', stateVersion: 1 }, + }, 1_000); + const argv = createCodexCommand(createPhasePolicy(invocation), 'Plan this.').argv; + expect(argv.slice(argv.indexOf('--output-last-message'), argv.indexOf('--output-last-message') + 2)) + .toEqual(['--output-last-message', CODEX_OUTPUT_FILE]); + }); + + it('publishes immutable production output ceilings', () => { + expect(OUTPUT_LIMITS).toEqual({ stdoutBytes: 16 * 1024 * 1024, stderrBytes: 4 * 1024 * 1024, + combinedBytes: 20 * 1024 * 1024 }); + expect(Object.isFrozen(OUTPUT_LIMITS)).toBe(true); + }); +}); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts new file mode 100644 index 0000000..5e87f73 --- /dev/null +++ b/test/agent-supervisor.test.ts @@ -0,0 +1,175 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { chmodSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { startClaudeInvocation } from '../agents/adapters/claude.ts'; +import { readCodexOutput, startCodexInvocation } from '../agents/adapters/codex.ts'; +import { isInvocationActive, startProfileInvocation } from '../agents/adapters/supervisor.ts'; +import { captureInvocation, type InvocationInput } from '../agents/contract.ts'; +import { buildAgentImage } from '../agents/container/image.ts'; +import { createContainerProfile, disposeContainerProfile, type ContainerProfile } from '../agents/container/profile.ts'; +import { prepareTaskFilesystems, removeTaskFilesystems } from '../agents/container/run.ts'; +import { createVendorNetwork } from '../agents/network/network.ts'; +import { createIsolationProbeCommand, createPhasePolicy, type IsolationProbe } from '../agents/policy.ts'; +import { createTaskClone } from '../git/clone.ts'; + +const roots: string[] = [], profiles: ContainerProfile[] = []; +const allocations: ReturnType[] = []; +let imageId = ''; +const git = (cwd: string, ...args: string[]) => execFileSync('git', ['-c', 'core.hooksPath=/dev/null', ...args], + { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); + +function fixture() { + const root = mkdtempSync(join(tmpdir(), 'agent-supervisor-')); roots.push(root); + const source = join(root, 'source'), staging = join(root, 'staging'), input = join(root, 'input'); + mkdirSync(source); mkdirSync(staging); mkdirSync(input); + git(source, 'init'); git(source, 'config', 'user.name', 'Test'); git(source, 'config', 'user.email', 'test@example.com'); + writeFileSync(join(source, 'file.txt'), 'trusted\n'); git(source, 'add', '.'); git(source, 'commit', '-m', 'baseline'); + writeFileSync(join(input, 'schema.json'), '{}\n'); chmodSync(join(input, 'schema.json'), 0o444); chmodSync(input, 0o555); + const clone = createTaskClone({ source, parent: staging, taskId: 'supervisor', head: git(source, 'rev-parse', 'HEAD') }); + const filesystems = prepareTaskFilesystems(clone, { + workBytes: 16 * 1024 * 1024, workInodes: 512, metadataBytes: 16 * 1024 * 1024, metadataInodes: 512, + }, imageId); allocations.push(filesystems); + const auth = join(root, 'auth.json'); writeFileSync(auth, '{}', { mode: 0o600 }); + return { root, input, clone, filesystems, auth }; +} +function invocation(data: ReturnType, attemptId: string, deadlineMs = 2 * 60_000, + vendor: 'codex' | 'claude' = 'codex'): InvocationInput { + return captureInvocation({ clone: data.clone, phase: 'planning', vendor, approvedArgv: [], + deadline: Date.now() + deadlineMs, attemptId, + context: { snapshotId: 'snapshot', planId: 'plan', planRevision: 1, assignmentId: 'assignment', + referencedCodeHash: 'code', stateVersion: 1 } }); +} +function profile(data: ReturnType, probe: IsolationProbe, attemptId = `attempt-${Math.random()}`, + deadlineMs = 2 * 60_000, deferredOutput = false) { + const captured = invocation(data, attemptId, deadlineMs), policy = createPhasePolicy(captured); + const network = createVendorNetwork(captured, imageId); + const value = createContainerProfile({ invocation: captured, policy, network, filesystems: data.filesystems, + inputDirectory: data.input, command: createIsolationProbeCommand(policy, probe), imageId, codexAuthFile: data.auth, + deferredOutput }); + profiles.push(value); return value; +} + +beforeAll(() => { imageId = buildAgentImage(); }, 10 * 60_000); +afterAll(() => { + for (const profile of profiles) disposeContainerProfile(profile); + for (const allocation of allocations.reverse()) removeTaskFilesystems(allocation); + for (const root of roots.reverse()) { + chmodSync(join(root, 'input'), 0o700); + rmSync(root, { recursive: true, force: true, maxRetries: 3, retryDelay: 50 }); + } +}, 3 * 60_000); + +describe('container invocation supervisor', () => { + it('captures finite output and releases ownership only after cleanup', async () => { + const current = profile(fixture(), 'finite-output', 'finite'); + const handle = startProfileInvocation(current); + expect(isInvocationActive('finite')).toBe(true); + const result = await handle.settled; + expect(result).toMatchObject({ attemptId: 'finite', exitCode: 0, signal: null, + stdout: 'stdout-marker', stderr: 'stderr-marker' }); + expect(result.stopReason).toBeUndefined(); + expect(isInvocationActive('finite')).toBe(false); + expect(spawnSync('docker', ['container', 'inspect', current.name]).status).not.toBe(0); + expect(spawnSync('docker', ['network', 'inspect', current.network.name]).status).not.toBe(0); + }, 60_000); + + it.each([ + ['infinite-stdout', { stdoutBytes: 64 * 1024, stderrBytes: 32 * 1024, combinedBytes: 96 * 1024 }], + ['infinite-stderr', { stdoutBytes: 64 * 1024, stderrBytes: 32 * 1024, combinedBytes: 96 * 1024 }], + ['infinite-mixed', { stdoutBytes: 64 * 1024, stderrBytes: 64 * 1024, combinedBytes: 48 * 1024 }], + ] as const)('terminates %s at bounded output limits', async (probe, limits) => { + const attemptId = `limit-${probe}`, handle = startProfileInvocation(profile(fixture(), probe, attemptId), { limits }); + const result = await handle.settled; + expect(result.stopReason).toBe('output-limit'); + expect(Buffer.byteLength(result.stdout)).toBeLessThanOrEqual(limits.stdoutBytes); + expect(Buffer.byteLength(result.stderr)).toBeLessThanOrEqual(limits.stderrBytes); + expect(Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr)).toBeLessThanOrEqual(limits.combinedBytes); + expect(isInvocationActive(attemptId)).toBe(false); + }, 60_000); + + it('preserves the first cancellation reason until an ignored SIGTERM fully settles', async () => { + const current = profile(fixture(), 'ignore-term', 'cancelled'); + const handle = startProfileInvocation(current, { timeoutMs: 30_000 }); + let settled = false; void handle.settled.then(() => { settled = true; }); + handle.cancel('cancelled'); handle.cancel('shutdown'); + await Promise.resolve(); + expect(settled).toBe(false); + expect(isInvocationActive('cancelled')).toBe(true); + const result = await handle.settled; + expect(result.stopReason).toBe('cancelled'); + expect(result.stderr).toContain('[codeboost: cancelled]'); + expect(isInvocationActive('cancelled')).toBe(false); + }, 60_000); + + it('enforces a finite wall deadline and force-settles the container', async () => { + const started = Date.now(); + const handle = startProfileInvocation(profile(fixture(), 'ignore-term', 'timeout', 8_000), { timeoutMs: 30_000 }); + const result = await handle.settled; + expect(result.stopReason).toBe('timeout'); + expect(Date.now() - started).toBeLessThan(20_000); + expect(isInvocationActive('timeout')).toBe(false); + }, 60_000); + + it('blocks a duplicate attempt while the original container remains active', async () => { + const data = fixture(), first = startProfileInvocation(profile(data, 'ignore-term', 'duplicate'), { timeoutMs: 30_000 }); + expect(() => startProfileInvocation(profile(data, 'finite-output', 'duplicate'))).toThrow('still active'); + expect(isInvocationActive('duplicate')).toBe(true); + first.cancel('shutdown'); + expect((await first.settled).stopReason).toBe('shutdown'); + }, 60_000); + + it('records decoder failure without publishing a successful result', async () => { + const handle = startProfileInvocation(profile(fixture(), 'finite-output', 'capture-failure'), { + decode: () => { throw new Error('simulated capture failure'); }, + }); + const result = await handle.settled; + expect(result.stopReason).toBe('capture-failure'); + expect(result.stderr).toContain('[codeboost: capture-failure'); + expect(isInvocationActive('capture-failure')).toBe(false); + }, 60_000); + + it.each([ + ['symlink-output', 'capture-failure'], + ['oversized-output', 'output-limit'], + ] as const)('rejects unsafe Codex output from %s', async (probe, reason) => { + const handle = startProfileInvocation(profile(fixture(), probe, `file-${probe}`, 2 * 60_000, true), { + limits: { stdoutBytes: 64 * 1024, stderrBytes: 64 * 1024, combinedBytes: 128 * 1024 }, + decode: (current, _raw, maximum) => readCodexOutput(current.name, maximum), + }); + const result = await handle.settled; + expect(result.stopReason, result.stderr).toBe(reason); + }, 60_000); + + it('rejects limits above the production ceilings and cleans the unused profile', () => { + const current = profile(fixture(), 'finite-output', 'invalid-limit'); + expect(() => startProfileInvocation(current, { limits: { stdoutBytes: 16 * 1024 * 1024 + 1 } })) + .toThrow('production hard limits'); + expect(spawnSync('docker', ['network', 'inspect', current.network.name]).status).not.toBe(0); + }, 60_000); + + if (process.env.CODEBOOST_RUN_AUTH_PROBES === '1') { + it('runs the production Codex adapter and collects its bounded output file', async () => { + const data = fixture(), authFile = process.env.CODEBOOST_CODEX_AUTH_FILE; + if (!authFile) throw new Error('CODEBOOST_CODEX_AUTH_FILE is required.'); + const result = await startCodexInvocation({ invocation: invocation(data, 'live-codex', 6 * 60_000), + filesystems: data.filesystems, inputDirectory: data.input, imageId, + prompt: 'Reply only with this exact marker: codeboost-adapter-marker' }, authFile).settled; + expect(result.stopReason).toBeUndefined(); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('codeboost-adapter-marker'); + }, 8 * 60_000); + + it('runs the production Claude adapter and parses its bounded envelope', async () => { + const data = fixture(), token = process.env.CLAUDE_CODE_OAUTH_TOKEN; + if (!token) throw new Error('CLAUDE_CODE_OAUTH_TOKEN is required.'); + const result = await startClaudeInvocation({ invocation: invocation(data, 'live-claude', 6 * 60_000, 'claude'), + filesystems: data.filesystems, inputDirectory: data.input, imageId, + prompt: 'Reply only with this exact marker: codeboost-adapter-marker' }, token).settled; + expect(result.stopReason).toBeUndefined(); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('codeboost-adapter-marker'); + }, 8 * 60_000); + } +}); From 8e9f77a18efc73f83f2c5ee49530616c8ca2641a Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 16:53:51 -0700 Subject: [PATCH 02/32] Harden adapter capture settlement --- agents/adapters/codex.ts | 9 +- agents/adapters/supervisor.ts | 152 ++++++++++++++++++++-------------- agents/policy.ts | 4 +- test/agent-supervisor.test.ts | 19 ++++- 4 files changed, 117 insertions(+), 67 deletions(-) diff --git a/agents/adapters/codex.ts b/agents/adapters/codex.ts index deef695..70436b7 100644 --- a/agents/adapters/codex.ts +++ b/agents/adapters/codex.ts @@ -7,9 +7,10 @@ import type { AgentAdapterOptions, AgentAdapterRequest } from './types.ts'; export const CODEX_OUTPUT_FILE = '/tmp/codeboost-output/final.txt'; -export function readCodexOutput(container: string, maximumBytes: number) { - const output = readBoundedContainerFile(container, CODEX_OUTPUT_FILE, maximumBytes); - return Object.freeze({ text: output.toString('utf8'), additionalBytes: output.length }); +export async function readCodexOutput(container: string, maximumBytes: number, timeoutMs = 30_000) { + const output = await readBoundedContainerFile(container, CODEX_OUTPUT_FILE, maximumBytes, timeoutMs); + const text = new TextDecoder('utf-8', { fatal: true }).decode(output); + return Object.freeze({ text, additionalBytes: output.length }); } export function startCodexInvocation(request: AgentAdapterRequest, @@ -21,7 +22,7 @@ export function startCodexInvocation(request: AgentAdapterRequest, const profile = createContainerProfile({ ...request, policy, network, command: createCodexCommand(policy, request.prompt), codexAuthFile: authFile, deferredOutput: true }); return startProfileInvocation(profile, { ...options, - decode: (current, _raw, maximum) => readCodexOutput(current.name, maximum) }); + decode: (current, _raw, maximum, timeoutMs) => readCodexOutput(current.name, maximum, timeoutMs) }); } catch (error) { removeVendorNetwork(network); throw error; diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index d1d84c3..f78faf5 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -1,4 +1,4 @@ -import { execFileSync, spawn, type ChildProcess } from 'node:child_process'; +import { execFile, spawn, type ChildProcess } from 'node:child_process'; import type { InvocationHandle, InvocationResult, StopReason } from '../contract.ts'; import { assertPhasePolicy } from '../policy.ts'; import { createValidatedContainer, disposeValidatedContainer, validateContainer } from '../container/run.ts'; @@ -28,9 +28,11 @@ export interface SupervisorOptions { readonly secrets?: Readonly>; readonly timeoutMs?: number; readonly limits?: Partial; - readonly decode?: (profile: ContainerProfile, rawStdout: Buffer, maximumBytes: number) => DecodedOutput; + readonly decode?: (profile: ContainerProfile, rawStdout: Buffer, maximumBytes: number, + timeoutMs: number) => DecodedOutput | Promise; } export class OutputLimitError extends Error {} +export class CaptureDeadlineError extends Error {} const dockerEnvironment = () => ({ PATH: process.env.PATH, DOCKER_HOST: process.env.DOCKER_HOST }); const positiveInteger = (value: number, name: string) => { @@ -55,42 +57,42 @@ const withDiagnostic = (stderr: Buffer, stdoutBytes: number, reason: StopReason, if (maximum <= diagnostic.length) return diagnostic.subarray(0, maximum); return Buffer.concat([stderr.subarray(0, maximum - diagnostic.length), diagnostic]); }; -const runControl = (args: readonly string[], timeoutMs = 5_000): ChildProcess => { - const child = spawn('docker', [...args], { env: dockerEnvironment(), stdio: 'ignore' }); - const timer = setTimeout(() => child.kill('SIGKILL'), timeoutMs); - timer.unref(); - child.once('close', () => clearTimeout(timer)); - child.once('error', () => clearTimeout(timer)); - return child; -}; - /** Read a running container's tmpfs file with a pinned no-follow bounded reader. */ -export function readBoundedContainerFile(container: string, source: string, maximumBytes: number): Buffer { +export function readBoundedContainerFile(container: string, source: string, maximumBytes: number, + timeoutMs = 30_000): Promise { positiveInteger(maximumBytes, 'maximumBytes'); - if (!source.startsWith('/tmp/codeboost-output/') || source.includes('\0')) + positiveInteger(timeoutMs, 'timeoutMs'); + if (!/^\/tmp\/codeboost-output\/[A-Za-z0-9][A-Za-z0-9._-]*$/.test(source)) throw new Error('Adapter output must come from the bounded output directory.'); - try { - const reader = [ - "const fs=require('node:fs'),path=process.argv[1],maximum=Number(process.argv[2]);", - 'let fd;try{fd=fs.openSync(path,fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW);', - "const before=fs.fstatSync(fd);if(before.size>maximum)throw new Error('OUTPUT_LIMIT');", - "if(!before.isFile()||before.nlink!==1)throw new Error('UNSAFE_FILE');", - 'const output=Buffer.allocUnsafe(maximum+1);let length=0,count=0;', - 'do{count=fs.readSync(fd,output,length,output.length-length,null);length+=count}', - "while(count>0&&lengthmaximum)throw new Error('OUTPUT_LIMIT');", - 'const after=fs.fstatSync(fd);if(before.dev!==after.dev||before.ino!==after.ino||before.size!==after.size', - "||before.mtimeMs!==after.mtimeMs||before.ctimeMs!==after.ctimeMs)throw new Error('CHANGED_FILE');", - 'process.stdout.write(output.subarray(0,length))}finally{if(fd!==undefined)fs.closeSync(fd)}', - ].join(''); - return execFileSync('docker', ['exec', container, 'node', '-e', reader, source, String(maximumBytes)], { - env: dockerEnvironment(), timeout: 30_000, killSignal: 'SIGKILL', maxBuffer: maximumBytes + 1, - stdio: ['ignore', 'pipe', 'pipe'], + const reader = [ + "const fs=require('node:fs'),path=process.argv[1],maximum=Number(process.argv[2]);", + 'let fd;try{fd=fs.openSync(path,fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW|fs.constants.O_NONBLOCK);', + "const before=fs.fstatSync(fd);if(before.size>maximum)throw new Error('OUTPUT_LIMIT');", + "if(!before.isFile()||before.nlink!==1)throw new Error('UNSAFE_FILE');", + 'const output=Buffer.allocUnsafe(maximum+1);let length=0,count=0;', + 'do{count=fs.readSync(fd,output,length,output.length-length,null);length+=count}', + "while(count>0&&lengthmaximum)throw new Error('OUTPUT_LIMIT');", + 'const after=fs.fstatSync(fd);if(before.dev!==after.dev||before.ino!==after.ino||before.size!==after.size', + '||before.mtimeMs!==after.mtimeMs||before.ctimeMs!==after.ctimeMs||after.nlink!==1', + "||!after.isFile())throw new Error('CHANGED_FILE');", + "process.stdout.write(output.subarray(0,length))}catch(error){process.exitCode=error.message==='OUTPUT_LIMIT'?42:43}", + 'finally{if(fd!==undefined)fs.closeSync(fd)}', + ].join(''); + return new Promise((resolve, reject) => { + execFile('docker', ['exec', container, 'node', '-e', reader, source, String(maximumBytes)], { + env: dockerEnvironment(), timeout: timeoutMs, killSignal: 'SIGKILL', maxBuffer: maximumBytes + 1, + encoding: 'buffer', + }, (error, stdout, stderr) => { + if (!error) { resolve(stdout); return; } + if (error.code === 42) { + reject(new OutputLimitError('Adapter output exceeds its capture limit.')); return; + } + if ('killed' in error && error.killed) { + reject(new CaptureDeadlineError('Adapter output capture exceeded the invocation deadline.')); return; + } + reject(new Error('Adapter output is not a stable bounded unlinked regular file.')); }); - } catch (error) { - const diagnostic = error && typeof error === 'object' && 'stderr' in error ? String(error.stderr) : String(error); - if (diagnostic.includes('OUTPUT_LIMIT')) throw new OutputLimitError('Adapter output exceeds its capture limit.'); - throw new Error('Adapter output is not a stable bounded unlinked regular file.'); - } + }); } export function isInvocationActive(attemptId: string): boolean { @@ -133,12 +135,27 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super const stdoutChunks: Buffer[] = [], stderrChunks: Buffer[] = []; let stdoutBytes = 0, stderrBytes = 0, combinedBytes = 0; let stopReason: StopReason | undefined, failureDetail: string | undefined, closed = false, terminating = false; - let decodedOutput: DecodedOutput | undefined, protocolToken: string | undefined; + let decodedOutput: DecodedOutput | undefined, decodePromise: Promise | undefined; + let protocolToken: string | undefined; let protocolBuffer = Buffer.alloc(0); const child = spawn('docker', ['start', '--attach', profile.name], { env: dockerEnvironment(), stdio: ['ignore', 'pipe', 'pipe'], }); const timers = new Set>(); + const controls = new Set>(); + const runControl = (args: readonly string[], timeoutMs = 5_000) => { + const operation = new Promise(resolve => { + const control = spawn('docker', [...args], { env: dockerEnvironment(), stdio: 'ignore' }); + const timer = setTimeout(() => control.kill('SIGKILL'), timeoutMs); + timer.unref(); + const done = () => { clearTimeout(timer); resolve(); }; + control.once('close', done); + control.once('error', done); + }); + controls.add(operation); + void operation.finally(() => controls.delete(operation)); + return operation; + }; const later = (callback: () => void, delay: number) => { const timer = setTimeout(() => { timers.delete(timer); callback(); }, delay); timer.unref(); timers.add(timer); return timer; @@ -147,11 +164,11 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super if (terminating || closed) return; terminating = true; child.stdout?.resume(); child.stderr?.resume(); - runControl(['stop', '--signal=TERM', '--time=1', profile.name]); - later(() => { if (!closed) runControl(['kill', '--signal=KILL', profile.name]); }, 1_500); + void runControl(['stop', '--signal=TERM', '--time=1', profile.name]); + later(() => { if (!closed) void runControl(['kill', '--signal=KILL', profile.name]); }, 1_500); later(() => { if (!closed) { - runControl(['rm', '--force', profile.name]); + void runControl(['rm', '--force', profile.name]); child.kill('SIGKILL'); } }, 4_000); @@ -177,24 +194,35 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super if (chunk.length > available) stop('output-limit'); }; const decodeOutput = () => { - if (!options.decode || decodedOutput || stopReason) return; - try { - const raw = Buffer.concat(stdoutChunks, stdoutBytes); - const decoded = options.decode(profile, raw, - Math.max(1, Math.min(limits.stdoutBytes - stdoutBytes, limits.combinedBytes - combinedBytes))); - const additional = decoded.additionalBytes ?? 0; - if (!Number.isSafeInteger(additional) || additional < 0) throw new Error('Adapter returned an invalid byte count.'); - if (stdoutBytes + additional > limits.stdoutBytes || combinedBytes + additional > limits.combinedBytes) - throw new OutputLimitError('Adapter output exceeds its capture limit.'); - decodedOutput = decoded; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - const reason = error instanceof OutputLimitError || /exceeds its capture limit/i.test(message) - ? 'output-limit' : 'capture-failure'; - failureDetail ??= message; - if (closed) stopReason ??= reason; - else stop(reason); - } + if (decodePromise) return decodePromise; + if (!options.decode || decodedOutput || stopReason) return Promise.resolve(); + decodePromise = (async () => { + try { + const budget = deadline - Date.now(); + if (budget < 1) throw new CaptureDeadlineError('Invocation deadline expired before output capture.'); + const raw = Buffer.concat(stdoutChunks, stdoutBytes); + const decoded = await options.decode!(profile, raw, + Math.max(1, Math.min(limits.stdoutBytes - stdoutBytes, limits.combinedBytes - combinedBytes)), budget); + const additional = decoded.additionalBytes ?? 0; + const textBytes = Buffer.byteLength(decoded.text); + if (!Number.isSafeInteger(additional) || additional < 0) + throw new Error('Adapter returned an invalid byte count.'); + if ((additional > 0 && additional < textBytes) || textBytes > limits.stdoutBytes + || textBytes + stderrBytes > limits.combinedBytes) + throw new OutputLimitError('Decoded adapter output exceeds its capture limit.'); + if (stdoutBytes + additional > limits.stdoutBytes || combinedBytes + additional > limits.combinedBytes) + throw new OutputLimitError('Adapter output exceeds its capture limit.'); + decodedOutput = decoded; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const reason = error instanceof OutputLimitError || /exceeds its capture limit/i.test(message) + ? 'output-limit' : error instanceof CaptureDeadlineError ? 'timeout' : 'capture-failure'; + failureDetail ??= message; + if (closed) stopReason ??= reason; + else stop(reason); + } + })(); + return decodePromise; }; const protocolLine = (line: Buffer) => { const text = line.toString('utf8').trim(); @@ -206,8 +234,10 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super } const ready = /^\x1eCODEBOOST_READY:([0-9a-f-]{36}):([0-9]+)\x1e$/.exec(text); if (!ready || ready[1] !== protocolToken) return false; - decodeOutput(); - if (decodedOutput) runControl(['exec', profile.name, 'touch', `/tmp/codeboost-output/collected-${ready[1]}`]); + void decodeOutput().then(() => { + if (decodedOutput && !stopReason) + void runControl(['exec', profile.name, 'touch', `/tmp/codeboost-output/collected-${ready[1]}`]); + }); return true; }; const captureStderr = (value: Buffer | string) => { @@ -242,7 +272,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super }); active.set(invocation.attemptId, handle); - child.once('close', (code, signal) => { + child.once('close', async (code, signal) => { for (const timer of timers) clearTimeout(timer); timers.clear(); if (protocolBuffer.length) { @@ -252,7 +282,8 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super closed = true; let finalStdout = Buffer.concat(stdoutChunks, stdoutBytes), finalStderr = Buffer.concat(stderrChunks, stderrBytes); let exitCode = code, finalSignal = signal; - if (!stopReason && code === 0 && options.decode && !profile.deferredOutput) decodeOutput(); + if (!stopReason && code === 0 && options.decode && !profile.deferredOutput) await decodeOutput(); + if (decodePromise) await decodePromise; if (!stopReason && code === 0 && profile.deferredOutput && !decodedOutput) { stopReason = 'capture-failure'; failureDetail ??= 'Deferred output protocol did not complete.'; } @@ -260,6 +291,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super finalStdout = Buffer.from(decodedOutput.text); if (decodedOutput.providerFailed) exitCode = exitCode === 0 ? 1 : exitCode; } + await Promise.all([...controls]); try { disposeValidatedContainer(profile); } catch { diff --git a/agents/policy.ts b/agents/policy.ts index 59d585d..02a9758 100644 --- a/agents/policy.ts +++ b/agents/policy.ts @@ -91,7 +91,7 @@ export function createCodexCommand(policy: PhasePolicy, prompt: string): AgentCo export type IsolationProbe = 'noop' | 'phase-worktree' | 'read-only-isolation' | 'persist-write' | 'persist-read' | 'capacity' | 'metadata' | 'must-not-run' | 'input-marker' | 'finite-output' | 'infinite-stdout' | 'infinite-stderr' | 'infinite-mixed' | 'ignore-term' | 'symlink-output' - | 'oversized-output'; + | 'oversized-output' | 'fifo-output' | 'invalid-utf8-output'; /** Fixed startup probes validate the sandbox itself without granting an agent a process tool. */ export function createIsolationProbeCommand(policy: PhasePolicy, probe: IsolationProbe): AgentCommand { @@ -121,6 +121,8 @@ export function createIsolationProbeCommand(policy: PhasePolicy, probe: Isolatio 'ignore-term': "trap '' TERM; while :; do sleep 1; done", 'symlink-output': 'ln -s /etc/passwd /tmp/codeboost-output/final.txt', 'oversized-output': 'head -c 131072 /dev/zero > /tmp/codeboost-output/final.txt', + 'fifo-output': 'mkfifo /tmp/codeboost-output/final.txt', + 'invalid-utf8-output': "printf '\\377' > /tmp/codeboost-output/final.txt", }; return command(policy, probe === 'noop' ? ['true'] : ['sh', '-c', scripts[probe]]); } diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 5e87f73..0e190fb 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -5,7 +5,7 @@ import { join } from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { startClaudeInvocation } from '../agents/adapters/claude.ts'; import { readCodexOutput, startCodexInvocation } from '../agents/adapters/codex.ts'; -import { isInvocationActive, startProfileInvocation } from '../agents/adapters/supervisor.ts'; +import { isInvocationActive, readBoundedContainerFile, startProfileInvocation } from '../agents/adapters/supervisor.ts'; import { captureInvocation, type InvocationInput } from '../agents/contract.ts'; import { buildAgentImage } from '../agents/container/image.ts'; import { createContainerProfile, disposeContainerProfile, type ContainerProfile } from '../agents/container/profile.ts'; @@ -130,13 +130,28 @@ describe('container invocation supervisor', () => { expect(isInvocationActive('capture-failure')).toBe(false); }, 60_000); + it('bounds decoded text independently of adapter byte accounting', async () => { + const handle = startProfileInvocation(profile(fixture(), 'finite-output', 'decoded-limit'), { + limits: { stdoutBytes: 64 * 1024, stderrBytes: 64 * 1024, combinedBytes: 128 * 1024 }, + decode: () => ({ text: 'x'.repeat(64 * 1024 + 1), additionalBytes: 0 }), + }); + expect((await handle.settled).stopReason).toBe('output-limit'); + }, 60_000); + + it('rejects traversal before starting an output read', () => { + expect(() => readBoundedContainerFile('unused', + '/tmp/codeboost-output/../../run/codeboost-auth/codex/auth.json', 1024)).toThrow('bounded output directory'); + }); + it.each([ ['symlink-output', 'capture-failure'], ['oversized-output', 'output-limit'], + ['fifo-output', 'capture-failure'], + ['invalid-utf8-output', 'capture-failure'], ] as const)('rejects unsafe Codex output from %s', async (probe, reason) => { const handle = startProfileInvocation(profile(fixture(), probe, `file-${probe}`, 2 * 60_000, true), { limits: { stdoutBytes: 64 * 1024, stderrBytes: 64 * 1024, combinedBytes: 128 * 1024 }, - decode: (current, _raw, maximum) => readCodexOutput(current.name, maximum), + decode: (current, _raw, maximum, timeoutMs) => readCodexOutput(current.name, maximum, timeoutMs), }); const result = await handle.settled; expect(result.stopReason, result.stderr).toBe(reason); From 44d1304d68627c8824c9b82c98862bbd20a580f6 Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 17:10:13 -0700 Subject: [PATCH 03/32] Pin adapter output to dedicated tmpfs --- agents/adapters/claude.ts | 3 ++- agents/adapters/codex.ts | 2 +- agents/adapters/supervisor.ts | 27 +++++++++++++++++++-------- agents/container/probe.sh | 3 +-- agents/container/profile.ts | 1 + agents/container/run.ts | 4 +++- agents/policy.ts | 14 ++++++++------ test/agent-adapter.test.ts | 1 + test/agent-supervisor.test.ts | 4 +++- 9 files changed, 39 insertions(+), 20 deletions(-) diff --git a/agents/adapters/claude.ts b/agents/adapters/claude.ts index 41c7cc3..485f2f6 100644 --- a/agents/adapters/claude.ts +++ b/agents/adapters/claude.ts @@ -6,7 +6,8 @@ import { startProfileInvocation } from './supervisor.ts'; import type { AgentAdapterOptions, AgentAdapterRequest } from './types.ts'; export function parseClaudeOutput(raw: Buffer): { text: string; providerFailed: boolean } { - const envelope = JSON.parse(raw.toString('utf8')) as { result?: unknown; is_error?: unknown }; + const envelope = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(raw)) as + { result?: unknown; is_error?: unknown }; if (typeof envelope.result !== 'string' || typeof envelope.is_error !== 'boolean') throw new Error('Claude returned a malformed output envelope.'); return Object.freeze({ text: envelope.result, providerFailed: envelope.is_error }); diff --git a/agents/adapters/codex.ts b/agents/adapters/codex.ts index 70436b7..1e23d96 100644 --- a/agents/adapters/codex.ts +++ b/agents/adapters/codex.ts @@ -5,7 +5,7 @@ import { createCodexCommand, createPhasePolicy } from '../policy.ts'; import { readBoundedContainerFile, startProfileInvocation } from './supervisor.ts'; import type { AgentAdapterOptions, AgentAdapterRequest } from './types.ts'; -export const CODEX_OUTPUT_FILE = '/tmp/codeboost-output/final.txt'; +export const CODEX_OUTPUT_FILE = '/run/codeboost-output/final.txt'; export async function readCodexOutput(container: string, maximumBytes: number, timeoutMs = 30_000) { const output = await readBoundedContainerFile(container, CODEX_OUTPUT_FILE, maximumBytes, timeoutMs); diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index f78faf5..db5b1be 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -62,7 +62,7 @@ export function readBoundedContainerFile(container: string, source: string, maxi timeoutMs = 30_000): Promise { positiveInteger(maximumBytes, 'maximumBytes'); positiveInteger(timeoutMs, 'timeoutMs'); - if (!/^\/tmp\/codeboost-output\/[A-Za-z0-9][A-Za-z0-9._-]*$/.test(source)) + if (!/^\/run\/codeboost-output\/[A-Za-z0-9][A-Za-z0-9._-]*$/.test(source)) throw new Error('Adapter output must come from the bounded output directory.'); const reader = [ "const fs=require('node:fs'),path=process.argv[1],maximum=Number(process.argv[2]);", @@ -142,15 +142,19 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super env: dockerEnvironment(), stdio: ['ignore', 'pipe', 'pipe'], }); const timers = new Set>(); - const controls = new Set>(); + const controls = new Set>(); const runControl = (args: readonly string[], timeoutMs = 5_000) => { - const operation = new Promise(resolve => { + const operation = new Promise(resolve => { const control = spawn('docker', [...args], { env: dockerEnvironment(), stdio: 'ignore' }); const timer = setTimeout(() => control.kill('SIGKILL'), timeoutMs); timer.unref(); - const done = () => { clearTimeout(timer); resolve(); }; - control.once('close', done); - control.once('error', done); + let completed = false; + const done = (success: boolean) => { + if (completed) return; + completed = true; clearTimeout(timer); resolve(success); + }; + control.once('close', code => done(code === 0)); + control.once('error', () => done(false)); }); controls.add(operation); void operation.finally(() => controls.delete(operation)); @@ -235,8 +239,15 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super const ready = /^\x1eCODEBOOST_READY:([0-9a-f-]{36}):([0-9]+)\x1e$/.exec(text); if (!ready || ready[1] !== protocolToken) return false; void decodeOutput().then(() => { - if (decodedOutput && !stopReason) - void runControl(['exec', profile.name, 'touch', `/tmp/codeboost-output/collected-${ready[1]}`]); + if (decodedOutput && !stopReason) { + void runControl(['exec', profile.name, 'touch', `/run/codeboost-output/collected-${ready[1]}`]) + .then(success => { + if (!success) { + failureDetail ??= 'Deferred output acknowledgement failed.'; + stop('capture-failure'); + } + }); + } }); return true; }; diff --git a/agents/container/probe.sh b/agents/container/probe.sh index 40f2af1..e579903 100644 --- a/agents/container/probe.sh +++ b/agents/container/probe.sh @@ -81,7 +81,6 @@ esac [ "$(codex --version)" = 'codex-cli 0.153.4' ] || fail 'unexpected Codex version' [ "$(claude --version | awk '{print $1}')" = '2.1.281' ] || fail 'unexpected Claude version' -install --directory --owner=10001 --group=10001 --mode=0700 /tmp/codeboost-output if [ "${CODEBOOST_DEFERRED_OUTPUT:-}" = '1' ]; then token="$(cat /proc/sys/kernel/random/uuid)" printf '\036CODEBOOST_START:%s\036\n' "$token" >&2 @@ -90,7 +89,7 @@ if [ "${CODEBOOST_DEFERRED_OUTPUT:-}" = '1' ]; then status="$?" set -e printf '\036CODEBOOST_READY:%s:%s\036\n' "$token" "$status" >&2 - acknowledgement="/tmp/codeboost-output/collected-$token" + acknowledgement="/run/codeboost-output/collected-$token" while [ ! -e "$acknowledgement" ]; do sleep 0.05; done exit "$status" fi diff --git a/agents/container/profile.ts b/agents/container/profile.ts index b9bd742..ed9ee4b 100644 --- a/agents/container/profile.ts +++ b/agents/container/profile.ts @@ -237,6 +237,7 @@ export function createContainerProfile(options: ProfileOptions): ContainerProfil } if (invocation.vendor === 'codex') { args.push('--env', 'CODEX_HOME=/run/codeboost-auth/codex', + '--tmpfs', '/run/codeboost-output:rw,nosuid,nodev,noexec,size=20971520,nr_inodes=64,uid=10001,gid=10001,mode=0700', '--tmpfs', '/run/codeboost-auth/codex:rw,nosuid,nodev,size=4194304,nr_inodes=256,uid=10001,gid=10001,mode=0700', '--mount', mount({ type: 'bind', source: codexAuthFile!, target: '/run/codeboost-auth/codex/auth.json', readonly: true })); } else args.push('--env', 'CLAUDE_CODE_OAUTH_TOKEN'); diff --git a/agents/container/run.ts b/agents/container/run.ts index 38cebdd..ac36bfd 100644 --- a/agents/container/run.ts +++ b/agents/container/run.ts @@ -171,7 +171,9 @@ export function validateContainer(container: string, profile: ContainerProfile, ['/tmp', ['rw', 'nosuid', 'nodev', 'size=33554432', 'nr_inodes=4096', 'mode=1777']], ['/home/codeboost', ['rw', 'nosuid', 'nodev', 'size=1048576', 'nr_inodes=128', 'uid=10001', 'gid=10001', 'mode=0700']], ...(profile.vendor === 'codex' ? [['/run/codeboost-auth/codex', - ['rw', 'nosuid', 'nodev', 'size=4194304', 'nr_inodes=256', 'uid=10001', 'gid=10001', 'mode=0700']] as const] : []), + ['rw', 'nosuid', 'nodev', 'size=4194304', 'nr_inodes=256', 'uid=10001', 'gid=10001', 'mode=0700']] as const, + ['/run/codeboost-output', + ['rw', 'nosuid', 'nodev', 'noexec', 'size=20971520', 'nr_inodes=64', 'uid=10001', 'gid=10001', 'mode=0700']] as const] : []), ]); if (Object.keys(tmpfs).length !== expectedTmpfs.size) throw new Error('Container tmpfs mount set changed.'); for (const [path, expected] of expectedTmpfs) { diff --git a/agents/policy.ts b/agents/policy.ts index 02a9758..81bc222 100644 --- a/agents/policy.ts +++ b/agents/policy.ts @@ -85,13 +85,13 @@ export function createCodexCommand(policy: PhasePolicy, prompt: string): AgentCo const sandbox = policy.worktree === 'read-write' ? 'workspace-write' : 'read-only'; // `--` ends option parsing, so a prompt beginning with `-` stays prompt data. return command(policy, [...codexBaseArguments(policy), 'exec', '--sandbox', sandbox, '--skip-git-repo-check', - '--output-last-message', '/tmp/codeboost-output/final.txt', '--', prompt]); + '--output-last-message', '/run/codeboost-output/final.txt', '--', prompt]); } export type IsolationProbe = 'noop' | 'phase-worktree' | 'read-only-isolation' | 'persist-write' | 'persist-read' | 'capacity' | 'metadata' | 'must-not-run' | 'input-marker' | 'finite-output' | 'infinite-stdout' | 'infinite-stderr' | 'infinite-mixed' | 'ignore-term' | 'symlink-output' - | 'oversized-output' | 'fifo-output' | 'invalid-utf8-output'; + | 'oversized-output' | 'fifo-output' | 'invalid-utf8-output' | 'replace-output-directory' | 'ack-failure'; /** Fixed startup probes validate the sandbox itself without granting an agent a process tool. */ export function createIsolationProbeCommand(policy: PhasePolicy, probe: IsolationProbe): AgentCommand { @@ -119,10 +119,12 @@ export function createIsolationProbeCommand(policy: PhasePolicy, probe: Isolatio 'infinite-stderr': "while :; do head -c 4096 /dev/zero | tr '\\0' x >&2; done", 'infinite-mixed': "while :; do head -c 4096 /dev/zero | tr '\\0' x; head -c 4096 /dev/zero | tr '\\0' y >&2; done", 'ignore-term': "trap '' TERM; while :; do sleep 1; done", - 'symlink-output': 'ln -s /etc/passwd /tmp/codeboost-output/final.txt', - 'oversized-output': 'head -c 131072 /dev/zero > /tmp/codeboost-output/final.txt', - 'fifo-output': 'mkfifo /tmp/codeboost-output/final.txt', - 'invalid-utf8-output': "printf '\\377' > /tmp/codeboost-output/final.txt", + 'symlink-output': 'ln -s /etc/passwd /run/codeboost-output/final.txt', + 'oversized-output': 'head -c 131072 /dev/zero > /run/codeboost-output/final.txt', + 'fifo-output': 'mkfifo /run/codeboost-output/final.txt', + 'invalid-utf8-output': "printf '\\377' > /run/codeboost-output/final.txt", + 'replace-output-directory': 'rm -rf /run/codeboost-output; ln -s /etc /run/codeboost-output', + 'ack-failure': "printf captured > /run/codeboost-output/final.txt; chmod 0500 /run/codeboost-output", }; return command(policy, probe === 'noop' ? ['true'] : ['sh', '-c', scripts[probe]]); } diff --git a/test/agent-adapter.test.ts b/test/agent-adapter.test.ts index eddc772..c79af29 100644 --- a/test/agent-adapter.test.ts +++ b/test/agent-adapter.test.ts @@ -13,6 +13,7 @@ describe('production agent adapters', () => { .toEqual({ text: 'login required', providerFailed: true }); expect(() => parseClaudeOutput(Buffer.from('{"result":3,"is_error":false}'))).toThrow('malformed'); expect(() => parseClaudeOutput(Buffer.from('not json'))).toThrow(); + expect(() => parseClaudeOutput(Buffer.from([0xff]))).toThrow(); }); it('routes Codex final output to the bounded scratch directory', () => { diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 0e190fb..5b594b6 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -140,7 +140,7 @@ describe('container invocation supervisor', () => { it('rejects traversal before starting an output read', () => { expect(() => readBoundedContainerFile('unused', - '/tmp/codeboost-output/../../run/codeboost-auth/codex/auth.json', 1024)).toThrow('bounded output directory'); + '/run/codeboost-output/../../run/codeboost-auth/codex/auth.json', 1024)).toThrow('bounded output directory'); }); it.each([ @@ -148,6 +148,8 @@ describe('container invocation supervisor', () => { ['oversized-output', 'output-limit'], ['fifo-output', 'capture-failure'], ['invalid-utf8-output', 'capture-failure'], + ['replace-output-directory', 'capture-failure'], + ['ack-failure', 'capture-failure'], ] as const)('rejects unsafe Codex output from %s', async (probe, reason) => { const handle = startProfileInvocation(profile(fixture(), probe, `file-${probe}`, 2 * 60_000, true), { limits: { stdoutBytes: 64 * 1024, stderrBytes: 64 * 1024, combinedBytes: 128 * 1024 }, From cb6f3fb203390bc1f32aac2eadf82f0b233da893 Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 17:27:49 -0700 Subject: [PATCH 04/32] Close remaining adapter lifecycle races --- agents/adapters/supervisor.ts | 59 +++++++++++++++++++++++++++++------ agents/policy.ts | 4 ++- test/agent-supervisor.test.ts | 25 +++++++++++++++ 3 files changed, 77 insertions(+), 11 deletions(-) diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index db5b1be..7f2bc1c 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -50,6 +50,32 @@ const captureLimits = (override: Partial | undefined): CaptureLim }; const diagnosticFor = (reason: StopReason, detail?: string) => Buffer.from( `[codeboost: ${reason}${detail ? `: ${detail.replace(/[\r\n]+/g, ' ').slice(0, 512)}` : ''}]\n`); +const retainCleanupOwnership = (profile: ContainerProfile, detail: string): InvocationHandle => { + const invocation = assertPhasePolicy(profile.policy); + let resolveSettled!: (result: InvocationResult) => void, cleaning = false; + const settled = new Promise(resolve => { resolveSettled = resolve; }); + const retry = () => { + if (cleaning) return; + cleaning = true; + try { + disposeValidatedContainer(profile); + active.delete(invocation.attemptId); + resolveSettled(Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, + exitCode: null, signal: null, stopReason: 'capture-failure', stdout: '', + stderr: diagnosticFor('capture-failure', detail).toString('utf8') })); + } catch { + const timer = setTimeout(() => { cleaning = false; retry(); }, 1_000); + timer.unref(); + return; + } + cleaning = false; + }; + const handle: InvocationHandle = Object.freeze({ attemptId: invocation.attemptId, settled, + cancel: retry }); + active.set(invocation.attemptId, handle); + const timer = setTimeout(retry, 1_000); timer.unref(); + return handle; +}; const withDiagnostic = (stderr: Buffer, stdoutBytes: number, reason: StopReason, limits: CaptureLimits, detail?: string) => { const diagnostic = diagnosticFor(reason, detail).subarray(0, DIAGNOSTIC_BYTES); @@ -62,21 +88,27 @@ export function readBoundedContainerFile(container: string, source: string, maxi timeoutMs = 30_000): Promise { positiveInteger(maximumBytes, 'maximumBytes'); positiveInteger(timeoutMs, 'timeoutMs'); + if (maximumBytes > OUTPUT_LIMITS.stdoutBytes) + throw new Error('Adapter output read cannot exceed the production stdout limit.'); if (!/^\/run\/codeboost-output\/[A-Za-z0-9][A-Za-z0-9._-]*$/.test(source)) throw new Error('Adapter output must come from the bounded output directory.'); const reader = [ - "const fs=require('node:fs'),path=process.argv[1],maximum=Number(process.argv[2]);", - 'let fd;try{fd=fs.openSync(path,fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW|fs.constants.O_NONBLOCK);', - "const before=fs.fstatSync(fd);if(before.size>maximum)throw new Error('OUTPUT_LIMIT');", + "const fs=require('node:fs'),path=process.argv[1],maximum=Number(process.argv[2]),directory='/run/codeboost-output';", + 'let dirfd,fd;try{dirfd=fs.openSync(directory,fs.constants.O_RDONLY|fs.constants.O_DIRECTORY|fs.constants.O_NOFOLLOW);', + "fd=fs.openSync('/proc/self/fd/'+dirfd+'/'+path.slice(directory.length+1),", + 'fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW|fs.constants.O_NONBLOCK);', + "const before=fs.fstatSync(fd,{bigint:true});if(before.size>BigInt(maximum))throw new Error('OUTPUT_LIMIT');", "if(!before.isFile()||before.nlink!==1)throw new Error('UNSAFE_FILE');", 'const output=Buffer.allocUnsafe(maximum+1);let length=0,count=0;', 'do{count=fs.readSync(fd,output,length,output.length-length,null);length+=count}', "while(count>0&&lengthmaximum)throw new Error('OUTPUT_LIMIT');", - 'const after=fs.fstatSync(fd);if(before.dev!==after.dev||before.ino!==after.ino||before.size!==after.size', + 'const after=fs.fstatSync(fd,{bigint:true});if(before.dev!==after.dev||before.ino!==after.ino||before.size!==after.size', '||before.mtimeMs!==after.mtimeMs||before.ctimeMs!==after.ctimeMs||after.nlink!==1', "||!after.isFile())throw new Error('CHANGED_FILE');", + "const named=fs.statSync('/proc/self/fd/'+dirfd+'/'+path.slice(directory.length+1),{bigint:true,throwIfNoEntry:false});", + "if(!named||named.dev!==after.dev||named.ino!==after.ino||named.nlink!==1n)throw new Error('REPLACED_FILE');", "process.stdout.write(output.subarray(0,length))}catch(error){process.exitCode=error.message==='OUTPUT_LIMIT'?42:43}", - 'finally{if(fd!==undefined)fs.closeSync(fd)}', + 'finally{if(fd!==undefined)fs.closeSync(fd);if(dirfd!==undefined)fs.closeSync(dirfd)}', ].join(''); return new Promise((resolve, reject) => { execFile('docker', ['exec', container, 'node', '-e', reader, source, String(maximumBytes)], { @@ -128,13 +160,18 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super createValidatedContainer(profile, remaining(), options.secrets ?? {}); validateContainer(profile.name, profile, remaining()); } catch (error) { - try { disposeValidatedContainer(profile); } catch { /* createValidatedContainer already reports unsettled cleanup */ } + try { disposeValidatedContainer(profile); } + catch (cleanupError) { + const detail = `Container validation failed and cleanup remains unsettled: ${String(error)}; ${String(cleanupError)}`; + return retainCleanupOwnership(profile, detail); + } throw error; } const stdoutChunks: Buffer[] = [], stderrChunks: Buffer[] = []; let stdoutBytes = 0, stderrBytes = 0, combinedBytes = 0; - let stopReason: StopReason | undefined, failureDetail: string | undefined, closed = false, terminating = false; + let stopReason: StopReason | undefined, failureDetail: string | undefined; + let closed = false, terminating = false, settlementComplete = false; let decodedOutput: DecodedOutput | undefined, decodePromise: Promise | undefined; let protocolToken: string | undefined; let protocolBuffer = Buffer.alloc(0); @@ -178,9 +215,9 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super }, 4_000); }; const stop = (reason: StopReason) => { - if (closed || stopReason) return; + if (settlementComplete || stopReason) return; stopReason = reason; - terminate(); + if (!closed) terminate(); }; const capture = (stream: 'stdout' | 'stderr', value: Buffer | string) => { if (stopReason || closed) return; @@ -207,6 +244,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super const raw = Buffer.concat(stdoutChunks, stdoutBytes); const decoded = await options.decode!(profile, raw, Math.max(1, Math.min(limits.stdoutBytes - stdoutBytes, limits.combinedBytes - combinedBytes)), budget); + if (stopReason) return; const additional = decoded.additionalBytes ?? 0; const textBytes = Buffer.byteLength(decoded.text); if (!Number.isSafeInteger(additional) || additional < 0) @@ -293,7 +331,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super closed = true; let finalStdout = Buffer.concat(stdoutChunks, stdoutBytes), finalStderr = Buffer.concat(stderrChunks, stderrBytes); let exitCode = code, finalSignal = signal; - if (!stopReason && code === 0 && options.decode && !profile.deferredOutput) await decodeOutput(); + if (!stopReason && options.decode && !profile.deferredOutput) await decodeOutput(); if (decodePromise) await decodePromise; if (!stopReason && code === 0 && profile.deferredOutput && !decodedOutput) { stopReason = 'capture-failure'; failureDetail ??= 'Deferred output protocol did not complete.'; @@ -313,6 +351,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super const result = Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, exitCode, signal: finalSignal, ...(stopReason ? { stopReason } : {}), stdout: finalStdout.toString('utf8'), stderr: finalStderr.toString('utf8') }); + settlementComplete = true; active.delete(invocation.attemptId); resolveSettled(result); }); diff --git a/agents/policy.ts b/agents/policy.ts index 81bc222..89e3dd6 100644 --- a/agents/policy.ts +++ b/agents/policy.ts @@ -91,7 +91,8 @@ export function createCodexCommand(policy: PhasePolicy, prompt: string): AgentCo export type IsolationProbe = 'noop' | 'phase-worktree' | 'read-only-isolation' | 'persist-write' | 'persist-read' | 'capacity' | 'metadata' | 'must-not-run' | 'input-marker' | 'finite-output' | 'infinite-stdout' | 'infinite-stderr' | 'infinite-mixed' | 'ignore-term' | 'symlink-output' - | 'oversized-output' | 'fifo-output' | 'invalid-utf8-output' | 'replace-output-directory' | 'ack-failure'; + | 'oversized-output' | 'fifo-output' | 'invalid-utf8-output' | 'replace-output-directory' | 'ack-failure' + | 'nonzero-output'; /** Fixed startup probes validate the sandbox itself without granting an agent a process tool. */ export function createIsolationProbeCommand(policy: PhasePolicy, probe: IsolationProbe): AgentCommand { @@ -125,6 +126,7 @@ export function createIsolationProbeCommand(policy: PhasePolicy, probe: Isolatio 'invalid-utf8-output': "printf '\\377' > /run/codeboost-output/final.txt", 'replace-output-directory': 'rm -rf /run/codeboost-output; ln -s /etc /run/codeboost-output', 'ack-failure': "printf captured > /run/codeboost-output/final.txt; chmod 0500 /run/codeboost-output", + 'nonzero-output': 'printf encoded-output; exit 7', }; return command(policy, probe === 'noop' ? ['true'] : ['sh', '-c', scripts[probe]]); } diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 5b594b6..0eda4e5 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -130,6 +130,29 @@ describe('container invocation supervisor', () => { expect(isInvocationActive('capture-failure')).toBe(false); }, 60_000); + it('preserves cancellation while post-close decoding is still unsettled', async () => { + let begin!: () => void, release!: () => void; + const started = new Promise(resolve => { begin = resolve; }); + const gate = new Promise(resolve => { release = resolve; }); + const handle = startProfileInvocation(profile(fixture(), 'finite-output', 'cancel-during-decode'), { + decode: async () => { begin(); await gate; return { text: 'must-not-publish' }; }, + }); + await started; + handle.cancel('cancelled'); + release(); + const result = await handle.settled; + expect(result.stopReason).toBe('cancelled'); + expect(result.stdout).not.toContain('must-not-publish'); + }, 60_000); + + it('validates and decodes provider output even when the process exits nonzero', async () => { + const result = await startProfileInvocation(profile(fixture(), 'nonzero-output', 'nonzero-decode'), { + decode: (_current, raw) => ({ text: `decoded:${raw.toString('utf8')}` }), + }).settled; + expect(result).toMatchObject({ exitCode: 7, stdout: 'decoded:encoded-output' }); + expect(result.stopReason).toBeUndefined(); + }, 60_000); + it('bounds decoded text independently of adapter byte accounting', async () => { const handle = startProfileInvocation(profile(fixture(), 'finite-output', 'decoded-limit'), { limits: { stdoutBytes: 64 * 1024, stderrBytes: 64 * 1024, combinedBytes: 128 * 1024 }, @@ -141,6 +164,8 @@ describe('container invocation supervisor', () => { it('rejects traversal before starting an output read', () => { expect(() => readBoundedContainerFile('unused', '/run/codeboost-output/../../run/codeboost-auth/codex/auth.json', 1024)).toThrow('bounded output directory'); + expect(() => readBoundedContainerFile('unused', '/run/codeboost-output/final.txt', + 16 * 1024 * 1024 + 1)).toThrow('production stdout limit'); }); it.each([ From 0453c5dc9f04ff942275847d573f715d9f6ef96d Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 17:39:47 -0700 Subject: [PATCH 05/32] Validate pinned output identities --- agents/adapters/supervisor.ts | 10 ++++++---- test/agent-supervisor.test.ts | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index 7f2bc1c..f4f25b4 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -98,16 +98,16 @@ export function readBoundedContainerFile(container: string, source: string, maxi "fd=fs.openSync('/proc/self/fd/'+dirfd+'/'+path.slice(directory.length+1),", 'fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW|fs.constants.O_NONBLOCK);', "const before=fs.fstatSync(fd,{bigint:true});if(before.size>BigInt(maximum))throw new Error('OUTPUT_LIMIT');", - "if(!before.isFile()||before.nlink!==1)throw new Error('UNSAFE_FILE');", + "if(!before.isFile()||before.nlink!==1n)throw new Error('UNSAFE_FILE');", 'const output=Buffer.allocUnsafe(maximum+1);let length=0,count=0;', 'do{count=fs.readSync(fd,output,length,output.length-length,null);length+=count}', "while(count>0&&lengthmaximum)throw new Error('OUTPUT_LIMIT');", 'const after=fs.fstatSync(fd,{bigint:true});if(before.dev!==after.dev||before.ino!==after.ino||before.size!==after.size', - '||before.mtimeMs!==after.mtimeMs||before.ctimeMs!==after.ctimeMs||after.nlink!==1', + '||before.mtimeMs!==after.mtimeMs||before.ctimeMs!==after.ctimeMs||after.nlink!==1n', "||!after.isFile())throw new Error('CHANGED_FILE');", "const named=fs.statSync('/proc/self/fd/'+dirfd+'/'+path.slice(directory.length+1),{bigint:true,throwIfNoEntry:false});", "if(!named||named.dev!==after.dev||named.ino!==after.ino||named.nlink!==1n)throw new Error('REPLACED_FILE');", - "process.stdout.write(output.subarray(0,length))}catch(error){process.exitCode=error.message==='OUTPUT_LIMIT'?42:43}", + "process.stdout.write(output.subarray(0,length))}catch(error){const codes={OUTPUT_LIMIT:42,UNSAFE_FILE:43,CHANGED_FILE:44,REPLACED_FILE:45};process.exitCode=codes[error.message]||46}", 'finally{if(fd!==undefined)fs.closeSync(fd);if(dirfd!==undefined)fs.closeSync(dirfd)}', ].join(''); return new Promise((resolve, reject) => { @@ -122,7 +122,9 @@ export function readBoundedContainerFile(container: string, source: string, maxi if ('killed' in error && error.killed) { reject(new CaptureDeadlineError('Adapter output capture exceeded the invocation deadline.')); return; } - reject(new Error('Adapter output is not a stable bounded unlinked regular file.')); + const reason = error.code === 43 ? 'unsafe type or link count' : error.code === 44 ? 'changed while reading' + : error.code === 45 ? 'pathname identity changed' : 'reader failure'; + reject(new Error(`Adapter output is not a stable bounded unlinked regular file (${reason}).`)); }); }); } diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 0eda4e5..2bd8ed9 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -198,7 +198,7 @@ describe('container invocation supervisor', () => { const result = await startCodexInvocation({ invocation: invocation(data, 'live-codex', 6 * 60_000), filesystems: data.filesystems, inputDirectory: data.input, imageId, prompt: 'Reply only with this exact marker: codeboost-adapter-marker' }, authFile).settled; - expect(result.stopReason).toBeUndefined(); + expect(result.stopReason, result.stderr).toBeUndefined(); expect(result.exitCode).toBe(0); expect(result.stdout).toContain('codeboost-adapter-marker'); }, 8 * 60_000); From 485521da1ecee371218acfdb01a1f57670372ddd Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 17:54:17 -0700 Subject: [PATCH 06/32] Make cleanup and decode settlement retryable --- agents/adapters/supervisor.ts | 52 +++++++++++++++++++++++++++-------- test/agent-supervisor.test.ts | 11 ++++++++ 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index f4f25b4..e9f8c99 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -53,27 +53,35 @@ const diagnosticFor = (reason: StopReason, detail?: string) => Buffer.from( const retainCleanupOwnership = (profile: ContainerProfile, detail: string): InvocationHandle => { const invocation = assertPhasePolicy(profile.policy); let resolveSettled!: (result: InvocationResult) => void, cleaning = false; + let timer: ReturnType | undefined; const settled = new Promise(resolve => { resolveSettled = resolve; }); + const schedule = () => { + if (timer) return; + timer = setTimeout(() => { timer = undefined; retry(); }, 1_000); + timer.unref(); + }; const retry = () => { if (cleaning) return; cleaning = true; try { disposeValidatedContainer(profile); + if (timer) clearTimeout(timer); + timer = undefined; active.delete(invocation.attemptId); resolveSettled(Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, exitCode: null, signal: null, stopReason: 'capture-failure', stdout: '', stderr: diagnosticFor('capture-failure', detail).toString('utf8') })); } catch { - const timer = setTimeout(() => { cleaning = false; retry(); }, 1_000); - timer.unref(); + cleaning = false; + schedule(); return; } cleaning = false; }; const handle: InvocationHandle = Object.freeze({ attemptId: invocation.attemptId, settled, - cancel: retry }); + cancel: () => { if (timer) clearTimeout(timer); timer = undefined; retry(); } }); active.set(invocation.attemptId, handle); - const timer = setTimeout(retry, 1_000); timer.unref(); + schedule(); return handle; }; const withDiagnostic = (stderr: Buffer, stdoutBytes: number, reason: StopReason, limits: CaptureLimits, @@ -244,8 +252,17 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super const budget = deadline - Date.now(); if (budget < 1) throw new CaptureDeadlineError('Invocation deadline expired before output capture.'); const raw = Buffer.concat(stdoutChunks, stdoutBytes); - const decoded = await options.decode!(profile, raw, - Math.max(1, Math.min(limits.stdoutBytes - stdoutBytes, limits.combinedBytes - combinedBytes)), budget); + const operation = Promise.resolve(options.decode!(profile, raw, + Math.max(1, Math.min(limits.stdoutBytes - stdoutBytes, limits.combinedBytes - combinedBytes)), budget)); + let decodeTimer: ReturnType | undefined; + const timeout = new Promise((_resolve, reject) => { + decodeTimer = setTimeout(() => reject( + new CaptureDeadlineError('Adapter output capture exceeded the invocation deadline.')), budget); + decodeTimer.unref(); + }); + let decoded: DecodedOutput; + try { decoded = await Promise.race([operation, timeout]); } + finally { if (decodeTimer) clearTimeout(decodeTimer); } if (stopReason) return; const additional = decoded.additionalBytes ?? 0; const textBytes = Buffer.byteLength(decoded.text); @@ -315,11 +332,12 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super later(() => stop('timeout'), Math.max(1, deadline - Date.now())); let resolveSettled!: (result: InvocationResult) => void; + let wakeCleanup: (() => void) | undefined; const settled = new Promise(resolve => { resolveSettled = resolve; }); const handle: InvocationHandle = Object.freeze({ attemptId: invocation.attemptId, settled, - cancel: (reason: StopReason) => stop(reason), + cancel: (reason: StopReason) => { stop(reason); wakeCleanup?.(); }, }); active.set(invocation.attemptId, handle); @@ -343,11 +361,21 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super if (decodedOutput.providerFailed) exitCode = exitCode === 0 ? 1 : exitCode; } await Promise.all([...controls]); - try { - disposeValidatedContainer(profile); - } catch { - stopReason ??= 'capture-failure'; - return; // Ownership remains active because termination/cleanup was not confirmed. + while (true) { + try { + disposeValidatedContainer(profile); + wakeCleanup = undefined; + break; + } catch (error) { + stopReason ??= 'capture-failure'; + failureDetail ??= error instanceof Error ? error.message : String(error); + await new Promise(resolve => { + let finished = false; + const wake = () => { if (finished) return; finished = true; clearTimeout(timer); resolve(); }; + const timer = setTimeout(wake, 1_000); timer.unref(); + wakeCleanup = wake; + }); + } } if (stopReason) finalStderr = withDiagnostic(finalStderr, finalStdout.length, stopReason, limits, failureDetail); const result = Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 2bd8ed9..87cdcf9 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -145,6 +145,17 @@ describe('container invocation supervisor', () => { expect(result.stdout).not.toContain('must-not-publish'); }, 60_000); + it('keeps post-close decoding inside the invocation deadline', async () => { + const started = Date.now(); + const result = await startProfileInvocation(profile(fixture(), 'finite-output', 'decode-timeout', 30_000), { + timeoutMs: 3_000, + decode: () => new Promise(() => {}), + }).settled; + expect(result.stopReason).toBe('timeout'); + expect(Date.now() - started).toBeLessThan(10_000); + expect(isInvocationActive('decode-timeout')).toBe(false); + }, 30_000); + it('validates and decodes provider output even when the process exits nonzero', async () => { const result = await startProfileInvocation(profile(fixture(), 'nonzero-output', 'nonzero-decode'), { decode: (_current, raw) => ({ text: `decoded:${raw.toString('utf8')}` }), From a4322e8f9ad87935cf8879d4175c7e04890cce25 Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 18:11:55 -0700 Subject: [PATCH 07/32] Abort and await bounded adapter capture --- agents/adapters/codex.ts | 8 +++--- agents/adapters/supervisor.ts | 50 +++++++++++++++++++++++++---------- test/agent-supervisor.test.ts | 18 ++++++++++++- 3 files changed, 58 insertions(+), 18 deletions(-) diff --git a/agents/adapters/codex.ts b/agents/adapters/codex.ts index 1e23d96..f0e4135 100644 --- a/agents/adapters/codex.ts +++ b/agents/adapters/codex.ts @@ -7,8 +7,9 @@ import type { AgentAdapterOptions, AgentAdapterRequest } from './types.ts'; export const CODEX_OUTPUT_FILE = '/run/codeboost-output/final.txt'; -export async function readCodexOutput(container: string, maximumBytes: number, timeoutMs = 30_000) { - const output = await readBoundedContainerFile(container, CODEX_OUTPUT_FILE, maximumBytes, timeoutMs); +export async function readCodexOutput(container: string, maximumBytes: number, timeoutMs = 30_000, + signal?: AbortSignal) { + const output = await readBoundedContainerFile(container, CODEX_OUTPUT_FILE, maximumBytes, timeoutMs, signal); const text = new TextDecoder('utf-8', { fatal: true }).decode(output); return Object.freeze({ text, additionalBytes: output.length }); } @@ -22,7 +23,8 @@ export function startCodexInvocation(request: AgentAdapterRequest, const profile = createContainerProfile({ ...request, policy, network, command: createCodexCommand(policy, request.prompt), codexAuthFile: authFile, deferredOutput: true }); return startProfileInvocation(profile, { ...options, - decode: (current, _raw, maximum, timeoutMs) => readCodexOutput(current.name, maximum, timeoutMs) }); + decode: (current, _raw, maximum, timeoutMs, signal) => + readCodexOutput(current.name, maximum, timeoutMs, signal) }); } catch (error) { removeVendorNetwork(network); throw error; diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index e9f8c99..6a0dbc3 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -29,7 +29,7 @@ export interface SupervisorOptions { readonly timeoutMs?: number; readonly limits?: Partial; readonly decode?: (profile: ContainerProfile, rawStdout: Buffer, maximumBytes: number, - timeoutMs: number) => DecodedOutput | Promise; + timeoutMs: number, signal: AbortSignal) => DecodedOutput | Promise; } export class OutputLimitError extends Error {} export class CaptureDeadlineError extends Error {} @@ -93,7 +93,7 @@ const withDiagnostic = (stderr: Buffer, stdoutBytes: number, reason: StopReason, }; /** Read a running container's tmpfs file with a pinned no-follow bounded reader. */ export function readBoundedContainerFile(container: string, source: string, maximumBytes: number, - timeoutMs = 30_000): Promise { + timeoutMs = 30_000, signal?: AbortSignal): Promise { positiveInteger(maximumBytes, 'maximumBytes'); positiveInteger(timeoutMs, 'timeoutMs'); if (maximumBytes > OUTPUT_LIMITS.stdoutBytes) @@ -113,15 +113,15 @@ export function readBoundedContainerFile(container: string, source: string, maxi 'const after=fs.fstatSync(fd,{bigint:true});if(before.dev!==after.dev||before.ino!==after.ino||before.size!==after.size', '||before.mtimeMs!==after.mtimeMs||before.ctimeMs!==after.ctimeMs||after.nlink!==1n', "||!after.isFile())throw new Error('CHANGED_FILE');", - "const named=fs.statSync('/proc/self/fd/'+dirfd+'/'+path.slice(directory.length+1),{bigint:true,throwIfNoEntry:false});", - "if(!named||named.dev!==after.dev||named.ino!==after.ino||named.nlink!==1n)throw new Error('REPLACED_FILE');", + "const named=fs.lstatSync('/proc/self/fd/'+dirfd+'/'+path.slice(directory.length+1),{bigint:true,throwIfNoEntry:false});", + "if(!named||!named.isFile()||named.dev!==after.dev||named.ino!==after.ino||named.nlink!==1n)throw new Error('REPLACED_FILE');", "process.stdout.write(output.subarray(0,length))}catch(error){const codes={OUTPUT_LIMIT:42,UNSAFE_FILE:43,CHANGED_FILE:44,REPLACED_FILE:45};process.exitCode=codes[error.message]||46}", 'finally{if(fd!==undefined)fs.closeSync(fd);if(dirfd!==undefined)fs.closeSync(dirfd)}', ].join(''); return new Promise((resolve, reject) => { execFile('docker', ['exec', container, 'node', '-e', reader, source, String(maximumBytes)], { env: dockerEnvironment(), timeout: timeoutMs, killSignal: 'SIGKILL', maxBuffer: maximumBytes + 1, - encoding: 'buffer', + encoding: 'buffer', signal, }, (error, stdout, stderr) => { if (!error) { resolve(stdout); return; } if (error.code === 42) { @@ -143,6 +143,14 @@ export function isInvocationActive(attemptId: string): boolean { export function startProfileInvocation(profile: ContainerProfile, options: SupervisorOptions = {}): InvocationHandle { const invocation = assertPhasePolicy(profile.policy); + const rejectWithCleanup = (error: unknown): InvocationHandle => { + try { disposeValidatedContainer(profile); } + catch (cleanupError) { + return retainCleanupOwnership(profile, + `Invocation was rejected and cleanup remains unsettled: ${String(error)}; ${String(cleanupError)}`); + } + throw error; + }; if (active.has(invocation.attemptId)) { disposeValidatedContainer(profile); throw new Error('An invocation with this attempt ID is still active.'); @@ -152,14 +160,14 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super limits = captureLimits(options.limits); configuredTimeout = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; positiveInteger(configuredTimeout, 'timeoutMs'); + if (configuredTimeout > DEFAULT_TIMEOUT_MS) + throw new Error('timeoutMs cannot exceed the production ten-minute ceiling.'); } catch (error) { - disposeValidatedContainer(profile); - throw error; + return rejectWithCleanup(error); } const now = Date.now(), deadline = Math.min(invocation.deadline, now + configuredTimeout); if (!Number.isSafeInteger(deadline) || deadline <= now) { - disposeValidatedContainer(profile); - throw new Error('Invocation deadline has already expired.'); + return rejectWithCleanup(new Error('Invocation deadline has already expired.')); } const remaining = () => { const value = deadline - Date.now(); @@ -183,6 +191,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super let stopReason: StopReason | undefined, failureDetail: string | undefined; let closed = false, terminating = false, settlementComplete = false; let decodedOutput: DecodedOutput | undefined, decodePromise: Promise | undefined; + let decodeAbort: AbortController | undefined; let protocolToken: string | undefined; let protocolBuffer = Buffer.alloc(0); const child = spawn('docker', ['start', '--attach', profile.name], { @@ -227,6 +236,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super const stop = (reason: StopReason) => { if (settlementComplete || stopReason) return; stopReason = reason; + decodeAbort?.abort(); if (!closed) terminate(); }; const capture = (stream: 'stdout' | 'stderr', value: Buffer | string) => { @@ -251,18 +261,30 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super try { const budget = deadline - Date.now(); if (budget < 1) throw new CaptureDeadlineError('Invocation deadline expired before output capture.'); + const controller = new AbortController(); + decodeAbort = controller; const raw = Buffer.concat(stdoutChunks, stdoutBytes); const operation = Promise.resolve(options.decode!(profile, raw, - Math.max(1, Math.min(limits.stdoutBytes - stdoutBytes, limits.combinedBytes - combinedBytes)), budget)); + Math.max(1, Math.min(limits.stdoutBytes - stdoutBytes, limits.combinedBytes - combinedBytes)), + budget, controller.signal)); let decodeTimer: ReturnType | undefined; const timeout = new Promise((_resolve, reject) => { - decodeTimer = setTimeout(() => reject( - new CaptureDeadlineError('Adapter output capture exceeded the invocation deadline.')), budget); + decodeTimer = setTimeout(() => { + stop('timeout'); + reject(new CaptureDeadlineError('Adapter output capture exceeded the invocation deadline.')); + }, budget); decodeTimer.unref(); }); let decoded: DecodedOutput; try { decoded = await Promise.race([operation, timeout]); } - finally { if (decodeTimer) clearTimeout(decodeTimer); } + catch (error) { + controller.abort(); + try { await operation; } catch { /* termination is confirmed by operation settlement */ } + throw error; + } finally { + if (decodeTimer) clearTimeout(decodeTimer); + if (decodeAbort === controller) decodeAbort = undefined; + } if (stopReason) return; const additional = decoded.additionalBytes ?? 0; const textBytes = Buffer.byteLength(decoded.text); @@ -353,7 +375,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super let exitCode = code, finalSignal = signal; if (!stopReason && options.decode && !profile.deferredOutput) await decodeOutput(); if (decodePromise) await decodePromise; - if (!stopReason && code === 0 && profile.deferredOutput && !decodedOutput) { + if (!stopReason && profile.deferredOutput && !decodedOutput) { stopReason = 'capture-failure'; failureDetail ??= 'Deferred output protocol did not complete.'; } if (decodedOutput) { diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 87cdcf9..aa84752 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -149,7 +149,8 @@ describe('container invocation supervisor', () => { const started = Date.now(); const result = await startProfileInvocation(profile(fixture(), 'finite-output', 'decode-timeout', 30_000), { timeoutMs: 3_000, - decode: () => new Promise(() => {}), + decode: (_current, _raw, _maximum, _timeout, signal) => new Promise((_resolve, reject) => + signal.addEventListener('abort', () => reject(new Error('decoder aborted')), { once: true })), }).settled; expect(result.stopReason).toBe('timeout'); expect(Date.now() - started).toBeLessThan(10_000); @@ -202,6 +203,21 @@ describe('container invocation supervisor', () => { expect(spawnSync('docker', ['network', 'inspect', current.network.name]).status).not.toBe(0); }, 60_000); + it('rejects timeouts above the production ceiling and cleans the unused profile', () => { + const current = profile(fixture(), 'finite-output', 'invalid-timeout'); + expect(() => startProfileInvocation(current, { timeoutMs: 10 * 60_000 + 1 })) + .toThrow('ten-minute ceiling'); + expect(spawnSync('docker', ['network', 'inspect', current.network.name]).status).not.toBe(0); + }, 60_000); + + it('fails closed when deferred output is never produced', async () => { + const handle = startProfileInvocation(profile(fixture(), 'nonzero-output', 'missing-deferred', 2 * 60_000, true), { + decode: (current, _raw, maximum, timeoutMs, signal) => + readCodexOutput(current.name, maximum, timeoutMs, signal), + }); + expect((await handle.settled).stopReason).toBe('capture-failure'); + }, 60_000); + if (process.env.CODEBOOST_RUN_AUTH_PROBES === '1') { it('runs the production Codex adapter and collects its bounded output file', async () => { const data = fixture(), authFile = process.env.CODEBOOST_CODEX_AUTH_FILE; From 963ca1683d64e2eb271775d940a7d243eccda5f9 Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 18:32:20 -0700 Subject: [PATCH 08/32] Retain adapter cleanup ownership --- agents/adapters/claude.ts | 5 ++-- agents/adapters/codex.ts | 5 ++-- agents/adapters/supervisor.ts | 49 +++++++++++++++++++++++++++++++++-- test/agent-supervisor.test.ts | 11 ++++++++ 4 files changed, 64 insertions(+), 6 deletions(-) diff --git a/agents/adapters/claude.ts b/agents/adapters/claude.ts index 485f2f6..6b4c48e 100644 --- a/agents/adapters/claude.ts +++ b/agents/adapters/claude.ts @@ -2,7 +2,7 @@ import type { InvocationHandle } from '../contract.ts'; import { createContainerProfile } from '../container/profile.ts'; import { createVendorNetwork, removeVendorNetwork } from '../network/network.ts'; import { createClaudeCommand, createPhasePolicy } from '../policy.ts'; -import { startProfileInvocation } from './supervisor.ts'; +import { retainNetworkCleanup, startProfileInvocation } from './supervisor.ts'; import type { AgentAdapterOptions, AgentAdapterRequest } from './types.ts'; export function parseClaudeOutput(raw: Buffer): { text: string; providerFailed: boolean } { @@ -24,7 +24,8 @@ export function startClaudeInvocation(request: AgentAdapterRequest, return startProfileInvocation(profile, { ...options, secrets: { CLAUDE_CODE_OAUTH_TOKEN: oauthToken }, decode: (_profile, raw) => parseClaudeOutput(raw) }); } catch (error) { - removeVendorNetwork(network); + try { removeVendorNetwork(network); } + catch (cleanupError) { return retainNetworkCleanup(request.invocation, network, error, cleanupError); } throw error; } } diff --git a/agents/adapters/codex.ts b/agents/adapters/codex.ts index f0e4135..777f9ef 100644 --- a/agents/adapters/codex.ts +++ b/agents/adapters/codex.ts @@ -2,7 +2,7 @@ import type { InvocationHandle } from '../contract.ts'; import { createContainerProfile } from '../container/profile.ts'; import { createVendorNetwork, removeVendorNetwork } from '../network/network.ts'; import { createCodexCommand, createPhasePolicy } from '../policy.ts'; -import { readBoundedContainerFile, startProfileInvocation } from './supervisor.ts'; +import { readBoundedContainerFile, retainNetworkCleanup, startProfileInvocation } from './supervisor.ts'; import type { AgentAdapterOptions, AgentAdapterRequest } from './types.ts'; export const CODEX_OUTPUT_FILE = '/run/codeboost-output/final.txt'; @@ -26,7 +26,8 @@ export function startCodexInvocation(request: AgentAdapterRequest, decode: (current, _raw, maximum, timeoutMs, signal) => readCodexOutput(current.name, maximum, timeoutMs, signal) }); } catch (error) { - removeVendorNetwork(network); + try { removeVendorNetwork(network); } + catch (cleanupError) { return retainNetworkCleanup(request.invocation, network, error, cleanupError); } throw error; } } diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index 6a0dbc3..eec900f 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -1,8 +1,9 @@ import { execFile, spawn, type ChildProcess } from 'node:child_process'; -import type { InvocationHandle, InvocationResult, StopReason } from '../contract.ts'; +import type { InvocationHandle, InvocationInput, InvocationResult, StopReason } from '../contract.ts'; import { assertPhasePolicy } from '../policy.ts'; import { createValidatedContainer, disposeValidatedContainer, validateContainer } from '../container/run.ts'; import type { ContainerProfile } from '../container/profile.ts'; +import { removeVendorNetwork, type VendorNetwork } from '../network/network.ts'; export const OUTPUT_LIMITS = Object.freeze({ stdoutBytes: 16 * 1024 * 1024, @@ -10,6 +11,7 @@ export const OUTPUT_LIMITS = Object.freeze({ combinedBytes: 20 * 1024 * 1024, }); const DEFAULT_TIMEOUT_MS = 10 * 60_000; +const CAPTURE_ABORT_GRACE_MS = 1_000; const DIAGNOSTIC_BYTES = 1024; const active = new Map(); @@ -84,6 +86,42 @@ const retainCleanupOwnership = (profile: ContainerProfile, detail: string): Invo schedule(); return handle; }; + +/** Retain attempt ownership while retrying a network allocated before profile construction failed. */ +export function retainNetworkCleanup(invocation: InvocationInput, network: VendorNetwork, + startupError: unknown, cleanupError: unknown): InvocationHandle { + if (active.has(invocation.attemptId)) throw cleanupError; + let resolveSettled!: (result: InvocationResult) => void, cleaning = false; + let timer: ReturnType | undefined; + const settled = new Promise(resolve => { resolveSettled = resolve; }); + const detail = `Adapter startup failed and network cleanup remains unsettled: ${String(startupError)}; ${String(cleanupError)}`; + const retry = () => { + if (cleaning) return; + cleaning = true; + try { + removeVendorNetwork(network); + if (timer) clearTimeout(timer); + timer = undefined; + active.delete(invocation.attemptId); + resolveSettled(Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, + exitCode: null, signal: null, stopReason: 'capture-failure', stdout: '', + stderr: diagnosticFor('capture-failure', detail).toString('utf8') })); + } catch { + cleaning = false; + if (!timer) { + timer = setTimeout(() => { timer = undefined; retry(); }, 1_000); + timer.unref(); + } + return; + } + cleaning = false; + }; + const handle: InvocationHandle = Object.freeze({ attemptId: invocation.attemptId, settled, + cancel: () => { if (timer) clearTimeout(timer); timer = undefined; retry(); } }); + active.set(invocation.attemptId, handle); + timer = setTimeout(() => { timer = undefined; retry(); }, 1_000); timer.unref(); + return handle; +} const withDiagnostic = (stderr: Buffer, stdoutBytes: number, reason: StopReason, limits: CaptureLimits, detail?: string) => { const diagnostic = diagnosticFor(reason, detail).subarray(0, DIAGNOSTIC_BYTES); @@ -279,7 +317,14 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super try { decoded = await Promise.race([operation, timeout]); } catch (error) { controller.abort(); - try { await operation; } catch { /* termination is confirmed by operation settlement */ } + let graceTimer: ReturnType | undefined; + const grace = new Promise(resolve => { + graceTimer = setTimeout(resolve, CAPTURE_ABORT_GRACE_MS); + graceTimer.unref(); + }); + await Promise.race([operation.then(() => undefined, () => undefined), grace]); + if (graceTimer) clearTimeout(graceTimer); + void operation.catch(() => { /* prevent a detached noncooperative decoder from becoming unhandled */ }); throw error; } finally { if (decodeTimer) clearTimeout(decodeTimer); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index aa84752..25d172c 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -157,6 +157,17 @@ describe('container invocation supervisor', () => { expect(isInvocationActive('decode-timeout')).toBe(false); }, 30_000); + it('does not wedge when an injected decoder ignores abort', async () => { + const started = Date.now(); + const result = await startProfileInvocation(profile(fixture(), 'finite-output', 'decode-ignores-abort', 30_000), { + timeoutMs: 3_000, + decode: () => new Promise(() => {}), + }).settled; + expect(result.stopReason).toBe('timeout'); + expect(Date.now() - started).toBeLessThan(10_000); + expect(isInvocationActive('decode-ignores-abort')).toBe(false); + }, 30_000); + it('validates and decodes provider output even when the process exits nonzero', async () => { const result = await startProfileInvocation(profile(fixture(), 'nonzero-output', 'nonzero-decode'), { decode: (_current, raw) => ({ text: `decoded:${raw.toString('utf8')}` }), From ef40f26e0a3e093a0b5bcb3f67470cc8e5791938 Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 18:45:14 -0700 Subject: [PATCH 09/32] Allow loaded Docker cleanup observation --- test/agent-supervisor.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 25d172c..492efd5 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -153,7 +153,7 @@ describe('container invocation supervisor', () => { signal.addEventListener('abort', () => reject(new Error('decoder aborted')), { once: true })), }).settled; expect(result.stopReason).toBe('timeout'); - expect(Date.now() - started).toBeLessThan(10_000); + expect(Date.now() - started).toBeLessThan(15_000); expect(isInvocationActive('decode-timeout')).toBe(false); }, 30_000); @@ -164,7 +164,7 @@ describe('container invocation supervisor', () => { decode: () => new Promise(() => {}), }).settled; expect(result.stopReason).toBe('timeout'); - expect(Date.now() - started).toBeLessThan(10_000); + expect(Date.now() - started).toBeLessThan(15_000); expect(isInvocationActive('decode-ignores-abort')).toBe(false); }, 30_000); From 97ca736e5a3dd85462ef6ae3e46eb47f7a876ebc Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 19:07:51 -0700 Subject: [PATCH 10/32] Secure deferred output acknowledgement --- agents/adapters/supervisor.ts | 75 +++++++++++++++++++++++++++-------- agents/container/probe.sh | 20 +++++++++- agents/container/profile.ts | 3 +- agents/container/run.ts | 2 + agents/policy.ts | 6 +-- test/agent-supervisor.test.ts | 15 ++++++- 6 files changed, 97 insertions(+), 24 deletions(-) diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index eec900f..46059a3 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -52,9 +52,10 @@ const captureLimits = (override: Partial | undefined): CaptureLim }; const diagnosticFor = (reason: StopReason, detail?: string) => Buffer.from( `[codeboost: ${reason}${detail ? `: ${detail.replace(/[\r\n]+/g, ' ').slice(0, 512)}` : ''}]\n`); -const retainCleanupOwnership = (profile: ContainerProfile, detail: string): InvocationHandle => { +const retainCleanupOwnership = (profile: ContainerProfile, detail: string, register = true): InvocationHandle => { const invocation = assertPhasePolicy(profile.policy); - let resolveSettled!: (result: InvocationResult) => void, cleaning = false; + let resolveSettled!: (result: InvocationResult) => void, cleaning = false, complete = false; + let handle!: InvocationHandle; let timer: ReturnType | undefined; const settled = new Promise(resolve => { resolveSettled = resolve; }); const schedule = () => { @@ -63,13 +64,14 @@ const retainCleanupOwnership = (profile: ContainerProfile, detail: string): Invo timer.unref(); }; const retry = () => { - if (cleaning) return; + if (cleaning || complete) return; cleaning = true; try { disposeValidatedContainer(profile); if (timer) clearTimeout(timer); timer = undefined; - active.delete(invocation.attemptId); + complete = true; + if (active.get(invocation.attemptId) === handle) active.delete(invocation.attemptId); resolveSettled(Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, exitCode: null, signal: null, stopReason: 'capture-failure', stdout: '', stderr: diagnosticFor('capture-failure', detail).toString('utf8') })); @@ -80,9 +82,9 @@ const retainCleanupOwnership = (profile: ContainerProfile, detail: string): Invo } cleaning = false; }; - const handle: InvocationHandle = Object.freeze({ attemptId: invocation.attemptId, settled, + handle = Object.freeze({ attemptId: invocation.attemptId, settled, cancel: () => { if (timer) clearTimeout(timer); timer = undefined; retry(); } }); - active.set(invocation.attemptId, handle); + if (register) active.set(invocation.attemptId, handle); schedule(); return handle; }; @@ -91,18 +93,20 @@ const retainCleanupOwnership = (profile: ContainerProfile, detail: string): Invo export function retainNetworkCleanup(invocation: InvocationInput, network: VendorNetwork, startupError: unknown, cleanupError: unknown): InvocationHandle { if (active.has(invocation.attemptId)) throw cleanupError; - let resolveSettled!: (result: InvocationResult) => void, cleaning = false; + let resolveSettled!: (result: InvocationResult) => void, cleaning = false, complete = false; + let handle!: InvocationHandle; let timer: ReturnType | undefined; const settled = new Promise(resolve => { resolveSettled = resolve; }); const detail = `Adapter startup failed and network cleanup remains unsettled: ${String(startupError)}; ${String(cleanupError)}`; const retry = () => { - if (cleaning) return; + if (cleaning || complete) return; cleaning = true; try { removeVendorNetwork(network); if (timer) clearTimeout(timer); timer = undefined; - active.delete(invocation.attemptId); + complete = true; + if (active.get(invocation.attemptId) === handle) active.delete(invocation.attemptId); resolveSettled(Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, exitCode: null, signal: null, stopReason: 'capture-failure', stdout: '', stderr: diagnosticFor('capture-failure', detail).toString('utf8') })); @@ -116,7 +120,7 @@ export function retainNetworkCleanup(invocation: InvocationInput, network: Vendo } cleaning = false; }; - const handle: InvocationHandle = Object.freeze({ attemptId: invocation.attemptId, settled, + handle = Object.freeze({ attemptId: invocation.attemptId, settled, cancel: () => { if (timer) clearTimeout(timer); timer = undefined; retry(); } }); active.set(invocation.attemptId, handle); timer = setTimeout(() => { timer = undefined; retry(); }, 1_000); timer.unref(); @@ -181,17 +185,16 @@ export function isInvocationActive(attemptId: string): boolean { export function startProfileInvocation(profile: ContainerProfile, options: SupervisorOptions = {}): InvocationHandle { const invocation = assertPhasePolicy(profile.policy); - const rejectWithCleanup = (error: unknown): InvocationHandle => { + const rejectWithCleanup = (error: unknown, register = true): InvocationHandle => { try { disposeValidatedContainer(profile); } catch (cleanupError) { return retainCleanupOwnership(profile, - `Invocation was rejected and cleanup remains unsettled: ${String(error)}; ${String(cleanupError)}`); + `Invocation was rejected and cleanup remains unsettled: ${String(error)}; ${String(cleanupError)}`, register); } throw error; }; if (active.has(invocation.attemptId)) { - disposeValidatedContainer(profile); - throw new Error('An invocation with this attempt ID is still active.'); + return rejectWithCleanup(new Error('An invocation with this attempt ID is still active.'), false); } let limits: CaptureLimits, configuredTimeout: number; try { @@ -230,7 +233,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super let closed = false, terminating = false, settlementComplete = false; let decodedOutput: DecodedOutput | undefined, decodePromise: Promise | undefined; let decodeAbort: AbortController | undefined; - let protocolToken: string | undefined; + let protocolToken: string | undefined, protocolStarted = false, protocolReady = false; let protocolBuffer = Buffer.alloc(0); const child = spawn('docker', ['start', '--attach', profile.name], { env: dockerEnvironment(), stdio: ['ignore', 'pipe', 'pipe'], @@ -254,6 +257,18 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super void operation.finally(() => controls.delete(operation)); return operation; }; + const acknowledgeDeferredOutput = (token: string) => { + const script = [ + "const fs=require('node:fs'),token=process.argv[1],directory='/run/codeboost-control';", + 'let dirfd,fd;try{dirfd=fs.openSync(directory,fs.constants.O_RDONLY|fs.constants.O_DIRECTORY|fs.constants.O_NOFOLLOW);', + "fd=fs.openSync('/proc/self/fd/'+dirfd+'/collected-'+token,", + 'fs.constants.O_WRONLY|fs.constants.O_CREAT|fs.constants.O_EXCL|fs.constants.O_NOFOLLOW,0o444);', + "fs.writeFileSync(fd,token);fs.fsyncSync(fd);const stat=fs.fstatSync(fd,{bigint:true});", + "if(!stat.isFile()||stat.nlink!==1n)throw new Error('UNSAFE_ACK')}finally{if(fd!==undefined)fs.closeSync(fd);", + 'if(dirfd!==undefined)fs.closeSync(dirfd)}', + ].join(''); + return runControl(['exec', '--user', '0', profile.name, 'node', '-e', script, token]); + }; const later = (callback: () => void, delay: number) => { const timer = setTimeout(() => { timers.delete(timer); callback(); }, delay); timer.unref(); timers.add(timer); return timer; @@ -292,6 +307,14 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super } if (chunk.length > available) stop('output-limit'); }; + const consumeProtocol = (length: number) => { + const available = Math.max(0, Math.min(limits.stderrBytes - stderrBytes, + limits.combinedBytes - combinedBytes)); + const consumed = Math.min(length, available); + stderrBytes += consumed; + combinedBytes += consumed; + if (length > available) stop('output-limit'); + }; const decodeOutput = () => { if (decodePromise) return decodePromise; if (!options.decode || decodedOutput || stopReason) return Promise.resolve(); @@ -356,15 +379,32 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super const text = line.toString('utf8').trim(); const started = /^\x1eCODEBOOST_START:([0-9a-f-]{36})\x1e$/.exec(text); if (started) { + consumeProtocol(line.length); + if (stopReason) return true; + if (protocolStarted) { + failureDetail ??= 'Deferred output emitted a duplicate START frame.'; + stop('capture-failure'); + return true; + } if (protocolToken && protocolToken !== started[1]) return false; + protocolStarted = true; protocolToken = started[1]; return true; } const ready = /^\x1eCODEBOOST_READY:([0-9a-f-]{36}):([0-9]+)\x1e$/.exec(text); if (!ready || ready[1] !== protocolToken) return false; + consumeProtocol(line.length); + if (stopReason) return true; + if (protocolReady) { + failureDetail ??= 'Deferred output emitted a duplicate READY frame.'; + stop('capture-failure'); + return true; + } + protocolReady = true; + const readyToken = ready[1]!; void decodeOutput().then(() => { if (decodedOutput && !stopReason) { - void runControl(['exec', profile.name, 'touch', `/run/codeboost-output/collected-${ready[1]}`]) + void acknowledgeDeferredOutput(readyToken) .then(success => { if (!success) { failureDetail ??= 'Deferred output acknowledgement failed.'; @@ -376,6 +416,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super return true; }; const captureStderr = (value: Buffer | string) => { + if (stopReason || closed) return; if (!profile.deferredOutput) { capture('stderr', value); return; } const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value); protocolBuffer = Buffer.concat([protocolBuffer, chunk]); @@ -416,7 +457,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super protocolBuffer = Buffer.alloc(0); } closed = true; - let finalStdout = Buffer.concat(stdoutChunks, stdoutBytes), finalStderr = Buffer.concat(stderrChunks, stderrBytes); + let finalStdout = Buffer.concat(stdoutChunks, stdoutBytes), finalStderr = Buffer.concat(stderrChunks); let exitCode = code, finalSignal = signal; if (!stopReason && options.decode && !profile.deferredOutput) await decodeOutput(); if (decodePromise) await decodePromise; diff --git a/agents/container/probe.sh b/agents/container/probe.sh index e579903..37d8582 100644 --- a/agents/container/probe.sh +++ b/agents/container/probe.sh @@ -70,6 +70,13 @@ case "$CODEBOOST_VENDOR" in require_option "$CODEX_HOME" nodev require_option "$CODEX_HOME/auth.json" ro require_ceiling "$CODEX_HOME" 4194304 256 + [ "$(findmnt --noheadings --output FSTYPE --target /run/codeboost-output)" = 'tmpfs' ] \ + || fail 'Codex output must use tmpfs' + require_option /run/codeboost-output rw + require_option /run/codeboost-output nosuid + require_option /run/codeboost-output nodev + require_option /run/codeboost-output noexec + require_ceiling /run/codeboost-output 20971520 64 ;; claude) [ -n "${CLAUDE_CODE_OAUTH_TOKEN:-}" ] || fail 'Claude credential is missing' @@ -82,6 +89,14 @@ esac [ "$(claude --version | awk '{print $1}')" = '2.1.281' ] || fail 'unexpected Claude version' if [ "${CODEBOOST_DEFERRED_OUTPUT:-}" = '1' ]; then + [ "$(findmnt --noheadings --output FSTYPE --target /run/codeboost-control)" = 'tmpfs' ] \ + || fail 'deferred control must use tmpfs' + require_option /run/codeboost-control rw + require_option /run/codeboost-control nosuid + require_option /run/codeboost-control nodev + require_option /run/codeboost-control noexec + require_ceiling /run/codeboost-control 65536 16 + [ ! -w /run/codeboost-control ] || fail 'agent must not write deferred control markers' token="$(cat /proc/sys/kernel/random/uuid)" printf '\036CODEBOOST_START:%s\036\n' "$token" >&2 set +e @@ -89,8 +104,9 @@ if [ "${CODEBOOST_DEFERRED_OUTPUT:-}" = '1' ]; then status="$?" set -e printf '\036CODEBOOST_READY:%s:%s\036\n' "$token" "$status" >&2 - acknowledgement="/run/codeboost-output/collected-$token" - while [ ! -e "$acknowledgement" ]; do sleep 0.05; done + acknowledgement="/run/codeboost-control/collected-$token" + while [ ! -f "$acknowledgement" ] || [ -L "$acknowledgement" ] \ + || [ "$(cat "$acknowledgement" 2>/dev/null || true)" != "$token" ]; do sleep 0.05; done exit "$status" fi exec "$@" diff --git a/agents/container/profile.ts b/agents/container/profile.ts index ed9ee4b..a1086b5 100644 --- a/agents/container/profile.ts +++ b/agents/container/profile.ts @@ -233,7 +233,8 @@ export function createContainerProfile(options: ProfileOptions): ContainerProfil '--mount', mount({ type: 'bind', source: inputIdentity.inputDirectory, target: '/run/codeboost-input', readonly: true })]; if (options.deferredOutput) { if (invocation.vendor !== 'codex') throw new Error('Deferred output is available only for Codex.'); - args.push('--env', 'CODEBOOST_DEFERRED_OUTPUT=1'); + args.push('--env', 'CODEBOOST_DEFERRED_OUTPUT=1', + '--tmpfs', '/run/codeboost-control:rw,nosuid,nodev,noexec,size=65536,nr_inodes=16,uid=0,gid=0,mode=0711'); } if (invocation.vendor === 'codex') { args.push('--env', 'CODEX_HOME=/run/codeboost-auth/codex', diff --git a/agents/container/run.ts b/agents/container/run.ts index ac36bfd..f809fe3 100644 --- a/agents/container/run.ts +++ b/agents/container/run.ts @@ -174,6 +174,8 @@ export function validateContainer(container: string, profile: ContainerProfile, ['rw', 'nosuid', 'nodev', 'size=4194304', 'nr_inodes=256', 'uid=10001', 'gid=10001', 'mode=0700']] as const, ['/run/codeboost-output', ['rw', 'nosuid', 'nodev', 'noexec', 'size=20971520', 'nr_inodes=64', 'uid=10001', 'gid=10001', 'mode=0700']] as const] : []), + ...(profile.deferredOutput ? [['/run/codeboost-control', + ['rw', 'nosuid', 'nodev', 'noexec', 'size=65536', 'nr_inodes=16', 'uid=0', 'gid=0', 'mode=0711']] as const] : []), ]); if (Object.keys(tmpfs).length !== expectedTmpfs.size) throw new Error('Container tmpfs mount set changed.'); for (const [path, expected] of expectedTmpfs) { diff --git a/agents/policy.ts b/agents/policy.ts index 89e3dd6..6fc5f73 100644 --- a/agents/policy.ts +++ b/agents/policy.ts @@ -91,8 +91,8 @@ export function createCodexCommand(policy: PhasePolicy, prompt: string): AgentCo export type IsolationProbe = 'noop' | 'phase-worktree' | 'read-only-isolation' | 'persist-write' | 'persist-read' | 'capacity' | 'metadata' | 'must-not-run' | 'input-marker' | 'finite-output' | 'infinite-stdout' | 'infinite-stderr' | 'infinite-mixed' | 'ignore-term' | 'symlink-output' - | 'oversized-output' | 'fifo-output' | 'invalid-utf8-output' | 'replace-output-directory' | 'ack-failure' - | 'nonzero-output'; + | 'oversized-output' | 'fifo-output' | 'invalid-utf8-output' | 'replace-output-directory' + | 'nonzero-output' | 'duplicate-protocol'; /** Fixed startup probes validate the sandbox itself without granting an agent a process tool. */ export function createIsolationProbeCommand(policy: PhasePolicy, probe: IsolationProbe): AgentCommand { @@ -125,8 +125,8 @@ export function createIsolationProbeCommand(policy: PhasePolicy, probe: Isolatio 'fifo-output': 'mkfifo /run/codeboost-output/final.txt', 'invalid-utf8-output': "printf '\\377' > /run/codeboost-output/final.txt", 'replace-output-directory': 'rm -rf /run/codeboost-output; ln -s /etc /run/codeboost-output', - 'ack-failure': "printf captured > /run/codeboost-output/final.txt; chmod 0500 /run/codeboost-output", 'nonzero-output': 'printf encoded-output; exit 7', + 'duplicate-protocol': "printf '\\036CODEBOOST_START:00000000-0000-0000-0000-000000000000\\036\\n' >&2", }; return command(policy, probe === 'noop' ? ['true'] : ['sh', '-c', scripts[probe]]); } diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 492efd5..9033b90 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -89,6 +89,19 @@ describe('container invocation supervisor', () => { expect(isInvocationActive(attemptId)).toBe(false); }, 60_000); + it('stops buffering deferred newline-free stderr after the limit is reached', async () => { + const attemptId = 'deferred-stderr-limit'; + const handle = startProfileInvocation(profile(fixture(), 'infinite-stderr', attemptId, 2 * 60_000, true), { + limits: { stdoutBytes: 64 * 1024, stderrBytes: 32 * 1024, combinedBytes: 64 * 1024 }, + decode: (current, _raw, maximum, timeoutMs, signal) => + readCodexOutput(current.name, maximum, timeoutMs, signal), + }); + const result = await handle.settled; + expect(result.stopReason).toBe('output-limit'); + expect(Buffer.byteLength(result.stderr)).toBeLessThanOrEqual(32 * 1024); + expect(isInvocationActive(attemptId)).toBe(false); + }, 60_000); + it('preserves the first cancellation reason until an ignored SIGTERM fully settles', async () => { const current = profile(fixture(), 'ignore-term', 'cancelled'); const handle = startProfileInvocation(current, { timeoutMs: 30_000 }); @@ -197,7 +210,7 @@ describe('container invocation supervisor', () => { ['fifo-output', 'capture-failure'], ['invalid-utf8-output', 'capture-failure'], ['replace-output-directory', 'capture-failure'], - ['ack-failure', 'capture-failure'], + ['duplicate-protocol', 'capture-failure'], ] as const)('rejects unsafe Codex output from %s', async (probe, reason) => { const handle = startProfileInvocation(profile(fixture(), probe, `file-${probe}`, 2 * 60_000, true), { limits: { stdoutBytes: 64 * 1024, stderrBytes: 64 * 1024, combinedBytes: 128 * 1024 }, From 1fed991c823471502736a46af73ccf232e6f937d Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 19:21:51 -0700 Subject: [PATCH 11/32] Retain colliding cleanup recovery --- agents/adapters/supervisor.ts | 9 +++++++-- agents/container/probe.sh | 2 +- agents/policy.ts | 3 ++- test/agent-supervisor.test.ts | 11 +++++++++++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index 46059a3..ee901f4 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -14,6 +14,7 @@ const DEFAULT_TIMEOUT_MS = 10 * 60_000; const CAPTURE_ABORT_GRACE_MS = 1_000; const DIAGNOSTIC_BYTES = 1024; const active = new Map(); +const cleanupRecoveries = new Set(); export interface CaptureLimits { readonly stdoutBytes: number; @@ -72,6 +73,7 @@ const retainCleanupOwnership = (profile: ContainerProfile, detail: string, regis timer = undefined; complete = true; if (active.get(invocation.attemptId) === handle) active.delete(invocation.attemptId); + cleanupRecoveries.delete(handle); resolveSettled(Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, exitCode: null, signal: null, stopReason: 'capture-failure', stdout: '', stderr: diagnosticFor('capture-failure', detail).toString('utf8') })); @@ -85,6 +87,7 @@ const retainCleanupOwnership = (profile: ContainerProfile, detail: string, regis handle = Object.freeze({ attemptId: invocation.attemptId, settled, cancel: () => { if (timer) clearTimeout(timer); timer = undefined; retry(); } }); if (register) active.set(invocation.attemptId, handle); + else cleanupRecoveries.add(handle); schedule(); return handle; }; @@ -92,7 +95,7 @@ const retainCleanupOwnership = (profile: ContainerProfile, detail: string, regis /** Retain attempt ownership while retrying a network allocated before profile construction failed. */ export function retainNetworkCleanup(invocation: InvocationInput, network: VendorNetwork, startupError: unknown, cleanupError: unknown): InvocationHandle { - if (active.has(invocation.attemptId)) throw cleanupError; + const register = !active.has(invocation.attemptId); let resolveSettled!: (result: InvocationResult) => void, cleaning = false, complete = false; let handle!: InvocationHandle; let timer: ReturnType | undefined; @@ -107,6 +110,7 @@ export function retainNetworkCleanup(invocation: InvocationInput, network: Vendo timer = undefined; complete = true; if (active.get(invocation.attemptId) === handle) active.delete(invocation.attemptId); + cleanupRecoveries.delete(handle); resolveSettled(Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, exitCode: null, signal: null, stopReason: 'capture-failure', stdout: '', stderr: diagnosticFor('capture-failure', detail).toString('utf8') })); @@ -122,7 +126,8 @@ export function retainNetworkCleanup(invocation: InvocationInput, network: Vendo }; handle = Object.freeze({ attemptId: invocation.attemptId, settled, cancel: () => { if (timer) clearTimeout(timer); timer = undefined; retry(); } }); - active.set(invocation.attemptId, handle); + if (register) active.set(invocation.attemptId, handle); + else cleanupRecoveries.add(handle); timer = setTimeout(() => { timer = undefined; retry(); }, 1_000); timer.unref(); return handle; } diff --git a/agents/container/probe.sh b/agents/container/probe.sh index 37d8582..06a4290 100644 --- a/agents/container/probe.sh +++ b/agents/container/probe.sh @@ -103,7 +103,7 @@ if [ "${CODEBOOST_DEFERRED_OUTPUT:-}" = '1' ]; then "$@" status="$?" set -e - printf '\036CODEBOOST_READY:%s:%s\036\n' "$token" "$status" >&2 + printf '\n\036CODEBOOST_READY:%s:%s\036\n' "$token" "$status" >&2 acknowledgement="/run/codeboost-control/collected-$token" while [ ! -f "$acknowledgement" ] || [ -L "$acknowledgement" ] \ || [ "$(cat "$acknowledgement" 2>/dev/null || true)" != "$token" ]; do sleep 0.05; done diff --git a/agents/policy.ts b/agents/policy.ts index 6fc5f73..ae09dd1 100644 --- a/agents/policy.ts +++ b/agents/policy.ts @@ -92,7 +92,7 @@ export type IsolationProbe = 'noop' | 'phase-worktree' | 'read-only-isolation' | | 'persist-read' | 'capacity' | 'metadata' | 'must-not-run' | 'input-marker' | 'finite-output' | 'infinite-stdout' | 'infinite-stderr' | 'infinite-mixed' | 'ignore-term' | 'symlink-output' | 'oversized-output' | 'fifo-output' | 'invalid-utf8-output' | 'replace-output-directory' - | 'nonzero-output' | 'duplicate-protocol'; + | 'nonzero-output' | 'duplicate-protocol' | 'newline-free-deferred-output'; /** Fixed startup probes validate the sandbox itself without granting an agent a process tool. */ export function createIsolationProbeCommand(policy: PhasePolicy, probe: IsolationProbe): AgentCommand { @@ -127,6 +127,7 @@ export function createIsolationProbeCommand(policy: PhasePolicy, probe: Isolatio 'replace-output-directory': 'rm -rf /run/codeboost-output; ln -s /etc /run/codeboost-output', 'nonzero-output': 'printf encoded-output; exit 7', 'duplicate-protocol': "printf '\\036CODEBOOST_START:00000000-0000-0000-0000-000000000000\\036\\n' >&2", + 'newline-free-deferred-output': "printf captured > /run/codeboost-output/final.txt; printf trailing-diagnostic >&2", }; return command(policy, probe === 'noop' ? ['true'] : ['sh', '-c', scripts[probe]]); } diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 9033b90..17552c9 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -242,6 +242,17 @@ describe('container invocation supervisor', () => { expect((await handle.settled).stopReason).toBe('capture-failure'); }, 60_000); + it('delimits READY after finite newline-free stderr', async () => { + const result = await startProfileInvocation( + profile(fixture(), 'newline-free-deferred-output', 'newline-free-ready', 2 * 60_000, true), { + decode: (current, _raw, maximum, timeoutMs, signal) => + readCodexOutput(current.name, maximum, timeoutMs, signal), + }).settled; + expect(result.stopReason, result.stderr).toBeUndefined(); + expect(result.stdout).toBe('captured'); + expect(result.stderr).toContain('trailing-diagnostic'); + }, 60_000); + if (process.env.CODEBOOST_RUN_AUTH_PROBES === '1') { it('runs the production Codex adapter and collects its bounded output file', async () => { const data = fixture(), authFile = process.env.CODEBOOST_CODEX_AUTH_FILE; From ae99db9d8556591a3c29e90d0dad61c2b73ba493 Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 19:35:10 -0700 Subject: [PATCH 12/32] Preserve adapter setup ownership --- agents/adapters/claude.ts | 36 +++++++++++++++++++++++---- agents/adapters/codex.ts | 38 ++++++++++++++++++++++++---- agents/adapters/supervisor.ts | 11 ++++++-- agents/container/profile.ts | 27 ++++++++++++++++---- agents/network/network.ts | 47 ++++++++++++++++++++++++----------- test/agent-adapter.test.ts | 42 ++++++++++++++++++++++++++++--- 6 files changed, 166 insertions(+), 35 deletions(-) diff --git a/agents/adapters/claude.ts b/agents/adapters/claude.ts index 6b4c48e..834991e 100644 --- a/agents/adapters/claude.ts +++ b/agents/adapters/claude.ts @@ -1,8 +1,9 @@ import type { InvocationHandle } from '../contract.ts'; -import { createContainerProfile } from '../container/profile.ts'; -import { createVendorNetwork, removeVendorNetwork } from '../network/network.ts'; +import { createContainerProfile, ProfileCreationCleanupError } from '../container/profile.ts'; +import { createVendorNetwork, removeVendorNetwork, VendorNetworkCreationCleanupError, + type VendorNetwork } from '../network/network.ts'; import { createClaudeCommand, createPhasePolicy } from '../policy.ts'; -import { retainNetworkCleanup, startProfileInvocation } from './supervisor.ts'; +import { retainNetworkCleanup, retainSetupCleanup, startProfileInvocation } from './supervisor.ts'; import type { AgentAdapterOptions, AgentAdapterRequest } from './types.ts'; export function parseClaudeOutput(raw: Buffer): { text: string; providerFailed: boolean } { @@ -17,13 +18,38 @@ export function startClaudeInvocation(request: AgentAdapterRequest, oauthToken: string, options: AgentAdapterOptions = {}): InvocationHandle { if (!oauthToken || oauthToken.includes('\0')) throw new Error('Claude OAuth token is malformed.'); const policy = createPhasePolicy(request.invocation); - const network = createVendorNetwork(request.invocation, request.imageId); + const remaining = () => { + const value = request.invocation.deadline - Date.now(); + if (!Number.isSafeInteger(value) || value < 1) + throw new Error('Invocation deadline expired during adapter setup.'); + return Math.min(60_000, value); + }; + let network: VendorNetwork; + try { network = createVendorNetwork(request.invocation, request.imageId, remaining()); } + catch (error) { + if (error instanceof VendorNetworkCreationCleanupError) + return retainSetupCleanup(request.invocation, error.retryCleanup, error.startupError, error, + 'network creation cleanup'); + throw error; + } try { const profile = createContainerProfile({ ...request, policy, network, - command: createClaudeCommand(policy, request.prompt), claudeToken: oauthToken }); + command: createClaudeCommand(policy, request.prompt), claudeToken: oauthToken, timeoutMs: remaining() }); return startProfileInvocation(profile, { ...options, secrets: { CLAUDE_CODE_OAUTH_TOKEN: oauthToken }, decode: (_profile, raw) => parseClaudeOutput(raw) }); } catch (error) { + if (error instanceof ProfileCreationCleanupError) { + const retryCleanup = () => { + const failures: unknown[] = []; + try { error.retryCleanup(); } catch (cleanupError) { failures.push(cleanupError); } + try { removeVendorNetwork(network); } catch (cleanupError) { failures.push(cleanupError); } + if (failures.length) throw new AggregateError(failures, 'Adapter setup cleanup did not settle.'); + }; + try { retryCleanup(); } + catch (cleanupError) { return retainSetupCleanup(request.invocation, retryCleanup, + error.startupError, cleanupError, 'profile and network cleanup'); } + throw error.startupError; + } try { removeVendorNetwork(network); } catch (cleanupError) { return retainNetworkCleanup(request.invocation, network, error, cleanupError); } throw error; diff --git a/agents/adapters/codex.ts b/agents/adapters/codex.ts index 777f9ef..8c5d1f6 100644 --- a/agents/adapters/codex.ts +++ b/agents/adapters/codex.ts @@ -1,8 +1,10 @@ import type { InvocationHandle } from '../contract.ts'; -import { createContainerProfile } from '../container/profile.ts'; -import { createVendorNetwork, removeVendorNetwork } from '../network/network.ts'; +import { createContainerProfile, ProfileCreationCleanupError } from '../container/profile.ts'; +import { createVendorNetwork, removeVendorNetwork, VendorNetworkCreationCleanupError, + type VendorNetwork } from '../network/network.ts'; import { createCodexCommand, createPhasePolicy } from '../policy.ts'; -import { readBoundedContainerFile, retainNetworkCleanup, startProfileInvocation } from './supervisor.ts'; +import { readBoundedContainerFile, retainNetworkCleanup, retainSetupCleanup, + startProfileInvocation } from './supervisor.ts'; import type { AgentAdapterOptions, AgentAdapterRequest } from './types.ts'; export const CODEX_OUTPUT_FILE = '/run/codeboost-output/final.txt'; @@ -18,14 +20,40 @@ export function startCodexInvocation(request: AgentAdapterRequest, authFile: string, options: AgentAdapterOptions = {}): InvocationHandle { if (!authFile || authFile.includes('\0')) throw new Error('Codex auth path is malformed.'); const policy = createPhasePolicy(request.invocation); - const network = createVendorNetwork(request.invocation, request.imageId); + const remaining = () => { + const value = request.invocation.deadline - Date.now(); + if (!Number.isSafeInteger(value) || value < 1) + throw new Error('Invocation deadline expired during adapter setup.'); + return Math.min(60_000, value); + }; + let network: VendorNetwork; + try { network = createVendorNetwork(request.invocation, request.imageId, remaining()); } + catch (error) { + if (error instanceof VendorNetworkCreationCleanupError) + return retainSetupCleanup(request.invocation, error.retryCleanup, error.startupError, error, + 'network creation cleanup'); + throw error; + } try { const profile = createContainerProfile({ ...request, policy, network, - command: createCodexCommand(policy, request.prompt), codexAuthFile: authFile, deferredOutput: true }); + command: createCodexCommand(policy, request.prompt), codexAuthFile: authFile, deferredOutput: true, + timeoutMs: remaining() }); return startProfileInvocation(profile, { ...options, decode: (current, _raw, maximum, timeoutMs, signal) => readCodexOutput(current.name, maximum, timeoutMs, signal) }); } catch (error) { + if (error instanceof ProfileCreationCleanupError) { + const retryCleanup = () => { + const failures: unknown[] = []; + try { error.retryCleanup(); } catch (cleanupError) { failures.push(cleanupError); } + try { removeVendorNetwork(network); } catch (cleanupError) { failures.push(cleanupError); } + if (failures.length) throw new AggregateError(failures, 'Adapter setup cleanup did not settle.'); + }; + try { retryCleanup(); } + catch (cleanupError) { return retainSetupCleanup(request.invocation, retryCleanup, + error.startupError, cleanupError, 'profile and network cleanup'); } + throw error.startupError; + } try { removeVendorNetwork(network); } catch (cleanupError) { return retainNetworkCleanup(request.invocation, network, error, cleanupError); } throw error; diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index ee901f4..f8d8988 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -95,17 +95,24 @@ const retainCleanupOwnership = (profile: ContainerProfile, detail: string, regis /** Retain attempt ownership while retrying a network allocated before profile construction failed. */ export function retainNetworkCleanup(invocation: InvocationInput, network: VendorNetwork, startupError: unknown, cleanupError: unknown): InvocationHandle { + return retainSetupCleanup(invocation, () => removeVendorNetwork(network), startupError, cleanupError, + 'network cleanup'); +} + +/** Retain attempt ownership while retrying resources allocated during synchronous adapter setup. */ +export function retainSetupCleanup(invocation: InvocationInput, retryCleanup: () => void, + startupError: unknown, cleanupError: unknown, kind = 'setup cleanup'): InvocationHandle { const register = !active.has(invocation.attemptId); let resolveSettled!: (result: InvocationResult) => void, cleaning = false, complete = false; let handle!: InvocationHandle; let timer: ReturnType | undefined; const settled = new Promise(resolve => { resolveSettled = resolve; }); - const detail = `Adapter startup failed and network cleanup remains unsettled: ${String(startupError)}; ${String(cleanupError)}`; + const detail = `Adapter startup failed and ${kind} remains unsettled: ${String(startupError)}; ${String(cleanupError)}`; const retry = () => { if (cleaning || complete) return; cleaning = true; try { - removeVendorNetwork(network); + retryCleanup(); if (timer) clearTimeout(timer); timer = undefined; complete = true; diff --git a/agents/container/profile.ts b/agents/container/profile.ts index a1086b5..f47e5f8 100644 --- a/agents/container/profile.ts +++ b/agents/container/profile.ts @@ -34,6 +34,19 @@ export interface ProfileOptions { readonly network: VendorNetwork; readonly policy: PhasePolicy; readonly deferredOutput?: boolean; + /** Remaining invocation budget for Docker-backed profile validation. */ + readonly timeoutMs?: number; +} + +export class ProfileCreationCleanupError extends AggregateError { + readonly startupError: unknown; + readonly retryCleanup: () => void; + + constructor(startupError: unknown, cleanupError: unknown, retryCleanup: () => void) { + super([startupError, cleanupError], 'Profile creation and cleanup both failed.'); + this.startupError = startupError; + this.retryCleanup = retryCleanup; + } } interface FileIdentity { @@ -176,7 +189,7 @@ export function createContainerProfile(options: ProfileOptions): ContainerProfil assertTaskFilesystems(filesystems, invocation.clone); const invocationLeft = Math.floor(invocation.deadline - Date.now()); if (invocationLeft < 1) throw new Error('Invocation deadline has passed.'); - assertVendorNetwork(options.network, invocation, undefined, Math.min(30_000, invocationLeft)); + assertVendorNetwork(options.network, invocation, undefined, Math.min(options.timeoutMs ?? 30_000, invocationLeft)); if (claimedNetworks.has(options.network)) throw new Error('Vendor network already belongs to another container profile.'); // Own the network from here on, so any later failure removes it rather than leaking it. claimedNetworks.add(options.network); @@ -255,10 +268,14 @@ export function createContainerProfile(options: ProfileOptions): ContainerProfil deadline: invocation.deadline, network: options.network, policy: options.policy, invocation })); return profile; } catch (error) { - const failures: unknown[] = []; - try { removeOwnedDirectories(cleanupDirectories); } catch (cleanupError) { failures.push(cleanupError); } - try { removeVendorNetwork(options.network); } catch (cleanupError) { failures.push(cleanupError); } - if (failures.length) throw new AggregateError([error, ...failures], 'Profile creation and cleanup both failed.'); + const cleanupProfileResources = () => { + const failures: unknown[] = []; + try { removeOwnedDirectories(cleanupDirectories); } catch (cleanupError) { failures.push(cleanupError); } + try { removeVendorNetwork(options.network); } catch (cleanupError) { failures.push(cleanupError); } + if (failures.length) throw new AggregateError(failures, 'Profile resource cleanup did not settle.'); + }; + try { cleanupProfileResources(); } + catch (cleanupError) { throw new ProfileCreationCleanupError(error, cleanupError, cleanupProfileResources); } throw error; } } diff --git a/agents/network/network.ts b/agents/network/network.ts index 56c451a..9756134 100644 --- a/agents/network/network.ts +++ b/agents/network/network.ts @@ -18,6 +18,16 @@ interface NetworkIdentity { readonly allocationId: string; readonly imageId: str readonly subnet: string; readonly proxyIp: string; /** Daemon object IDs captured at creation; a same-named replacement has a different ID. */ readonly networkId: string; readonly proxyId: string } +export class VendorNetworkCreationCleanupError extends AggregateError { + readonly startupError: unknown; + readonly retryCleanup: () => void; + + constructor(startupError: unknown, cleanupError: unknown, retryCleanup: () => void) { + super([startupError, cleanupError], 'Vendor network creation and cleanup failed.'); + this.startupError = startupError; + this.retryCleanup = retryCleanup; + } +} const identities = new WeakMap(); const removedNetworks = new WeakSet(); const environment = () => ({ PATH: process.env.PATH, DOCKER_HOST: process.env.DOCKER_HOST }); @@ -168,6 +178,24 @@ export function createVendorNetwork(invocation: InvocationInput, imageId: string throw error; } }; + // The first cleanup shares the caller's overall deadline; a later retry gets its own budget. Killed + // creates get a settle window, bounded by whatever that budget has left. + const cleanupPlannedResources = (budget: () => number = deadline(30_000)) => { + let budgetLeft = 0; + try { budgetLeft = budget(); } catch { /* the budget is spent */ } + const settleBy = (object: string) => unsettled.has(object) + ? performance.now() + Math.min(CREATE_SETTLE_MS, budgetLeft) : 0; + const failures: unknown[] = []; + // Target the created IDs; names only for a create whose ID never came back, which alone gets a settle window. + const proxyTarget = proxyId ?? proxyContainer, networkTarget = networkId ?? name; + if (proxyPlanned) try { remove(['rm', '--force', proxyTarget], ['container', 'inspect', proxyTarget], + budget, 'vendor proxy', allocationId, proxyId ? 0 : settleBy(proxyContainer)); } + catch (cleanupError) { failures.push(cleanupError); } + if (networkPlanned) try { remove(['network', 'rm', networkTarget], ['network', 'inspect', networkTarget], + budget, 'vendor network', allocationId, networkId ? 0 : settleBy(name)); } + catch (cleanupError) { failures.push(cleanupError); } + if (failures.length) throw new AggregateError(failures, 'Vendor network cleanup did not settle.'); + }; try { networkPlanned = true; networkId = createdId(create(name, ['network', 'create', '--internal', '--driver', 'bridge', '--subnet', subnet, @@ -196,21 +224,10 @@ export function createVendorNetwork(invocation: InvocationInput, imageId: string remaining(); return network; } catch (error) { - const failures: unknown[] = []; - // Killed creates get a settle window, but only inside the cleanup reserve of the caller's budget. - let reserveLeft = 0; - try { reserveLeft = overall(); } catch { /* the overall budget is spent */ } - const settleBy = (object: string) => unsettled.has(object) - ? performance.now() + Math.min(CREATE_SETTLE_MS, reserveLeft) : 0; - const cleanupBudget = overall; - const proxyTarget = proxyId ?? proxyContainer, networkTarget = networkId ?? name; - if (proxyPlanned) try { remove(['rm', '--force', proxyTarget], ['container', 'inspect', proxyTarget], - cleanupBudget, 'vendor proxy', allocationId, proxyId ? 0 : settleBy(proxyContainer)); } - catch (cleanupError) { failures.push(cleanupError); } - if (networkPlanned) try { remove(['network', 'rm', networkTarget], ['network', 'inspect', networkTarget], - cleanupBudget, 'vendor network', allocationId, networkId ? 0 : settleBy(name)); } - catch (cleanupError) { failures.push(cleanupError); } - if (failures.length) throw new AggregateError([error, ...failures], 'Vendor network creation and cleanup failed.'); + try { cleanupPlannedResources(overall); } + catch (cleanupError) { + throw new VendorNetworkCreationCleanupError(error, cleanupError, () => cleanupPlannedResources()); + } throw error; } } diff --git a/test/agent-adapter.test.ts b/test/agent-adapter.test.ts index c79af29..799ecc0 100644 --- a/test/agent-adapter.test.ts +++ b/test/agent-adapter.test.ts @@ -1,11 +1,19 @@ import { describe, expect, it } from 'vitest'; -import { parseClaudeOutput } from '../agents/adapters/claude.ts'; -import { CODEX_OUTPUT_FILE } from '../agents/adapters/codex.ts'; -import { OUTPUT_LIMITS } from '../agents/adapters/supervisor.ts'; +import { parseClaudeOutput, startClaudeInvocation } from '../agents/adapters/claude.ts'; +import { CODEX_OUTPUT_FILE, startCodexInvocation } from '../agents/adapters/codex.ts'; +import { isInvocationActive, OUTPUT_LIMITS, retainSetupCleanup } from '../agents/adapters/supervisor.ts'; import { captureInvocation } from '../agents/contract.ts'; import { createCodexCommand, createPhasePolicy } from '../agents/policy.ts'; describe('production agent adapters', () => { + const capturedInvocation = (attemptId: string, deadline: number, vendor: 'codex' | 'claude' = 'codex') => + captureInvocation({ + clone: { id: 'clone', taskId: 'task', directory: '/tmp/task', head: 'a'.repeat(40) }, + phase: 'planning', vendor, approvedArgv: [], deadline, attemptId, + context: { snapshotId: 's', planId: 'p', planRevision: 1, assignmentId: 'a', + referencedCodeHash: 'c', stateVersion: 1 }, + }, deadline - 1); + it('parses recorded Claude success and failure envelopes', () => { expect(parseClaudeOutput(Buffer.from('{"result":"planned","is_error":false}'))) .toEqual({ text: 'planned', providerFailed: false }); @@ -32,4 +40,32 @@ describe('production agent adapters', () => { combinedBytes: 20 * 1024 * 1024 }); expect(Object.isFrozen(OUTPUT_LIMITS)).toBe(true); }); + + it.each(['codex', 'claude'] as const)('rejects expired %s setup before allocating a network', vendor => { + const invocation = capturedInvocation(`expired-${vendor}`, Date.now() - 1, vendor); + const request = { invocation, filesystems: {} as never, inputDirectory: '/unused', + imageId: `sha256:${'a'.repeat(64)}`, prompt: 'unused' }; + const start = () => vendor === 'codex' + ? startCodexInvocation(request, '/unused/auth.json') + : startClaudeInvocation(request, 'token'); + expect(start).toThrow('deadline expired during adapter setup'); + expect(isInvocationActive(invocation.attemptId)).toBe(false); + }); + + it('retains setup cleanup ownership until a retry succeeds', async () => { + const invocation = capturedInvocation('setup-recovery', Date.now() + 60_000); + let attempts = 0; + const handle = retainSetupCleanup(invocation, () => { + attempts += 1; + if (attempts === 1) throw new Error('still busy'); + }, new Error('startup failed'), new Error('cleanup failed')); + expect(isInvocationActive(invocation.attemptId)).toBe(true); + handle.cancel('cancelled'); + expect(isInvocationActive(invocation.attemptId)).toBe(true); + handle.cancel('cancelled'); + const result = await handle.settled; + expect(result.stopReason).toBe('capture-failure'); + expect(result.stderr).toContain('setup cleanup remains unsettled'); + expect(isInvocationActive(invocation.attemptId)).toBe(false); + }); }); From ee579177a5a554fc741e0bf57fd36ada19c45937 Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 20:00:02 -0700 Subject: [PATCH 13/32] Bound cancellation with monotonic deadlines --- agents/adapters/claude.ts | 9 ++------- agents/adapters/codex.ts | 9 ++------- agents/adapters/supervisor.ts | 18 ++++++++++++------ agents/adapters/types.ts | 17 +++++++++++++++++ test/agent-adapter.test.ts | 14 +++++++++++++- test/agent-supervisor.test.ts | 16 ++++++++++++++++ 6 files changed, 62 insertions(+), 21 deletions(-) diff --git a/agents/adapters/claude.ts b/agents/adapters/claude.ts index 834991e..a79f47e 100644 --- a/agents/adapters/claude.ts +++ b/agents/adapters/claude.ts @@ -4,7 +4,7 @@ import { createVendorNetwork, removeVendorNetwork, VendorNetworkCreationCleanupE type VendorNetwork } from '../network/network.ts'; import { createClaudeCommand, createPhasePolicy } from '../policy.ts'; import { retainNetworkCleanup, retainSetupCleanup, startProfileInvocation } from './supervisor.ts'; -import type { AgentAdapterOptions, AgentAdapterRequest } from './types.ts'; +import { createInvocationBudget, type AgentAdapterOptions, type AgentAdapterRequest } from './types.ts'; export function parseClaudeOutput(raw: Buffer): { text: string; providerFailed: boolean } { const envelope = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(raw)) as @@ -18,12 +18,7 @@ export function startClaudeInvocation(request: AgentAdapterRequest, oauthToken: string, options: AgentAdapterOptions = {}): InvocationHandle { if (!oauthToken || oauthToken.includes('\0')) throw new Error('Claude OAuth token is malformed.'); const policy = createPhasePolicy(request.invocation); - const remaining = () => { - const value = request.invocation.deadline - Date.now(); - if (!Number.isSafeInteger(value) || value < 1) - throw new Error('Invocation deadline expired during adapter setup.'); - return Math.min(60_000, value); - }; + const remaining = createInvocationBudget(request.invocation, 60_000); let network: VendorNetwork; try { network = createVendorNetwork(request.invocation, request.imageId, remaining()); } catch (error) { diff --git a/agents/adapters/codex.ts b/agents/adapters/codex.ts index 8c5d1f6..f0efc26 100644 --- a/agents/adapters/codex.ts +++ b/agents/adapters/codex.ts @@ -5,7 +5,7 @@ import { createVendorNetwork, removeVendorNetwork, VendorNetworkCreationCleanupE import { createCodexCommand, createPhasePolicy } from '../policy.ts'; import { readBoundedContainerFile, retainNetworkCleanup, retainSetupCleanup, startProfileInvocation } from './supervisor.ts'; -import type { AgentAdapterOptions, AgentAdapterRequest } from './types.ts'; +import { createInvocationBudget, type AgentAdapterOptions, type AgentAdapterRequest } from './types.ts'; export const CODEX_OUTPUT_FILE = '/run/codeboost-output/final.txt'; @@ -20,12 +20,7 @@ export function startCodexInvocation(request: AgentAdapterRequest, authFile: string, options: AgentAdapterOptions = {}): InvocationHandle { if (!authFile || authFile.includes('\0')) throw new Error('Codex auth path is malformed.'); const policy = createPhasePolicy(request.invocation); - const remaining = () => { - const value = request.invocation.deadline - Date.now(); - if (!Number.isSafeInteger(value) || value < 1) - throw new Error('Invocation deadline expired during adapter setup.'); - return Math.min(60_000, value); - }; + const remaining = createInvocationBudget(request.invocation, 60_000); let network: VendorNetwork; try { network = createVendorNetwork(request.invocation, request.imageId, remaining()); } catch (error) { diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index f8d8988..8e74399 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -218,12 +218,13 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super } catch (error) { return rejectWithCleanup(error); } - const now = Date.now(), deadline = Math.min(invocation.deadline, now + configuredTimeout); - if (!Number.isSafeInteger(deadline) || deadline <= now) { + const wallRemaining = invocation.deadline - Date.now(); + if (!Number.isSafeInteger(wallRemaining) || wallRemaining < 1) { return rejectWithCleanup(new Error('Invocation deadline has already expired.')); } + const duration = Math.min(wallRemaining, configuredTimeout), deadline = performance.now() + duration; const remaining = () => { - const value = deadline - Date.now(); + const value = Math.ceil(deadline - performance.now()); if (value < 1) throw new Error('Invocation deadline has already expired.'); return value; }; @@ -332,10 +333,15 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super if (!options.decode || decodedOutput || stopReason) return Promise.resolve(); decodePromise = (async () => { try { - const budget = deadline - Date.now(); + const budget = Math.ceil(deadline - performance.now()); if (budget < 1) throw new CaptureDeadlineError('Invocation deadline expired before output capture.'); const controller = new AbortController(); decodeAbort = controller; + const aborted = new Promise((_resolve, reject) => { + controller.signal.addEventListener('abort', () => reject(stopReason === 'timeout' + ? new CaptureDeadlineError('Adapter output capture exceeded the invocation deadline.') + : new Error('Adapter output capture was cancelled.')), { once: true }); + }); const raw = Buffer.concat(stdoutChunks, stdoutBytes); const operation = Promise.resolve(options.decode!(profile, raw, Math.max(1, Math.min(limits.stdoutBytes - stdoutBytes, limits.combinedBytes - combinedBytes)), @@ -349,7 +355,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super decodeTimer.unref(); }); let decoded: DecodedOutput; - try { decoded = await Promise.race([operation, timeout]); } + try { decoded = await Promise.race([operation, timeout, aborted]); } catch (error) { controller.abort(); let graceTimer: ReturnType | undefined; @@ -449,7 +455,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super child.stdout?.once('error', error => { failureDetail ??= error.message; stop('capture-failure'); }); child.stderr?.once('error', error => { failureDetail ??= error.message; stop('capture-failure'); }); child.once('error', error => { failureDetail ??= error.message; stop('capture-failure'); }); - later(() => stop('timeout'), Math.max(1, deadline - Date.now())); + later(() => stop('timeout'), Math.max(1, Math.ceil(deadline - performance.now()))); let resolveSettled!: (result: InvocationResult) => void; let wakeCleanup: (() => void) | undefined; diff --git a/agents/adapters/types.ts b/agents/adapters/types.ts index ab942c7..bcbaeaf 100644 --- a/agents/adapters/types.ts +++ b/agents/adapters/types.ts @@ -13,3 +13,20 @@ export interface AgentAdapterOptions { readonly timeoutMs?: number; readonly limits?: Partial; } + +/** Convert an absolute wall-clock deadline once, then enforce it with a monotonic clock. */ +export function createInvocationBudget(invocation: InvocationInput, maximumMs: number): () => number { + if (!Number.isSafeInteger(maximumMs) || maximumMs < 1) + throw new Error('Invocation setup budget must be a positive integer.'); + const wallRemaining = invocation.deadline - Date.now(); + if (!Number.isSafeInteger(wallRemaining) || wallRemaining < 1) + throw new Error('Invocation deadline expired during adapter setup.'); + const duration = Math.min(maximumMs, wallRemaining); + const end = performance.now() + duration; + return () => { + const value = Math.ceil(end - performance.now()); + if (!Number.isSafeInteger(value) || value < 1) + throw new Error('Invocation deadline expired during adapter setup.'); + return value; + }; +} diff --git a/test/agent-adapter.test.ts b/test/agent-adapter.test.ts index 799ecc0..ddda579 100644 --- a/test/agent-adapter.test.ts +++ b/test/agent-adapter.test.ts @@ -1,7 +1,8 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { parseClaudeOutput, startClaudeInvocation } from '../agents/adapters/claude.ts'; import { CODEX_OUTPUT_FILE, startCodexInvocation } from '../agents/adapters/codex.ts'; import { isInvocationActive, OUTPUT_LIMITS, retainSetupCleanup } from '../agents/adapters/supervisor.ts'; +import { createInvocationBudget } from '../agents/adapters/types.ts'; import { captureInvocation } from '../agents/contract.ts'; import { createCodexCommand, createPhasePolicy } from '../agents/policy.ts'; @@ -68,4 +69,15 @@ describe('production agent adapters', () => { expect(result.stderr).toContain('setup cleanup remains unsettled'); expect(isInvocationActive(invocation.attemptId)).toBe(false); }); + + it('does not extend an invocation budget when the wall clock moves backward', () => { + const wall = Date.now(); + const invocation = capturedInvocation('monotonic-budget', wall + 5_000); + const clock = vi.spyOn(Date, 'now').mockReturnValueOnce(wall).mockReturnValue(wall - 60_000); + try { + const remaining = createInvocationBudget(invocation, 1_000); + expect(remaining()).toBeGreaterThan(0); + expect(remaining()).toBeLessThanOrEqual(1_000); + } finally { clock.mockRestore(); } + }); }); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 17552c9..f4e72ec 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -181,6 +181,22 @@ describe('container invocation supervisor', () => { expect(isInvocationActive('decode-ignores-abort')).toBe(false); }, 30_000); + it('settles cancellation promptly when an injected decoder ignores abort', async () => { + let begin!: () => void; + const started = new Promise(resolve => { begin = resolve; }); + const handle = startProfileInvocation(profile(fixture(), 'finite-output', 'cancel-ignored-decode', 30_000), { + timeoutMs: 30_000, + decode: () => { begin(); return new Promise(() => {}); }, + }); + await started; + const cancelledAt = performance.now(); + handle.cancel('cancelled'); + const result = await handle.settled; + expect(result.stopReason).toBe('cancelled'); + expect(performance.now() - cancelledAt).toBeLessThan(5_000); + expect(isInvocationActive('cancel-ignored-decode')).toBe(false); + }, 15_000); + it('validates and decodes provider output even when the process exits nonzero', async () => { const result = await startProfileInvocation(profile(fixture(), 'nonzero-output', 'nonzero-decode'), { decode: (_current, raw) => ({ text: `decoded:${raw.toString('utf8')}` }), From fe8c11fcf39d3178f01804a58bbaeac9fcfdfc52 Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 20:19:32 -0700 Subject: [PATCH 14/32] Carry invocation ownership through settlement --- agents/adapters/claude.ts | 8 +++++--- agents/adapters/codex.ts | 7 ++++--- agents/adapters/supervisor.ts | 29 ++++++++++++++++++++--------- test/agent-adapter.test.ts | 21 +++++++++++++-------- test/agent-supervisor.test.ts | 13 +++++++++++++ 5 files changed, 55 insertions(+), 23 deletions(-) diff --git a/agents/adapters/claude.ts b/agents/adapters/claude.ts index a79f47e..5e3f965 100644 --- a/agents/adapters/claude.ts +++ b/agents/adapters/claude.ts @@ -18,9 +18,9 @@ export function startClaudeInvocation(request: AgentAdapterRequest, oauthToken: string, options: AgentAdapterOptions = {}): InvocationHandle { if (!oauthToken || oauthToken.includes('\0')) throw new Error('Claude OAuth token is malformed.'); const policy = createPhasePolicy(request.invocation); - const remaining = createInvocationBudget(request.invocation, 60_000); + const remaining = createInvocationBudget(request.invocation, 10 * 60_000); let network: VendorNetwork; - try { network = createVendorNetwork(request.invocation, request.imageId, remaining()); } + try { network = createVendorNetwork(request.invocation, request.imageId, Math.min(60_000, remaining())); } catch (error) { if (error instanceof VendorNetworkCreationCleanupError) return retainSetupCleanup(request.invocation, error.retryCleanup, error.startupError, error, @@ -29,8 +29,10 @@ export function startClaudeInvocation(request: AgentAdapterRequest, } try { const profile = createContainerProfile({ ...request, policy, network, - command: createClaudeCommand(policy, request.prompt), claudeToken: oauthToken, timeoutMs: remaining() }); + command: createClaudeCommand(policy, request.prompt), claudeToken: oauthToken, + timeoutMs: Math.min(60_000, remaining()) }); return startProfileInvocation(profile, { ...options, secrets: { CLAUDE_CODE_OAUTH_TOKEN: oauthToken }, + invocationBudget: remaining, decode: (_profile, raw) => parseClaudeOutput(raw) }); } catch (error) { if (error instanceof ProfileCreationCleanupError) { diff --git a/agents/adapters/codex.ts b/agents/adapters/codex.ts index f0efc26..8bd1672 100644 --- a/agents/adapters/codex.ts +++ b/agents/adapters/codex.ts @@ -20,9 +20,9 @@ export function startCodexInvocation(request: AgentAdapterRequest, authFile: string, options: AgentAdapterOptions = {}): InvocationHandle { if (!authFile || authFile.includes('\0')) throw new Error('Codex auth path is malformed.'); const policy = createPhasePolicy(request.invocation); - const remaining = createInvocationBudget(request.invocation, 60_000); + const remaining = createInvocationBudget(request.invocation, 10 * 60_000); let network: VendorNetwork; - try { network = createVendorNetwork(request.invocation, request.imageId, remaining()); } + try { network = createVendorNetwork(request.invocation, request.imageId, Math.min(60_000, remaining())); } catch (error) { if (error instanceof VendorNetworkCreationCleanupError) return retainSetupCleanup(request.invocation, error.retryCleanup, error.startupError, error, @@ -32,8 +32,9 @@ export function startCodexInvocation(request: AgentAdapterRequest, try { const profile = createContainerProfile({ ...request, policy, network, command: createCodexCommand(policy, request.prompt), codexAuthFile: authFile, deferredOutput: true, - timeoutMs: remaining() }); + timeoutMs: Math.min(60_000, remaining()) }); return startProfileInvocation(profile, { ...options, + invocationBudget: remaining, decode: (current, _raw, maximum, timeoutMs, signal) => readCodexOutput(current.name, maximum, timeoutMs, signal) }); } catch (error) { diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index 8e74399..4401fe9 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -15,6 +15,9 @@ const CAPTURE_ABORT_GRACE_MS = 1_000; const DIAGNOSTIC_BYTES = 1024; const active = new Map(); const cleanupRecoveries = new Set(); +const hasCleanupRecovery = (attemptId: string) => + Array.from(cleanupRecoveries).some(handle => handle.attemptId === attemptId); +const ownsAttempt = (attemptId: string) => active.has(attemptId) || hasCleanupRecovery(attemptId); export interface CaptureLimits { readonly stdoutBytes: number; @@ -31,6 +34,8 @@ export interface SupervisorOptions { readonly secrets?: Readonly>; readonly timeoutMs?: number; readonly limits?: Partial; + /** Trusted monotonic budget carried from synchronous adapter setup. */ + readonly invocationBudget?: () => number; readonly decode?: (profile: ContainerProfile, rawStdout: Buffer, maximumBytes: number, timeoutMs: number, signal: AbortSignal) => DecodedOutput | Promise; } @@ -62,7 +67,6 @@ const retainCleanupOwnership = (profile: ContainerProfile, detail: string, regis const schedule = () => { if (timer) return; timer = setTimeout(() => { timer = undefined; retry(); }, 1_000); - timer.unref(); }; const retry = () => { if (cleaning || complete) return; @@ -102,7 +106,7 @@ export function retainNetworkCleanup(invocation: InvocationInput, network: Vendo /** Retain attempt ownership while retrying resources allocated during synchronous adapter setup. */ export function retainSetupCleanup(invocation: InvocationInput, retryCleanup: () => void, startupError: unknown, cleanupError: unknown, kind = 'setup cleanup'): InvocationHandle { - const register = !active.has(invocation.attemptId); + const register = !ownsAttempt(invocation.attemptId); let resolveSettled!: (result: InvocationResult) => void, cleaning = false, complete = false; let handle!: InvocationHandle; let timer: ReturnType | undefined; @@ -125,7 +129,6 @@ export function retainSetupCleanup(invocation: InvocationInput, retryCleanup: () cleaning = false; if (!timer) { timer = setTimeout(() => { timer = undefined; retry(); }, 1_000); - timer.unref(); } return; } @@ -135,7 +138,7 @@ export function retainSetupCleanup(invocation: InvocationInput, retryCleanup: () cancel: () => { if (timer) clearTimeout(timer); timer = undefined; retry(); } }); if (register) active.set(invocation.attemptId, handle); else cleanupRecoveries.add(handle); - timer = setTimeout(() => { timer = undefined; retry(); }, 1_000); timer.unref(); + timer = setTimeout(() => { timer = undefined; retry(); }, 1_000); return handle; } const withDiagnostic = (stderr: Buffer, stdoutBytes: number, reason: StopReason, limits: CaptureLimits, @@ -192,7 +195,7 @@ export function readBoundedContainerFile(container: string, source: string, maxi } export function isInvocationActive(attemptId: string): boolean { - return active.has(attemptId); + return ownsAttempt(attemptId); } export function startProfileInvocation(profile: ContainerProfile, options: SupervisorOptions = {}): InvocationHandle { @@ -205,16 +208,20 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super } throw error; }; - if (active.has(invocation.attemptId)) { + if (ownsAttempt(invocation.attemptId)) { return rejectWithCleanup(new Error('An invocation with this attempt ID is still active.'), false); } - let limits: CaptureLimits, configuredTimeout: number; + let limits: CaptureLimits, configuredTimeout: number, carriedBudget: number; try { limits = captureLimits(options.limits); configuredTimeout = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; positiveInteger(configuredTimeout, 'timeoutMs'); if (configuredTimeout > DEFAULT_TIMEOUT_MS) throw new Error('timeoutMs cannot exceed the production ten-minute ceiling.'); + carriedBudget = options.invocationBudget?.() ?? DEFAULT_TIMEOUT_MS; + positiveInteger(carriedBudget, 'invocationBudget'); + if (carriedBudget > DEFAULT_TIMEOUT_MS) + throw new Error('invocationBudget cannot exceed the production ten-minute ceiling.'); } catch (error) { return rejectWithCleanup(error); } @@ -222,7 +229,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super if (!Number.isSafeInteger(wallRemaining) || wallRemaining < 1) { return rejectWithCleanup(new Error('Invocation deadline has already expired.')); } - const duration = Math.min(wallRemaining, configuredTimeout), deadline = performance.now() + duration; + const duration = Math.min(wallRemaining, configuredTimeout, carriedBudget), deadline = performance.now() + duration; const remaining = () => { const value = Math.ceil(deadline - performance.now()); if (value < 1) throw new Error('Invocation deadline has already expired.'); @@ -372,6 +379,8 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super if (decodeAbort === controller) decodeAbort = undefined; } if (stopReason) return; + if (performance.now() >= deadline) + throw new CaptureDeadlineError('Invocation deadline expired while decoding output.'); const additional = decoded.additionalBytes ?? 0; const textBytes = Buffer.byteLength(decoded.text); if (!Number.isSafeInteger(additional) || additional < 0) @@ -381,6 +390,8 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super throw new OutputLimitError('Decoded adapter output exceeds its capture limit.'); if (stdoutBytes + additional > limits.stdoutBytes || combinedBytes + additional > limits.combinedBytes) throw new OutputLimitError('Adapter output exceeds its capture limit.'); + if (performance.now() >= deadline) + throw new CaptureDeadlineError('Invocation deadline expired while validating decoded output.'); decodedOutput = decoded; } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -498,7 +509,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super await new Promise(resolve => { let finished = false; const wake = () => { if (finished) return; finished = true; clearTimeout(timer); resolve(); }; - const timer = setTimeout(wake, 1_000); timer.unref(); + const timer = setTimeout(wake, 1_000); wakeCleanup = wake; }); } diff --git a/test/agent-adapter.test.ts b/test/agent-adapter.test.ts index ddda579..97c2b1c 100644 --- a/test/agent-adapter.test.ts +++ b/test/agent-adapter.test.ts @@ -53,18 +53,23 @@ describe('production agent adapters', () => { expect(isInvocationActive(invocation.attemptId)).toBe(false); }); - it('retains setup cleanup ownership until a retry succeeds', async () => { + it('retains every colliding setup cleanup owner until all retries succeed', async () => { const invocation = capturedInvocation('setup-recovery', Date.now() + 60_000); - let attempts = 0; - const handle = retainSetupCleanup(invocation, () => { - attempts += 1; - if (attempts === 1) throw new Error('still busy'); + let releaseFirst = false, releaseSecond = false; + const first = retainSetupCleanup(invocation, () => { + if (!releaseFirst) throw new Error('first still busy'); }, new Error('startup failed'), new Error('cleanup failed')); + const second = retainSetupCleanup(invocation, () => { + if (!releaseSecond) throw new Error('second still busy'); + }, new Error('duplicate startup failed'), new Error('duplicate cleanup failed')); expect(isInvocationActive(invocation.attemptId)).toBe(true); - handle.cancel('cancelled'); + releaseFirst = true; + first.cancel('cancelled'); + await first.settled; expect(isInvocationActive(invocation.attemptId)).toBe(true); - handle.cancel('cancelled'); - const result = await handle.settled; + releaseSecond = true; + second.cancel('cancelled'); + const result = await second.settled; expect(result.stopReason).toBe('capture-failure'); expect(result.stderr).toContain('setup cleanup remains unsettled'); expect(isInvocationActive(invocation.attemptId)).toBe(false); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index f4e72ec..9af74bd 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -197,6 +197,19 @@ describe('container invocation supervisor', () => { expect(isInvocationActive('cancel-ignored-decode')).toBe(false); }, 15_000); + it('does not publish a synchronous decode that finishes after the monotonic deadline', async () => { + const result = await startProfileInvocation(profile(fixture(), 'finite-output', 'decode-over-deadline', 30_000), { + timeoutMs: 3_000, + decode: (_current, _raw, _maximum, timeoutMs) => { + const end = performance.now() + timeoutMs + 50; + while (performance.now() < end) { /* deliberately block the timer queue */ } + return { text: 'must-not-publish' }; + }, + }).settled; + expect(result.stopReason).toBe('timeout'); + expect(result.stdout).not.toContain('must-not-publish'); + }, 15_000); + it('validates and decodes provider output even when the process exits nonzero', async () => { const result = await startProfileInvocation(profile(fixture(), 'nonzero-output', 'nonzero-decode'), { decode: (_current, raw) => ({ text: `decoded:${raw.toString('utf8')}` }), From 82dbc837e514edc2878c078b3b0b051638aa11f6 Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 20:42:03 -0700 Subject: [PATCH 15/32] Guard active profile and close deadline --- agents/adapters/supervisor.ts | 8 +++++++- test/agent-supervisor.test.ts | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index 4401fe9..8887c71 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -14,6 +14,7 @@ const DEFAULT_TIMEOUT_MS = 10 * 60_000; const CAPTURE_ABORT_GRACE_MS = 1_000; const DIAGNOSTIC_BYTES = 1024; const active = new Map(); +const activeProfiles = new WeakSet(); const cleanupRecoveries = new Set(); const hasCleanupRecovery = (attemptId: string) => Array.from(cleanupRecoveries).some(handle => handle.attemptId === attemptId); @@ -209,6 +210,8 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super throw error; }; if (ownsAttempt(invocation.attemptId)) { + if (activeProfiles.has(profile)) + throw new Error('This container profile already owns the active invocation.'); return rejectWithCleanup(new Error('An invocation with this attempt ID is still active.'), false); } let limits: CaptureLimits, configuredTimeout: number, carriedBudget: number; @@ -477,10 +480,12 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super cancel: (reason: StopReason) => { stop(reason); wakeCleanup?.(); }, }); active.set(invocation.attemptId, handle); + activeProfiles.add(profile); child.once('close', async (code, signal) => { for (const timer of timers) clearTimeout(timer); timers.clear(); + if (!stopReason && performance.now() >= deadline) stopReason = 'timeout'; if (protocolBuffer.length) { if (!protocolLine(protocolBuffer)) capture('stderr', protocolBuffer); protocolBuffer = Buffer.alloc(0); @@ -493,7 +498,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super if (!stopReason && profile.deferredOutput && !decodedOutput) { stopReason = 'capture-failure'; failureDetail ??= 'Deferred output protocol did not complete.'; } - if (decodedOutput) { + if (decodedOutput && !stopReason) { finalStdout = Buffer.from(decodedOutput.text); if (decodedOutput.providerFailed) exitCode = exitCode === 0 ? 1 : exitCode; } @@ -519,6 +524,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super exitCode, signal: finalSignal, ...(stopReason ? { stopReason } : {}), stdout: finalStdout.toString('utf8'), stderr: finalStderr.toString('utf8') }); settlementComplete = true; + activeProfiles.delete(profile); active.delete(invocation.attemptId); resolveSettled(result); }); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 9af74bd..35f76e3 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -125,6 +125,15 @@ describe('container invocation supervisor', () => { expect(isInvocationActive('timeout')).toBe(false); }, 60_000); + it('records timeout when close delivery resumes after the monotonic deadline', async () => { + const handle = startProfileInvocation(profile(fixture(), 'finite-output', 'late-close-delivery', 30_000), + { timeoutMs: 3_000 }); + const end = performance.now() + 3_500; + while (performance.now() < end) { /* delay both close and timer delivery */ } + const result = await handle.settled; + expect(result.stopReason).toBe('timeout'); + }, 15_000); + it('blocks a duplicate attempt while the original container remains active', async () => { const data = fixture(), first = startProfileInvocation(profile(data, 'ignore-term', 'duplicate'), { timeoutMs: 30_000 }); expect(() => startProfileInvocation(profile(data, 'finite-output', 'duplicate'))).toThrow('still active'); @@ -133,6 +142,16 @@ describe('container invocation supervisor', () => { expect((await first.settled).stopReason).toBe('shutdown'); }, 60_000); + it('rejects reuse of the same active profile without disposing its container', async () => { + const current = profile(fixture(), 'ignore-term', 'same-profile-duplicate'); + const first = startProfileInvocation(current, { timeoutMs: 30_000 }); + expect(() => startProfileInvocation(current)).toThrow('already owns the active invocation'); + expect(isInvocationActive('same-profile-duplicate')).toBe(true); + expect(spawnSync('docker', ['container', 'inspect', current.name]).status).toBe(0); + first.cancel('shutdown'); + expect((await first.settled).stopReason).toBe('shutdown'); + }, 60_000); + it('records decoder failure without publishing a successful result', async () => { const handle = startProfileInvocation(profile(fixture(), 'finite-output', 'capture-failure'), { decode: () => { throw new Error('simulated capture failure'); }, From bee7c6429332480063ca79ab68e633be405756bf Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 20:52:05 -0700 Subject: [PATCH 16/32] Apply adapter timeout across setup --- agents/adapters/claude.ts | 4 ++-- agents/adapters/codex.ts | 4 ++-- agents/adapters/types.ts | 10 ++++++++++ test/agent-adapter.test.ts | 10 +++++++++- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/agents/adapters/claude.ts b/agents/adapters/claude.ts index 5e3f965..319f6eb 100644 --- a/agents/adapters/claude.ts +++ b/agents/adapters/claude.ts @@ -4,7 +4,7 @@ import { createVendorNetwork, removeVendorNetwork, VendorNetworkCreationCleanupE type VendorNetwork } from '../network/network.ts'; import { createClaudeCommand, createPhasePolicy } from '../policy.ts'; import { retainNetworkCleanup, retainSetupCleanup, startProfileInvocation } from './supervisor.ts'; -import { createInvocationBudget, type AgentAdapterOptions, type AgentAdapterRequest } from './types.ts'; +import { createAdapterInvocationBudget, type AgentAdapterOptions, type AgentAdapterRequest } from './types.ts'; export function parseClaudeOutput(raw: Buffer): { text: string; providerFailed: boolean } { const envelope = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(raw)) as @@ -18,7 +18,7 @@ export function startClaudeInvocation(request: AgentAdapterRequest, oauthToken: string, options: AgentAdapterOptions = {}): InvocationHandle { if (!oauthToken || oauthToken.includes('\0')) throw new Error('Claude OAuth token is malformed.'); const policy = createPhasePolicy(request.invocation); - const remaining = createInvocationBudget(request.invocation, 10 * 60_000); + const remaining = createAdapterInvocationBudget(request.invocation, options.timeoutMs); let network: VendorNetwork; try { network = createVendorNetwork(request.invocation, request.imageId, Math.min(60_000, remaining())); } catch (error) { diff --git a/agents/adapters/codex.ts b/agents/adapters/codex.ts index 8bd1672..21d9e94 100644 --- a/agents/adapters/codex.ts +++ b/agents/adapters/codex.ts @@ -5,7 +5,7 @@ import { createVendorNetwork, removeVendorNetwork, VendorNetworkCreationCleanupE import { createCodexCommand, createPhasePolicy } from '../policy.ts'; import { readBoundedContainerFile, retainNetworkCleanup, retainSetupCleanup, startProfileInvocation } from './supervisor.ts'; -import { createInvocationBudget, type AgentAdapterOptions, type AgentAdapterRequest } from './types.ts'; +import { createAdapterInvocationBudget, type AgentAdapterOptions, type AgentAdapterRequest } from './types.ts'; export const CODEX_OUTPUT_FILE = '/run/codeboost-output/final.txt'; @@ -20,7 +20,7 @@ export function startCodexInvocation(request: AgentAdapterRequest, authFile: string, options: AgentAdapterOptions = {}): InvocationHandle { if (!authFile || authFile.includes('\0')) throw new Error('Codex auth path is malformed.'); const policy = createPhasePolicy(request.invocation); - const remaining = createInvocationBudget(request.invocation, 10 * 60_000); + const remaining = createAdapterInvocationBudget(request.invocation, options.timeoutMs); let network: VendorNetwork; try { network = createVendorNetwork(request.invocation, request.imageId, Math.min(60_000, remaining())); } catch (error) { diff --git a/agents/adapters/types.ts b/agents/adapters/types.ts index bcbaeaf..01291b9 100644 --- a/agents/adapters/types.ts +++ b/agents/adapters/types.ts @@ -13,6 +13,7 @@ export interface AgentAdapterOptions { readonly timeoutMs?: number; readonly limits?: Partial; } +const MAXIMUM_INVOCATION_MS = 10 * 60_000; /** Convert an absolute wall-clock deadline once, then enforce it with a monotonic clock. */ export function createInvocationBudget(invocation: InvocationInput, maximumMs: number): () => number { @@ -30,3 +31,12 @@ export function createInvocationBudget(invocation: InvocationInput, maximumMs: n return value; }; } + +export function createAdapterInvocationBudget(invocation: InvocationInput, + timeoutMs = MAXIMUM_INVOCATION_MS): () => number { + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1) + throw new Error('timeoutMs must be a positive integer.'); + if (timeoutMs > MAXIMUM_INVOCATION_MS) + throw new Error('timeoutMs cannot exceed the production ten-minute ceiling.'); + return createInvocationBudget(invocation, timeoutMs); +} diff --git a/test/agent-adapter.test.ts b/test/agent-adapter.test.ts index 97c2b1c..2c0af9f 100644 --- a/test/agent-adapter.test.ts +++ b/test/agent-adapter.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it, vi } from 'vitest'; import { parseClaudeOutput, startClaudeInvocation } from '../agents/adapters/claude.ts'; import { CODEX_OUTPUT_FILE, startCodexInvocation } from '../agents/adapters/codex.ts'; import { isInvocationActive, OUTPUT_LIMITS, retainSetupCleanup } from '../agents/adapters/supervisor.ts'; -import { createInvocationBudget } from '../agents/adapters/types.ts'; +import { createAdapterInvocationBudget, createInvocationBudget } from '../agents/adapters/types.ts'; import { captureInvocation } from '../agents/contract.ts'; import { createCodexCommand, createPhasePolicy } from '../agents/policy.ts'; @@ -85,4 +85,12 @@ describe('production agent adapters', () => { expect(remaining()).toBeLessThanOrEqual(1_000); } finally { clock.mockRestore(); } }); + + it('applies the configured timeout to the original adapter setup budget', () => { + const invocation = capturedInvocation('configured-budget', Date.now() + 60_000); + const remaining = createAdapterInvocationBudget(invocation, 250); + expect(remaining()).toBeGreaterThan(0); + expect(remaining()).toBeLessThanOrEqual(250); + expect(() => createAdapterInvocationBudget(invocation, 10 * 60_000 + 1)).toThrow('ten-minute ceiling'); + }); }); From 0988f12a389f318f91e60727acec4913914d9768 Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 21:04:52 -0700 Subject: [PATCH 17/32] Bound adapter cleanup and final stderr --- agents/adapters/claude.ts | 10 +++++----- agents/adapters/codex.ts | 10 +++++----- agents/adapters/supervisor.ts | 13 ++++++++----- agents/network/network.ts | 4 ++-- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/agents/adapters/claude.ts b/agents/adapters/claude.ts index 319f6eb..902d3dd 100644 --- a/agents/adapters/claude.ts +++ b/agents/adapters/claude.ts @@ -36,18 +36,18 @@ export function startClaudeInvocation(request: AgentAdapterRequest, decode: (_profile, raw) => parseClaudeOutput(raw) }); } catch (error) { if (error instanceof ProfileCreationCleanupError) { - const retryCleanup = () => { + const retryCleanup = (networkTimeoutMs = 30_000) => { const failures: unknown[] = []; try { error.retryCleanup(); } catch (cleanupError) { failures.push(cleanupError); } - try { removeVendorNetwork(network); } catch (cleanupError) { failures.push(cleanupError); } + try { removeVendorNetwork(network, networkTimeoutMs); } catch (cleanupError) { failures.push(cleanupError); } if (failures.length) throw new AggregateError(failures, 'Adapter setup cleanup did not settle.'); }; - try { retryCleanup(); } - catch (cleanupError) { return retainSetupCleanup(request.invocation, retryCleanup, + try { retryCleanup(Math.min(30_000, remaining())); } + catch (cleanupError) { return retainSetupCleanup(request.invocation, () => retryCleanup(), error.startupError, cleanupError, 'profile and network cleanup'); } throw error.startupError; } - try { removeVendorNetwork(network); } + try { removeVendorNetwork(network, Math.min(30_000, remaining())); } catch (cleanupError) { return retainNetworkCleanup(request.invocation, network, error, cleanupError); } throw error; } diff --git a/agents/adapters/codex.ts b/agents/adapters/codex.ts index 21d9e94..158f8fe 100644 --- a/agents/adapters/codex.ts +++ b/agents/adapters/codex.ts @@ -39,18 +39,18 @@ export function startCodexInvocation(request: AgentAdapterRequest, readCodexOutput(current.name, maximum, timeoutMs, signal) }); } catch (error) { if (error instanceof ProfileCreationCleanupError) { - const retryCleanup = () => { + const retryCleanup = (networkTimeoutMs = 30_000) => { const failures: unknown[] = []; try { error.retryCleanup(); } catch (cleanupError) { failures.push(cleanupError); } - try { removeVendorNetwork(network); } catch (cleanupError) { failures.push(cleanupError); } + try { removeVendorNetwork(network, networkTimeoutMs); } catch (cleanupError) { failures.push(cleanupError); } if (failures.length) throw new AggregateError(failures, 'Adapter setup cleanup did not settle.'); }; - try { retryCleanup(); } - catch (cleanupError) { return retainSetupCleanup(request.invocation, retryCleanup, + try { retryCleanup(Math.min(30_000, remaining())); } + catch (cleanupError) { return retainSetupCleanup(request.invocation, () => retryCleanup(), error.startupError, cleanupError, 'profile and network cleanup'); } throw error.startupError; } - try { removeVendorNetwork(network); } + try { removeVendorNetwork(network, Math.min(30_000, remaining())); } catch (cleanupError) { return retainNetworkCleanup(request.invocation, network, error, cleanupError); } throw error; } diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index 8887c71..3cc33fb 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -315,8 +315,8 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super decodeAbort?.abort(); if (!closed) terminate(); }; - const capture = (stream: 'stdout' | 'stderr', value: Buffer | string) => { - if (stopReason || closed) return; + const capture = (stream: 'stdout' | 'stderr', value: Buffer | string, final = false) => { + if (!final && (stopReason || closed)) return; const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value); const streamBytes = stream === 'stdout' ? stdoutBytes : stderrBytes; const streamLimit = stream === 'stdout' ? limits.stdoutBytes : limits.stderrBytes; @@ -328,7 +328,10 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super else stderrBytes += retained.length; combinedBytes += retained.length; } - if (chunk.length > available) stop('output-limit'); + if (chunk.length > available) { + if (final) stopReason ??= 'output-limit'; + else stop('output-limit'); + } }; const consumeProtocol = (length: number) => { const available = Math.max(0, Math.min(limits.stderrBytes - stderrBytes, @@ -486,11 +489,11 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super for (const timer of timers) clearTimeout(timer); timers.clear(); if (!stopReason && performance.now() >= deadline) stopReason = 'timeout'; + closed = true; if (protocolBuffer.length) { - if (!protocolLine(protocolBuffer)) capture('stderr', protocolBuffer); + if (!protocolLine(protocolBuffer)) capture('stderr', protocolBuffer, true); protocolBuffer = Buffer.alloc(0); } - closed = true; let finalStdout = Buffer.concat(stdoutChunks, stdoutBytes), finalStderr = Buffer.concat(stderrChunks); let exitCode = code, finalSignal = signal; if (!stopReason && options.decode && !profile.deferredOutput) await decodeOutput(); diff --git a/agents/network/network.ts b/agents/network/network.ts index 9756134..e62fb08 100644 --- a/agents/network/network.ts +++ b/agents/network/network.ts @@ -232,7 +232,7 @@ export function createVendorNetwork(invocation: InvocationInput, imageId: string } } -export function removeVendorNetwork(network: VendorNetwork): void { +export function removeVendorNetwork(network: VendorNetwork, timeoutMs = 30_000): void { const identity = identities.get(network); if (!identity) { if (removedNetworks.has(network)) return; @@ -240,7 +240,7 @@ export function removeVendorNetwork(network: VendorNetwork): void { } assertBuiltAgentImage(identity.imageId); const allocationId = identity.allocationId; - const remaining = deadline(30_000), failures: unknown[] = []; + const remaining = deadline(timeoutMs), failures: unknown[] = []; // Remove by the captured IDs; a same-named replacement is not ours to delete and keeps the network busy. try { remove(['rm', '--force', identity.proxyId], ['container', 'inspect', identity.proxyId], remaining, 'vendor proxy', allocationId); } catch (error) { failures.push(error); } From 1f7cf366672b66001978d4eb9f176a616290658d Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 21:16:52 -0700 Subject: [PATCH 18/32] Keep decoder settlement timers alive --- agents/adapters/supervisor.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index 3cc33fb..2f0edca 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -365,7 +365,6 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super stop('timeout'); reject(new CaptureDeadlineError('Adapter output capture exceeded the invocation deadline.')); }, budget); - decodeTimer.unref(); }); let decoded: DecodedOutput; try { decoded = await Promise.race([operation, timeout, aborted]); } @@ -374,7 +373,6 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super let graceTimer: ReturnType | undefined; const grace = new Promise(resolve => { graceTimer = setTimeout(resolve, CAPTURE_ABORT_GRACE_MS); - graceTimer.unref(); }); await Promise.race([operation.then(() => undefined, () => undefined), grace]); if (graceTimer) clearTimeout(graceTimer); From 4c1dc2e7677b2aea675507c86773c84e56c2a88b Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 21:29:22 -0700 Subject: [PATCH 19/32] Retain recovery profile ownership --- agents/adapters/supervisor.ts | 7 +++++-- test/agent-supervisor.test.ts | 12 ++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index 2f0edca..29e84e9 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -78,6 +78,7 @@ const retainCleanupOwnership = (profile: ContainerProfile, detail: string, regis timer = undefined; complete = true; if (active.get(invocation.attemptId) === handle) active.delete(invocation.attemptId); + activeProfiles.delete(profile); cleanupRecoveries.delete(handle); resolveSettled(Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, exitCode: null, signal: null, stopReason: 'capture-failure', stdout: '', @@ -91,6 +92,7 @@ const retainCleanupOwnership = (profile: ContainerProfile, detail: string, regis }; handle = Object.freeze({ attemptId: invocation.attemptId, settled, cancel: () => { if (timer) clearTimeout(timer); timer = undefined; retry(); } }); + activeProfiles.add(profile); if (register) active.set(invocation.attemptId, handle); else cleanupRecoveries.add(handle); schedule(); @@ -399,8 +401,9 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super decodedOutput = decoded; } catch (error) { const message = error instanceof Error ? error.message : String(error); - const reason = error instanceof OutputLimitError || /exceeds its capture limit/i.test(message) - ? 'output-limit' : error instanceof CaptureDeadlineError ? 'timeout' : 'capture-failure'; + const reason: StopReason = stopReason ?? (performance.now() >= deadline ? 'timeout' + : error instanceof OutputLimitError || /exceeds its capture limit/i.test(message) + ? 'output-limit' : error instanceof CaptureDeadlineError ? 'timeout' : 'capture-failure'); failureDetail ??= message; if (closed) stopReason ??= reason; else stop(reason); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 35f76e3..c03f0cf 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -229,6 +229,18 @@ describe('container invocation supervisor', () => { expect(result.stdout).not.toContain('must-not-publish'); }, 15_000); + it('classifies a decoder failure after the monotonic deadline as timeout', async () => { + const result = await startProfileInvocation(profile(fixture(), 'finite-output', 'decode-fails-late', 30_000), { + timeoutMs: 3_000, + decode: (_current, _raw, _maximum, timeoutMs) => { + const end = performance.now() + timeoutMs + 50; + while (performance.now() < end) { /* deliberately block the timer queue */ } + throw new Error('late decoder failure'); + }, + }).settled; + expect(result.stopReason).toBe('timeout'); + }, 15_000); + it('validates and decodes provider output even when the process exits nonzero', async () => { const result = await startProfileInvocation(profile(fixture(), 'nonzero-output', 'nonzero-decode'), { decode: (_current, raw) => ({ text: `decoded:${raw.toString('utf8')}` }), From 8b963a4c43f3669f32968b1dcfaaec49d3af6e88 Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 21:42:15 -0700 Subject: [PATCH 20/32] Revalidate container at launch boundary --- agents/adapters/supervisor.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index 29e84e9..3bc1f41 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -242,7 +242,6 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super }; try { createValidatedContainer(profile, remaining(), options.secrets ?? {}); - validateContainer(profile.name, profile, remaining()); } catch (error) { try { disposeValidatedContainer(profile); } catch (cleanupError) { @@ -260,6 +259,15 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super let decodeAbort: AbortController | undefined; let protocolToken: string | undefined, protocolStarted = false, protocolReady = false; let protocolBuffer = Buffer.alloc(0); + try { validateContainer(profile.name, profile, remaining()); } + catch (error) { + try { disposeValidatedContainer(profile); } + catch (cleanupError) { + const detail = `Final container validation failed and cleanup remains unsettled: ${String(error)}; ${String(cleanupError)}`; + return retainCleanupOwnership(profile, detail); + } + throw error; + } const child = spawn('docker', ['start', '--attach', profile.name], { env: dockerEnvironment(), stdio: ['ignore', 'pipe', 'pipe'], }); From bf65dec45afa8fdecb580a100991a73bb79a0708 Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 21:58:32 -0700 Subject: [PATCH 21/32] Validate profile capability before cleanup --- agents/adapters/supervisor.ts | 3 ++- agents/container/profile.ts | 5 +++++ test/agent-supervisor.test.ts | 11 +++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index 3bc1f41..7380502 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -2,7 +2,7 @@ import { execFile, spawn, type ChildProcess } from 'node:child_process'; import type { InvocationHandle, InvocationInput, InvocationResult, StopReason } from '../contract.ts'; import { assertPhasePolicy } from '../policy.ts'; import { createValidatedContainer, disposeValidatedContainer, validateContainer } from '../container/run.ts'; -import type { ContainerProfile } from '../container/profile.ts'; +import { assertContainerProfileAuthenticity, type ContainerProfile } from '../container/profile.ts'; import { removeVendorNetwork, type VendorNetwork } from '../network/network.ts'; export const OUTPUT_LIMITS = Object.freeze({ @@ -202,6 +202,7 @@ export function isInvocationActive(attemptId: string): boolean { } export function startProfileInvocation(profile: ContainerProfile, options: SupervisorOptions = {}): InvocationHandle { + assertContainerProfileAuthenticity(profile); const invocation = assertPhasePolicy(profile.policy); const rejectWithCleanup = (error: unknown, register = true): InvocationHandle => { try { disposeValidatedContainer(profile); } diff --git a/agents/container/profile.ts b/agents/container/profile.ts index f47e5f8..821aba9 100644 --- a/agents/container/profile.ts +++ b/agents/container/profile.ts @@ -127,6 +127,11 @@ const captureInput = (directory: string): InputCapture => { return Object.freeze({ inputDirectory: canonical, schema, content: captured.content }); }; +/** Prove that a profile object is the exact capability issued by this module. */ +export function assertContainerProfileAuthenticity(profile: ContainerProfile): void { + if (!identities.has(profile)) throw new Error('Container profile was not created by the trusted profile builder.'); +} + /** Internal authenticity and host-file revalidation used at every launch boundary. */ export function assertContainerProfile(profile: ContainerProfile, timeoutMs = 30_000): void { const expected = identities.get(profile); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index c03f0cf..7e6cdae 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -152,6 +152,17 @@ describe('container invocation supervisor', () => { expect((await first.settled).stopReason).toBe('shutdown'); }, 60_000); + it('rejects a cloned profile without disposing the authentic active container', async () => { + const current = profile(fixture(), 'ignore-term', 'cloned-profile'); + const first = startProfileInvocation(current, { timeoutMs: 30_000 }); + const clone = Object.freeze({ ...current }); + expect(() => startProfileInvocation(clone)).toThrow('not created by the trusted profile builder'); + expect(isInvocationActive('cloned-profile')).toBe(true); + expect(spawnSync('docker', ['container', 'inspect', current.name]).status).toBe(0); + first.cancel('shutdown'); + expect((await first.settled).stopReason).toBe('shutdown'); + }, 60_000); + it('records decoder failure without publishing a successful result', async () => { const handle = startProfileInvocation(profile(fixture(), 'finite-output', 'capture-failure'), { decode: () => { throw new Error('simulated capture failure'); }, From 27059e3f3b596b932d2d6fb61c23b818e0cf2d06 Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 22:14:43 -0700 Subject: [PATCH 22/32] Preserve cleanup cancellation reasons --- agents/adapters/supervisor.ts | 24 ++++++++++++++++++------ test/agent-supervisor.test.ts | 20 +++++++++++++++++++- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index 7380502..dfc05a3 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -62,6 +62,7 @@ const diagnosticFor = (reason: StopReason, detail?: string) => Buffer.from( const retainCleanupOwnership = (profile: ContainerProfile, detail: string, register = true): InvocationHandle => { const invocation = assertPhasePolicy(profile.policy); let resolveSettled!: (result: InvocationResult) => void, cleaning = false, complete = false; + let cancelReason: StopReason | undefined; let handle!: InvocationHandle; let timer: ReturnType | undefined; const settled = new Promise(resolve => { resolveSettled = resolve; }); @@ -81,8 +82,8 @@ const retainCleanupOwnership = (profile: ContainerProfile, detail: string, regis activeProfiles.delete(profile); cleanupRecoveries.delete(handle); resolveSettled(Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, - exitCode: null, signal: null, stopReason: 'capture-failure', stdout: '', - stderr: diagnosticFor('capture-failure', detail).toString('utf8') })); + exitCode: null, signal: null, stopReason: cancelReason ?? 'capture-failure', stdout: '', + stderr: diagnosticFor(cancelReason ?? 'capture-failure', detail).toString('utf8') })); } catch { cleaning = false; schedule(); @@ -91,7 +92,12 @@ const retainCleanupOwnership = (profile: ContainerProfile, detail: string, regis cleaning = false; }; handle = Object.freeze({ attemptId: invocation.attemptId, settled, - cancel: () => { if (timer) clearTimeout(timer); timer = undefined; retry(); } }); + cancel: (reason: StopReason) => { + cancelReason ??= reason; + if (timer) clearTimeout(timer); + timer = undefined; + retry(); + } }); activeProfiles.add(profile); if (register) active.set(invocation.attemptId, handle); else cleanupRecoveries.add(handle); @@ -111,6 +117,7 @@ export function retainSetupCleanup(invocation: InvocationInput, retryCleanup: () startupError: unknown, cleanupError: unknown, kind = 'setup cleanup'): InvocationHandle { const register = !ownsAttempt(invocation.attemptId); let resolveSettled!: (result: InvocationResult) => void, cleaning = false, complete = false; + let cancelReason: StopReason | undefined; let handle!: InvocationHandle; let timer: ReturnType | undefined; const settled = new Promise(resolve => { resolveSettled = resolve; }); @@ -126,8 +133,8 @@ export function retainSetupCleanup(invocation: InvocationInput, retryCleanup: () if (active.get(invocation.attemptId) === handle) active.delete(invocation.attemptId); cleanupRecoveries.delete(handle); resolveSettled(Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, - exitCode: null, signal: null, stopReason: 'capture-failure', stdout: '', - stderr: diagnosticFor('capture-failure', detail).toString('utf8') })); + exitCode: null, signal: null, stopReason: cancelReason ?? 'capture-failure', stdout: '', + stderr: diagnosticFor(cancelReason ?? 'capture-failure', detail).toString('utf8') })); } catch { cleaning = false; if (!timer) { @@ -138,7 +145,12 @@ export function retainSetupCleanup(invocation: InvocationInput, retryCleanup: () cleaning = false; }; handle = Object.freeze({ attemptId: invocation.attemptId, settled, - cancel: () => { if (timer) clearTimeout(timer); timer = undefined; retry(); } }); + cancel: (reason: StopReason) => { + cancelReason ??= reason; + if (timer) clearTimeout(timer); + timer = undefined; + retry(); + } }); if (register) active.set(invocation.attemptId, handle); else cleanupRecoveries.add(handle); timer = setTimeout(() => { timer = undefined; retry(); }, 1_000); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 7e6cdae..0211f21 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -5,7 +5,8 @@ import { join } from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { startClaudeInvocation } from '../agents/adapters/claude.ts'; import { readCodexOutput, startCodexInvocation } from '../agents/adapters/codex.ts'; -import { isInvocationActive, readBoundedContainerFile, startProfileInvocation } from '../agents/adapters/supervisor.ts'; +import { isInvocationActive, readBoundedContainerFile, retainSetupCleanup, + startProfileInvocation } from '../agents/adapters/supervisor.ts'; import { captureInvocation, type InvocationInput } from '../agents/contract.ts'; import { buildAgentImage } from '../agents/container/image.ts'; import { createContainerProfile, disposeContainerProfile, type ContainerProfile } from '../agents/container/profile.ts'; @@ -62,6 +63,23 @@ afterAll(() => { }, 3 * 60_000); describe('container invocation supervisor', () => { + it('preserves the first cancellation reason while retained setup cleanup settles', async () => { + const data = fixture(), captured = invocation(data, 'cancel-setup-cleanup'); + let attempts = 0; + const handle = retainSetupCleanup(captured, () => { + attempts += 1; + if (attempts === 1) return; + throw new Error('unexpected repeated cleanup'); + }, new Error('startup failed'), new Error('cleanup failed')); + handle.cancel('shutdown'); + handle.cancel('cancelled'); + const result = await handle.settled; + expect(result.stopReason).toBe('shutdown'); + expect(result.stderr).toContain('[codeboost: shutdown:'); + expect(attempts).toBe(1); + expect(isInvocationActive('cancel-setup-cleanup')).toBe(false); + }); + it('captures finite output and releases ownership only after cleanup', async () => { const current = profile(fixture(), 'finite-output', 'finite'); const handle = startProfileInvocation(current); From a5dfaba66b624f10c1694459ecae3ccd2f9efe9e Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 22:21:06 -0700 Subject: [PATCH 23/32] Update cleanup cancellation regression --- test/agent-adapter.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/agent-adapter.test.ts b/test/agent-adapter.test.ts index 2c0af9f..0d727c4 100644 --- a/test/agent-adapter.test.ts +++ b/test/agent-adapter.test.ts @@ -70,7 +70,8 @@ describe('production agent adapters', () => { releaseSecond = true; second.cancel('cancelled'); const result = await second.settled; - expect(result.stopReason).toBe('capture-failure'); + expect(result.stopReason).toBe('cancelled'); + expect(result.stderr).toContain('[codeboost: cancelled:'); expect(result.stderr).toContain('setup cleanup remains unsettled'); expect(isInvocationActive(invocation.attemptId)).toBe(false); }); From 10c8d527374b764cd0c3e68e0dc544b3fa5a2a04 Mon Sep 17 00:00:00 2001 From: mchwang Date: Thu, 24 Sep 2026 22:34:22 -0700 Subject: [PATCH 24/32] Authenticate profiles at disposal boundary --- agents/container/run.ts | 4 +++- test/agent-supervisor.test.ts | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/agents/container/run.ts b/agents/container/run.ts index f809fe3..cdc09a6 100644 --- a/agents/container/run.ts +++ b/agents/container/run.ts @@ -1,6 +1,7 @@ import { execFileSync, spawnSync } from 'node:child_process'; import { realpathSync } from 'node:fs'; -import { assertContainerProfile, disposeContainerProfile, isContainerProfileAuthentic, profileTimeout, +import { assertContainerProfile, assertContainerProfileAuthenticity, disposeContainerProfile, + isContainerProfileAuthentic, profileTimeout, type ContainerProfile } from './profile.ts'; import { BASE_IMAGE, CLAUDE_VERSION, CODEX_VERSION } from './image.ts'; import { taskFilesystemAllocationId } from './storage.ts'; @@ -90,6 +91,7 @@ const removeContainerOrThrow = (profile: ContainerProfile, createUnsettled = fal /** Remove a validated invocation container, then its profile-owned staging and network resources. */ export function disposeValidatedContainer(profile: ContainerProfile): void { + assertContainerProfileAuthenticity(profile); removeContainerOrThrow(profile); } diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 0211f21..f69b21a 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -10,7 +10,7 @@ import { isInvocationActive, readBoundedContainerFile, retainSetupCleanup, import { captureInvocation, type InvocationInput } from '../agents/contract.ts'; import { buildAgentImage } from '../agents/container/image.ts'; import { createContainerProfile, disposeContainerProfile, type ContainerProfile } from '../agents/container/profile.ts'; -import { prepareTaskFilesystems, removeTaskFilesystems } from '../agents/container/run.ts'; +import { disposeValidatedContainer, prepareTaskFilesystems, removeTaskFilesystems } from '../agents/container/run.ts'; import { createVendorNetwork } from '../agents/network/network.ts'; import { createIsolationProbeCommand, createPhasePolicy, type IsolationProbe } from '../agents/policy.ts'; import { createTaskClone } from '../git/clone.ts'; @@ -174,6 +174,7 @@ describe('container invocation supervisor', () => { const current = profile(fixture(), 'ignore-term', 'cloned-profile'); const first = startProfileInvocation(current, { timeoutMs: 30_000 }); const clone = Object.freeze({ ...current }); + expect(() => disposeValidatedContainer(clone)).toThrow('not created by the trusted profile builder'); expect(() => startProfileInvocation(clone)).toThrow('not created by the trusted profile builder'); expect(isInvocationActive('cloned-profile')).toBe(true); expect(spawnSync('docker', ['container', 'inspect', current.name]).status).toBe(0); From d9539fbb2b2e6acb19352ebdb38749dfe6eb68c9 Mon Sep 17 00:00:00 2001 From: mchwang Date: Fri, 25 Sep 2026 10:05:42 -0700 Subject: [PATCH 25/32] Skip cleanup for unauthenticated profiles and stop on late close Commit pending work left in the D4 worktree: - Add isContainerProfileAuthentic and skip disposal when container validation fails for a profile whose identity was already removed. - Route a close observed after the deadline through stop('timeout') so a running decoder is aborted. Co-Authored-By: Claude Opus 5.5 --- agents/adapters/supervisor.ts | 6 ++++-- agents/container/profile.ts | 8 ++++---- test/agent-container.test.ts | 4 +++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index dfc05a3..18d1e19 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -2,7 +2,8 @@ import { execFile, spawn, type ChildProcess } from 'node:child_process'; import type { InvocationHandle, InvocationInput, InvocationResult, StopReason } from '../contract.ts'; import { assertPhasePolicy } from '../policy.ts'; import { createValidatedContainer, disposeValidatedContainer, validateContainer } from '../container/run.ts'; -import { assertContainerProfileAuthenticity, type ContainerProfile } from '../container/profile.ts'; +import { assertContainerProfileAuthenticity, isContainerProfileAuthentic, + type ContainerProfile } from '../container/profile.ts'; import { removeVendorNetwork, type VendorNetwork } from '../network/network.ts'; export const OUTPUT_LIMITS = Object.freeze({ @@ -256,6 +257,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super try { createValidatedContainer(profile, remaining(), options.secrets ?? {}); } catch (error) { + if (!isContainerProfileAuthentic(profile)) throw error; try { disposeValidatedContainer(profile); } catch (cleanupError) { const detail = `Container validation failed and cleanup remains unsettled: ${String(error)}; ${String(cleanupError)}`; @@ -510,8 +512,8 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super child.once('close', async (code, signal) => { for (const timer of timers) clearTimeout(timer); timers.clear(); - if (!stopReason && performance.now() >= deadline) stopReason = 'timeout'; closed = true; + if (!stopReason && performance.now() >= deadline) stop('timeout'); if (protocolBuffer.length) { if (!protocolLine(protocolBuffer)) capture('stderr', protocolBuffer, true); protocolBuffer = Buffer.alloc(0); diff --git a/agents/container/profile.ts b/agents/container/profile.ts index 821aba9..cb8754d 100644 --- a/agents/container/profile.ts +++ b/agents/container/profile.ts @@ -132,6 +132,10 @@ export function assertContainerProfileAuthenticity(profile: ContainerProfile): v if (!identities.has(profile)) throw new Error('Container profile was not created by the trusted profile builder.'); } +export function isContainerProfileAuthentic(profile: ContainerProfile): boolean { + return identities.has(profile); +} + /** Internal authenticity and host-file revalidation used at every launch boundary. */ export function assertContainerProfile(profile: ContainerProfile, timeoutMs = 30_000): void { const expected = identities.get(profile); @@ -149,10 +153,6 @@ export function assertContainerProfile(profile: ContainerProfile, timeoutMs = 30 } } -export function isContainerProfileAuthentic(profile: ContainerProfile): boolean { - return identities.has(profile); -} - /** Clamp a Docker budget to the captured invocation deadline, which no launch may outlive. */ export function profileTimeout(profile: ContainerProfile, timeoutMs: number, now = Date.now()): number { const expected = identities.get(profile); diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index dd2295f..4eb5525 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -6,7 +6,8 @@ import { join } from 'node:path'; import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; import { captureInvocation, type InvocationInput, type Phase } from '../agents/contract.ts'; import { AGENT_IMAGE, assertBuiltAgentImage, buildAgentImage } from '../agents/container/image.ts'; -import { assertContainerProfile, createContainerProfile, disposeContainerProfile } from '../agents/container/profile.ts'; +import { assertContainerProfile, createContainerProfile, disposeContainerProfile, + isContainerProfileAuthentic } from '../agents/container/profile.ts'; import { createValidatedContainer, prepareTaskFilesystems, removeTaskFilesystems, runContainer, startValidatedContainer, hasExactOptions, validateContainer } from '../agents/container/run.ts'; import { createTaskClone } from '../git/clone.ts'; @@ -272,6 +273,7 @@ describe('real Docker agent isolation', () => { docker(...first.args); containers.add(first.name); expect(() => createValidatedContainer(duplicate)).toThrow('Container creation failed and cleanup did not settle.'); expect(existsSync(duplicate.codexAuthFile!)).toBe(true); + expect(isContainerProfileAuthentic(duplicate)).toBe(true); const state = JSON.parse(docker('container', 'inspect', first.name))[0] as { State: { Status: string } }; expect(state.State.Status).toBe('created'); docker('rm', '--force', first.name); containers.delete(first.name); From cd1fc31149cfbf0d2def1058732dd9025f9ecc39 Mon Sep 17 00:00:00 2001 From: mchwang Date: Fri, 25 Sep 2026 10:50:03 -0700 Subject: [PATCH 26/32] Reject duplicate attempts without touching the active container A duplicate attempt's profile never created a container, and its deterministic container name belongs to the active invocation, so rejection now releases only the duplicate profile's staging and network. Previously it tried to remove the container by name, which D2 now refuses for another invocation's container, leaving a cleanup handle that never settled. The regression reuses one captured invocation, since an attempt can only be captured once. Co-Authored-By: Claude Opus 5.5 --- agents/adapters/supervisor.ts | 12 ++++++++++-- test/agent-supervisor.test.ts | 13 ++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index 18d1e19..0219a96 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -2,7 +2,7 @@ import { execFile, spawn, type ChildProcess } from 'node:child_process'; import type { InvocationHandle, InvocationInput, InvocationResult, StopReason } from '../contract.ts'; import { assertPhasePolicy } from '../policy.ts'; import { createValidatedContainer, disposeValidatedContainer, validateContainer } from '../container/run.ts'; -import { assertContainerProfileAuthenticity, isContainerProfileAuthentic, +import { assertContainerProfileAuthenticity, disposeContainerProfile, isContainerProfileAuthentic, type ContainerProfile } from '../container/profile.ts'; import { removeVendorNetwork, type VendorNetwork } from '../network/network.ts'; @@ -228,7 +228,15 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super if (ownsAttempt(invocation.attemptId)) { if (activeProfiles.has(profile)) throw new Error('This container profile already owns the active invocation.'); - return rejectWithCleanup(new Error('An invocation with this attempt ID is still active.'), false); + // This profile never created a container; the name belongs to the active invocation, so only + // release this profile's own staging and network. + const error = new Error('An invocation with this attempt ID is still active.'); + try { disposeContainerProfile(profile); } + catch (cleanupError) { + return retainCleanupOwnership(profile, + `Invocation was rejected and cleanup remains unsettled: ${String(error)}; ${String(cleanupError)}`, false); + } + throw error; } let limits: CaptureLimits, configuredTimeout: number, carriedBudget: number; try { diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index f69b21a..3e18489 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -42,9 +42,11 @@ function invocation(data: ReturnType, attemptId: string, deadlin context: { snapshotId: 'snapshot', planId: 'plan', planRevision: 1, assignmentId: 'assignment', referencedCodeHash: 'code', stateVersion: 1 } }); } -function profile(data: ReturnType, probe: IsolationProbe, attemptId = `attempt-${Math.random()}`, - deadlineMs = 2 * 60_000, deferredOutput = false) { - const captured = invocation(data, attemptId, deadlineMs), policy = createPhasePolicy(captured); +function profile(data: ReturnType, probe: IsolationProbe, + attempt: string | InvocationInput = `attempt-${Math.random()}`, deadlineMs = 2 * 60_000, deferredOutput = false) { + // An attempt can be captured once, so a duplicate-attempt profile reuses the captured invocation. + const captured = typeof attempt === 'string' ? invocation(data, attempt, deadlineMs) : attempt; + const policy = createPhasePolicy(captured); const network = createVendorNetwork(captured, imageId); const value = createContainerProfile({ invocation: captured, policy, network, filesystems: data.filesystems, inputDirectory: data.input, command: createIsolationProbeCommand(policy, probe), imageId, codexAuthFile: data.auth, @@ -153,8 +155,9 @@ describe('container invocation supervisor', () => { }, 15_000); it('blocks a duplicate attempt while the original container remains active', async () => { - const data = fixture(), first = startProfileInvocation(profile(data, 'ignore-term', 'duplicate'), { timeoutMs: 30_000 }); - expect(() => startProfileInvocation(profile(data, 'finite-output', 'duplicate'))).toThrow('still active'); + const data = fixture(), duplicate = invocation(data, 'duplicate'); + const first = startProfileInvocation(profile(data, 'ignore-term', duplicate), { timeoutMs: 30_000 }); + expect(() => startProfileInvocation(profile(data, 'finite-output', duplicate))).toThrow('still active'); expect(isInvocationActive('duplicate')).toBe(true); first.cancel('shutdown'); expect((await first.settled).stopReason).toBe('shutdown'); From 691df93f03d8f8f67c8c36d83296c39b19aac6f5 Mon Sep 17 00:00:00 2001 From: mchwang Date: Fri, 25 Sep 2026 11:07:23 -0700 Subject: [PATCH 27/32] Keep D4 Docker suites out of the parallel CI run The adapter and supervisor suites use Docker and already run one file at a time in the Agent isolation workflow. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 330b0dc..fc5a962 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,6 @@ jobs: - run: npm run typecheck # The Docker agent suites run one file at a time in the Agent isolation workflow; running them here # would put them in parallel against the same image tag and daemon. - - run: npm test -- --exclude test/agent-container.test.ts --exclude test/agent-network.test.ts + - run: npm test -- --exclude test/agent-container.test.ts --exclude test/agent-network.test.ts --exclude test/agent-adapter.test.ts --exclude test/agent-supervisor.test.ts - run: npx playwright install --with-deps chromium - run: npm run test:browser From dd0af274ac9d00807d120eedc25c9a49b097534d Mon Sep 17 00:00:00 2001 From: mchwang Date: Fri, 25 Sep 2026 11:50:22 -0700 Subject: [PATCH 28/32] Retain captured output in fixed-size blocks Each output event was stored as its own Buffer slice, so a container could emit the byte ceiling as millions of one-byte writes and exhaust supervisor memory before the byte limit applied. Output is now copied into 64 KiB blocks, so retained objects scale with bytes rather than with write events, and slices no longer pin their original chunks. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/agent-isolation.yml | 2 +- agents/adapters/supervisor.ts | 35 ++++++++++++++++++++++++--- test/agent-output.test.ts | 22 +++++++++++++++++ 3 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 test/agent-output.test.ts diff --git a/.github/workflows/agent-isolation.yml b/.github/workflows/agent-isolation.yml index 9e66e35..5cd1af0 100644 --- a/.github/workflows/agent-isolation.yml +++ b/.github/workflows/agent-isolation.yml @@ -29,4 +29,4 @@ jobs: - run: npm ci --ignore-scripts - run: npm run typecheck # The Docker suites share one image tag and daemon, so run test files one at a time. - - run: npx vitest run --no-file-parallelism test/agent-contract.test.ts test/agent-clone.test.ts test/agent-container.test.ts test/agent-network.test.ts test/agent-policy.test.ts test/agent-proxy.test.ts test/agent-adapter.test.ts test/agent-supervisor.test.ts + - run: npx vitest run --no-file-parallelism test/agent-contract.test.ts test/agent-clone.test.ts test/agent-container.test.ts test/agent-network.test.ts test/agent-policy.test.ts test/agent-proxy.test.ts test/agent-adapter.test.ts test/agent-supervisor.test.ts test/agent-output.test.ts diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index 0219a96..6c11446 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -60,6 +60,35 @@ const captureLimits = (override: Partial | undefined): CaptureLim }; const diagnosticFor = (reason: StopReason, detail?: string) => Buffer.from( `[codeboost: ${reason}${detail ? `: ${detail.replace(/[\r\n]+/g, ' ').slice(0, 512)}` : ''}]\n`); +/** + * Retains captured output in fixed-size blocks, so memory scales with retained bytes rather than with the number + * of write events a container emits. + */ +export class ByteCollector { + static readonly BLOCK_BYTES = 64 * 1024; + private readonly blocks: Buffer[] = []; + private used = 0; + push(data: Buffer): void { + for (let offset = 0; offset < data.length;) { + let block = this.blocks.at(-1); + if (!block || this.used === block.length) { + block = Buffer.allocUnsafe(ByteCollector.BLOCK_BYTES); + this.blocks.push(block); + this.used = 0; + } + const count = Math.min(data.length - offset, block.length - this.used); + data.copy(block, this.used, offset, offset + count); + this.used += count; + offset += count; + } + } + toBuffer(): Buffer { + if (!this.blocks.length) return Buffer.alloc(0); + return Buffer.concat([...this.blocks.slice(0, -1), this.blocks.at(-1)!.subarray(0, this.used)]); + } + get blockCount(): number { return this.blocks.length; } +} + const retainCleanupOwnership = (profile: ContainerProfile, detail: string, register = true): InvocationHandle => { const invocation = assertPhasePolicy(profile.policy); let resolveSettled!: (result: InvocationResult) => void, cleaning = false, complete = false; @@ -274,7 +303,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super throw error; } - const stdoutChunks: Buffer[] = [], stderrChunks: Buffer[] = []; + const stdoutChunks = new ByteCollector(), stderrChunks = new ByteCollector(); let stdoutBytes = 0, stderrBytes = 0, combinedBytes = 0; let stopReason: StopReason | undefined, failureDetail: string | undefined; let closed = false, terminating = false, settlementComplete = false; @@ -388,7 +417,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super ? new CaptureDeadlineError('Adapter output capture exceeded the invocation deadline.') : new Error('Adapter output capture was cancelled.')), { once: true }); }); - const raw = Buffer.concat(stdoutChunks, stdoutBytes); + const raw = stdoutChunks.toBuffer(); const operation = Promise.resolve(options.decode!(profile, raw, Math.max(1, Math.min(limits.stdoutBytes - stdoutBytes, limits.combinedBytes - combinedBytes)), budget, controller.signal)); @@ -526,7 +555,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super if (!protocolLine(protocolBuffer)) capture('stderr', protocolBuffer, true); protocolBuffer = Buffer.alloc(0); } - let finalStdout = Buffer.concat(stdoutChunks, stdoutBytes), finalStderr = Buffer.concat(stderrChunks); + let finalStdout = stdoutChunks.toBuffer(), finalStderr = stderrChunks.toBuffer(); let exitCode = code, finalSignal = signal; if (!stopReason && options.decode && !profile.deferredOutput) await decodeOutput(); if (decodePromise) await decodePromise; diff --git a/test/agent-output.test.ts b/test/agent-output.test.ts new file mode 100644 index 0000000..51b4c90 --- /dev/null +++ b/test/agent-output.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest'; +import { ByteCollector } from '../agents/adapters/supervisor.ts'; + +describe('captured output storage', () => { + it('keeps storage proportional to bytes, not to the number of writes', () => { + const collector = new ByteCollector(); + for (let i = 0; i < 100_000; i++) collector.push(Buffer.from([i % 256])); + expect(collector.blockCount).toBeLessThanOrEqual(2); + const output = collector.toBuffer(); + expect(output.length).toBe(100_000); + expect(output.every((byte, index) => byte === index % 256)).toBe(true); + }); + + it('splits a large write across blocks without losing or reordering bytes', () => { + const collector = new ByteCollector(), large = Buffer.alloc(3 * ByteCollector.BLOCK_BYTES + 17, 7); + collector.push(Buffer.from('head')); + collector.push(large); + expect(collector.blockCount).toBe(4); + expect(collector.toBuffer()).toEqual(Buffer.concat([Buffer.from('head'), large])); + expect(new ByteCollector().toBuffer().length).toBe(0); + }); +}); From 8bc00da0c1e295ce65e083535a778b548f445e7e Mon Sep 17 00:00:00 2001 From: mchwang Date: Fri, 25 Sep 2026 12:54:52 -0700 Subject: [PATCH 29/32] Keep killed creates unsettled across cleanups and publish strict UTF-8 - Record, per profile, when a killed docker create stops counting as in flight. Later cleanup (for example the supervisor's dispose after an early create-path cleanup failure) no longer treats absence inside that window as proof and releases the profile; it reports "did not settle" so ownership is retained and retried. Also round the create-path wait to whole milliseconds, which the deadline helper requires. - Decode final stdout and stderr with a fatal UTF-8 policy. Invalid output becomes a capture-failure with that stream withheld, so replacement characters cannot grow the result past its byte ceilings; an incomplete trailing character cut at a limit is dropped. Co-Authored-By: Claude Opus 5.5 --- agents/adapters/supervisor.ts | 12 ++++++++++++ agents/container/run.ts | 19 +++++++++++++------ agents/policy.ts | 3 ++- test/agent-container.test.ts | 27 +++++++++++++++++++++++++-- test/agent-supervisor.test.ts | 7 +++++++ 5 files changed, 59 insertions(+), 9 deletions(-) diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index 6c11446..1a706db 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -583,6 +583,18 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super }); } } + // Publish only strictly valid UTF-8: replacement characters would grow the result past the byte ceilings. + // A multi-byte character cut at a capture limit is an incomplete tail, which the streaming decode drops. + const strictText = (value: Buffer) => { + try { return new TextDecoder('utf-8', { fatal: true }).decode(value, { stream: true }); } + catch { return undefined; } + }; + const stdoutText = strictText(finalStdout), stderrText = strictText(finalStderr); + if (stdoutText === undefined || stderrText === undefined) { + stopReason ??= 'capture-failure'; + failureDetail ??= 'Captured output is not valid UTF-8.'; + } + finalStdout = Buffer.from(stdoutText ?? ''); finalStderr = Buffer.from(stderrText ?? ''); if (stopReason) finalStderr = withDiagnostic(finalStderr, finalStdout.length, stopReason, limits, failureDetail); const result = Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, exitCode, signal: finalSignal, ...(stopReason ? { stopReason } : {}), diff --git a/agents/container/run.ts b/agents/container/run.ts index cdc09a6..a00dd99 100644 --- a/agents/container/run.ts +++ b/agents/container/run.ts @@ -49,12 +49,14 @@ const canonicalDockerBindSource = (source: string) => { /** How long a killed `docker create` may still materialize its container in the daemon. */ const CREATE_SETTLE_MS = 10_000; const sleep = (ms: number) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); -const removeContainerOrThrow = (profile: ContainerProfile, createUnsettled = false) => { +// When a killed `docker create` for a profile stops counting as possibly in flight (performance.now() timestamp). +const unsettledCreates = new WeakMap(); +const removeContainerOrThrow = (profile: ContainerProfile, waitForSettle = false) => { // Destructive cleanup acts only for the builder-registered profile; a copy's name and label are not a capability. if (!isContainerProfileAuthentic(profile)) throw new Error('Container profile was not created by the trusted profile builder.'); - const remaining = createDeadline(30_000 + (createUnsettled ? CREATE_SETTLE_MS : 0)); - const settleBy = performance.now() + (createUnsettled ? CREATE_SETTLE_MS : 0); + const settleUntil = unsettledCreates.get(profile) ?? 0; + const remaining = createDeadline(30_000 + (waitForSettle ? Math.max(0, Math.ceil(settleUntil - performance.now())) : 0)); let before: ReturnType; for (;;) { before = spawnSync('docker', ['container', 'inspect', profile.name], { @@ -63,12 +65,15 @@ const removeContainerOrThrow = (profile: ContainerProfile, createUnsettled = fal if (before.status === 0) break; const missing = !before.error && /No such (?:object|container)/i.test(`${before.stdout ?? ''}\n${before.stderr ?? ''}`); if (!missing) throw new Error('Failed to establish ownership of the agent container; staged credentials were retained.'); - if (!createUnsettled) { + // A killed create may still land in the daemon; absence only counts once its settle window has passed. Only the + // create path waits here; later cleanup (such as a supervisor recovery) reports "not settled" and retries later. + const settled = performance.now() >= settleUntil; + if (settled && !(waitForSettle && settleUntil)) { + unsettledCreates.delete(profile); disposeContainerProfile(profile); return; } - // A killed create may still land in the daemon; absence is not proof until the settle window passes. - if (performance.now() >= settleBy) + if (settled || !waitForSettle) throw new Error('Agent container creation did not settle; staged credentials were retained.'); sleep(250); } @@ -86,6 +91,7 @@ const removeContainerOrThrow = (profile: ContainerProfile, createUnsettled = fal && /No such (?:object|container)/i.test(`${inspect.stdout ?? ''}\n${inspect.stderr ?? ''}`); if (!absent) throw new Error('Failed to confirm removal of the agent container; staged credentials were retained.'); } + unsettledCreates.delete(profile); disposeContainerProfile(profile); }; @@ -299,6 +305,7 @@ export function createValidatedContainer(profile: ContainerProfile, timeoutMs = remaining(); return profile.name; } catch (error) { + if (createUnsettled) unsettledCreates.set(profile, performance.now() + CREATE_SETTLE_MS); try { removeContainerOrThrow(profile, createUnsettled); } catch (cleanupError) { throw new AggregateError([error, cleanupError], 'Container creation failed and cleanup did not settle.'); } throw error; diff --git a/agents/policy.ts b/agents/policy.ts index ae09dd1..e7584c1 100644 --- a/agents/policy.ts +++ b/agents/policy.ts @@ -91,7 +91,7 @@ export function createCodexCommand(policy: PhasePolicy, prompt: string): AgentCo export type IsolationProbe = 'noop' | 'phase-worktree' | 'read-only-isolation' | 'persist-write' | 'persist-read' | 'capacity' | 'metadata' | 'must-not-run' | 'input-marker' | 'finite-output' | 'infinite-stdout' | 'infinite-stderr' | 'infinite-mixed' | 'ignore-term' | 'symlink-output' - | 'oversized-output' | 'fifo-output' | 'invalid-utf8-output' | 'replace-output-directory' + | 'oversized-output' | 'fifo-output' | 'invalid-utf8-output' | 'invalid-utf8-stderr' | 'replace-output-directory' | 'nonzero-output' | 'duplicate-protocol' | 'newline-free-deferred-output'; /** Fixed startup probes validate the sandbox itself without granting an agent a process tool. */ @@ -116,6 +116,7 @@ export function createIsolationProbeCommand(policy: PhasePolicy, probe: Isolatio 'input-marker': 'set -eu; grep -q codeboost-schema-marker /run/codeboost-input/schema.json; ' + 'test ! -e /run/codeboost-input/extra.json', 'finite-output': 'printf stdout-marker; printf stderr-marker >&2', + 'invalid-utf8-stderr': "printf 'bad-\\377\\377-stderr' >&2", 'infinite-stdout': "while :; do head -c 4096 /dev/zero | tr '\\0' x; done", 'infinite-stderr': "while :; do head -c 4096 /dev/zero | tr '\\0' x >&2; done", 'infinite-mixed': "while :; do head -c 4096 /dev/zero | tr '\\0' x; head -c 4096 /dev/zero | tr '\\0' y >&2; done", diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index 4eb5525..f4a13c7 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -8,8 +8,8 @@ import { captureInvocation, type InvocationInput, type Phase } from '../agents/c import { AGENT_IMAGE, assertBuiltAgentImage, buildAgentImage } from '../agents/container/image.ts'; import { assertContainerProfile, createContainerProfile, disposeContainerProfile, isContainerProfileAuthentic } from '../agents/container/profile.ts'; -import { createValidatedContainer, prepareTaskFilesystems, removeTaskFilesystems, runContainer, startValidatedContainer, - hasExactOptions, validateContainer } from '../agents/container/run.ts'; +import { createValidatedContainer, disposeValidatedContainer, prepareTaskFilesystems, removeTaskFilesystems, runContainer, + startValidatedContainer, hasExactOptions, validateContainer } from '../agents/container/run.ts'; import { createTaskClone } from '../git/clone.ts'; import { createVendorNetwork, removeVendorNetwork, type VendorNetwork } from '../agents/network/network.ts'; import { createClaudeCommand, createCodexCommand, createIsolationProbeCommand, createPhasePolicy, @@ -293,6 +293,29 @@ describe('real Docker agent isolation', () => { expect(existsSync(unsettled.codexAuthFile!)).toBe(true); }, 60_000); + it('keeps a killed create unsettled for later cleanup until its settle window passes', () => { + const data = fixture(), unsettled = profile(data, 'planning', 'noop'); + const shim = join(data.root, 'docker-shim'); mkdirSync(shim); + const created = join(shim, 'created'), failed = join(shim, 'failed'); + const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + // The create client hangs until killed, and the first inspect after it fails for an unrelated reason, so the + // create path gives up early, inside the settle window. + writeFileSync(join(shim, 'docker'), ['#!/bin/sh', + `if [ "$1" = create ]; then touch '${created}'; exec sleep 30; fi`, + `if [ "$1" = container ] && [ "$2" = inspect ] && [ -e '${created}' ] && [ ! -e '${failed}' ]; then`, + ` touch '${failed}'; echo 'daemon unavailable' >&2; exit 1`, 'fi', + `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); + const path = process.env.PATH; + process.env.PATH = `${shim}:${path}`; + try { + expect(() => createValidatedContainer(unsettled, 3_000)).toThrow('cleanup did not settle'); + // A follow-up cleanup inside the window must not treat absence as proof and release the profile. + expect(() => disposeValidatedContainer(unsettled)).toThrow('did not settle'); + } finally { process.env.PATH = path; } + expect(isContainerProfileAuthentic(unsettled)).toBe(true); + expect(existsSync(unsettled.codexAuthFile!)).toBe(true); + }, 60_000); + it('refuses to seed a clone whose staging directory was replaced after creation', () => { const data = fixture(); const clone = createTaskClone({ source: data.source, parent: join(data.root, 'staging'), taskId: 'task-2', diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 3e18489..088d3f2 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -154,6 +154,13 @@ describe('container invocation supervisor', () => { expect(result.stopReason).toBe('timeout'); }, 15_000); + it('fails capture instead of publishing replacement characters for invalid UTF-8 stderr', async () => { + const result = await startProfileInvocation(profile(fixture(), 'invalid-utf8-stderr'), { timeoutMs: 30_000 }).settled; + expect(result.stopReason).toBe('capture-failure'); + expect(result.stderr).not.toContain('\uFFFD'); + expect(result.stderr).not.toContain('bad-'); + }, 60_000); + it('blocks a duplicate attempt while the original container remains active', async () => { const data = fixture(), duplicate = invocation(data, 'duplicate'); const first = startProfileInvocation(profile(data, 'ignore-term', duplicate), { timeoutMs: 30_000 }); From 6c757e11adaabab882f9057b8389bca4f4899ec8 Mon Sep 17 00:00:00 2001 From: mchwang Date: Fri, 25 Sep 2026 13:21:10 -0700 Subject: [PATCH 30/32] Treat an absent container as settled once the create window passes After a killed docker create, the create path waited out the settle window but still reported "did not settle" when no container appeared, leaving the profile's staging and network owned. Callers such as runContainer have no later cleanup, so those resources leaked. Absence now counts as settled on every path once the window has passed, matching storage and network cleanup; only inside the window is it reported as unsettled. Co-Authored-By: Claude Opus 5.5 --- agents/container/run.ts | 11 +++++------ test/agent-container.test.ts | 10 +++++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/agents/container/run.ts b/agents/container/run.ts index a00dd99..85f7d44 100644 --- a/agents/container/run.ts +++ b/agents/container/run.ts @@ -65,16 +65,15 @@ const removeContainerOrThrow = (profile: ContainerProfile, waitForSettle = false if (before.status === 0) break; const missing = !before.error && /No such (?:object|container)/i.test(`${before.stdout ?? ''}\n${before.stderr ?? ''}`); if (!missing) throw new Error('Failed to establish ownership of the agent container; staged credentials were retained.'); - // A killed create may still land in the daemon; absence only counts once its settle window has passed. Only the - // create path waits here; later cleanup (such as a supervisor recovery) reports "not settled" and retries later. - const settled = performance.now() >= settleUntil; - if (settled && !(waitForSettle && settleUntil)) { + // A killed create may still land in the daemon; absence only counts once its settle window has passed, on every + // path. Only the create path waits here; later cleanup (such as a supervisor recovery) reports "not settled" + // inside the window and retries later. + if (performance.now() >= settleUntil) { unsettledCreates.delete(profile); disposeContainerProfile(profile); return; } - if (settled || !waitForSettle) - throw new Error('Agent container creation did not settle; staged credentials were retained.'); + if (!waitForSettle) throw new Error('Agent container creation did not settle; staged credentials were retained.'); sleep(250); } const inspected = JSON.parse(String(before.stdout || '[]'))[0] as { Config?: { Labels?: Record } } | undefined; diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index f4a13c7..0320c88 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -279,7 +279,7 @@ describe('real Docker agent isolation', () => { docker('rm', '--force', first.name); containers.delete(first.name); }, 60_000); - it('retains credentials when a killed create cannot be proven absent', () => { + it('releases a killed create once its settle window passes with no container', () => { const data = fixture(), unsettled = profile(data, 'planning', 'noop'); const shim = join(data.root, 'docker-shim'); mkdirSync(shim); const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); @@ -288,9 +288,13 @@ describe('real Docker agent isolation', () => { `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); const path = process.env.PATH; process.env.PATH = `${shim}:${path}`; - try { expect(() => createValidatedContainer(unsettled, 1_000)).toThrow('cleanup did not settle'); } + const started = performance.now(); + // The create path waits out the settle window, then treats absence as settled and releases the profile. + try { expect(() => createValidatedContainer(unsettled, 3_000)).toThrow('ETIMEDOUT'); } finally { process.env.PATH = path; } - expect(existsSync(unsettled.codexAuthFile!)).toBe(true); + expect(performance.now() - started).toBeGreaterThanOrEqual(10_000); + expect(isContainerProfileAuthentic(unsettled)).toBe(false); + expect(existsSync(unsettled.codexAuthFile!)).toBe(false); }, 60_000); it('keeps a killed create unsettled for later cleanup until its settle window passes', () => { From a7a6eb9b26912a6cc53e12365dfafc6e47afbb83 Mon Sep 17 00:00:00 2001 From: mchwang Date: Fri, 25 Sep 2026 13:35:37 -0700 Subject: [PATCH 31/32] Fail closed on a trailing incomplete UTF-8 character unless truncated The final output decode used streaming mode on every stream, so output ending in a lone lead byte was silently trimmed and published as a success. Only output cut at a capture limit may now end in an incomplete character; otherwise the decode flushes and fails closed as a capture-failure. Co-Authored-By: Claude Opus 5.5 --- agents/adapters/supervisor.ts | 6 ++++-- agents/policy.ts | 4 +++- test/agent-supervisor.test.ts | 6 ++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index 1a706db..a81c4ea 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -584,9 +584,11 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super } } // Publish only strictly valid UTF-8: replacement characters would grow the result past the byte ceilings. - // A multi-byte character cut at a capture limit is an incomplete tail, which the streaming decode drops. + // Only output cut at a capture limit may end in an incomplete character, which the streaming decode then drops; + // otherwise the decode flushes, so a trailing lone lead byte fails closed. + const truncated = stopReason === 'output-limit'; const strictText = (value: Buffer) => { - try { return new TextDecoder('utf-8', { fatal: true }).decode(value, { stream: true }); } + try { return new TextDecoder('utf-8', { fatal: true }).decode(value, truncated ? { stream: true } : undefined); } catch { return undefined; } }; const stdoutText = strictText(finalStdout), stderrText = strictText(finalStderr); diff --git a/agents/policy.ts b/agents/policy.ts index e7584c1..0d1ebaf 100644 --- a/agents/policy.ts +++ b/agents/policy.ts @@ -91,7 +91,8 @@ export function createCodexCommand(policy: PhasePolicy, prompt: string): AgentCo export type IsolationProbe = 'noop' | 'phase-worktree' | 'read-only-isolation' | 'persist-write' | 'persist-read' | 'capacity' | 'metadata' | 'must-not-run' | 'input-marker' | 'finite-output' | 'infinite-stdout' | 'infinite-stderr' | 'infinite-mixed' | 'ignore-term' | 'symlink-output' - | 'oversized-output' | 'fifo-output' | 'invalid-utf8-output' | 'invalid-utf8-stderr' | 'replace-output-directory' + | 'oversized-output' | 'fifo-output' | 'invalid-utf8-output' | 'invalid-utf8-stderr' | 'truncated-utf8-stderr' + | 'replace-output-directory' | 'nonzero-output' | 'duplicate-protocol' | 'newline-free-deferred-output'; /** Fixed startup probes validate the sandbox itself without granting an agent a process tool. */ @@ -117,6 +118,7 @@ export function createIsolationProbeCommand(policy: PhasePolicy, probe: Isolatio + 'test ! -e /run/codeboost-input/extra.json', 'finite-output': 'printf stdout-marker; printf stderr-marker >&2', 'invalid-utf8-stderr': "printf 'bad-\\377\\377-stderr' >&2", + 'truncated-utf8-stderr': "printf 'cut-\\342' >&2", 'infinite-stdout': "while :; do head -c 4096 /dev/zero | tr '\\0' x; done", 'infinite-stderr': "while :; do head -c 4096 /dev/zero | tr '\\0' x >&2; done", 'infinite-mixed': "while :; do head -c 4096 /dev/zero | tr '\\0' x; head -c 4096 /dev/zero | tr '\\0' y >&2; done", diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 088d3f2..ee9607c 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -161,6 +161,12 @@ describe('container invocation supervisor', () => { expect(result.stderr).not.toContain('bad-'); }, 60_000); + it('fails capture when output ends in an incomplete character without reaching a limit', async () => { + const result = await startProfileInvocation(profile(fixture(), 'truncated-utf8-stderr'), { timeoutMs: 30_000 }).settled; + expect(result.stopReason).toBe('capture-failure'); + expect(result.stderr).not.toContain('cut-'); + }, 60_000); + it('blocks a duplicate attempt while the original container remains active', async () => { const data = fixture(), duplicate = invocation(data, 'duplicate'); const first = startProfileInvocation(profile(data, 'ignore-term', duplicate), { timeoutMs: 30_000 }); From d309dd7d101c837593890a1b606edc30cd60b18e Mon Sep 17 00:00:00 2001 From: mchwang Date: Fri, 25 Sep 2026 13:52:01 -0700 Subject: [PATCH 32/32] Release only the profile when rejecting an invocation before creation Rejections before container creation (invalid limits, an expired deadline, a duplicate attempt) own no container, but they cleaned up with container-level disposal. If the deterministic name was held by another invocation or Docker inspect failed, that cleanup could never settle, and the recovery retried it forever while the profile's staging and network leaked. These paths now dispose only the profile, and recovery retries that same profile-only cleanup. Co-Authored-By: Claude Opus 5.5 --- agents/adapters/supervisor.ts | 22 +++++++++++----------- test/agent-supervisor.test.ts | 15 ++++++++++++++- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index a81c4ea..502f38f 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -89,7 +89,10 @@ export class ByteCollector { get blockCount(): number { return this.blocks.length; } } -const retainCleanupOwnership = (profile: ContainerProfile, detail: string, register = true): InvocationHandle => { +// `cleanup` is what recovery retries: container-level disposal by default, or profile-only disposal for a +// rejection that happened before this profile created any container. +const retainCleanupOwnership = (profile: ContainerProfile, detail: string, register = true, + cleanup: (profile: ContainerProfile) => void = disposeValidatedContainer): InvocationHandle => { const invocation = assertPhasePolicy(profile.policy); let resolveSettled!: (result: InvocationResult) => void, cleaning = false, complete = false; let cancelReason: StopReason | undefined; @@ -104,7 +107,7 @@ const retainCleanupOwnership = (profile: ContainerProfile, detail: string, regis if (cleaning || complete) return; cleaning = true; try { - disposeValidatedContainer(profile); + cleanup(profile); if (timer) clearTimeout(timer); timer = undefined; complete = true; @@ -246,11 +249,14 @@ export function isInvocationActive(attemptId: string): boolean { export function startProfileInvocation(profile: ContainerProfile, options: SupervisorOptions = {}): InvocationHandle { assertContainerProfileAuthenticity(profile); const invocation = assertPhasePolicy(profile.policy); + // Rejections before container creation own no container, so they release (and retry) only the profile's own + // staging and network; a name held by another invocation or a failing inspect cannot block that. const rejectWithCleanup = (error: unknown, register = true): InvocationHandle => { - try { disposeValidatedContainer(profile); } + try { disposeContainerProfile(profile); } catch (cleanupError) { return retainCleanupOwnership(profile, - `Invocation was rejected and cleanup remains unsettled: ${String(error)}; ${String(cleanupError)}`, register); + `Invocation was rejected and cleanup remains unsettled: ${String(error)}; ${String(cleanupError)}`, register, + disposeContainerProfile); } throw error; }; @@ -259,13 +265,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super throw new Error('This container profile already owns the active invocation.'); // This profile never created a container; the name belongs to the active invocation, so only // release this profile's own staging and network. - const error = new Error('An invocation with this attempt ID is still active.'); - try { disposeContainerProfile(profile); } - catch (cleanupError) { - return retainCleanupOwnership(profile, - `Invocation was rejected and cleanup remains unsettled: ${String(error)}; ${String(cleanupError)}`, false); - } - throw error; + return rejectWithCleanup(new Error('An invocation with this attempt ID is still active.'), false); } let limits: CaptureLimits, configuredTimeout: number, carriedBudget: number; try { diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index ee9607c..7204045 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -9,7 +9,8 @@ import { isInvocationActive, readBoundedContainerFile, retainSetupCleanup, startProfileInvocation } from '../agents/adapters/supervisor.ts'; import { captureInvocation, type InvocationInput } from '../agents/contract.ts'; import { buildAgentImage } from '../agents/container/image.ts'; -import { createContainerProfile, disposeContainerProfile, type ContainerProfile } from '../agents/container/profile.ts'; +import { createContainerProfile, disposeContainerProfile, isContainerProfileAuthentic, + type ContainerProfile } from '../agents/container/profile.ts'; import { disposeValidatedContainer, prepareTaskFilesystems, removeTaskFilesystems } from '../agents/container/run.ts'; import { createVendorNetwork } from '../agents/network/network.ts'; import { createIsolationProbeCommand, createPhasePolicy, type IsolationProbe } from '../agents/policy.ts'; @@ -167,6 +168,18 @@ describe('container invocation supervisor', () => { expect(result.stderr).not.toContain('cut-'); }, 60_000); + it('releases only the profile when rejecting before creation, even if its name is held elsewhere', () => { + const current = profile(fixture(), 'noop'); + // A foreign container occupies the deterministic name, so container-level cleanup could never settle. + execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=someone-else', + '--entrypoint', 'true', imageId], { stdio: 'ignore' }); + try { + expect(() => startProfileInvocation(current, { timeoutMs: 10 * 60_000 + 1 })).toThrow('ceiling'); + expect(isContainerProfileAuthentic(current)).toBe(false); + expect(spawnSync('docker', ['container', 'inspect', current.name], { stdio: 'ignore' }).status).toBe(0); + } finally { spawnSync('docker', ['rm', '--force', current.name], { stdio: 'ignore' }); } + }, 60_000); + it('blocks a duplicate attempt while the original container remains active', async () => { const data = fixture(), duplicate = invocation(data, 'duplicate'); const first = startProfileInvocation(profile(data, 'ignore-term', duplicate), { timeoutMs: 30_000 });