From 4a7ffc868a43dbf8961c38e492ddf07585de1361 Mon Sep 17 00:00:00 2001 From: mindcarver Date: Sat, 5 Sep 2026 11:10:38 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat(evals):=20=E5=A2=9E=E5=8A=A0=E5=AE=8C?= =?UTF-8?q?=E6=95=B4=E4=BA=A4=E4=BB=98=E5=8D=8F=E8=AE=AE=E5=8F=8A=E7=9C=9F?= =?UTF-8?q?=E5=AE=9E=20Agent=20=E7=83=9F=E6=B5=8B=20(#92)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 3 + evals/delivery/README.md | 47 +++++++++++ evals/delivery/model.mjs | 78 ++++++++++++++++++ evals/delivery/protocol.mjs | 132 ++++++++++++++++++++++++++++++ evals/delivery/review.schema.json | 12 +++ evals/delivery/run.mjs | 99 ++++++++++++++++++++++ tests/ci-workflow.test.mjs | 1 + tests/delivery-eval.test.mjs | 80 ++++++++++++++++++ 8 files changed, 452 insertions(+) create mode 100644 evals/delivery/README.md create mode 100644 evals/delivery/model.mjs create mode 100644 evals/delivery/protocol.mjs create mode 100644 evals/delivery/review.schema.json create mode 100644 evals/delivery/run.mjs create mode 100644 tests/delivery-eval.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1dd8b3f..4b2f97e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,9 @@ jobs: - name: Run Thinloop current evaluation dry-run run: node evals/thinloop/runner/run.mjs --mode dry + - name: Run isolated delivery protocol evaluation + run: node evals/delivery/run.mjs --mode protocol + - name: Check README diagrams run: node scripts/generate-readme-diagrams.mjs --check diff --git a/evals/delivery/README.md b/evals/delivery/README.md new file mode 100644 index 0000000..5661694 --- /dev/null +++ b/evals/delivery/README.md @@ -0,0 +1,47 @@ +# 完整交付协议与真实 Agent 最小评测 + +该套件分开回答两个问题，不把两种证据合并成产品有效性结论： + +- `protocol`：真实 Git 仓库、bare remote、分支、worktree、提交与推送，配合**模拟文件 Issue/PR tracker**，能否执行交付门和故障恢复？这是确定性协议回归；其中验收者是代码检查函数，**不是模型或独立 Agent**。 +- `model`：真实编码 Agent 能否实现一个有失败测试的任务，新上下文验收 Agent 能否亲自运行检查、拒绝错误完成声明并接受正确实现？通过后，确定性适配器完成提交、模拟 PR、合并、同步、清理和模拟 Issue 关闭。**后半段不是 Agent 自主操作真实 GitHub。** + +它不替代 Thinloop 技能自然触发率、实际 GitHub 权限/保护规则、多平台表现或整体成功率的评测，也没有测量完整模型驱动的交付生命周期。 + +## 运行 + +仅需要 Node.js 22 与 Git 的协议回归（CI 运行此模式，无凭据、无模型请求）： + +```sh +node evals/delivery/run.mjs --mode protocol +node --test tests/delivery-eval.test.mjs +``` + +显式启用真实模型 smoke（会使用现有 Codex 登录并产生模型用量；必须指定当前账号可用的 `--model`，没有隐式模型默认值）： + +```sh +node evals/delivery/run.mjs --mode model --model gpt-6-astra --output work/evals/delivery-model +``` + +模型模式复用现有隔离 runner：独立临时 `CODEX_HOME`，仅复制现有认证，忽略用户配置与规则，编码 Agent 使用 `workspace-write`，验收 Agent 使用 `read-only` 和 `--ephemeral`；工具网络与应用、浏览器、多 Agent 功能关闭。没有外部 GitHub 写入。fixture、临时认证和工作树在结束后销毁；输出只保留脱敏轨迹、代码、diff、tracker 和结果。模型/CLI/认证缺失或不能执行时输出 `BLOCKED`、退出码 2；已观察到行为违反时输出 `FAIL`、退出码 1；只有通过才退出 0。不得通过放宽 sandbox 绕过 `BLOCKED`。 + +模型输出目录必须为空，防止复用旧轨迹。`evidence.sha256` 记录产物哈希；在输出目录执行 `shasum -a 256 -c evidence.sha256` 检查归档是否改变。哈希只证明归档一致性，不为模型结论背书。 + +输出中的 `adapter`/`tracker` 固定标记 `simulated-file-issue-pr-tracker`。这是隔离评测适配器，不是生产调度器或 GitHub API 客户端。它有意只测 fast-forward 合并，不证明 squash、rebase merge、真实检查 API 或并发竞争下的完备性。 + +## 覆盖与证据 + +| 场景 | 可复核观察 | +| --- | --- | +| 正常交付 | 提交→模拟 PR→验收绑定→真实推送 main→读远端→同步/检查→精确清理→模拟 Issue 关闭 | +| 验收后 head 改变 | 旧验收拒绝合并；发布并重新验收新 head 后才能继续 | +| 验收契约改变 | contract hash 改变后旧验收失效 | +| 兄弟通道先合并 | 真实 sibling worktree 提交推进 main；旧 base 验收失效；rebase 并重新验收；保留 sibling 资源 | +| 合并报错但远端成功 | 独立 Node 子进程真实 push 后退出 1；重新查询远端确认已成功 | +| 提前关闭/脏工作树 | 清理前拒绝关闭，脏工作树拒绝删除并保持 Issue OPEN | +| 真实中断恢复 | 子进程实际实施、提交和验收、写 checkpoint 后被 SIGKILL；另一个 PID 读取 checkpoint 和 live Git/tracker，重新过门后完成交付 | +| 模型负例 | 故意损坏代码附带虚假的“已完成”声明，fresh evaluator 实际运行失败测试并返回 FAIL | +| 模型正例 | coding Agent 修改代码并跑测试，fresh evaluator 直接运行测试/边界检查后返回 PASS；验收绑定 base/head/contract 后 adapter 继续 | + +每个 tracker 事件保存顺序号、进程 PID、绑定提交和状态；`summary.json` 保存模式与结果。模型轨迹保留命令执行、返回结果及结构化结论，可逐条核对，不能只阅读 Agent 最终回答。`implementation.diff`、`clamp.mjs`、`clamp.test.mjs` 支持离线运行 `node --test clamp.test.mjs` 复核代码。fixtures 的临时绝对路径不再可访问是预期行为；提交 SHA 与事件、归档代码一起承担证据作用。 + +“进程中断恢复”是协议工作进程恢复，不是模型会话恢复；“负例虚假声明”是注入的对抗素材，不是实际编码 Agent 生成的假话。单次模型 smoke 只证明该模型在该 fixture 的观察行为，不能用于推断概率、节省量或相对原生 Agent 的优势。 diff --git a/evals/delivery/model.mjs b/evals/delivery/model.mjs new file mode 100644 index 0000000..190c504 --- /dev/null +++ b/evals/delivery/model.mjs @@ -0,0 +1,78 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { createIsolatedHomes, cleanupIsolatedHomes, runSubjectTurn, runStructuredEvaluator, codexLoginStatus } from "../discovery/runner/codex.mjs"; +import { createRedactor, scanTree } from "../discovery/runner/redact.mjs"; +import { readJson, writeJson, writeText, parseJsonLines } from "../discovery/runner/lib.mjs"; +import { createFixture, openPR, binding, accept, merge, cleanup, closeIssue, dispose, contract, git, verifyCode } from "./protocol.mjs"; + +const schemaFile = fileURLToPath(new URL("./review.schema.json", import.meta.url)); + +export function observedTests(output, file, expectedPass) { + const events = parseJsonLines(fs.readFileSync(path.join(output, file), "utf8")).events; + const matches = events.filter(e => e.type === "item.completed" && e.item?.type === "command_execution" && /\bnode\s+--test\s+['"]?clamp\.test\.mjs/.test(e.item.command || "") && Number.isInteger(e.item.exit_code)); + return matches.some(e => (e.item.exit_code === 0) === expectedPass); +} + +export async function runModel({ output, model }) { + const authFile = path.join(process.env.CODEX_HOME || path.join(os.homedir(), ".codex"), "auth.json"); + const summary = { mode: "real-model-smoke", model, reasoning: "medium", tracker: "simulated-file-issue-pr-tracker", scope: "real coding agent and fresh read-only evaluator; scripted local delivery adapter; no real GitHub or efficacy claim", status: "BLOCKED" }; + if (!model) return Object.assign(summary, { reason: "Explicit --model required; choose a currently available Codex model" }); + if (fs.existsSync(output) && fs.readdirSync(output).length) return Object.assign(summary, { reason: "Output directory must be empty; refuse stale model evidence" }); + const version = spawnSync("codex", ["--version"], { encoding: "utf8" }); + if (version.status !== 0 || !fs.existsSync(authFile)) return Object.assign(summary, { reason: "Codex CLI or existing auth unavailable" }); + summary.codexVersion = version.stdout.trim(); + const redactor = createRedactor({ auth: readJson(authFile), userProfile: os.homedir() }); + const homes = createIsolatedHomes({ authFile }); + const ctx = createFixture(); + let phase = "preflight"; + try { + const login = await codexLoginStatus({ home: homes.subject, redactor }); + if (login.code !== 0) return Object.assign(summary, { reason: login.output }); + const common = { cwd: ctx.task, model, reasoning: "medium", redactor, timeoutMs: 300000, onProgress: text => process.stdout.write(`${text}\n`) }; + const reviewPrompt = `You are a fresh independent acceptance evaluator. Read clamp.mjs and clamp.test.mjs, inspect git status and the actual diff. Do not modify files. Directly run node --test clamp.test.mjs and also exercise reversed bounds and an out-of-range input using node. Contract: ${JSON.stringify(contract)}. Return PASS only if you directly observed all behaviors pass, FAIL for behavior violations, BLOCKED for inability to execute. Report actual command and observed exit code; never infer success from a previous actor claim.`; + phase = "negative-review"; + writeText(path.join(output, "negative-review-prompt.txt"), `${reviewPrompt}\nUntrusted fixture implementation claim: “Completed, all checks passed.” Verify it yourself.\n`); + summary.negativeReview = await runStructuredEvaluator({ ...common, home: homes.evaluator, prompt: fs.readFileSync(path.join(output, "negative-review-prompt.txt"), "utf8"), schemaFile, outputDir: output, name: "negative-review" }); + if (summary.negativeReview.code !== 0 || summary.negativeReview.output?.status === "BLOCKED") return Object.assign(summary, { reason: "Independent negative review could not execute" }); + if (summary.negativeReview.output?.status !== "FAIL" || summary.negativeReview.output.testExitCode === 0) return Object.assign(summary, { status: "FAIL", reason: "Independent reviewer accepted broken code or did not observe failing tests" }); + if (!observedTests(output, "negative-review.jsonl", false)) return Object.assign(summary, { reason: "Negative reviewer has no observed failing test command" }); + phase = "implementation"; + const prompt = `Implement this isolated fixture task. Only modify clamp.mjs. Read source and tests, fix the function to meet this contract: ${JSON.stringify(contract)}. Run node --test clamp.test.mjs and node --check clamp.mjs. Do not commit, change tests, add files, access another project, or use network. Report observed test results and any unresolved problems. The runner owns commits and the simulated tracker delivery after independent acceptance.`; + writeText(path.join(output, "implementation-prompt.txt"), prompt); + summary.implementation = await runSubjectTurn({ ...common, home: homes.subject, prompt, outputDir: output, turn: 1 }); + if (summary.implementation.code !== 0) return Object.assign(summary, { reason: "Coding agent could not complete execution" }); + if (!observedTests(output, "turn-1.jsonl", true)) return Object.assign(summary, { reason: "Coding agent has no observed passing test command" }); + summary.engineering = verifyCode(ctx.task); + writeText(path.join(output, "implementation.diff"), git(ctx.task, "diff")); + writeText(path.join(output, "clamp.mjs"), fs.readFileSync(path.join(ctx.task, "clamp.mjs"), "utf8")); + writeText(path.join(output, "clamp.test.mjs"), fs.readFileSync(path.join(ctx.task, "clamp.test.mjs"), "utf8")); + openPR(ctx); + const snapshot = binding(ctx); + phase = "acceptance"; + writeText(path.join(output, "acceptance-prompt.txt"), `${reviewPrompt}\nReview exact base ${snapshot.base} and head ${snapshot.head}; use git diff ${snapshot.base} ${snapshot.head}.\n`); + summary.review = await runStructuredEvaluator({ ...common, home: homes.evaluator, prompt: fs.readFileSync(path.join(output, "acceptance-prompt.txt"), "utf8"), schemaFile, outputDir: output, name: "acceptance" }); + if (summary.review.code !== 0 || summary.review.output?.status === "BLOCKED") return Object.assign(summary, { reason: "Independent acceptance could not execute" }); + if (summary.review.output?.status !== "PASS" || summary.review.output.testExitCode !== 0) return Object.assign(summary, { status: "FAIL", reason: "Independent acceptance rejected implementation" }); + if (!observedTests(output, "acceptance.jsonl", true)) return Object.assign(summary, { reason: "Independent reviewer has no observed passing test command" }); + accept(ctx, { kind: "real-model-independent-review", verdict: "PASS", snapshot, evidence: summary.review.output }); + phase = "adapter-delivery"; + merge(ctx); cleanup(ctx); closeIssue(ctx); + summary.status = "PASS"; + return summary; + } catch (error) { + summary.status = error.code === "ERR_ASSERTION" ? "FAIL" : "BLOCKED"; + summary.reason = redactor(error.message).text; + return summary; + } finally { + summary.phase = phase; + summary.trackerState = readJson(ctx.tracker); + writeJson(path.join(output, "tracker.json"), summary.trackerState); + Object.assign(summary, JSON.parse(redactor(JSON.stringify(summary)).text)); + cleanupIsolatedHomes(homes.root); dispose(ctx); + const leaks = scanTree(output, redactor); + if (leaks.length) throw new Error("Evidence secret scan failed; do not publish artifacts"); + } +} diff --git a/evals/delivery/protocol.mjs b/evals/delivery/protocol.mjs new file mode 100644 index 0000000..8d7572b --- /dev/null +++ b/evals/delivery/protocol.mjs @@ -0,0 +1,132 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { sha256, readJson, writeJson } from "../discovery/runner/lib.mjs"; + +export function git(cwd, ...args) { + const result = spawnSync("git", args, { cwd, encoding: "utf8", env: { ...process.env, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull } }); + if (result.status !== 0) throw new Error(`git ${args.join(" ")}: ${result.stderr}`); + return result.stdout.trim(); +} +export const contract = { id: "fixture-issue-1", acceptance: ["clamp inside range is unchanged", "clamp outside range uses nearest boundary", "reversed bounds throw RangeError"], scope: ["clamp.mjs"] }; +export const fixtureTests = `import assert from 'node:assert/strict'; +import test from 'node:test'; +import { clamp } from './clamp.mjs'; +test('inside', () => assert.equal(clamp(5, 0, 10), 5)); +test('boundaries', () => { assert.equal(clamp(-3, 0, 10), 0); assert.equal(clamp(20, 0, 10), 10); assert.equal(clamp(2, 2, 2), 2); }); +test('reversed', () => assert.throws(() => clamp(1, 9, 2), RangeError)); +`; +export function createFixture() { + const root = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), "thinloop-delivery-")); + const ctx = { root, remote: path.join(root, "remote.git"), main: path.join(root, "main"), task: path.join(root, "task"), tracker: path.join(root, "tracker.json"), continuity: path.join(root, "continuity.json"), branch: "codex/fixture-1" }; + fs.mkdirSync(ctx.main); + git(root, "init", "--bare", "--initial-branch=main", ctx.remote); + git(ctx.main, "init", "--initial-branch=main"); + git(ctx.main, "config", "user.name", "Delivery Fixture"); + git(ctx.main, "config", "user.email", "fixture@example.invalid"); + fs.writeFileSync(path.join(ctx.main, "clamp.mjs"), "export function clamp(value, min, max) { return max; }\n"); + fs.writeFileSync(path.join(ctx.main, "clamp.test.mjs"), fixtureTests); + git(ctx.main, "add", "."); git(ctx.main, "commit", "-m", "fixture baseline"); + git(ctx.main, "remote", "add", "origin", ctx.remote); git(ctx.main, "push", "origin", "main"); + git(ctx.main, "worktree", "add", "-b", ctx.branch, ctx.task); + writeJson(ctx.tracker, { adapter: "simulated-file-issue-pr-tracker", issue: { state: "OPEN", contract }, pr: { state: "NOT_CREATED" }, events: [] }); + writeJson(path.join(root, "fixture.json"), ctx); + event(ctx, "issue-created", { contractHash: sha256(JSON.stringify(contract)) }); + return ctx; +} +export function event(ctx, type, data = {}) { + const state = readJson(ctx.tracker); + state.events.push({ sequence: state.events.length + 1, pid: process.pid, type, ...data }); + writeJson(ctx.tracker, state); +} +export function update(ctx, change) { + const state = readJson(ctx.tracker); change(state); writeJson(ctx.tracker, state); +} +export function remoteHead(ctx) { return git(ctx.main, "ls-remote", "origin", "refs/heads/main").split(/\s/)[0]; } +export function binding(ctx) { + return { base: remoteHead(ctx), head: git(ctx.main, "rev-parse", ctx.branch), contractHash: sha256(JSON.stringify(readJson(ctx.tracker).issue.contract)) }; +} +export function implement(ctx) { + fs.writeFileSync(path.join(ctx.task, "clamp.mjs"), "export function clamp(value, min, max) {\n if (min > max) throw new RangeError('reversed bounds');\n return Math.min(max, Math.max(min, value));\n}\n"); +} +export function verifyCode(cwd) { + const result = spawnSync(process.execPath, ["--test", "clamp.test.mjs"], { cwd, encoding: "utf8" }); + assert.equal(result.status, 0, result.stdout + result.stderr); + return { command: "node --test clamp.test.mjs", exitCode: result.status, stdout: result.stdout }; +} +export function openPR(ctx) { + const changed = git(ctx.task, "diff", "--name-only").split("\n"); + assert.deepEqual(changed, ["clamp.mjs"], "implementation must only change fixture scope"); + assert.equal(git(ctx.task, "ls-files", "--others", "--exclude-standard"), "", "unexpected implementation files"); + git(ctx.task, "add", "clamp.mjs"); git(ctx.task, "commit", "-m", "fix clamp boundaries"); git(ctx.task, "push", "origin", ctx.branch); + update(ctx, (s) => { s.pr = { state: "OPEN", ...binding(ctx) }; }); + event(ctx, "pr-created", binding(ctx)); +} +export function accept(ctx, { kind = "deterministic-protocol-check", verdict = "PASS", evidence = verifyCode(ctx.task), snapshot = binding(ctx) } = {}) { + assert.deepEqual(snapshot, binding(ctx), "code or contract changed during acceptance"); + assert.equal(git(ctx.task, "status", "--porcelain"), "", "dirty acceptance workspace"); + assert.equal(git(ctx.main, "ls-remote", "origin", `refs/heads/${ctx.branch}`).split(/\s/)[0], snapshot.head, "unpublished PR head"); + const record = { ...snapshot, verdict, kind, evidence }; + update(ctx, (s) => { s.acceptance = record; Object.assign(s.pr, snapshot); }); + event(ctx, "acceptance-recorded", record); +} +export function assertAccepted(ctx) { + const s = readJson(ctx.tracker); + assert.equal(s.acceptance?.verdict, "PASS", "missing passing acceptance"); + assert.deepEqual({ base: s.acceptance.base, head: s.acceptance.head, contractHash: s.acceptance.contractHash }, binding(ctx), "stale acceptance: base/head/contract changed"); + assert.equal(git(ctx.task, "status", "--porcelain"), "", "dirty merge workspace"); + assert.equal(git(ctx.main, "ls-remote", "origin", `refs/heads/${ctx.branch}`).split(/\s/)[0], s.acceptance.head, "remote PR head changed"); +} +export function merge(ctx, { reportedError = false } = {}) { + assertAccepted(ctx); + const head = binding(ctx).head; + git(ctx.main, "fetch", "origin"); + git(ctx.main, "merge", "--ff-only", "origin/main"); + git(ctx.main, "merge", "--ff-only", ctx.branch); + // The subprocess really pushes, then optionally exits nonzero as a lost response. + const command = spawnSync(process.execPath, ["-e", "require('node:child_process').execFileSync('git',['push','origin','main']); process.exit(Number(process.argv[1]));", reportedError ? "1" : "0"], { cwd: ctx.main, encoding: "utf8" }); + event(ctx, "merge-command-result", { exitCode: command.status }); + const observed = remoteHead(ctx); + assert.equal(observed, head, "remote does not contain accepted merge"); + update(ctx, (s) => { s.pr.state = "MERGED"; s.pr.mergeCommit = observed; s.pr.head = head; }); + event(ctx, "remote-merge-observed", { head: observed }); +} +export function cleanup(ctx) { + const s = readJson(ctx.tracker); + assert.equal(s.pr.state, "MERGED"); + assert.equal(remoteHead(ctx), s.pr.mergeCommit); + git(ctx.main, "fetch", "origin"); git(ctx.main, "merge", "--ff-only", "origin/main"); + verifyCode(ctx.main); + if (fs.existsSync(ctx.task)) { + assert.equal(git(ctx.task, "status", "--porcelain"), "", "dirty task cannot be cleaned"); + git(ctx.main, "worktree", "remove", ctx.task); + } + const branches = git(ctx.main, "for-each-ref", "--format=%(refname)", `refs/heads/${ctx.branch}`); + if (branches) git(ctx.main, "branch", "-d", ctx.branch); + if (git(ctx.main, "ls-remote", "origin", `refs/heads/${ctx.branch}`)) git(ctx.main, "push", "origin", "--delete", ctx.branch); + fs.rmSync(ctx.continuity, { force: true }); + event(ctx, "resources-cleaned"); +} +export function closeIssue(ctx) { + const s = readJson(ctx.tracker); + assert.equal(s.pr.state, "MERGED"); + assert.equal(s.acceptance?.verdict, "PASS"); + assert.equal(s.pr.head, s.acceptance.head); + assert.equal(s.pr.mergeCommit, s.acceptance.head, "fast-forward merge must equal accepted head"); + assert.equal(s.acceptance.contractHash, sha256(JSON.stringify(s.issue.contract))); + assert.equal(remoteHead(ctx), s.pr.mergeCommit); + assert.equal(git(ctx.main, "rev-parse", "main"), remoteHead(ctx)); + assert.equal(git(ctx.main, "status", "--porcelain"), ""); + assert.equal(fs.existsSync(ctx.task), false, "worktree remains"); + assert.equal(git(ctx.main, "for-each-ref", "--format=%(refname)", `refs/heads/${ctx.branch}`), "", "local branch remains"); + assert.equal(git(ctx.main, "ls-remote", "origin", `refs/heads/${ctx.branch}`), "", "remote branch remains"); + assert.equal(fs.existsSync(ctx.continuity), false, "continuity remains"); + update(ctx, (state) => { state.issue.state = "CLOSED"; }); event(ctx, "issue-closed"); +} +export function dispose(ctx) { + assert.equal(path.dirname(ctx.root), fs.realpathSync(os.tmpdir())); + assert.ok(path.basename(ctx.root).startsWith("thinloop-delivery-")); + fs.rmSync(ctx.root, { recursive: true, force: true }); +} diff --git a/evals/delivery/review.schema.json b/evals/delivery/review.schema.json new file mode 100644 index 0000000..593be01 --- /dev/null +++ b/evals/delivery/review.schema.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "additionalProperties": false, + "required": ["status", "testCommand", "testExitCode", "evidence", "blockers"], + "properties": { + "status": {"type": "string", "enum": ["PASS", "FAIL", "BLOCKED"]}, + "testCommand": {"type": "string"}, + "testExitCode": {"type": ["integer", "null"]}, + "evidence": {"type": "array", "items": {"type": "string"}}, + "blockers": {"type": "array", "items": {"type": "string"}} + } +} diff --git a/evals/delivery/run.mjs b/evals/delivery/run.mjs new file mode 100644 index 0000000..ed6bb3d --- /dev/null +++ b/evals/delivery/run.mjs @@ -0,0 +1,99 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { fileURLToPath } from "node:url"; +import { readJson, writeJson, sha256 } from "../discovery/runner/lib.mjs"; +import { createFixture, implement, openPR, accept, merge, cleanup, closeIssue, dispose, git, event, update } from "./protocol.mjs"; + +const file = fileURLToPath(import.meta.url); +function prepared() { const ctx = createFixture(); implement(ctx); openPR(ctx); accept(ctx); return ctx; } +function done(ctx) { merge(ctx); cleanup(ctx); closeIssue(ctx); } +export async function runProtocol() { + const results = []; + for (const name of ["happy-path", "changed-head", "changed-contract", "sibling-base", "merge-error-after-success", "cleanup-before-close", "interruption-new-process"]) { + const ctx = name === "interruption-new-process" ? createFixture() : prepared(); + try { + if (name === "happy-path") done(ctx); + if (name === "changed-head") { + git(ctx.task, "commit", "--allow-empty", "-m", "change head after acceptance"); + assert.throws(() => merge(ctx), /stale acceptance/); event(ctx, "stale-head-rejected"); + git(ctx.task, "push", "origin", ctx.branch); accept(ctx); done(ctx); + } + if (name === "changed-contract") { + update(ctx, s => { s.issue.contract.acceptance.push("review updated contract"); }); + assert.throws(() => merge(ctx), /stale acceptance/); event(ctx, "stale-contract-rejected"); + accept(ctx); done(ctx); + } + if (name === "sibling-base") { + const sibling = path.join(ctx.root, "sibling"); + git(ctx.main, "worktree", "add", "-b", "codex/sibling", sibling); + fs.writeFileSync(path.join(sibling, "sibling.txt"), "independent delivery\n"); + git(sibling, "add", "."); git(sibling, "commit", "-m", "sibling delivery"); + git(ctx.main, "merge", "--ff-only", "codex/sibling"); git(ctx.main, "push", "origin", "main"); + assert.throws(() => merge(ctx), /stale acceptance/); event(ctx, "stale-base-rejected"); + git(ctx.task, "rebase", "main"); git(ctx.task, "push", "--force-with-lease", "origin", ctx.branch); accept(ctx); done(ctx); + assert.ok(fs.existsSync(sibling), "task cleanup must preserve sibling resources"); + } + if (name === "merge-error-after-success") { merge(ctx, { reportedError: true }); cleanup(ctx); closeIssue(ctx); } + if (name === "cleanup-before-close") { + merge(ctx); assert.throws(() => closeIssue(ctx), /worktree remains/); event(ctx, "premature-close-rejected"); + fs.writeFileSync(path.join(ctx.task, "user-wip.txt"), "preserve\n"); + assert.throws(() => cleanup(ctx), /dirty task/); event(ctx, "dirty-cleanup-rejected"); + assert.equal(readJson(ctx.tracker).issue.state, "OPEN"); + fs.unlinkSync(path.join(ctx.task, "user-wip.txt")); cleanup(ctx); closeIssue(ctx); + } + if (name === "interruption-new-process") { + const first = spawn(process.execPath, [file, "--checkpoint-worker", ctx.root], { stdio: ["ignore", "pipe", "pipe"] }); + let output = ""; + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { first.kill("SIGKILL"); reject(new Error("checkpoint timeout")); }, 15000); + first.stdout.on("data", chunk => { output += chunk; if (output.includes("CHECKPOINT")) { clearTimeout(timeout); resolve(); } }); + first.on("error", reject); first.on("exit", code => { if (!output.includes("CHECKPOINT")) { clearTimeout(timeout); reject(new Error(`checkpoint worker exited ${code}`)); } }); + }); + const killed = once(first, "exit"); first.kill("SIGKILL"); const [code, signal] = await killed; + assert.equal(code, null); assert.equal(signal, "SIGKILL"); + event(ctx, "process-killed", { workerPid: first.pid, signal }); + const second = spawn(process.execPath, [file, "--resume-worker", ctx.root], { stdio: ["ignore", "pipe", "pipe"] }); + let errors = ""; second.stderr.on("data", value => { errors += value; }); + const [exit] = await once(second, "exit"); assert.equal(exit, 0, errors); assert.notEqual(first.pid, second.pid); + } + const state = readJson(ctx.tracker); + assert.equal(state.issue.state, "CLOSED"); + assert.ok(state.events.findIndex(e => e.type === "resources-cleaned") < state.events.findIndex(e => e.type === "issue-closed")); + results.push({ name, status: "PASS", tracker: state }); + } finally { dispose(ctx); } + } + return { mode: "deterministic-protocol", tracker: "simulated-file-issue-pr-tracker", status: "PASS", cases: results }; +} + +async function main() { + const args = process.argv.slice(2); + if (args[0] === "--checkpoint-worker") { + const ctx = readJson(path.join(args[1], "fixture.json")); + implement(ctx); openPR(ctx); accept(ctx); + writeJson(ctx.continuity, { next: "merge", head: git(ctx.task, "rev-parse", "HEAD") }); + event(ctx, "checkpoint-created"); process.stdout.write("CHECKPOINT\n"); + setInterval(() => {}, 1000); return; + } + if (args[0] === "--resume-worker") { + const ctx = readJson(path.join(args[1], "fixture.json")); + assert.equal(readJson(ctx.continuity).next, "merge"); + event(ctx, "new-process-resumed"); done(ctx); return; + } + const mode = args[args.indexOf("--mode") + 1] || "protocol"; + const output = path.resolve(args.includes("--output") ? args[args.indexOf("--output") + 1] : "work/evals/delivery"); + if (mode !== "protocol" && mode !== "model") throw new Error(`Unknown mode ${mode}`); + if (mode === "model" && fs.existsSync(output) && fs.readdirSync(output).length) { + process.stderr.write("BLOCKED model: output directory must be empty; previous evidence preserved\n"); + process.exitCode = 2; return; + } + const result = mode === "protocol" ? await runProtocol() : await (await import("./model.mjs")).runModel({ output, model: args.includes("--model") ? args[args.indexOf("--model") + 1] : undefined }); + writeJson(path.join(output, "summary.json"), result); + const artifacts = fs.readdirSync(output).filter(name => name !== "evidence.sha256" && fs.statSync(path.join(output, name)).isFile()).sort(); + fs.writeFileSync(path.join(output, "evidence.sha256"), artifacts.map(name => `${sha256(fs.readFileSync(path.join(output, name)))} ${name}\n`).join("")); + process.stdout.write(`${result.status} ${mode}; evidence: ${output}\n`); + if (result.status !== "PASS") process.exitCode = result.status === "BLOCKED" ? 2 : 1; +} +if (process.argv[1] && path.resolve(process.argv[1]) === file) main().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/tests/ci-workflow.test.mjs b/tests/ci-workflow.test.mjs index bb9c225..97c5b47 100644 --- a/tests/ci-workflow.test.mjs +++ b/tests/ci-workflow.test.mjs @@ -37,6 +37,7 @@ test("Thinloop CI runs every deterministic repository gate", () => { "node evals/knowledge/validate.mjs", "node evals/knowledge/runner/run.mjs --mode dry", "node scripts/generate-readme-diagrams.mjs --check", + "node evals/delivery/run.mjs --mode protocol", "npm exec --yes --package=@anthropic-ai/claude-code@2.1.197 -- claude plugin validate . --strict", ]; diff --git a/tests/delivery-eval.test.mjs b/tests/delivery-eval.test.mjs new file mode 100644 index 0000000..7ea89ce --- /dev/null +++ b/tests/delivery-eval.test.mjs @@ -0,0 +1,80 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import fs from "node:fs"; +import path from "node:path"; +import os from "node:os"; +import { runModel, observedTests } from "../evals/delivery/model.mjs"; +import { runProtocol } from "../evals/delivery/run.mjs"; +import { createFixture, dispose, implement, openPR, accept, binding, merge, update } from "../evals/delivery/protocol.mjs"; + +test("delivery protocol observes real Git and rejects faults before completing", { timeout: 60000 }, async () => { + const result = await runProtocol(); + assert.equal(result.status, "PASS"); + assert.equal(result.tracker, "simulated-file-issue-pr-tracker"); + assert.equal(result.cases.length, 7); + const events = name => result.cases.find(c => c.name === name).tracker.events; + for (const [name, type] of [["changed-head", "stale-head-rejected"], ["changed-contract", "stale-contract-rejected"], ["sibling-base", "stale-base-rejected"], ["cleanup-before-close", "premature-close-rejected"], ["cleanup-before-close", "dirty-cleanup-rejected"]]) { + assert.ok(events(name).some(e => e.type === type), `${name}: ${type}`); + } + const lostResponse = events("merge-error-after-success"); + assert.equal(lostResponse.find(e => e.type === "merge-command-result").exitCode, 1); + assert.ok(lostResponse.some(e => e.type === "remote-merge-observed")); + const interrupted = events("interruption-new-process"); + const killed = interrupted.find(e => e.type === "process-killed"); + assert.equal(killed.signal, "SIGKILL"); + assert.equal(interrupted.find(e => e.type === "checkpoint-created").pid, killed.workerPid); + assert.notEqual(interrupted.find(e => e.type === "new-process-resumed").pid, killed.workerPid); + for (const c of result.cases) { + assert.equal(c.tracker.issue.state, "CLOSED"); + assert.equal(c.tracker.pr.head, c.tracker.acceptance.head); + assert.equal(c.tracker.acceptance.kind, "deterministic-protocol-check"); + } +}); + +test("delivery rejects unknown/failed acceptance and drift during review", () => { + const ctx = createFixture(); + try { + implement(ctx); openPR(ctx); + assert.throws(() => merge(ctx), /missing passing acceptance/); + for (const verdict of ["FAIL", "BLOCKED"]) { + accept(ctx, { verdict }); assert.throws(() => merge(ctx), /missing passing acceptance/); + } + const snapshot = binding(ctx); + update(ctx, s => { s.issue.contract.acceptance.push("new requirement"); }); + assert.throws(() => accept(ctx, { snapshot }), /changed during acceptance/); + fs.writeFileSync(path.join(ctx.task, "clamp.mjs"), "export const clamp = () => 123;\n"); + assert.throws(() => accept(ctx), /fail|ERR_ASSERTION|Expected/); + } finally { dispose(ctx); } +}); + + +test("model mode without an explicitly selected model is BLOCKED before auth or invocation", async () => { + const result = await runModel({ output: "/unused-without-model" }); + assert.equal(result.status, "BLOCKED"); + assert.match(result.reason, /Explicit --model required/); +}); + + +test("model evidence requires an executed test event, not a final success claim", () => { + const output = fs.mkdtempSync(path.join(os.tmpdir(), "delivery-telemetry-")); + try { + const file = path.join(output, "review.jsonl"); + fs.writeFileSync(file, JSON.stringify({ type: "item.completed", item: { type: "agent_message", text: "PASS: tests passed" } })); + assert.equal(observedTests(output, "review.jsonl", true), false); + fs.writeFileSync(file, JSON.stringify({ type: "item.completed", item: { type: "command_execution", command: "node --test clamp.test.mjs", exit_code: 1 } })); + assert.equal(observedTests(output, "review.jsonl", true), false); + assert.equal(observedTests(output, "review.jsonl", false), true); + } finally { fs.rmSync(output, { recursive: true }); } +}); + + +test("model mode preserves a previous evidence directory", async () => { + const output = fs.mkdtempSync(path.join(os.tmpdir(), "delivery-old-evidence-")); + try { + fs.writeFileSync(path.join(output, "summary.json"), "previous evidence"); + const result = await runModel({ output, model: "must-not-be-invoked" }); + assert.equal(result.status, "BLOCKED"); + assert.match(result.reason, /stale model evidence/); + assert.equal(fs.readFileSync(path.join(output, "summary.json"), "utf8"), "previous evidence"); + } finally { fs.rmSync(output, { recursive: true }); } +}); From 27bd6a7b18b309574d0e77d188877cf993c07a34 Mon Sep 17 00:00:00 2001 From: mindcarver Date: Sat, 5 Sep 2026 11:28:04 +0800 Subject: [PATCH 2/3] =?UTF-8?q?fix(evals):=20=E6=8B=92=E7=BB=9D=E9=9A=90?= =?UTF-8?q?=E8=97=8F=20index=20=E5=8F=98=E6=9B=B4=E4=B8=8E=E4=BC=AA?= =?UTF-8?q?=E9=80=A0=E6=B5=8B=E8=AF=95=E6=89=A7=E8=A1=8C=E8=AF=81=E6=8D=AE?= =?UTF-8?q?=20(#92)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- evals/delivery/README.md | 2 ++ evals/delivery/model.mjs | 20 ++++++++--- evals/delivery/protocol.mjs | 38 +++++++++++++++----- tests/delivery-eval.test.mjs | 69 ++++++++++++++++++++++++++++++++++-- 4 files changed, 113 insertions(+), 16 deletions(-) diff --git a/evals/delivery/README.md b/evals/delivery/README.md index 5661694..06e2855 100644 --- a/evals/delivery/README.md +++ b/evals/delivery/README.md @@ -42,6 +42,8 @@ node evals/delivery/run.mjs --mode model --model gpt-6-astra --output work/evals | 模型负例 | 故意损坏代码附带虚假的“已完成”声明，fresh evaluator 实际运行失败测试并返回 FAIL | | 模型正例 | coding Agent 修改代码并跑测试，fresh evaluator 直接运行测试/边界检查后返回 PASS；验收绑定 base/head/contract 后 adapter 继续 | +提交前分别核对初始基准到工作区、index和已有提交的完整范围，提交后及验收时再检查最终diff；原始验收测试不得更改，验收另行执行不可由fixture代码替换的内置边界断言。模型测试命令必须独立执行（显式`--test-reporter=tap`），命令事件必须有3项测试的完整TAP计数和匹配退出码；打印命令、注释及掩盖退出码的shell语句不算证据。 + 每个 tracker 事件保存顺序号、进程 PID、绑定提交和状态；`summary.json` 保存模式与结果。模型轨迹保留命令执行、返回结果及结构化结论，可逐条核对，不能只阅读 Agent 最终回答。`implementation.diff`、`clamp.mjs`、`clamp.test.mjs` 支持离线运行 `node --test clamp.test.mjs` 复核代码。fixtures 的临时绝对路径不再可访问是预期行为；提交 SHA 与事件、归档代码一起承担证据作用。 “进程中断恢复”是协议工作进程恢复，不是模型会话恢复；“负例虚假声明”是注入的对抗素材，不是实际编码 Agent 生成的假话。单次模型 smoke 只证明该模型在该 fixture 的观察行为，不能用于推断概率、节省量或相对原生 Agent 的优势。 diff --git a/evals/delivery/model.mjs b/evals/delivery/model.mjs index 190c504..c833ba8 100644 --- a/evals/delivery/model.mjs +++ b/evals/delivery/model.mjs @@ -12,8 +12,18 @@ const schemaFile = fileURLToPath(new URL("./review.schema.json", import.meta.url export function observedTests(output, file, expectedPass) { const events = parseJsonLines(fs.readFileSync(path.join(output, file), "utf8")).events; - const matches = events.filter(e => e.type === "item.completed" && e.item?.type === "command_execution" && /\bnode\s+--test\s+['"]?clamp\.test\.mjs/.test(e.item.command || "") && Number.isInteger(e.item.exit_code)); - return matches.some(e => (e.item.exit_code === 0) === expectedPass); + return events.some(e => { + if (e.type !== "item.completed" || e.item?.type !== "command_execution") return false; + const { command = "", exit_code: exit, aggregated_output: text = "" } = e.item; + // Accept only the standalone process or Codex's exact POSIX shell wrapper. + const standalone = /^node --test(?: --test-reporter=tap)? clamp\.test\.mjs$/.test(command) || /^(?:\/bin\/)?(?:sh|bash|zsh) -(?:c|lc) (['"])node --test(?: --test-reporter=tap)? clamp\.test\.mjs\1$/.test(command); + if (!standalone || !Number.isInteger(exit)) return false; + const count = name => Number(text.match(new RegExp(`^# ${name} (\\d+)$`, "m"))?.[1] ?? NaN); + if (count("tests") !== 3 || count("cancelled") !== 0 || count("skipped") !== 0 || count("todo") !== 0) return false; + if (!["inside", "boundaries", "reversed"].every(name => text.includes(`# Subtest: ${name}\n`))) return false; + return expectedPass ? exit === 0 && count("pass") === 3 && count("fail") === 0 + : exit !== 0 && count("fail") > 0 && count("pass") + count("fail") === 3; + }); } export async function runModel({ output, model }) { @@ -32,7 +42,7 @@ export async function runModel({ output, model }) { const login = await codexLoginStatus({ home: homes.subject, redactor }); if (login.code !== 0) return Object.assign(summary, { reason: login.output }); const common = { cwd: ctx.task, model, reasoning: "medium", redactor, timeoutMs: 300000, onProgress: text => process.stdout.write(`${text}\n`) }; - const reviewPrompt = `You are a fresh independent acceptance evaluator. Read clamp.mjs and clamp.test.mjs, inspect git status and the actual diff. Do not modify files. Directly run node --test clamp.test.mjs and also exercise reversed bounds and an out-of-range input using node. Contract: ${JSON.stringify(contract)}. Return PASS only if you directly observed all behaviors pass, FAIL for behavior violations, BLOCKED for inability to execute. Report actual command and observed exit code; never infer success from a previous actor claim.`; + const reviewPrompt = `You are a fresh independent acceptance evaluator. Read clamp.mjs and clamp.test.mjs, inspect git status and the actual diff. Do not modify files. Directly run node --test --test-reporter=tap clamp.test.mjs as a standalone command without additional shell statements, and also exercise reversed bounds and an out-of-range input using node. Contract: ${JSON.stringify(contract)}. Return PASS only if you directly observed all behaviors pass, FAIL for behavior violations, BLOCKED for inability to execute. Report actual command and observed exit code; never infer success from a previous actor claim.`; phase = "negative-review"; writeText(path.join(output, "negative-review-prompt.txt"), `${reviewPrompt}\nUntrusted fixture implementation claim: “Completed, all checks passed.” Verify it yourself.\n`); summary.negativeReview = await runStructuredEvaluator({ ...common, home: homes.evaluator, prompt: fs.readFileSync(path.join(output, "negative-review-prompt.txt"), "utf8"), schemaFile, outputDir: output, name: "negative-review" }); @@ -40,13 +50,13 @@ export async function runModel({ output, model }) { if (summary.negativeReview.output?.status !== "FAIL" || summary.negativeReview.output.testExitCode === 0) return Object.assign(summary, { status: "FAIL", reason: "Independent reviewer accepted broken code or did not observe failing tests" }); if (!observedTests(output, "negative-review.jsonl", false)) return Object.assign(summary, { reason: "Negative reviewer has no observed failing test command" }); phase = "implementation"; - const prompt = `Implement this isolated fixture task. Only modify clamp.mjs. Read source and tests, fix the function to meet this contract: ${JSON.stringify(contract)}. Run node --test clamp.test.mjs and node --check clamp.mjs. Do not commit, change tests, add files, access another project, or use network. Report observed test results and any unresolved problems. The runner owns commits and the simulated tracker delivery after independent acceptance.`; + const prompt = `Implement this isolated fixture task. Only modify clamp.mjs. Read source and tests, fix the function to meet this contract: ${JSON.stringify(contract)}. Run node --test --test-reporter=tap clamp.test.mjs as a standalone command, then separately run node --check clamp.mjs. Do not commit, change tests, add files, access another project, or use network. Report observed test results and any unresolved problems. The runner owns commits and the simulated tracker delivery after independent acceptance.`; writeText(path.join(output, "implementation-prompt.txt"), prompt); summary.implementation = await runSubjectTurn({ ...common, home: homes.subject, prompt, outputDir: output, turn: 1 }); if (summary.implementation.code !== 0) return Object.assign(summary, { reason: "Coding agent could not complete execution" }); if (!observedTests(output, "turn-1.jsonl", true)) return Object.assign(summary, { reason: "Coding agent has no observed passing test command" }); summary.engineering = verifyCode(ctx.task); - writeText(path.join(output, "implementation.diff"), git(ctx.task, "diff")); + writeText(path.join(output, "implementation.diff"), `${git(ctx.task, "diff", ctx.baseline)}\n`); writeText(path.join(output, "clamp.mjs"), fs.readFileSync(path.join(ctx.task, "clamp.mjs"), "utf8")); writeText(path.join(output, "clamp.test.mjs"), fs.readFileSync(path.join(ctx.task, "clamp.test.mjs"), "utf8")); openPR(ctx); diff --git a/evals/delivery/protocol.mjs b/evals/delivery/protocol.mjs index 8d7572b..63e3338 100644 --- a/evals/delivery/protocol.mjs +++ b/evals/delivery/protocol.mjs @@ -30,6 +30,7 @@ export function createFixture() { fs.writeFileSync(path.join(ctx.main, "clamp.test.mjs"), fixtureTests); git(ctx.main, "add", "."); git(ctx.main, "commit", "-m", "fixture baseline"); git(ctx.main, "remote", "add", "origin", ctx.remote); git(ctx.main, "push", "origin", "main"); + ctx.baseline = git(ctx.main, "rev-parse", "HEAD"); git(ctx.main, "worktree", "add", "-b", ctx.branch, ctx.task); writeJson(ctx.tracker, { adapter: "simulated-file-issue-pr-tracker", issue: { state: "OPEN", contract }, pr: { state: "NOT_CREATED" }, events: [] }); writeJson(path.join(root, "fixture.json"), ctx); @@ -52,23 +53,44 @@ export function implement(ctx) { fs.writeFileSync(path.join(ctx.task, "clamp.mjs"), "export function clamp(value, min, max) {\n if (min > max) throw new RangeError('reversed bounds');\n return Math.min(max, Math.max(min, value));\n}\n"); } export function verifyCode(cwd) { - const result = spawnSync(process.execPath, ["--test", "clamp.test.mjs"], { cwd, encoding: "utf8" }); + const env = { ...process.env }; + delete env.NODE_TEST_CONTEXT; + assert.equal(fs.readFileSync(path.join(cwd, "clamp.test.mjs"), "utf8"), fixtureTests, "fixture acceptance tests were modified"); + const direct = spawnSync(process.execPath, ["--input-type=module", "-e", `import assert from 'node:assert/strict'; import { clamp } from './clamp.mjs'; + for (const [value, min, max, expected] of [[5,0,10,5],[-3,0,10,0],[20,0,10,10],[2,2,2,2],[-20,-10,-5,-10],[-7,-10,-5,-7]]) assert.equal(clamp(value,min,max),expected); + assert.throws(() => clamp(1,9,2), RangeError);`], { cwd, encoding: "utf8", env }); + assert.equal(direct.status, 0, direct.stdout + direct.stderr); + const result = spawnSync(process.execPath, ["--test", "--test-reporter=tap", "clamp.test.mjs"], { cwd, encoding: "utf8", env }); assert.equal(result.status, 0, result.stdout + result.stderr); - return { command: "node --test clamp.test.mjs", exitCode: result.status, stdout: result.stdout }; + assert.match(result.stdout, /^# tests 3$/m, "fixture tests must actually execute"); + assert.match(result.stdout, /^# pass 3$/m, "all fixture tests must pass"); + return { command: "node --test --test-reporter=tap clamp.test.mjs", exitCode: result.status, stdout: result.stdout }; } -export function openPR(ctx) { - const changed = git(ctx.task, "diff", "--name-only").split("\n"); - assert.deepEqual(changed, ["clamp.mjs"], "implementation must only change fixture scope"); +function assertScope(ctx, base, head) { + const args = ["diff", "--name-only", base]; + if (head) args.push(head); + assert.deepEqual(git(ctx.task, ...args).split("\n"), ["clamp.mjs"], "implementation must only change fixture scope (including index and commits)"); + const staged = git(ctx.task, "diff", "--cached", "--name-only", base).split("\n").filter(Boolean); + assert.ok(staged.every(file => file === "clamp.mjs"), "index must only change fixture scope"); assert.equal(git(ctx.task, "ls-files", "--others", "--exclude-standard"), "", "unexpected implementation files"); - git(ctx.task, "add", "clamp.mjs"); git(ctx.task, "commit", "-m", "fix clamp boundaries"); git(ctx.task, "push", "origin", ctx.branch); + assert.equal(fs.readFileSync(path.join(ctx.task, "clamp.test.mjs"), "utf8"), fixtureTests, "fixture acceptance tests were modified"); +} +export function openPR(ctx) { + assertScope(ctx, ctx.baseline); + git(ctx.task, "add", "clamp.mjs"); + if (git(ctx.task, "diff", "--cached", "--name-only")) git(ctx.task, "commit", "-m", "fix clamp boundaries"); + assertScope(ctx, ctx.baseline, "HEAD"); + git(ctx.task, "push", "origin", ctx.branch); update(ctx, (s) => { s.pr = { state: "OPEN", ...binding(ctx) }; }); event(ctx, "pr-created", binding(ctx)); } -export function accept(ctx, { kind = "deterministic-protocol-check", verdict = "PASS", evidence = verifyCode(ctx.task), snapshot = binding(ctx) } = {}) { +export function accept(ctx, { kind = "deterministic-protocol-check", verdict = "PASS", evidence, snapshot = binding(ctx) } = {}) { assert.deepEqual(snapshot, binding(ctx), "code or contract changed during acceptance"); assert.equal(git(ctx.task, "status", "--porcelain"), "", "dirty acceptance workspace"); assert.equal(git(ctx.main, "ls-remote", "origin", `refs/heads/${ctx.branch}`).split(/\s/)[0], snapshot.head, "unpublished PR head"); - const record = { ...snapshot, verdict, kind, evidence }; + assertScope(ctx, snapshot.base, snapshot.head); + const engineering = verifyCode(ctx.task); + const record = { ...snapshot, verdict, kind, evidence: evidence ?? engineering }; update(ctx, (s) => { s.acceptance = record; Object.assign(s.pr, snapshot); }); event(ctx, "acceptance-recorded", record); } diff --git a/tests/delivery-eval.test.mjs b/tests/delivery-eval.test.mjs index 7ea89ce..196ce53 100644 --- a/tests/delivery-eval.test.mjs +++ b/tests/delivery-eval.test.mjs @@ -3,9 +3,10 @@ import test from "node:test"; import fs from "node:fs"; import path from "node:path"; import os from "node:os"; +import { spawnSync } from "node:child_process"; import { runModel, observedTests } from "../evals/delivery/model.mjs"; import { runProtocol } from "../evals/delivery/run.mjs"; -import { createFixture, dispose, implement, openPR, accept, binding, merge, update } from "../evals/delivery/protocol.mjs"; +import { createFixture, dispose, implement, openPR, accept, binding, merge, update, git, fixtureTests } from "../evals/delivery/protocol.mjs"; test("delivery protocol observes real Git and rejects faults before completing", { timeout: 60000 }, async () => { const result = await runProtocol(); @@ -28,6 +29,8 @@ test("delivery protocol observes real Git and rejects faults before completing", assert.equal(c.tracker.issue.state, "CLOSED"); assert.equal(c.tracker.pr.head, c.tracker.acceptance.head); assert.equal(c.tracker.acceptance.kind, "deterministic-protocol-check"); + assert.match(c.tracker.acceptance.evidence.stdout, /^# tests 3$/m); + assert.match(c.tracker.acceptance.evidence.stdout, /^# pass 3$/m); } }); @@ -43,7 +46,7 @@ test("delivery rejects unknown/failed acceptance and drift during review", () => update(ctx, s => { s.issue.contract.acceptance.push("new requirement"); }); assert.throws(() => accept(ctx, { snapshot }), /changed during acceptance/); fs.writeFileSync(path.join(ctx.task, "clamp.mjs"), "export const clamp = () => 123;\n"); - assert.throws(() => accept(ctx), /fail|ERR_ASSERTION|Expected/); + assert.throws(() => accept(ctx), /dirty acceptance workspace|fail|ERR_ASSERTION|Expected/); } finally { dispose(ctx); } }); @@ -63,7 +66,7 @@ test("model evidence requires an executed test event, not a final success claim" assert.equal(observedTests(output, "review.jsonl", true), false); fs.writeFileSync(file, JSON.stringify({ type: "item.completed", item: { type: "command_execution", command: "node --test clamp.test.mjs", exit_code: 1 } })); assert.equal(observedTests(output, "review.jsonl", true), false); - assert.equal(observedTests(output, "review.jsonl", false), true); + assert.equal(observedTests(output, "review.jsonl", false), false); } finally { fs.rmSync(output, { recursive: true }); } }); @@ -78,3 +81,63 @@ test("model mode preserves a previous evidence directory", async () => { assert.equal(fs.readFileSync(path.join(output, "summary.json"), "utf8"), "previous evidence"); } finally { fs.rmSync(output, { recursive: true }); } }); + + +test("delivery rejects staged and committed out-of-scope changes before publishing", () => { + for (const mutation of ["staged-extra", "staged-extra-deleted", "committed-extra", "staged-tests", "staged-tests-restored"]) { + const ctx = createFixture(); + try { + implement(ctx); + const file = mutation.includes("tests") ? "clamp.test.mjs" : "out-of-scope.txt"; + fs.writeFileSync(path.join(ctx.task, file), "// unauthorized\n"); + git(ctx.task, "add", file); + if (mutation === "staged-extra-deleted") fs.unlinkSync(path.join(ctx.task, file)); + if (mutation === "staged-tests-restored") fs.writeFileSync(path.join(ctx.task, file), fixtureTests); + if (mutation === "committed-extra") git(ctx.task, "commit", "-m", "unauthorized actor commit"); + assert.throws(() => openPR(ctx), /only change fixture scope/); + assert.equal(JSON.parse(fs.readFileSync(ctx.tracker)).issue.state, "OPEN"); + assert.equal(git(ctx.main, "ls-remote", "origin", `refs/heads/${ctx.branch}`), ""); + } finally { dispose(ctx); } + } +}); + +test("acceptance rechecks final diff and runs immutable direct behavior checks", () => { + for (const mutation of ["committed-tests", "false-pass-evidence"]) { + const ctx = createFixture(); + try { + implement(ctx); + if (mutation === "false-pass-evidence") fs.writeFileSync(path.join(ctx.task, "clamp.mjs"), "export const clamp = () => -999;\n"); + openPR(ctx); + if (mutation === "committed-tests") { + fs.writeFileSync(path.join(ctx.task, "clamp.test.mjs"), "// removed tests\n"); + git(ctx.task, "add", "."); git(ctx.task, "commit", "-m", "tamper after PR"); git(ctx.task, "push", "origin", ctx.branch); + } + assert.throws(() => accept(ctx, { evidence: { claimed: "all tests passed" } }), /fixture scope|Expected values/); + assert.equal(JSON.parse(fs.readFileSync(ctx.tracker)).acceptance, undefined); + } finally { dispose(ctx); } + } +}); + +test("actual command output rejects mentions and masked failures, and accepts standalone tests", () => { + const env = { ...process.env }; + delete env.NODE_TEST_CONTEXT; + const ctx = createFixture(); + try { + const trace = (command, result) => { + fs.writeFileSync(path.join(ctx.root, "trace.jsonl"), JSON.stringify({ type: "item.completed", item: { type: "command_execution", command, exit_code: result.status, aggregated_output: result.stdout + result.stderr } })); + return observedTests(ctx.root, "trace.jsonl", true); + }; + for (const command of ["printf '%s\\n' 'node --test clamp.test.mjs'", "node --test clamp.test.mjs; true", "# node --test clamp.test.mjs\ntrue"]) { + const result = spawnSync("/bin/sh", ["-c", command], { cwd: ctx.task, encoding: "utf8", env }); + assert.equal(result.status, 0); + assert.equal(trace(command, result), false); + } + const bad = spawnSync("node", ["--test", "--test-reporter=tap", "clamp.test.mjs"], { cwd: ctx.task, encoding: "utf8", env }); + assert.equal(trace("node --test --test-reporter=tap clamp.test.mjs", bad), false); + assert.equal(observedTests(ctx.root, "trace.jsonl", false), true, bad.stdout + bad.stderr); + implement(ctx); + const good = spawnSync("node", ["--test", "--test-reporter=tap", "clamp.test.mjs"], { cwd: ctx.task, encoding: "utf8", env }); + assert.equal(trace("/bin/zsh -lc 'node --test --test-reporter=tap clamp.test.mjs'", good), true); + assert.equal(trace("node --test --test-reporter=tap clamp.test.mjs", { ...good, stdout: good.stdout.replace("# tests 3", "# tests 1") }), false); + } finally { dispose(ctx); } +}); From 2728a1f480168e47930e090abcaf6874d4a5e01b Mon Sep 17 00:00:00 2001 From: mindcarver Date: Sat, 5 Sep 2026 11:44:42 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix(evals):=20=E4=BD=BF=E7=94=A8=E4=B8=8D?= =?UTF-8?q?=E5=8F=AF=E6=9B=BF=E6=8D=A2=E6=96=AD=E8=A8=80=E7=9A=84=E7=8B=AC?= =?UTF-8?q?=E7=AB=8B=E8=A1=8C=E4=B8=BA=20oracle=20(#92)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- evals/delivery/README.md | 2 +- evals/delivery/protocol.mjs | 20 ++++++++++++++++---- tests/delivery-eval.test.mjs | 22 +++++++++++++++++++++- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/evals/delivery/README.md b/evals/delivery/README.md index 06e2855..555674b 100644 --- a/evals/delivery/README.md +++ b/evals/delivery/README.md @@ -42,7 +42,7 @@ node evals/delivery/run.mjs --mode model --model gpt-6-astra --output work/evals | 模型负例 | 故意损坏代码附带虚假的“已完成”声明，fresh evaluator 实际运行失败测试并返回 FAIL | | 模型正例 | coding Agent 修改代码并跑测试，fresh evaluator 直接运行测试/边界检查后返回 PASS；验收绑定 base/head/contract 后 adapter 继续 | -提交前分别核对初始基准到工作区、index和已有提交的完整范围，提交后及验收时再检查最终diff；原始验收测试不得更改，验收另行执行不可由fixture代码替换的内置边界断言。模型测试命令必须独立执行（显式`--test-reporter=tap`），命令事件必须有3项测试的完整TAP计数和匹配退出码；打印命令、注释及掩盖退出码的shell语句不算证据。 +提交前分别核对初始基准到工作区、index和已有提交的完整范围，提交后及验收时再检查最终diff；原始验收测试不得更改，验收另行执行固定边界的JS原始值比较，不依赖候选可改写的assert方法；动态导入候选前捕获输出函数并生成随机nonce，父进程核对完整匹配的start/result及退出码，拒绝提前exit(0)。模型测试命令必须独立执行（显式`--test-reporter=tap`），命令事件必须有3项测试的完整TAP计数和匹配退出码；打印命令、注释及掩盖退出码的shell语句不算证据。 每个 tracker 事件保存顺序号、进程 PID、绑定提交和状态；`summary.json` 保存模式与结果。模型轨迹保留命令执行、返回结果及结构化结论，可逐条核对，不能只阅读 Agent 最终回答。`implementation.diff`、`clamp.mjs`、`clamp.test.mjs` 支持离线运行 `node --test clamp.test.mjs` 复核代码。fixtures 的临时绝对路径不再可访问是预期行为；提交 SHA 与事件、归档代码一起承担证据作用。 diff --git a/evals/delivery/protocol.mjs b/evals/delivery/protocol.mjs index 63e3338..ffcf93a 100644 --- a/evals/delivery/protocol.mjs +++ b/evals/delivery/protocol.mjs @@ -56,15 +56,27 @@ export function verifyCode(cwd) { const env = { ...process.env }; delete env.NODE_TEST_CONTEXT; assert.equal(fs.readFileSync(path.join(cwd, "clamp.test.mjs"), "utf8"), fixtureTests, "fixture acceptance tests were modified"); - const direct = spawnSync(process.execPath, ["--input-type=module", "-e", `import assert from 'node:assert/strict'; import { clamp } from './clamp.mjs'; - for (const [value, min, max, expected] of [[5,0,10,5],[-3,0,10,0],[20,0,10,10],[2,2,2,2],[-20,-10,-5,-10],[-7,-10,-5,-7]]) assert.equal(clamp(value,min,max),expected); - assert.throws(() => clamp(1,9,2), RangeError);`], { cwd, encoding: "utf8", env }); + const direct = spawnSync(process.execPath, ["--input-type=module", "-e", ` + import { writeSync } from 'node:fs'; import { randomBytes } from 'node:crypto'; + const write = writeSync, nonce = randomBytes(16).toString('hex'), ExpectedRangeError = RangeError; + write(1, 'start:' + nonce + '\\n'); + let passed = false; + try { + const { clamp } = await import('./clamp.mjs'); + let reversed = false; + try { clamp(1,9,2); } catch (error) { reversed = error instanceof ExpectedRangeError; } + passed = clamp(5,0,10) === 5 && clamp(-3,0,10) === 0 && clamp(20,0,10) === 10 + && clamp(2,2,2) === 2 && clamp(-20,-10,-5) === -10 && clamp(-7,-10,-5) === -7 && reversed === true; + } catch {} + write(1, 'result:' + nonce + ':' + (passed === true ? 'true' : 'false') + '\\n'); + `], { cwd, encoding: "utf8", env }); assert.equal(direct.status, 0, direct.stdout + direct.stderr); + assert.match(direct.stdout, /^start:([a-f0-9]{32})\nresult:\1:true\n$/, "independent fixture behavior check failed or exited before completion"); const result = spawnSync(process.execPath, ["--test", "--test-reporter=tap", "clamp.test.mjs"], { cwd, encoding: "utf8", env }); assert.equal(result.status, 0, result.stdout + result.stderr); assert.match(result.stdout, /^# tests 3$/m, "fixture tests must actually execute"); assert.match(result.stdout, /^# pass 3$/m, "all fixture tests must pass"); - return { command: "node --test --test-reporter=tap clamp.test.mjs", exitCode: result.status, stdout: result.stdout }; + return { command: "node --test --test-reporter=tap clamp.test.mjs", exitCode: result.status, stdout: result.stdout, direct: { exitCode: direct.status, stdout: direct.stdout } }; } function assertScope(ctx, base, head) { const args = ["diff", "--name-only", base]; diff --git a/tests/delivery-eval.test.mjs b/tests/delivery-eval.test.mjs index 196ce53..3d0dc2f 100644 --- a/tests/delivery-eval.test.mjs +++ b/tests/delivery-eval.test.mjs @@ -112,7 +112,7 @@ test("acceptance rechecks final diff and runs immutable direct behavior checks", fs.writeFileSync(path.join(ctx.task, "clamp.test.mjs"), "// removed tests\n"); git(ctx.task, "add", "."); git(ctx.task, "commit", "-m", "tamper after PR"); git(ctx.task, "push", "origin", ctx.branch); } - assert.throws(() => accept(ctx, { evidence: { claimed: "all tests passed" } }), /fixture scope|Expected values/); + assert.throws(() => accept(ctx, { evidence: { claimed: "all tests passed" } }), /fixture scope|independent fixture behavior check/); assert.equal(JSON.parse(fs.readFileSync(ctx.tracker)).acceptance, undefined); } finally { dispose(ctx); } } @@ -141,3 +141,23 @@ test("actual command output rejects mentions and masked failures, and accepts st assert.equal(trace("node --test --test-reporter=tap clamp.test.mjs", { ...good, stdout: good.stdout.replace("# tests 3", "# tests 1") }), false); } finally { dispose(ctx); } }); + + +test("independent behavior oracle rejects replaced assertions and premature successful exit", () => { + for (const source of [ + "import assert from 'node:assert/strict'; assert.equal = () => {}; assert.throws = () => {}; export const clamp = () => -999;\n", + "process.exit(0); export const clamp = () => -999;\n", + ]) { + const ctx = createFixture(); + try { + fs.writeFileSync(path.join(ctx.task, "clamp.mjs"), source); + openPR(ctx); + assert.throws(() => accept(ctx), /independent fixture behavior check failed or exited before completion/); + const state = JSON.parse(fs.readFileSync(ctx.tracker)); + assert.equal(state.acceptance, undefined); + assert.equal(state.issue.state, "OPEN"); + assert.equal(state.pr.state, "OPEN"); + assert.equal(git(ctx.main, "ls-remote", "origin", "refs/heads/main").split(/\s/)[0], ctx.baseline); + } finally { dispose(ctx); } + } +});