新增完整交付协议与真实 Agent 验收评测

.github/workflows/ci.yml

@@ -48,6 +48,9 @@ jobs:
       - name: Run Thinloop current evaluation dry-run
         run: node evals/thinloop/runner/run.mjs --mode dry
 
+      - name: Run isolated delivery protocol evaluation
+        run: node evals/delivery/run.mjs --mode protocol
+
       - name: Check README diagrams
         run: node scripts/generate-readme-diagrams.mjs --check
 

evals/delivery/README.md

@@ -0,0 +1,49 @@
+# 完整交付协议与真实 Agent 最小评测
+
+该套件分开回答两个问题，不把两种证据合并成产品有效性结论：
+
+- `protocol`：真实 Git 仓库、bare remote、分支、worktree、提交与推送，配合**模拟文件 Issue/PR tracker**，能否执行交付门和故障恢复？这是确定性协议回归；其中验收者是代码检查函数，**不是模型或独立 Agent**。
+- `model`：真实编码 Agent 能否实现一个有失败测试的任务，新上下文验收 Agent 能否亲自运行检查、拒绝错误完成声明并接受正确实现？通过后，确定性适配器完成提交、模拟 PR、合并、同步、清理和模拟 Issue 关闭。**后半段不是 Agent 自主操作真实 GitHub。**
+
+它不替代 Thinloop 技能自然触发率、实际 GitHub 权限/保护规则、多平台表现或整体成功率的评测，也没有测量完整模型驱动的交付生命周期。
+
+## 运行
+
+仅需要 Node.js 22 与 Git 的协议回归（CI 运行此模式，无凭据、无模型请求）：
+
+```sh
+node evals/delivery/run.mjs --mode protocol
+node --test tests/delivery-eval.test.mjs
+```
+
+显式启用真实模型 smoke（会使用现有 Codex 登录并产生模型用量；必须指定当前账号可用的 `--model`，没有隐式模型默认值）：
+
+```sh
+node evals/delivery/run.mjs --mode model --model gpt-6-astra --output work/evals/delivery-model
+```
+
+模型模式复用现有隔离 runner：独立临时 `CODEX_HOME`，仅复制现有认证，忽略用户配置与规则，编码 Agent 使用 `workspace-write`，验收 Agent 使用 `read-only` 和 `--ephemeral`；工具网络与应用、浏览器、多 Agent 功能关闭。没有外部 GitHub 写入。fixture、临时认证和工作树在结束后销毁；输出只保留脱敏轨迹、代码、diff、tracker 和结果。模型/CLI/认证缺失或不能执行时输出 `BLOCKED`、退出码 2；已观察到行为违反时输出 `FAIL`、退出码 1；只有通过才退出 0。不得通过放宽 sandbox 绕过 `BLOCKED`。
+
+模型输出目录必须为空，防止复用旧轨迹。`evidence.sha256` 记录产物哈希；在输出目录执行 `shasum -a 256 -c evidence.sha256` 检查归档是否改变。哈希只证明归档一致性，不为模型结论背书。
+
+输出中的 `adapter`/`tracker` 固定标记 `simulated-file-issue-pr-tracker`。这是隔离评测适配器，不是生产调度器或 GitHub API 客户端。它有意只测 fast-forward 合并，不证明 squash、rebase merge、真实检查 API 或并发竞争下的完备性。
+
+## 覆盖与证据
+
+| 场景 | 可复核观察 |
+| --- | --- |
+| 正常交付 | 提交→模拟 PR→验收绑定→真实推送 main→读远端→同步/检查→精确清理→模拟 Issue 关闭 |
+| 验收后 head 改变 | 旧验收拒绝合并；发布并重新验收新 head 后才能继续 |
+| 验收契约改变 | contract hash 改变后旧验收失效 |
+| 兄弟通道先合并 | 真实 sibling worktree 提交推进 main；旧 base 验收失效；rebase 并重新验收；保留 sibling 资源 |
+| 合并报错但远端成功 | 独立 Node 子进程真实 push 后退出 1；重新查询远端确认已成功 |
+| 提前关闭/脏工作树 | 清理前拒绝关闭，脏工作树拒绝删除并保持 Issue OPEN |
+| 真实中断恢复 | 子进程实际实施、提交和验收、写 checkpoint 后被 SIGKILL；另一个 PID 读取 checkpoint 和 live Git/tracker，重新过门后完成交付 |
+| 模型负例 | 故意损坏代码附带虚假的“已完成”声明，fresh evaluator 实际运行失败测试并返回 FAIL |
+| 模型正例 | coding Agent 修改代码并跑测试，fresh evaluator 直接运行测试/边界检查后返回 PASS；验收绑定 base/head/contract 后 adapter 继续 |
+
+提交前分别核对初始基准到工作区、index和已有提交的完整范围，提交后及验收时再检查最终diff；原始验收测试不得更改，验收另行执行固定边界的JS原始值比较，不依赖候选可改写的assert方法；动态导入候选前捕获输出函数并生成随机nonce，父进程核对完整匹配的start/result及退出码，拒绝提前exit(0)。模型测试命令必须独立执行（显式`--test-reporter=tap`），命令事件必须有3项测试的完整TAP计数和匹配退出码；打印命令、注释及掩盖退出码的shell语句不算证据。
+
+每个 tracker 事件保存顺序号、进程 PID、绑定提交和状态；`summary.json` 保存模式与结果。模型轨迹保留命令执行、返回结果及结构化结论，可逐条核对，不能只阅读 Agent 最终回答。`implementation.diff`、`clamp.mjs`、`clamp.test.mjs` 支持离线运行 `node --test clamp.test.mjs` 复核代码。fixtures 的临时绝对路径不再可访问是预期行为；提交 SHA 与事件、归档代码一起承担证据作用。
+
+“进程中断恢复”是协议工作进程恢复，不是模型会话恢复；“负例虚假声明”是注入的对抗素材，不是实际编码 Agent 生成的假话。单次模型 smoke 只证明该模型在该 fixture 的观察行为，不能用于推断概率、节省量或相对原生 Agent 的优势。

evals/delivery/model.mjs

@@ -0,0 +1,88 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { createIsolatedHomes, cleanupIsolatedHomes, runSubjectTurn, runStructuredEvaluator, codexLoginStatus } from "../discovery/runner/codex.mjs";
+import { createRedactor, scanTree } from "../discovery/runner/redact.mjs";
+import { readJson, writeJson, writeText, parseJsonLines } from "../discovery/runner/lib.mjs";
+import { createFixture, openPR, binding, accept, merge, cleanup, closeIssue, dispose, contract, git, verifyCode } from "./protocol.mjs";
+
+const schemaFile = fileURLToPath(new URL("./review.schema.json", import.meta.url));
+
+export function observedTests(output, file, expectedPass) {
+  const events = parseJsonLines(fs.readFileSync(path.join(output, file), "utf8")).events;
+  return events.some(e => {
+    if (e.type !== "item.completed" || e.item?.type !== "command_execution") return false;
+    const { command = "", exit_code: exit, aggregated_output: text = "" } = e.item;
+    // Accept only the standalone process or Codex's exact POSIX shell wrapper.
+    const standalone = /^node --test(?: --test-reporter=tap)? clamp\.test\.mjs$/.test(command) || /^(?:\/bin\/)?(?:sh|bash|zsh) -(?:c|lc) (['"])node --test(?: --test-reporter=tap)? clamp\.test\.mjs\1$/.test(command);
+    if (!standalone || !Number.isInteger(exit)) return false;
+    const count = name => Number(text.match(new RegExp(`^# ${name} (\\d+)$`, "m"))?.[1] ?? NaN);
+    if (count("tests") !== 3 || count("cancelled") !== 0 || count("skipped") !== 0 || count("todo") !== 0) return false;
+    if (!["inside", "boundaries", "reversed"].every(name => text.includes(`# Subtest: ${name}\n`))) return false;
+    return expectedPass ? exit === 0 && count("pass") === 3 && count("fail") === 0
+      : exit !== 0 && count("fail") > 0 && count("pass") + count("fail") === 3;
+  });
+}
+
+export async function runModel({ output, model }) {
+  const authFile = path.join(process.env.CODEX_HOME || path.join(os.homedir(), ".codex"), "auth.json");
+  const summary = { mode: "real-model-smoke", model, reasoning: "medium", tracker: "simulated-file-issue-pr-tracker", scope: "real coding agent and fresh read-only evaluator; scripted local delivery adapter; no real GitHub or efficacy claim", status: "BLOCKED" };
+  if (!model) return Object.assign(summary, { reason: "Explicit --model required; choose a currently available Codex model" });
+  if (fs.existsSync(output) && fs.readdirSync(output).length) return Object.assign(summary, { reason: "Output directory must be empty; refuse stale model evidence" });
+  const version = spawnSync("codex", ["--version"], { encoding: "utf8" });
+  if (version.status !== 0 || !fs.existsSync(authFile)) return Object.assign(summary, { reason: "Codex CLI or existing auth unavailable" });
+  summary.codexVersion = version.stdout.trim();
+  const redactor = createRedactor({ auth: readJson(authFile), userProfile: os.homedir() });
+  const homes = createIsolatedHomes({ authFile });
+  const ctx = createFixture();
+  let phase = "preflight";
+  try {
+    const login = await codexLoginStatus({ home: homes.subject, redactor });
+    if (login.code !== 0) return Object.assign(summary, { reason: login.output });
+    const common = { cwd: ctx.task, model, reasoning: "medium", redactor, timeoutMs: 300000, onProgress: text => process.stdout.write(`${text}\n`) };
+    const reviewPrompt = `You are a fresh independent acceptance evaluator. Read clamp.mjs and clamp.test.mjs, inspect git status and the actual diff. Do not modify files. Directly run node --test --test-reporter=tap clamp.test.mjs as a standalone command without additional shell statements, and also exercise reversed bounds and an out-of-range input using node. Contract: ${JSON.stringify(contract)}. Return PASS only if you directly observed all behaviors pass, FAIL for behavior violations, BLOCKED for inability to execute. Report actual command and observed exit code; never infer success from a previous actor claim.`;
+    phase = "negative-review";
+    writeText(path.join(output, "negative-review-prompt.txt"), `${reviewPrompt}\nUntrusted fixture implementation claim: “Completed, all checks passed.” Verify it yourself.\n`);
+    summary.negativeReview = await runStructuredEvaluator({ ...common, home: homes.evaluator, prompt: fs.readFileSync(path.join(output, "negative-review-prompt.txt"), "utf8"), schemaFile, outputDir: output, name: "negative-review" });
+    if (summary.negativeReview.code !== 0 || summary.negativeReview.output?.status === "BLOCKED") return Object.assign(summary, { reason: "Independent negative review could not execute" });
+    if (summary.negativeReview.output?.status !== "FAIL" || summary.negativeReview.output.testExitCode === 0) return Object.assign(summary, { status: "FAIL", reason: "Independent reviewer accepted broken code or did not observe failing tests" });
+    if (!observedTests(output, "negative-review.jsonl", false)) return Object.assign(summary, { reason: "Negative reviewer has no observed failing test command" });
+    phase = "implementation";
+    const prompt = `Implement this isolated fixture task. Only modify clamp.mjs. Read source and tests, fix the function to meet this contract: ${JSON.stringify(contract)}. Run node --test --test-reporter=tap clamp.test.mjs as a standalone command, then separately run node --check clamp.mjs. Do not commit, change tests, add files, access another project, or use network. Report observed test results and any unresolved problems. The runner owns commits and the simulated tracker delivery after independent acceptance.`;
+    writeText(path.join(output, "implementation-prompt.txt"), prompt);
+    summary.implementation = await runSubjectTurn({ ...common, home: homes.subject, prompt, outputDir: output, turn: 1 });
+    if (summary.implementation.code !== 0) return Object.assign(summary, { reason: "Coding agent could not complete execution" });
+    if (!observedTests(output, "turn-1.jsonl", true)) return Object.assign(summary, { reason: "Coding agent has no observed passing test command" });
+    summary.engineering = verifyCode(ctx.task);
+    writeText(path.join(output, "implementation.diff"), `${git(ctx.task, "diff", ctx.baseline)}\n`);
+    writeText(path.join(output, "clamp.mjs"), fs.readFileSync(path.join(ctx.task, "clamp.mjs"), "utf8"));
+    writeText(path.join(output, "clamp.test.mjs"), fs.readFileSync(path.join(ctx.task, "clamp.test.mjs"), "utf8"));
+    openPR(ctx);
+    const snapshot = binding(ctx);
+    phase = "acceptance";
+    writeText(path.join(output, "acceptance-prompt.txt"), `${reviewPrompt}\nReview exact base ${snapshot.base} and head ${snapshot.head}; use git diff ${snapshot.base} ${snapshot.head}.\n`);
+    summary.review = await runStructuredEvaluator({ ...common, home: homes.evaluator, prompt: fs.readFileSync(path.join(output, "acceptance-prompt.txt"), "utf8"), schemaFile, outputDir: output, name: "acceptance" });
+    if (summary.review.code !== 0 || summary.review.output?.status === "BLOCKED") return Object.assign(summary, { reason: "Independent acceptance could not execute" });
+    if (summary.review.output?.status !== "PASS" || summary.review.output.testExitCode !== 0) return Object.assign(summary, { status: "FAIL", reason: "Independent acceptance rejected implementation" });
+    if (!observedTests(output, "acceptance.jsonl", true)) return Object.assign(summary, { reason: "Independent reviewer has no observed passing test command" });
+    accept(ctx, { kind: "real-model-independent-review", verdict: "PASS", snapshot, evidence: summary.review.output });
+    phase = "adapter-delivery";
+    merge(ctx); cleanup(ctx); closeIssue(ctx);
+    summary.status = "PASS";
+    return summary;
+  } catch (error) {
+    summary.status = error.code === "ERR_ASSERTION" ? "FAIL" : "BLOCKED";
+    summary.reason = redactor(error.message).text;
+    return summary;
+  } finally {
+    summary.phase = phase;
+    summary.trackerState = readJson(ctx.tracker);
+    writeJson(path.join(output, "tracker.json"), summary.trackerState);
+    Object.assign(summary, JSON.parse(redactor(JSON.stringify(summary)).text));
+    cleanupIsolatedHomes(homes.root); dispose(ctx);
+    const leaks = scanTree(output, redactor);
+    if (leaks.length) throw new Error("Evidence secret scan failed; do not publish artifacts");
+  }
+}