diff --git a/.gitattributes b/.gitattributes index 60897224b..80a112dcd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -12,3 +12,6 @@ # Same drift, caught on the native helper's build file: an edit from Windows # rewrote all 67 lines as CRLF and buried a 22-line change in a 156-line diff. CMakeLists.txt text eol=lf + +# Adopted measurement manifests hash exact bytes; keep their artifacts unchanged. +workbench/measurements/** -text diff --git a/.gitignore b/.gitignore index 24ad425eb..db0c43ef3 100644 --- a/.gitignore +++ b/.gitignore @@ -121,6 +121,7 @@ poc/out/ workbench/reports/ workbench/.build/ workbench/runs/ +workbench/.agent-evidence/ # Fixture d'évaluation — enregistrement réel de l'utilisateur, jamais versionné workbench/fixtures/ diff --git a/biome.json b/biome.json index 8954e61bf..bb51ce78e 100644 --- a/biome.json +++ b/biome.json @@ -107,6 +107,10 @@ }, "javascript": { "formatter": { "quoteStyle": "double" } }, "overrides": [ + { + "includes": ["workbench/measurements/**/*.json"], + "formatter": { "enabled": false } + }, { "includes": ["*.ts", "*.tsx", "*.mts", "*.cts"], "linter": { diff --git a/electron/ai-edition/chat-service.ts b/electron/ai-edition/chat-service.ts index 20e635062..067b957ed 100644 --- a/electron/ai-edition/chat-service.ts +++ b/electron/ai-edition/chat-service.ts @@ -270,6 +270,8 @@ export interface ChatRunEnv { /** Reads recorded cursor telemetry for an asset. Built in `electron/ipc/ * handlers.ts`, where the path allow-list lives. */ cursor?: CursorTelemetryReader; + /** Runtime-only model retry override used by bounded measurement harnesses. */ + maxRetries?: number; } // ponytail: zero-config noop for sink callbacks that the caller did not provide. @@ -405,6 +407,7 @@ export async function runChat( apiKey: apiKey ?? undefined, baseUrl: config.baseUrl, reasoningEffort: config.reasoningEffort, + maxRetries: env.maxRetries, }, history, userMessage: message, diff --git a/electron/ai-edition/deep-agent/chat-model.test.ts b/electron/ai-edition/deep-agent/chat-model.test.ts index 477f5bece..2b917ca5b 100644 --- a/electron/ai-edition/deep-agent/chat-model.test.ts +++ b/electron/ai-edition/deep-agent/chat-model.test.ts @@ -91,6 +91,33 @@ describe("createOpenScreenChatModel — Anthropic-wire output budget", () => { }); }); +describe("createOpenScreenChatModel — runtime retry override", () => { + function retries(model: unknown): number | undefined { + return (model as { caller?: { maxRetries?: number } }).caller?.maxRetries; + } + + it("passes an explicit zero to LangChain", async () => { + const model = await createOpenScreenChatModel({ + provider: "openai-compatible", + model: "gpt-5-test", + apiKey: "test-key", + baseUrl: "http://127.0.0.1:1/v1", + maxRetries: 0, + }); + expect(retries(model)).toBe(0); + }); + + it("keeps LangChain's default when the override is omitted", async () => { + const model = await createOpenScreenChatModel({ + provider: "openai-compatible", + model: "gpt-5-test", + apiKey: "test-key", + baseUrl: "http://127.0.0.1:1/v1", + }); + expect(retries(model)).toBe(6); + }); +}); + describe("messageContentToText", () => { it("passes a plain string through", () => { expect(messageContentToText("hello")).toBe("hello"); diff --git a/electron/ai-edition/deep-agent/chat-model.ts b/electron/ai-edition/deep-agent/chat-model.ts index 037343262..3b57f379e 100644 --- a/electron/ai-edition/deep-agent/chat-model.ts +++ b/electron/ai-edition/deep-agent/chat-model.ts @@ -283,6 +283,8 @@ export interface OpenScreenChatModelConfig { apiKey?: string; baseUrl?: string; reasoningEffort?: string; + /** Runtime-only retry override. Omitted callers keep LangChain's default. */ + maxRetries?: number; } // ponytail: placeholder API key for self-hosted OpenAI-compatible endpoints @@ -422,6 +424,7 @@ export async function createOpenScreenChatModel( ...(reasoningOptions.reasoning ? { reasoning: reasoningOptions.reasoning } : {}), ...(reasoningOptions.useResponsesApi ? { useResponsesApi: true } : {}), ...(reasoningOptions.modelKwargs ? { modelKwargs: reasoningOptions.modelKwargs } : {}), + ...(config.maxRetries !== undefined ? { maxRetries: config.maxRetries } : {}), // ponytail: Gemini's OpenAI-compat path can't stream + tool-call at once // — disable streaming so the ChatOpenAI compat layer buffers and returns // cleanly. axcut does the same. diff --git a/package.json b/package.json index 94a1401ec..b8bec6e01 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "wb:judge": "npm run wb:build --silent && node --env-file=.env.workbench workbench/.build/cli.cjs judge", "wb:judge:replay": "npm run wb:build --silent && node workbench/.build/cli.cjs judge --replay", "wb:compare": "npm run wb:build --silent && node workbench/.build/cli.cjs compare", + "wb:measurement": "esbuild workbench/measurement-entry.ts --bundle --platform=node --format=cjs --packages=external --outfile=workbench/.build/measurement-cli.cjs --log-level=warning && node workbench/.build/measurement-cli.cjs", "test:watch": "vitest", "test:cursor-native:win": "node scripts/test-windows-native-cursor.mjs", "test:wgc-helper:win": "node scripts/test-windows-wgc-helper.mjs", diff --git a/workbench/README.md b/workbench/README.md index 32131b47e..0aaca20ee 100644 --- a/workbench/README.md +++ b/workbench/README.md @@ -66,8 +66,36 @@ par `node --env-file`. Aucun parseur maison, jamais `dotenv`. OPENSCREEN_WORKBENCH_API_KEY=… OPENSCREEN_WORKBENCH_BASE_URL=… OPENSCREEN_WORKBENCH_MODEL=… +OPENSCREEN_WORKBENCH_WIRE_API=responses +OPENSCREEN_WORKBENCH_USER_AGENT=codex_exec/… +OPENSCREEN_WORKBENCH_ORIGINATOR=codex_exec ``` +`WIRE_API` vaut `chat-completions` par défaut ou `responses`. Les deux en-têtes sont facultatifs, +validés comme valeurs publiques ordinaires, et leur profil normalisé est lié à l'identité de +transport sans enregistrer la clé. En mode Responses, le modèle configuré doit réellement +sélectionner le client Responses natif ; un désaccord échoue avant tout transfert. + +Un run Responses borné nomme toujours ses deux plafonds : + +```bash +node workbench/.build/cli.cjs run --scenario target-right-clip --reps 1 \ + --label