feat: add tool-call scorer

CHANGELOG.md

@@ -8,6 +8,9 @@ All notable changes to this project are documented here. The format is based on
 
 ### Added
 
+- New `tool-call` scorer: passes when the output is a JSON tool call whose
+  `name` is on the allowlist and whose `arguments` is a plain object, with
+  optional per-tool argument schemas (`allowedTools`, `schemas`).
 - New `json-nonempty` scorer: fails schema-valid-but-empty outputs (`{}`,
   `{"answer": ""}`, all-null payloads) unless the output has at least one
   non-empty leaf value (`minKeys`, `rejectBlankStrings`, `rejectNulls`).

README.md

@@ -136,6 +136,7 @@ score is the weighted mean of the scorer scores.
 | `contains` | all substrings present (partial credit) | `value` / `values`, `caseSensitive` |
 | `not-contains` | no banned substring present | `value` / `values`, `caseSensitive` |
 | `json-schema` | output is valid JSON matching a schema | `schema` |
+| `tool-call` | output is a JSON tool call with a name on the allowlist | `allowedTools`, `schemas` |
 | `json-nonempty` | output is valid JSON with non-empty leaf values | `schema`, `minKeys`, `rejectBlankStrings`, `rejectNulls` |
 | `embedding-similarity` | cosine similarity >= threshold | `expected`, `threshold` |
 | `llm-judge` | a judge model scores >= threshold | `criteria`, `expected`, `threshold`, `model` |

src/scorers/registry.ts

@@ -3,6 +3,7 @@ import { exactMatchScorer } from "./exact-match.js";
 import { regexScorer } from "./regex.js";
 import { containsScorer, notContainsScorer } from "./contains.js";
 import { jsonSchemaScorer } from "./json-schema.js";
+import { toolCallScorer } from "./tool-call.js";
 import { jsonNonemptyScorer } from "./json-nonempty.js";
 import { embeddingSimilarityScorer } from "./embedding-similarity.js";
 import { llmJudgeScorer } from "./llm-judge.js";
@@ -48,6 +49,7 @@ export const builtinScorers: Scorer[] = [
   containsScorer,
   notContainsScorer,
   jsonSchemaScorer,
+  toolCallScorer,
   jsonNonemptyScorer,
   embeddingSimilarityScorer,
   llmJudgeScorer,

src/scorers/tool-call.ts

@@ -0,0 +1,89 @@
+import type { Scorer, ScoreContext, ScorerSpec } from "../types.js";
+import { result } from "./util.js";
+import { validate, type JsonSchema } from "./json-schema.js";
+
+/**
+ * Passes when the output parses as JSON describing a callable tool action,
+ * with a `name` on an allowlist and an `arguments` plain object.
+ *
+ * Options:
+ *  - `allowedTools`: list of tool names the model may call (required)
+ *  - `schemas`: per-tool JSON Schema applied to `arguments` (optional)
+ */
+export const toolCallScorer: Scorer = {
+  type: "tool-call",
+  score(spec: ScorerSpec, ctx: ScoreContext) {
+    const allowedTools = spec.allowedTools as string[] | undefined;
+    if (!Array.isArray(allowedTools) || allowedTools.length === 0) {
+      return result(spec, {
+        score: 0,
+        passed: false,
+        reason: "allowedTools must be a non-empty list",
+      });
+    }
+
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(ctx.output);
+    } catch (err) {
+      return result(spec, {
+        score: 0,
+        passed: false,
+        reason: `output is not valid JSON: ${(err as Error).message}`,
+      });
+    }
+
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+      return result(spec, {
+        score: 0,
+        passed: false,
+        reason: "output must be a JSON object with name and arguments",
+      });
+    }
+
+    const call = parsed as Record<string, unknown>;
+    const name = call.name;
+    if (typeof name !== "string") {
+      return result(spec, {
+        score: 0,
+        passed: false,
+        reason: "output must have a string name",
+      });
+    }
+    if (!allowedTools.includes(name)) {
+      return result(spec, {
+        score: 0,
+        passed: false,
+        reason: `tool "${name}" is not in allowedTools`,
+      });
+    }
+
+    const args = call.arguments;
+    if (typeof args !== "object" || args === null || Array.isArray(args)) {
+      return result(spec, {
+        score: 0,
+        passed: false,
+        reason: "arguments must be a plain object",
+      });
+    }
+
+    const schemas = (spec.schemas ?? {}) as Record<string, JsonSchema>;
+    const schema = schemas[name];
+    if (schema) {
+      const errors = validate(args, schema);
+      if (errors.length > 0) {
+        return result(spec, {
+          score: 0,
+          passed: false,
+          reason: errors.join("; "),
+        });
+      }
+    }
+
+    return result(spec, {
+      score: 1,
+      passed: true,
+      reason: `calls allowed tool "${name}"`,
+    });
+  },
+};

tests/scorers.test.ts

@@ -5,6 +5,7 @@ import { exactMatchScorer } from "../src/scorers/exact-match.js";
 import { regexScorer } from "../src/scorers/regex.js";
 import { containsScorer, notContainsScorer } from "../src/scorers/contains.js";
 import { jsonSchemaScorer, validate } from "../src/scorers/json-schema.js";
+import { toolCallScorer } from "../src/scorers/tool-call.js";
 import { jsonNonemptyScorer } from "../src/scorers/json-nonempty.js";
 import { embeddingSimilarityScorer, cosineSimilarity } from "../src/scorers/embedding-similarity.js";
 import { llmJudgeScorer } from "../src/scorers/llm-judge.js";
@@ -203,6 +204,74 @@ describe("json-nonempty", () => {
   });
 });
 
+describe("tool-call", () => {
+  it("passes a tool call on the allowlist", async () => {
+    const r = await run(
+      toolCallScorer,
+      { type: "tool-call", allowedTools: ["get_weather"] },
+      ctx('{"name": "get_weather", "arguments": {"city": "London"}}'),
+    );
+    expect(r.passed).toBe(true);
+    expect(r.score).toBe(1);
+  });
+
+  it("fails a tool call not on the allowlist", async () => {
+    const r = await run(
+      toolCallScorer,
+      { type: "tool-call", allowedTools: ["get_weather"] },
+      ctx('{"name": "rm_rf", "arguments": {}}'),
+    );
+    expect(r.passed).toBe(false);
+    expect(r.reason).toMatch(/not in allowedTools/);
+  });
+
+  it("fails non-JSON output", async () => {
+    const r = await run(
+      toolCallScorer,
+      { type: "tool-call", allowedTools: ["get_weather"] },
+      ctx("{not json"),
+    );
+    expect(r.passed).toBe(false);
+    expect(r.reason).toMatch(/not valid JSON/);
+  });
+
+  it("fails when arguments is not a plain object", async () => {
+    const r = await run(
+      toolCallScorer,
+      { type: "tool-call", allowedTools: ["get_weather"] },
+      ctx('{"name": "get_weather", "arguments": "London"}'),
+    );
+    expect(r.passed).toBe(false);
+    expect(r.reason).toMatch(/plain object/);
+  });
+
+  it("fails when allowedTools is missing", async () => {
+    const r = await run(toolCallScorer, { type: "tool-call" }, ctx('{"name": "get_weather", "arguments": {}}'));
+    expect(r.passed).toBe(false);
+    expect(r.reason).toMatch(/allowedTools/);
+  });
+
+  it("validates arguments against a per-tool schema", async () => {
+    const r = await run(
+      toolCallScorer,
+      {
+        type: "tool-call",
+        allowedTools: ["get_weather"],
+        schemas: {
+          get_weather: {
+            type: "object",
+            required: ["city"],
+            properties: { city: { type: "string" } },
+          },
+        },
+      },
+      ctx('{"name": "get_weather", "arguments": {}}'),
+    );
+    expect(r.passed).toBe(false);
+    expect(r.reason).toContain("required");
+  });
+});
+
 describe("embedding-similarity", () => {
   it("scores identical text near 1", async () => {
     const r = await run(