diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dcec3a..c1d7369 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/), and this project adheres to [Semantic Versioning](https://semver.org/). +## [Unreleased] + +### Added + +- New `json-nonempty` scorer: fails schema-valid-but-empty outputs (`{}`, + `{"answer": ""}`, all-null payloads) unless the output has at least one + non-empty leaf value (`minKeys`, `rejectBlankStrings`, `rejectNulls`). + ## [0.1.2] - 2026-09-05 ### Fixed diff --git a/README.md b/README.md index b744fce..33318aa 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,7 @@ score is the weighted mean of the scorer scores. | `contains` | all substrings present (partial credit) | `value` / `values`, `caseSensitive` | | `not-contains` | no banned substring present | `value` / `values`, `caseSensitive` | | `json-schema` | output is valid JSON matching a schema | `schema` | +| `json-nonempty` | output is valid JSON with non-empty leaf values | `schema`, `minKeys`, `rejectBlankStrings`, `rejectNulls` | | `embedding-similarity` | cosine similarity >= threshold | `expected`, `threshold` | | `llm-judge` | a judge model scores >= threshold | `criteria`, `expected`, `threshold`, `model` | | `latency` | call latency within budget | `budgetMs` | diff --git a/src/scorers/json-nonempty.ts b/src/scorers/json-nonempty.ts new file mode 100644 index 0000000..5514bdb --- /dev/null +++ b/src/scorers/json-nonempty.ts @@ -0,0 +1,83 @@ +import type { Scorer, ScoreContext, ScorerSpec } from "../types.js"; +import { result } from "./util.js"; +import { validate, type JsonSchema } from "./json-schema.js"; + +/** + * Passes when the output parses as JSON, validates against an optional schema, + * and carries at least one non-empty leaf value. + * + * Catches models gaming structured-output evals with vacuous payloads like + * `{}`, `{"answer": ""}`, or all-null objects that still validate against the + * schema. + * + * Options: + * - `schema`: a JSON Schema object to validate against (optional) + * - `minKeys`: minimum number of non-empty leaf values required (default 1) + * - `rejectBlankStrings`: treat `""` and whitespace-only strings as empty values (default true) + * - `rejectNulls`: treat `null` as an empty value (default true) + */ +export const jsonNonemptyScorer: Scorer = { + type: "json-nonempty", + score(spec: ScorerSpec, ctx: ScoreContext) { + let parsed: unknown; + try { + parsed = JSON.parse(ctx.output); + } catch (err) { + return result(spec, { + score: 0, + passed: false, + reason: `output is not valid JSON: ${(err as Error).message}`, + }); + } + const schema = spec.schema as JsonSchema | undefined; + if (schema) { + const errors = validate(parsed, schema); + if (errors.length > 0) { + return result(spec, { + score: 0, + passed: false, + reason: errors.join("; "), + }); + } + } + + const minKeys = typeof spec.minKeys === "number" ? spec.minKeys : 1; + const rejectBlankStrings = spec.rejectBlankStrings !== false; + const rejectNulls = spec.rejectNulls !== false; + const nonEmpty = countNonEmptyLeaves(parsed, rejectBlankStrings, rejectNulls); + + if (nonEmpty < minKeys) { + return result(spec, { + score: 0, + passed: false, + reason: `output has ${nonEmpty} non-empty value(s), need at least ${minKeys}`, + }); + } + return result(spec, { + score: 1, + passed: true, + reason: "output is non-empty JSON", + }); + }, +}; + +function countNonEmptyLeaves(value: unknown, rejectBlankStrings: boolean, rejectNulls: boolean): number { + if (typeof value === "string") { + const blank = rejectBlankStrings ? value.trim().length === 0 : false; + return blank ? 0 : 1; + } + if (value === null) return rejectNulls ? 0 : 1; + if (Array.isArray(value)) { + return value.reduce((n, item) => n + countNonEmptyLeaves(item, rejectBlankStrings, rejectNulls), 0); + } + if (typeof value === "object") { + const obj = value as Record; + const keys = Object.keys(obj); + if (keys.length === 0) return 0; + return keys.reduce( + (n, key) => n + countNonEmptyLeaves(obj[key], rejectBlankStrings, rejectNulls), + 0, + ); + } + return 1; // booleans and numbers are non-empty +} \ No newline at end of file diff --git a/src/scorers/registry.ts b/src/scorers/registry.ts index 4f6a4cf..d230dc5 100644 --- a/src/scorers/registry.ts +++ b/src/scorers/registry.ts @@ -3,6 +3,7 @@ import { exactMatchScorer } from "./exact-match.js"; import { regexScorer } from "./regex.js"; import { containsScorer, notContainsScorer } from "./contains.js"; import { jsonSchemaScorer } from "./json-schema.js"; +import { jsonNonemptyScorer } from "./json-nonempty.js"; import { embeddingSimilarityScorer } from "./embedding-similarity.js"; import { llmJudgeScorer } from "./llm-judge.js"; import { latencyScorer } from "./latency.js"; @@ -47,6 +48,7 @@ export const builtinScorers: Scorer[] = [ containsScorer, notContainsScorer, jsonSchemaScorer, + jsonNonemptyScorer, embeddingSimilarityScorer, llmJudgeScorer, latencyScorer, diff --git a/tests/scorers.test.ts b/tests/scorers.test.ts index 6f4afe7..a0f1e65 100644 --- a/tests/scorers.test.ts +++ b/tests/scorers.test.ts @@ -5,6 +5,7 @@ import { exactMatchScorer } from "../src/scorers/exact-match.js"; import { regexScorer } from "../src/scorers/regex.js"; import { containsScorer, notContainsScorer } from "../src/scorers/contains.js"; import { jsonSchemaScorer, validate } from "../src/scorers/json-schema.js"; +import { jsonNonemptyScorer } from "../src/scorers/json-nonempty.js"; import { embeddingSimilarityScorer, cosineSimilarity } from "../src/scorers/embedding-similarity.js"; import { llmJudgeScorer } from "../src/scorers/llm-judge.js"; import { latencyScorer } from "../src/scorers/latency.js"; @@ -136,6 +137,72 @@ describe("json-schema", () => { }); }); +describe("json-nonempty", () => { + it("fails a schema-valid but empty object before any leaf values", async () => { + const r = await run( + jsonNonemptyScorer, + { type: "json-nonempty", schema: { type: "object" }, minKeys: 1 }, + ctx("{}"), + ); + expect(r.passed).toBe(false); + }); + + it("fails blank-string and all-null outputs", async () => { + const blank = await run( + jsonNonemptyScorer, + { type: "json-nonempty" }, + ctx('{"answer": ""}'), + ); + expect(blank.passed).toBe(false); + + const nulls = await run( + jsonNonemptyScorer, + { type: "json-nonempty" }, + ctx('{"answer": null, "extra": null}'), + ); + expect(nulls.passed).toBe(false); + }); + + it("passes an output with at least one non-empty leaf value", async () => { + const r = await run( + jsonNonemptyScorer, + { type: "json-nonempty" }, + ctx('{"answer": "Paris"}'), + ); + expect(r.passed).toBe(true); + expect(r.score).toBe(1); + }); + + it("honors minKeys over the default of 1", async () => { + const r = await run( + jsonNonemptyScorer, + { type: "json-nonempty", minKeys: 2 }, + ctx('{"a": "one", "b": ""}'), + ); + expect(r.passed).toBe(false); + }); + + it("fails non-JSON output with a parse reason", async () => { + const r = await run( + jsonNonemptyScorer, + { type: "json-nonempty" }, + ctx("{not json"), + ); + expect(r.passed).toBe(false); + expect(r.reason).toMatch(/not valid JSON/i); + }); + + it("fails before emptiness checks when the schema rejects the output", async () => { + const r = await run( + jsonNonemptyScorer, + { type: "json-nonempty", schema: { type: "object", required: ["n"] } }, + ctx('{"answer": "Paris"}'), + ); + expect(r.passed).toBe(false); + expect(r.reason).toContain("required"); + }); +}); + describe("embedding-similarity", () => { it("scores identical text near 1", async () => { const r = await run(