feat: add json-nonempty scorer

CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project are documented here. The format is based on
 [Keep a Changelog](https://keepachangelog.com/), and this project adheres to
 [Semantic Versioning](https://semver.org/).
 
+## [Unreleased]
+
+### Added
+
+- New `json-nonempty` scorer: fails schema-valid-but-empty outputs (`{}`,
+  `{"answer": ""}`, all-null payloads) unless the output has at least one
+  non-empty leaf value (`minKeys`, `rejectBlankStrings`, `rejectNulls`).
+
 ## [0.1.2] - 2026-09-05
 
 ### Fixed

README.md

@@ -136,6 +136,7 @@ score is the weighted mean of the scorer scores.
 | `contains` | all substrings present (partial credit) | `value` / `values`, `caseSensitive` |
 | `not-contains` | no banned substring present | `value` / `values`, `caseSensitive` |
 | `json-schema` | output is valid JSON matching a schema | `schema` |
+| `json-nonempty` | output is valid JSON with non-empty leaf values | `schema`, `minKeys`, `rejectBlankStrings`, `rejectNulls` |
 | `embedding-similarity` | cosine similarity >= threshold | `expected`, `threshold` |
 | `llm-judge` | a judge model scores >= threshold | `criteria`, `expected`, `threshold`, `model` |
 | `latency` | call latency within budget | `budgetMs` |

src/scorers/json-nonempty.ts

@@ -0,0 +1,83 @@
+import type { Scorer, ScoreContext, ScorerSpec } from "../types.js";
+import { result } from "./util.js";
+import { validate, type JsonSchema } from "./json-schema.js";
+
+/**
+ * Passes when the output parses as JSON, validates against an optional schema,
+ * and carries at least one non-empty leaf value.
+ *
+ * Catches models gaming structured-output evals with vacuous payloads like
+ * `{}`, `{"answer": ""}`, or all-null objects that still validate against the
+ * schema.
+ *
+ * Options:
+ *  - `schema`: a JSON Schema object to validate against (optional)
+ *  - `minKeys`: minimum number of non-empty leaf values required (default 1)
+ *  - `rejectBlankStrings`: treat `""` and whitespace-only strings as empty values (default true)
+ *  - `rejectNulls`: treat `null` as an empty value (default true)
+ */
+export const jsonNonemptyScorer: Scorer = {
+  type: "json-nonempty",
+  score(spec: ScorerSpec, ctx: ScoreContext) {
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(ctx.output);
+    } catch (err) {
+      return result(spec, {
+        score: 0,
+        passed: false,
+        reason: `output is not valid JSON: ${(err as Error).message}`,
+      });
+    }
+    const schema = spec.schema as JsonSchema | undefined;
+    if (schema) {
+      const errors = validate(parsed, schema);
+      if (errors.length > 0) {
+        return result(spec, {
+          score: 0,
+          passed: false,
+          reason: errors.join("; "),
+        });
+      }
+    }
+
+    const minKeys = typeof spec.minKeys === "number" ? spec.minKeys : 1;
+    const rejectBlankStrings = spec.rejectBlankStrings !== false;
+    const rejectNulls = spec.rejectNulls !== false;
+    const nonEmpty = countNonEmptyLeaves(parsed, rejectBlankStrings, rejectNulls);
+
+    if (nonEmpty < minKeys) {
+      return result(spec, {
+        score: 0,
+        passed: false,
+        reason: `output has ${nonEmpty} non-empty value(s), need at least ${minKeys}`,
+      });
+    }
+    return result(spec, {
+      score: 1,
+      passed: true,
+      reason: "output is non-empty JSON",
+    });
+  },
+};
+
+function countNonEmptyLeaves(value: unknown, rejectBlankStrings: boolean, rejectNulls: boolean): number {
+  if (typeof value === "string") {
+    const blank = rejectBlankStrings ? value.trim().length === 0 : false;
+    return blank ? 0 : 1;
+  }
+  if (value === null) return rejectNulls ? 0 : 1;
+  if (Array.isArray(value)) {
+    return value.reduce((n, item) => n + countNonEmptyLeaves(item, rejectBlankStrings, rejectNulls), 0);
+  }
+  if (typeof value === "object") {
+    const obj = value as Record<string, unknown>;
+    const keys = Object.keys(obj);
+    if (keys.length === 0) return 0;
+    return keys.reduce(
+      (n, key) => n + countNonEmptyLeaves(obj[key], rejectBlankStrings, rejectNulls),
+      0,
+    );
+  }
+  return 1; // booleans and numbers are non-empty
+}

src/scorers/registry.ts

@@ -3,6 +3,7 @@ import { exactMatchScorer } from "./exact-match.js";
 import { regexScorer } from "./regex.js";
 import { containsScorer, notContainsScorer } from "./contains.js";
 import { jsonSchemaScorer } from "./json-schema.js";
+import { jsonNonemptyScorer } from "./json-nonempty.js";
 import { embeddingSimilarityScorer } from "./embedding-similarity.js";
 import { llmJudgeScorer } from "./llm-judge.js";
 import { latencyScorer } from "./latency.js";
@@ -47,6 +48,7 @@ export const builtinScorers: Scorer[] = [
   containsScorer,
   notContainsScorer,
   jsonSchemaScorer,
+  jsonNonemptyScorer,
   embeddingSimilarityScorer,
   llmJudgeScorer,
   latencyScorer,

tests/scorers.test.ts

@@ -5,6 +5,7 @@ import { exactMatchScorer } from "../src/scorers/exact-match.js";
 import { regexScorer } from "../src/scorers/regex.js";
 import { containsScorer, notContainsScorer } from "../src/scorers/contains.js";
 import { jsonSchemaScorer, validate } from "../src/scorers/json-schema.js";
+import { jsonNonemptyScorer } from "../src/scorers/json-nonempty.js";
 import { embeddingSimilarityScorer, cosineSimilarity } from "../src/scorers/embedding-similarity.js";
 import { llmJudgeScorer } from "../src/scorers/llm-judge.js";
 import { latencyScorer } from "../src/scorers/latency.js";
@@ -136,6 +137,72 @@ describe("json-schema", () => {
   });
 });
 
+describe("json-nonempty", () => {
+  it("fails a schema-valid but empty object before any leaf values", async () => {
+    const r = await run(
+      jsonNonemptyScorer,
+      { type: "json-nonempty", schema: { type: "object" }, minKeys: 1 },
+      ctx("{}"),
+    );
+    expect(r.passed).toBe(false);
+  });
+
+  it("fails blank-string and all-null outputs", async () => {
+    const blank = await run(
+      jsonNonemptyScorer,
+      { type: "json-nonempty" },
+      ctx('{"answer": ""}'),
+    );
+    expect(blank.passed).toBe(false);
+
+    const nulls = await run(
+      jsonNonemptyScorer,
+      { type: "json-nonempty" },
+      ctx('{"answer": null, "extra": null}'),
+    );
+    expect(nulls.passed).toBe(false);
+  });
+
+  it("passes an output with at least one non-empty leaf value", async () => {
+    const r = await run(
+      jsonNonemptyScorer,
+      { type: "json-nonempty" },
+      ctx('{"answer": "Paris"}'),
+    );
+    expect(r.passed).toBe(true);
+    expect(r.score).toBe(1);
+  });
+
+  it("honors minKeys over the default of 1", async () => {
+    const r = await run(
+      jsonNonemptyScorer,
+      { type: "json-nonempty", minKeys: 2 },
+      ctx('{"a": "one", "b": ""}'),
+    );
+    expect(r.passed).toBe(false);
+  });
+
+  it("fails non-JSON output with a parse reason", async () => {
+    const r = await run(
+      jsonNonemptyScorer,
+      { type: "json-nonempty" },
+      ctx("{not json"),
+    );
+    expect(r.passed).toBe(false);
+    expect(r.reason).toMatch(/not valid JSON/i);
+  });
+
+  it("fails before emptiness checks when the schema rejects the output", async () => {
+    const r = await run(
+      jsonNonemptyScorer,
+      { type: "json-nonempty", schema: { type: "object", required: ["n"] } },
+      ctx('{"answer": "Paris"}'),
+    );
+    expect(r.passed).toBe(false);
+    expect(r.reason).toContain("required");
+  });
+});
+
 describe("embedding-similarity", () => {
   it("scores identical text near 1", async () => {
     const r = await run(