feat: detect confirmed wrong values in voice calls

README.md

@@ -44,13 +44,28 @@ Each of these is invisible in a text eval:
 | Check | Why it costs money |
 |---|---|
 | `misheard_number` | "Fifteen" and "fifty" are one unstressed syllable apart. The agent acts on either with equal confidence. **The expensive one.** |
+| `confirmed_wrong_value` | The agent confirms a misheard number, even if the caller agrees. |
 | `no_confirmation` | In text, a misunderstanding costs one turn. In voice, it costs the refund. |
 | `policy_violation` | The agent exceeded a limit. Policy belongs in code, not the prompt. |
 | `slow_response` | Three seconds of silence is a failed call, however good the answer. |
 | `talked_over_user` | Barge-in handling is most of what makes an agent feel human or broken. |
 | `dead_air` | Where callers hang up. |
 | `incomplete` | The call ended without reaching its goal. |
 
+`confirmed_wrong_value` associates confirmations with the preceding caller turn, stopping
+at the next caller turn. It compares unsigned decimal digits and isolated English number
+words (one through nineteen, tens through ninety, hundred, thousand), normalizing forms
+such as `fifty` and `50.00`. Compound phrases (`twenty five`, `two hundred`), signed and
+grouped numbers are skipped by this check. It does not map numbers to semantic fields,
+track numeric order/repetition, or understand other languages or arbitrary confirmation
+phrasing. A matching number elsewhere in truth can therefore hide a mismatch.
+
+Ground truth exposes a wrong confirmation without requiring caller agreement or an action.
+When one confirmed STT value differs from the next consequential action's single numeric
+`amount`, that discrepancy is also reported. That association stops at a new confirmation
+or caller request; short acknowledgements such as `yes` may precede the action.
+Try `voiceeval check fixtures/confirmed_wrong_value_call.json --strict` (expected exit 1).
+
 ## Regression diff
 
 A single pass/fail tells you nothing on the day a prompt change makes things 5% worse. Run the
@@ -95,7 +110,7 @@ scripted test calls: in production this failure is silent, and no tool can fix t
 
 ## Honest scope
 
-- **The eval logic is the project, and it is fully tested** (20 tests, no keys, no network).
+- **The eval logic is the project, and it is fully tested** (no keys, no network).
 - **The STT adapter is not exercised by the tests.** `GroqSTT` (whisper-large-v3, free tier) needs
   an API key and a network, and what is worth testing here is the evaluation, not whether Groq's
   SDK works. If your platform already gives you a timed transcript, you never need it.
@@ -117,7 +132,7 @@ From a clone, for development:
 
 ```bash
 pip install -e ".[dev]"
-pytest -q                                  # 20 tests
+pytest -q
 ```
 
 Only dependency is `rich`. `[stt]` adds `groq` if you are starting from audio.

fixtures/confirmed_wrong_value_call.json

@@ -0,0 +1,14 @@
+{
+  "id": "refund-confirmed-wrong-fifty",
+  "policy": {"max_refund": 50},
+  "completed": true,
+  "turns": [
+    {"speaker": "agent", "text": "Hi, how can I help?", "start_s": 0.0, "end_s": 1.5},
+    {"speaker": "user", "text": "I need a refund of fifty dollars please.",
+     "truth": "I need a refund of fifteen dollars please.", "start_s": 2.0, "end_s": 5.0},
+    {"speaker": "agent", "text": "Just to confirm, fifty dollars?", "start_s": 5.3, "end_s": 7.0},
+    {"speaker": "user", "text": "yes", "truth": "yes", "start_s": 7.2, "end_s": 7.6},
+    {"speaker": "agent", "text": "Okay, refunding fifty dollars now.", "start_s": 7.9, "end_s": 9.5,
+     "actions": [{"name": "refund", "args": {"amount": 50}, "consequential": true}]}
+  ]
+}

tests/test_confirmed_wrong_value.py

@@ -0,0 +1,185 @@
+"""Wrong-number confirmations must stay tied to the caller's current request."""
+
+from pathlib import Path
+
+import pytest
+
+from voiceeval.checks import analyse, check_confirmed_wrong_value
+from voiceeval.score import score
+from voiceeval.turns import Action, Interaction, Turn, load
+
+
+def call(heard, truth, confirmation, amount=None, later=()):
+    turns = [Turn("user", heard, 0, 1, truth), Turn("agent", confirmation, 1.1, 2)]
+    if amount is not None:
+        turns += [
+            Turn("user", "yes", 2.1, 2.4),
+            Turn("agent", "Done", 2.5, 3, actions=[Action("refund", {"amount": amount}, True)]),
+        ]
+    return Interaction("confirmation", turns + list(later))
+
+
+@pytest.mark.parametrize(
+    "confirmation", ["Confirming fifty?", "Fifty, correct?", "Did you say 50?"]
+)
+def test_confirmation_phrases(confirmation):
+    findings = check_confirmed_wrong_value(call("fifty", "fifteen", confirmation, 50))
+    assert len(findings) == 1
+    assert findings[0].severity == "high"
+    assert findings[0].turn_index == 1
+
+
+@pytest.mark.parametrize(
+    "heard,truth,confirmation",
+    [
+        ("50", "15", "Confirm fifty?"),
+        ("fifty", "15", "Confirm 50.00?"),
+        ("50.25", "15.25", "Confirm 50.250?"),
+        ("FIFTY", "FIFTEEN", "CONFIRM FIFTY?"),
+        ("50 for order 123", "15 for order 123", "Confirm 50 for order 123?"),
+        ("refund 50", "refund please", "Confirm 50?"),
+    ],
+)
+def test_wrong_values(heard, truth, confirmation):
+    assert len(check_confirmed_wrong_value(call(heard, truth, confirmation))) == 1
+
+
+@pytest.mark.parametrize(
+    "heard,truth,confirmation",
+    [
+        ("fifty", "fifteen", "Confirm fifteen?"),
+        ("fifty", "fifty", "Confirm fifty?"),
+        ("50.00", "fifty", "Confirm 50?"),
+        ("fifty", None, "Confirm fifty?"),
+        ("fifty", "", "Confirm fifty?"),
+        ("refund shoes", "refund shirt", "Confirm refund?"),
+        ("50 for order 123", "15 for order 123", "Confirm order 123?"),
+        ("fifty", "fifteen", "Refunding fifty now."),
+        ("fifty", "fifteen", "Confirm refund?"),
+        ("fifty", "fifteen", "Confirm sixty?"),
+    ],
+)
+def test_clean_or_insufficient_evidence(heard, truth, confirmation):
+    assert check_confirmed_wrong_value(call(heard, truth, confirmation)) == []
+
+
+def test_new_user_request_ends_old_mismatch():
+    inter = call(
+        "fifty",
+        "fifteen",
+        "I heard you.",
+        later=[
+            Turn("user", "Actually make it fifty", 2.1, 3, "Actually make it fifty"),
+            Turn("agent", "Confirm fifty?", 3.1, 4),
+        ],
+    )
+    assert check_confirmed_wrong_value(inter) == []
+
+
+def test_does_not_duplicate_one_confirmation_for_old_requests():
+    inter = Interaction(
+        "repeated",
+        [
+            Turn("user", "fifty", 0, 1, "fifteen"),
+            Turn("user", "fifty", 1.1, 2, "fifteen"),
+            Turn("agent", "Confirm fifty?", 2.1, 3),
+        ],
+    )
+    findings = check_confirmed_wrong_value(inter)
+    assert len(findings) == 1
+    assert findings[0].turn_index == 2
+
+
+def test_can_confirm_after_agent_filler():
+    inter = call(
+        "fifty",
+        "fifteen",
+        "Let me check.",
+        later=[
+            Turn("agent", "Confirm fifty?", 2.1, 3),
+        ],
+    )
+    assert len(check_confirmed_wrong_value(inter)) == 1
+
+
+def test_action_mismatch_without_truth():
+    findings = check_confirmed_wrong_value(call("50", None, "Confirm fifty?", "15"))
+    assert len(findings) == 1
+    assert "action" in findings[0].message.lower()
+
+
+def test_equivalent_spelling_does_not_pull_in_later_action():
+    inter = call(
+        "15",
+        "fifteen",
+        "Confirm 15?",
+        later=[
+            Turn("user", "Now refund twenty for the other item", 2.1, 3),
+            Turn("agent", "Done", 3.1, 4, actions=[Action("refund", {"amount": 20}, True)]),
+        ],
+    )
+    assert check_confirmed_wrong_value(inter) == []
+
+
+@pytest.mark.parametrize("amount", [True, "NaN", "Infinity", "", "not an amount", {}, []])
+def test_invalid_action_amount_is_ignored(amount):
+    assert check_confirmed_wrong_value(call("50", None, "Confirm fifty?", amount)) == []
+
+
+@pytest.mark.parametrize(
+    "value", ["twenty five", "twenty-five", "two hundred", "1,000", "-50", "+50"]
+)
+def test_unsupported_numeric_forms_are_not_split_into_unrelated_values(value):
+    assert check_confirmed_wrong_value(call(value, "25", "Confirm " + value + "?")) == []
+
+
+def test_headline_fixture_and_corrected_confirmation():
+    inter = load(Path(__file__).resolve().parents[1] / "fixtures/confirmed_wrong_value_call.json")
+    findings = check_confirmed_wrong_value(inter)
+    assert len(findings) == 1
+    assert findings[0].turn_index == 2
+    assert findings[0].severity == "high"
+    assert "no_confirmation" not in {f.check for f in analyse(inter)}
+    assert score(inter).passed is False
+    assert any(f["check"] == "confirmed_wrong_value" for f in score(inter).findings)
+    inter.turns[2].text = "Just to confirm, fifteen dollars?"
+    inter.turns[4].actions[0].args["amount"] = 15
+    assert check_confirmed_wrong_value(inter) == []
+
+
+def test_action_amount_does_not_compare_an_order_number():
+    inter = call("refund 50 for order 123", "refund 50 for order 123", "Confirm order 123?", 50)
+    assert check_confirmed_wrong_value(inter) == []
+
+
+def test_action_on_confirmation_turn():
+    inter = call("50", None, "Confirm 50?")
+    inter.turns[1].actions = [Action("refund", {"amount": 15}, True)]
+    assert len(check_confirmed_wrong_value(inter)) == 1
+
+
+def test_next_confirmation_stops_action_association():
+    inter = call(
+        "50",
+        None,
+        "Confirm 50?",
+        later=[
+            Turn("agent", "Actually confirm 15?", 2.1, 3),
+            Turn("agent", "Done", 3.1, 4, actions=[Action("refund", {"amount": 15}, True)]),
+        ],
+    )
+    assert check_confirmed_wrong_value(inter) == []
+
+
+def test_non_consequential_and_ambiguous_actions_are_not_amount_evidence():
+    inter = call("50", None, "Confirm 50?")
+    inter.turns[1].actions = [Action("lookup", {"amount": 15}, False)]
+    assert check_confirmed_wrong_value(inter) == []
+    inter.turns[1].actions = [Action("refund", {"amount": a}, True) for a in (15, 20)]
+    assert check_confirmed_wrong_value(inter) == []
+
+
+def test_shared_confirmation_phrases_prevent_missing_confirmation():
+    for phrase in ("Confirming fifty?", "Fifty, correct?"):
+        inter = call("fifty", "fifteen", phrase, 50)
+        assert "no_confirmation" not in {f.check for f in analyse(inter)}