System One: TypeSafe confidence formulas, strict schema 422s, bounded token counting

proxy/README.md

@@ -265,7 +265,8 @@ probabilities on Continuum GLM-5.3-Flash; request and response follow the
 TypeSafe System One schema, with optional `images` (`data:` URLs) and
 `temperature` fields. Rejections keep the backend's status and
 `x-opensecret-error-code` header, for example `422` with
-`system_one_too_many_questions`.
+`system_one_too_many_questions`, or `system_one_unknown_field` naming a
+misspelled key.
 
 This route needs the inference allowlist entry added in `maple-sdk` 4.1.0.
 `Cargo.toml` builds the in-tree SDK through a `[patch.crates-io]` entry, as do

sdk/rust/src/system_one.rs

@@ -233,7 +233,8 @@ pub enum SystemOneAnswer {
         choice: String,
         /// Probability per option label, in the order the options were given.
         probabilities: OrderedEntries<f64>,
-        /// Peakedness of the distribution, from 0 (uniform) to 1 (certain).
+        /// TypeSafe's choice confidence, `(p_max - 1/n) / (1 - 1/n)`: 0 when the options are
+        /// equally likely, 1 when one option takes all the probability.
         confidence: f64,
         temperature: f64,
         option_mass: f64,
@@ -245,6 +246,9 @@ pub enum SystemOneAnswer {
         legend: OrderedEntries<Value>,
         /// Probability per level number.
         probabilities: OrderedEntries<f64>,
+        /// TypeSafe's score confidence, `1 - sum(p_i |i - m|) / MAD_unif` with `m` the most
+        /// likely level: 0 when the mass is spread like a uniform distribution, 1 when it sits
+        /// on one level.
         confidence: f64,
         temperature: f64,
         option_mass: f64,

sdk/src/lib/systemOne.ts

@@ -75,7 +75,10 @@ export type SystemOneChoiceAnswer = {
   choice: string;
   /** Probability per option label, in the order the options were given. */
   probabilities: Record<string, number>;
-  /** Peakedness of the distribution, from 0 (uniform) to 1 (certain). */
+  /**
+   * TypeSafe's choice confidence, `(p_max - 1/n) / (1 - 1/n)`: 0 when the options are equally
+   * likely, 1 when one option takes all the probability.
+   */
   confidence: number;
   temperature: number;
   option_mass: number;
@@ -89,6 +92,10 @@ export type SystemOneScoreAnswer = {
   legend: Record<string, unknown>;
   /** Probability per level number. */
   probabilities: Record<string, number>;
+  /**
+   * TypeSafe's score confidence, `1 - sum(p_i |i - m|) / MAD_unif` with `m` the most likely
+   * level: 0 when the mass is spread like a uniform distribution, 1 when it sits on one level.
+   */
   confidence: number;
   temperature: number;
   option_mass: number;

services/opensecret/src/tokens.rs

@@ -10,9 +10,36 @@ use crate::model_config::model_context_window;
 static ENCODER: Lazy<tiktoken_rs::CoreBPE> =
     Lazy::new(|| cl100k_base().expect("init cl100k encoder"));
 
+/// Longest run without whitespace that is encoded in one go. BPE merging is quadratic in
+/// the length of a run: a 200k-character "word" takes about 16 s unsplit and under 100 ms in
+/// pieces this size. Splitting only inside such runs leaves every ordinary text's count
+/// unchanged and overcounts a pathological run by at most a token per piece.
+const MAX_RUN_BYTES: usize = 1024;
+
 /// Count tokens for a piece of UTF‑8 text.
 pub fn count_tokens(text: &str) -> usize {
-    ENCODER.encode_with_special_tokens(text).len()
+    split_long_runs(text)
+        .map(|piece| ENCODER.encode_with_special_tokens(piece).len())
+        .sum()
+}
+
+/// Yields `text` whole unless a whitespace-free run exceeds `MAX_RUN_BYTES`, in which case
+/// the run is cut into pieces of about that size, on char boundaries.
+fn split_long_runs(text: &str) -> impl Iterator<Item = &str> {
+    let mut pieces = Vec::new();
+    let mut piece_start = 0;
+    let mut run_start = 0;
+    for (index, ch) in text.char_indices() {
+        if ch.is_whitespace() {
+            run_start = index + ch.len_utf8();
+        } else if index - run_start >= MAX_RUN_BYTES {
+            pieces.push(&text[piece_start..index]);
+            piece_start = index;
+            run_start = index;
+        }
+    }
+    pieces.push(&text[piece_start..]);
+    pieces.into_iter()
 }
 
 pub fn model_max_ctx(model: &str) -> usize {
@@ -35,6 +62,29 @@ mod tests {
         assert_eq!(count_tokens("Hello"), 1);
     }
 
+    #[test]
+    fn long_runs_are_counted_in_bounded_pieces() {
+        // Unsplit, this one "word" takes seconds (quadratic merging); in pieces it is fast and
+        // the count stays within a token per piece of the exact one.
+        let run = "x".repeat(100_000);
+        let count = count_tokens(&run);
+        assert!((12_000..=13_000).contains(&count), "{count}");
+        // Ordinary text is encoded whole, so its count is exactly the encoder's.
+        let prose = "The quick brown fox jumps over the lazy dog. ".repeat(200);
+        assert_eq!(
+            count_tokens(&prose),
+            ENCODER.encode_with_special_tokens(&prose).len()
+        );
+        assert_eq!(split_long_runs(&prose).count(), 1);
+        assert_eq!(split_long_runs("").count(), 1);
+        // Pieces end on char boundaries and cover the text.
+        let wide = "é".repeat(2_000);
+        let pieces: Vec<&str> = split_long_runs(&wide).collect();
+        assert_eq!(pieces.iter().map(|p| p.len()).sum::<usize>(), wide.len());
+        assert!(pieces.iter().all(|p| p.len() <= MAX_RUN_BYTES + 4));
+        assert!(pieces.len() >= 3);
+    }
+
     #[test]
     fn test_count_tokens_consistency() {
         // Same text should always produce same count