From f5f064975347346c5e43f6239ebf2c473806b036 Mon Sep 17 00:00:00 2001
From: simba <simba@Mac-mini.local>
Date: Thu, 24 Sep 2026 19:01:43 -0700
Subject: [PATCH] fix(turbokv): warn when --turbo-kv can't apply (e.g. with
 --ctx-size)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On mlx-swift-lm 460ff81 a bounded context (--ctx-size) gives the attention
layers a RotatingKVCache, and the server only enables TurboQuant on
KVCacheSimple layers. So `--turbo-kv --ctx-size N` printed turbo_kv=enabled
and then silently compressed nothing. The M6 TurboKV benchmarks in #173 hit
exactly this (corrected in #180).

- Startup: warn when --turbo-kv and --ctx-size are both set, and say to drop
  --ctx-size to use --turbo-kv.
- Per request: count the layers TurboQuant was enabled on, and warn once if
  none were (covers any model or cache setup where TurboKV can't attach).

No change to cache selection or to TurboKV behaviour. Verified on a Mac mini M6
with Qwen3.8-27B-4bit, two requests per setup:
  --turbo-kv --ctx-size 16384 → both warnings, the per-request one once
  --turbo-kv                  → no warnings
  --ctx-size 16384            → no warnings

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 Sources/SwiftLM/Server.swift | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/Sources/SwiftLM/Server.swift b/Sources/SwiftLM/Server.swift
index 4e0e185..677f270 100644
--- a/Sources/SwiftLM/Server.swift
+++ b/Sources/SwiftLM/Server.swift
@@ -227,6 +227,11 @@ func isVLMCheckpointMismatch(_ error: any Error) -> Bool {
     }
 }
 
+/// One-shot flags for TurboKV configuration notices.
+enum TurboKVNotice {
+    nonisolated(unsafe) static var warnedNoLayers = false
+}
+
 func sanitizeForJinja(_ value: any Sendable) -> (any Sendable)? {
     if value is NSNull { return nil }
     let mirror = Mirror(reflecting: value)
@@ -1287,6 +1292,9 @@ struct MLXServer: AsyncParsableCommand {
         let turboKVStr = config.turboKV ? "enabled" : "disabled"
         let mtpStr = config.mtp ? "enabled (\(config.numMtpTokens) tokens/round)" : "disabled"
         print("[SwiftLM] Config: ctx_size=\(ctxSizeStr), temp=\(config.temp), top_p=\(config.topP), top_k=\(topKStr), min_p=\(minPStr), repeat_penalty=\(penaltyStr), parallel=\(parallelSlots), cors=\(corsStr), mem_limit=\(memLimitStr), auth=\(authStr), thinking=\(thinkingStr), ssd_stream=\(ssdStr), turbo_kv=\(turboKVStr), mtp=\(mtpStr)")
+        if config.turboKV, let ctx = config.ctxSize {
+            print("[SwiftLM] ⚠️  --turbo-kv has no effect with --ctx-size \(ctx): a bounded context gives the attention layers a RotatingKVCache, and TurboKV only compresses KVCacheSimple. Drop --ctx-size to use --turbo-kv.")
+        }
 
         // ── Build Hummingbird router ──
         let router = Router()
@@ -2038,11 +2046,18 @@ func handleChatCompletion(
         // This compresses cache history older than 8192 tokens into 3.5-bit Polar+QJL
         // form, halving KV RAM for long-context (100k+) requests.
         if config.turboKV {
+            var enabledLayers = 0
             for layer in cache {
                 if let simple = layer as? KVCacheSimple {
                     simple.turboQuantEnabled = true
+                    enabledLayers += 1
                 }
             }
+            if enabledLayers == 0 && !TurboKVNotice.warnedNoLayers {
+                // Warn once. A benign race here only means a duplicate log line.
+                TurboKVNotice.warnedNoLayers = true
+                print("[SwiftLM] ⚠️  --turbo-kv is enabled but this model's cache has no KVCacheSimple layers (\(cache.count) layers, e.g. RotatingKVCache from --ctx-size), so no KV compression is applied.")
+            }
         }
 
         // ── Prompt cache: bypass for multimodal inputs ──