From fdca5967dac8d805abf6b7fed09941b350890113 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sat, 26 Sep 2026 22:30:21 -0500 Subject: [PATCH] server /train: numeric inputs are validated before "examples" are prepared, and the prefix-stability refusal says what to do (Cormac's notes on #20) - window/rank/epochs/... are checked first, so a window of 0 is "window must be an integer in [16, 8192]" instead of "this window holds -4" from the example preparation that used it. - a template that rewrites earlier turns (Qwen3 strips earlier turns' ) refuses the multi-turn example; the 400 now says to send such a conversation as single-turn examples. On the 5090 (1.5B): window 0 -> the range message; the #20 mask run is unchanged (trainable_tokens 1979 of 2177, loss 4.733), as is the text path. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- tools/server/server-train.cpp | 55 +++++++++++++++++------------------ 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/tools/server/server-train.cpp b/tools/server/server-train.cpp index b3d59a46810c..3878578fc567 100644 --- a/tools/server/server-train.cpp +++ b/tools/server/server-train.cpp @@ -106,7 +106,8 @@ static bool prepare_examples(const json & examples, const server_trainer::render const std::string upto = render(head(k + 1), false); if (full.compare(0, upto.size(), upto) != 0 || upto.compare(0, pre.size(), pre) != 0 || pre.size() >= upto.size()) { why = at + ": the chat template does not render this conversation prefix-stably at message " + std::to_string(k) - + ", so the assistant turn cannot be located"; + + ", so the assistant turn cannot be located (templates that strip earlier turns' reasoning, e.g. Qwen3's" + + " , do this: send such a conversation as single-turn examples)"; return false; } spans.emplace_back(pre.size(), upto.size()); @@ -160,6 +161,31 @@ json server_trainer::start(const json & body_in) { if (params_base.train_dir.empty()) { return json::object({{"ok", false}, {"error", "training is disabled on this server (start it with --train-dir DIR to enable /train)"}}); } + // Every numeric input is checked first, before "examples" are prepared against "window" and + // before a thread starts: inside a serving process an + // assert in the training path would take the server down (Cormac on #14). + { + auto num = [&](const char * key, double def) { return body.contains(key) && body.at(key).is_number() ? body.at(key).get() : def; }; + for (const char * key : {"rank", "alpha", "window", "epochs", "lr", "val_split", "seed"}) { + if (body.contains(key) && !body.at(key).is_number()) { + return json::object({{"ok", false}, {"error", std::string("\"") + key + "\" must be a number"}}); + } + } + const double rank = num("rank", 8), alpha = num("alpha", 16), window = num("window", 256), epochs = num("epochs", 1); + const double lr = num("lr", 1e-5), val = num("val_split", 0.1); + std::string why; + if (rank < 1 || rank > 256 || rank != (int64_t) rank) why = "rank must be an integer in [1, 256]"; + else if (!(alpha > 0)) why = "alpha must be > 0"; + else if (window < 16 || window > 8192 || window != (int64_t) window) why = "window must be an integer in [16, 8192]"; + else if (epochs < 1 || epochs > 100 || epochs != (int64_t) epochs) why = "epochs must be an integer in [1, 100]"; + else if (!(lr > 0 && lr <= 1)) why = "lr must be in (0, 1]"; + else if (!(val >= 0 && val < 1)) why = "val_split must be in [0, 1)"; + else if (body.contains("targets") && (!body.at("targets").is_string() || split_targets(body.at("targets").get()).empty())) + why = "targets must be a non-empty comma-separated list of module names"; + if (!why.empty()) { + return json::object({{"ok", false}, {"error", why}}); + } + } if (body.contains("examples")) { if (body.contains("text")) { return json::object({{"ok", false}, {"error", "give \"text\" or \"examples\", not both"}}); @@ -173,9 +199,6 @@ json server_trainer::start(const json & body_in) { if (!render_chat) { return json::object({{"ok", false}, {"error", "this server has no chat template to render \"examples\" with; send \"text\""}}); } - if (!body.value("window", json(256)).is_number()) { - return json::object({{"ok", false}, {"error", "\"window\" must be a number"}}); - } std::string why; if (!prepare_examples(body.at("examples"), render_chat, llama_model_get_vocab(model), (int64_t) body.value("window", 256.0), prepared, why)) { @@ -193,30 +216,6 @@ json server_trainer::start(const json & body_in) { if (confine_out(params_base.train_dir, body.at("out").get()).empty()) { return json::object({{"ok", false}, {"error", "\"out\" must be a bare file name ending in .gguf (no directories, no \"..\"); it is written inside the server's --train-dir"}}); } - // Every numeric input is checked here, before a thread starts: inside a serving process an - // assert in the training path would take the server down (Cormac on #14). - { - auto num = [&](const char * key, double def) { return body.contains(key) && body.at(key).is_number() ? body.at(key).get() : def; }; - for (const char * key : {"rank", "alpha", "window", "epochs", "lr", "val_split", "seed"}) { - if (body.contains(key) && !body.at(key).is_number()) { - return json::object({{"ok", false}, {"error", std::string("\"") + key + "\" must be a number"}}); - } - } - const double rank = num("rank", 8), alpha = num("alpha", 16), window = num("window", 256), epochs = num("epochs", 1); - const double lr = num("lr", 1e-5), val = num("val_split", 0.1); - std::string why; - if (rank < 1 || rank > 256 || rank != (int64_t) rank) why = "rank must be an integer in [1, 256]"; - else if (!(alpha > 0)) why = "alpha must be > 0"; - else if (window < 16 || window > 8192 || window != (int64_t) window) why = "window must be an integer in [16, 8192]"; - else if (epochs < 1 || epochs > 100 || epochs != (int64_t) epochs) why = "epochs must be an integer in [1, 100]"; - else if (!(lr > 0 && lr <= 1)) why = "lr must be in (0, 1]"; - else if (!(val >= 0 && val < 1)) why = "val_split must be in [0, 1)"; - else if (body.contains("targets") && (!body.at("targets").is_string() || split_targets(body.at("targets").get()).empty())) - why = "targets must be a non-empty comma-separated list of module names"; - if (!why.empty()) { - return json::object({{"ok", false}, {"error", why}}); - } - } // REPACKED WEIGHTS GIVE WRONG GRADIENTS, SILENTLY (Cormac on the /train plan): the backward // reads quantized blocks in the standard layout, and a weight in an extra buffer type // (CPU_REPACK) is not in it — no crash, garbage gradients. Extra buffer types only apply to