Skip to content

server /train: validate numbers before examples; the prefix-stability refusal says what to do (Cormac's #20 notes) #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
joelteply merged 1 commit into from
Sep 27, 2026
Merged

server /train: validate numbers before examples; the prefix-stability refusal says what to do (Cormac's #20 notes) #21

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 27 additions & 28 deletions tools/server/server-train.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,8 @@ static bool prepare_examples(const json & examples, const server_trainer::render
const std::string upto = render(head(k + 1), false);
if (full.compare(0, upto.size(), upto) != 0 || upto.compare(0, pre.size(), pre) != 0 || pre.size() >= upto.size()) {
why = at + ": the chat template does not render this conversation prefix-stably at message " + std::to_string(k)
+ ", so the assistant turn cannot be located";
+ ", so the assistant turn cannot be located (templates that strip earlier turns' reasoning, e.g. Qwen3's"
+ " <think>, do this: send such a conversation as single-turn examples)";
return false;
}
spans.emplace_back(pre.size(), upto.size());
Expand Down Expand Up @@ -160,6 +161,31 @@ json server_trainer::start(const json & body_in) {
if (params_base.train_dir.empty()) {
return json::object({{"ok", false}, {"error", "training is disabled on this server (start it with --train-dir DIR to enable /train)"}});
}
// Every numeric input is checked first, before "examples" are prepared against "window" and
// before a thread starts: inside a serving process an
// assert in the training path would take the server down (Cormac on #14).
{
auto num = [&](const char * key, double def) { return body.contains(key) && body.at(key).is_number() ? body.at(key).get<double>() : def; };
for (const char * key : {"rank", "alpha", "window", "epochs", "lr", "val_split", "seed"}) {
if (body.contains(key) && !body.at(key).is_number()) {
return json::object({{"ok", false}, {"error", std::string("\"") + key + "\" must be a number"}});
}
}
const double rank = num("rank", 8), alpha = num("alpha", 16), window = num("window", 256), epochs = num("epochs", 1);
const double lr = num("lr", 1e-5), val = num("val_split", 0.1);
std::string why;
if (rank < 1 || rank > 256 || rank != (int64_t) rank) why = "rank must be an integer in [1, 256]";
else if (!(alpha > 0)) why = "alpha must be > 0";
else if (window < 16 || window > 8192 || window != (int64_t) window) why = "window must be an integer in [16, 8192]";
else if (epochs < 1 || epochs > 100 || epochs != (int64_t) epochs) why = "epochs must be an integer in [1, 100]";
else if (!(lr > 0 && lr <= 1)) why = "lr must be in (0, 1]";
else if (!(val >= 0 && val < 1)) why = "val_split must be in [0, 1)";
else if (body.contains("targets") && (!body.at("targets").is_string() || split_targets(body.at("targets").get<std::string>()).empty()))
why = "targets must be a non-empty comma-separated list of module names";
if (!why.empty()) {
return json::object({{"ok", false}, {"error", why}});
}
}
if (body.contains("examples")) {
if (body.contains("text")) {
return json::object({{"ok", false}, {"error", "give \"text\" or \"examples\", not both"}});
Expand All @@ -173,9 +199,6 @@ json server_trainer::start(const json & body_in) {
if (!render_chat) {
return json::object({{"ok", false}, {"error", "this server has no chat template to render \"examples\" with; send \"text\""}});
}
if (!body.value("window", json(256)).is_number()) {
return json::object({{"ok", false}, {"error", "\"window\" must be a number"}});
}
std::string why;
if (!prepare_examples(body.at("examples"), render_chat, llama_model_get_vocab(model),
(int64_t) body.value("window", 256.0), prepared, why)) {
Expand All @@ -193,30 +216,6 @@ json server_trainer::start(const json & body_in) {
if (confine_out(params_base.train_dir, body.at("out").get<std::string>()).empty()) {
return json::object({{"ok", false}, {"error", "\"out\" must be a bare file name ending in .gguf (no directories, no \"..\"); it is written inside the server's --train-dir"}});
}
// Every numeric input is checked here, before a thread starts: inside a serving process an
// assert in the training path would take the server down (Cormac on #14).
{
auto num = [&](const char * key, double def) { return body.contains(key) && body.at(key).is_number() ? body.at(key).get<double>() : def; };
for (const char * key : {"rank", "alpha", "window", "epochs", "lr", "val_split", "seed"}) {
if (body.contains(key) && !body.at(key).is_number()) {
return json::object({{"ok", false}, {"error", std::string("\"") + key + "\" must be a number"}});
}
}
const double rank = num("rank", 8), alpha = num("alpha", 16), window = num("window", 256), epochs = num("epochs", 1);
const double lr = num("lr", 1e-5), val = num("val_split", 0.1);
std::string why;
if (rank < 1 || rank > 256 || rank != (int64_t) rank) why = "rank must be an integer in [1, 256]";
else if (!(alpha > 0)) why = "alpha must be > 0";
else if (window < 16 || window > 8192 || window != (int64_t) window) why = "window must be an integer in [16, 8192]";
else if (epochs < 1 || epochs > 100 || epochs != (int64_t) epochs) why = "epochs must be an integer in [1, 100]";
else if (!(lr > 0 && lr <= 1)) why = "lr must be in (0, 1]";
else if (!(val >= 0 && val < 1)) why = "val_split must be in [0, 1)";
else if (body.contains("targets") && (!body.at("targets").is_string() || split_targets(body.at("targets").get<std::string>()).empty()))
why = "targets must be a non-empty comma-separated list of module names";
if (!why.empty()) {
return json::object({{"ok", false}, {"error", why}});
}
}
// REPACKED WEIGHTS GIVE WRONG GRADIENTS, SILENTLY (Cormac on the /train plan): the backward
// reads quantized blocks in the standard layout, and a weight in an extra buffer type
// (CPU_REPACK) is not in it — no crash, garbage gradients. Extra buffer types only apply to
Expand Down
Loading