diff --git a/common/common.cpp b/common/common.cpp index 9e2fdfc545dd..9b9bb74b3464 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -2136,6 +2136,27 @@ ggml_opt_dataset_t common_opt_dataset_init(struct llama_context * ctx, const std return result; } +ggml_opt_dataset_t common_opt_dataset_init_masked(int64_t window, const std::vector> & seqs, + const std::vector> & loss, llama_token pad) { + GGML_ASSERT(seqs.size() == loss.size()); + const int64_t ndata = (int64_t) seqs.size(); + ggml_opt_dataset_t result = ggml_opt_dataset_init(GGML_TYPE_I32, GGML_TYPE_I32, window, window, ndata, /*ndata_shard =*/ 1); + + llama_token * data = (llama_token *) ggml_opt_dataset_data(result)->data; + llama_token * labels = (llama_token *) ggml_opt_dataset_labels(result)->data; + + for (int64_t idata = 0; idata < ndata; ++idata) { + const auto & s = seqs[idata]; + const auto & m = loss[idata]; + GGML_ASSERT(s.size() == m.size() && (int64_t) s.size() <= window + 1); + for (int64_t j = 0; j < window; ++j) { + data [idata*window + j] = j < (int64_t) s.size() ? s[j] : pad; + labels[idata*window + j] = (j + 1 < (int64_t) s.size() && m[j + 1]) ? s[j + 1] : -1; + } + } + return result; +} + ggml_opt_optimizer_params common_opt_lr_pars(void * userdata) { ggml_opt_optimizer_params result = ggml_opt_get_default_optimizer_params(nullptr); const lr_opt & d = *(lr_opt *) userdata; diff --git a/common/common.h b/common/common.h index 4097564b98dd..941bd4f88f7b 100644 --- a/common/common.h +++ b/common/common.h @@ -1135,6 +1135,13 @@ inline llama_model_tensor_buft_override llm_ffn_exps_cpu_override() { ggml_opt_dataset_t common_opt_dataset_init(struct llama_context * ctx, const std::vector & tokens, int64_t stride); +// One datapoint per sequence: seqs[i] (at most window + 1 tokens) is the input, left-aligned and +// padded with `pad` to `window`; the label at position j is seqs[i][j + 1] where loss[i][j + 1] +// is set, and -1 (no loss) everywhere else, padding included. Completion-only training marks +// only the assistant's tokens. +ggml_opt_dataset_t common_opt_dataset_init_masked(int64_t window, const std::vector> & seqs, + const std::vector> & loss, llama_token pad); + // "adamw" or "sgd" (case insensitive) enum ggml_opt_optimizer_type common_opt_get_optimizer(const char *); diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 808266298800..e1b6f0e7bc60 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -11884,7 +11884,14 @@ static void ggml_compute_forward_cross_entropy_loss_back_f32( assert(sum > 0.0); ggml_vec_scale_f32(nc, ds0, 1.0/sum); - // grad(src0f) = (softmax(src0f) - src1f) * grad(cross_entropy_loss(src0f, src1f)) / nr + // grad(src0f) = (softmax(src0f) * sum(src1f) - src1f) * grad(cross_entropy_loss(src0f, src1f)) / nr + // The derivative of -sum(y log softmax(x)) is softmax(x) * sum(y) - y. Dropping sum(y) is + // exact only when every label row sums to 1; a MASKED row (all zero: a position that must + // carry no loss, e.g. a prompt token in completion-only training) would still push + // softmax(x) back through the graph. With the sum the masked row's gradient is zero. + float lsum = 0.0f; + ggml_vec_sum_f32(nc, &lsum, s1); + ggml_vec_scale_f32(nc, ds0, lsum); ggml_vec_sub_f32(nc, ds0, ds0, s1); ggml_vec_scale_f32(nc, ds0, d_by_nr); diff --git a/ggml/src/ggml-cuda/cross-entropy-loss.cu b/ggml/src/ggml-cuda/cross-entropy-loss.cu index 0c8b0819724e..9cfbedd93f3b 100644 --- a/ggml/src/ggml-cuda/cross-entropy-loss.cu +++ b/ggml/src/ggml-cuda/cross-entropy-loss.cu @@ -82,7 +82,15 @@ static __global__ void cross_entropy_loss_back_f32( } } sum = warp_reduce_sum(sum); - const float sm_scale = 1.0f/sum; + + // d/dx of -sum(y log softmax(x)) is softmax(x)*sum(y) - y: with sum(y) a masked (all-zero) + // label row gets a zero gradient; without it the row would still push softmax(x) back. + float lsum = 0.0f; + for (int i = threadIdx.x; i < nclasses; i += WARP_SIZE) { + lsum += labels[i]; + } + lsum = warp_reduce_sum(lsum); + const float sm_scale = lsum/sum; const float d_by_nrows = *grad/gridDim.x; for (int i = threadIdx.x; i < nclasses; i += WARP_SIZE) { diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 5febf5e6c983..24b36df8c51f 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -3483,6 +3483,9 @@ void llama_context::opt_epoch_iter( const float onef = 1.0f; for (uint32_t pos_ubatch = 0; pos_ubatch < n_ubatch; ++pos_ubatch) { const uint32_t ilabel = pos_ctx + pos_batch + pos_ubatch; + if (labels_sparse[ilabel] < 0) { + continue; // masked: carries no loss (the backward honours zero label rows) + } GGML_ASSERT(labels_sparse[ilabel] < labels->ne[0]); ggml_backend_tensor_set(labels, &onef, (pos_ubatch*labels->ne[0] + labels_sparse[ilabel])*sizeof(float), sizeof(float)); } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 0f915f65621a..9e02eca2ecf8 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -7363,17 +7363,41 @@ struct test_flash_attn_ext : public test_case { }; // GGML_OP_CROSS_ENTROPY_LOSS +// Completion-only training masks rows: a masked position's label row is all zero. Every other +// row sums to 1. "masked" zeroes every other row, so the finite-difference check sees whether the +// backward gives those rows a zero gradient (it must: they carry no loss). +static ggml_tensor * ce_test_labels(ggml_context * ctx, ggml_tensor * labels, bool masked) { + labels = ggml_soft_max(ctx, labels); + ggml_set_name(labels, "labels_normalized"); + if (masked) { + ggml_tensor * mask = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 1, labels->ne[1], labels->ne[2], labels->ne[3]); + ggml_set_name(mask, "mask"); + labels = ggml_mul(ctx, labels, mask); + ggml_set_name(labels, "labels_masked"); + } + return labels; +} + +static void ce_test_init_mask(ggml_tensor * t) { + std::vector m(ggml_nelements(t)); + for (size_t i = 0; i < m.size(); ++i) { + m[i] = (i % 2 == 0) ? 1.0f : 0.0f; + } + ggml_backend_tensor_set(t, m.data(), 0, m.size()*sizeof(float)); +} + struct test_cross_entropy_loss : public test_case { const ggml_type type; const std::array ne; + const bool masked; std::string vars() override { - return VARS_TO_STR2(type, ne); + return VARS_TO_STR3(type, ne, masked); } test_cross_entropy_loss(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 5, 4, 3}) - : type(type), ne(ne) {} + std::array ne = {10, 5, 4, 3}, bool masked = false) + : type(type), ne(ne), masked(masked) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * logits = ggml_new_tensor(ctx, type, 4, ne.data()); @@ -7384,9 +7408,8 @@ struct test_cross_entropy_loss : public test_case { // The labels are assumed to be constant -> no gradients. ggml_set_name(labels, "labels"); - // Ensure labels add up to 1: - labels = ggml_soft_max(ctx, labels); - ggml_set_name(labels, "labels_normalized"); + // Rows add up to 1 (or to 0 where masked): + labels = ce_test_labels(ctx, labels, masked); ggml_tensor * out = ggml_cross_entropy_loss(ctx, logits, labels); ggml_set_name(out, "out"); @@ -7397,12 +7420,21 @@ struct test_cross_entropy_loss : public test_case { void initialize_tensors(ggml_context * ctx) override { // For larger abs. diffs between logits softmax is more linear, therefore more precise num. gradients. for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -100.0f, 100.0f); + if (strcmp(t->name, "mask") == 0) { + ce_test_init_mask(t); + } else if (masked) { + // No exact zeros anywhere: the MAA metric is (a-b)/(a+b), so two exact zeros make + // it NaN and a NaN error reads as a pass. At +-100 softmax underflows to 0 and the + // check cannot fail; at +-3 every probability is > 1e-3 and it can. + init_tensor_uniform(t, -3.0f, 3.0f); + } else { + init_tensor_uniform(t, -100.0f, 100.0f); + } } } float grad_eps() override { - return 1.0f; + return masked ? 1e-1f : 1.0f; } bool grad_precise() override { @@ -7414,14 +7446,25 @@ struct test_cross_entropy_loss : public test_case { struct test_cross_entropy_loss_back : public test_case { const ggml_type type; const std::array ne; + const bool masked; std::string vars() override { - return VARS_TO_STR2(type, ne); + return VARS_TO_STR3(type, ne, masked); } test_cross_entropy_loss_back(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 5, 4, 3}) - : type(type), ne(ne) {} + std::array ne = {10, 5, 4, 3}, bool masked = false) + : type(type), ne(ne), masked(masked) {} + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (strcmp(t->name, "mask") == 0) { + ce_test_init_mask(t); + } else { + init_tensor_uniform(t); + } + } + } ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * grad = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); @@ -7433,9 +7476,8 @@ struct test_cross_entropy_loss_back : public test_case { ggml_tensor * labels = ggml_new_tensor(ctx, type, 4, ne.data()); ggml_set_name(labels, "labels"); - // Ensure labels add up to 1: - labels = ggml_soft_max(ctx, labels); - ggml_set_name(labels, "labels_normalized"); + // Rows add up to 1 (or to 0 where masked): + labels = ce_test_labels(ctx, labels, masked); ggml_tensor * out = ggml_cross_entropy_loss_back(ctx, grad, logits, labels); ggml_set_name(out, "out"); @@ -10168,6 +10210,10 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, {30000, 1, 1, 1})); test_cases.emplace_back(new test_cross_entropy_loss_back(GGML_TYPE_F32, { 10, 5, 4, 3})); test_cases.emplace_back(new test_cross_entropy_loss_back(GGML_TYPE_F32, {30000, 1, 1, 1})); + // masked rows (completion-only training): zero label rows must carry zero gradient + test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, { 10, 5, 4, 3}, true)); + test_cases.emplace_back(new test_cross_entropy_loss_back(GGML_TYPE_F32, { 10, 5, 4, 3}, true)); + test_cases.emplace_back(new test_cross_entropy_loss_back(GGML_TYPE_F32, {30000, 4, 1, 1}, true)); test_cases.emplace_back(new test_opt_step_adamw(GGML_TYPE_F32, {10, 5, 4, 3})); test_cases.emplace_back(new test_opt_step_sgd(GGML_TYPE_F32, {10, 5, 4, 3})); diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index f5dcfba496c8..0d811fafe0ac 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -4134,7 +4134,15 @@ bool server_context::load_model(common_params & params) { if (ok && impl->model_tgt != nullptr) { server_context_impl * p = impl.get(); impl->trainer = std::make_unique(impl->model_tgt, impl->params_base, - [p]() { return p->n_busy_slots.load(); }); + [p]() { return p->n_busy_slots.load(); }, + // the SAME templates serving frames turns with + [p](const json & messages, bool add_generation_prompt) { + common_chat_templates_inputs in; + in.messages = common_chat_msgs_parse_oaicompat(messages); + in.add_generation_prompt = add_generation_prompt; + in.use_jinja = p->params_base.use_jinja; + return common_chat_templates_apply(p->chat_params.tmpls.get(), in).prompt; + }); } return ok; } diff --git a/tools/server/server-train.cpp b/tools/server/server-train.cpp index 4f9b3884abac..b3d59a46810c 100644 --- a/tools/server/server-train.cpp +++ b/tools/server/server-train.cpp @@ -5,6 +5,7 @@ #include "llama.h" #include "ggml-opt.h" +#include #include #include #include @@ -29,8 +30,9 @@ static std::vector split_targets(const std::string & list) { return out; } -server_trainer::server_trainer(llama_model * model, const common_params & params_base, std::function busy_slots) - : model(model), params_base(params_base), busy_slots(std::move(busy_slots)) { +server_trainer::server_trainer(llama_model * model, const common_params & params_base, std::function busy_slots, + render_fn render_chat) + : model(model), params_base(params_base), busy_slots(std::move(busy_slots)), render_chat(std::move(render_chat)) { state = json::object({{"state", "idle"}}); } @@ -63,14 +65,127 @@ static std::string confine_out(const std::string & dir, const std::string & name return d + name; } -json server_trainer::start(const json & body) { +// "examples" -> one tokenized sequence per example with its loss mask. Each example is rendered +// through the model's chat template as a closed conversation; each assistant turn's span is +// located by rendering the conversation up to it WITH the generation prompt (where the turn's +// content starts) and through it without (where it ends, end-of-turn included). A template that +// does not render prefix-stably (e.g. one that rewrites earlier turns) is refused for that example: +// the span cannot be located, and guessing would train the wrong tokens. +static bool prepare_examples(const json & examples, const server_trainer::render_fn & render, const llama_vocab * vocab, + int64_t window, server_trainer::examples_data & out, std::string & why) { + for (size_t i = 0; i < examples.size(); ++i) { + const std::string at = "examples[" + std::to_string(i) + "]"; + const json & ex = examples[i]; + json messages; + if (ex.is_object() && ex.contains("messages") && ex.at("messages").is_array() && !ex.at("messages").empty()) { + messages = ex.at("messages"); + } else if (ex.is_object() && ex.contains("prompt") && ex.at("prompt").is_string() + && ex.contains("completion") && ex.at("completion").is_string()) { + messages = json::array({ json::object({{"role", "user"}, {"content", ex.at("prompt")}}), + json::object({{"role", "assistant"}, {"content", ex.at("completion")}}) }); + } else { + why = at + " needs \"prompt\" and \"completion\" strings, or a non-empty \"messages\" array"; + return false; + } + auto head = [&](size_t n) { // the first n messages + json h = json::array(); + for (size_t j = 0; j < n; ++j) { + h.push_back(messages[j]); + } + return h; + }; + std::string full; + std::vector> spans; // [begin, end) in chars of `full` + try { + full = render(messages, false); + for (size_t k = 0; k < messages.size(); ++k) { + if (!messages[k].is_object() || messages[k].value("role", std::string()) != "assistant") { + continue; + } + const std::string pre = render(head(k), true); + const std::string upto = render(head(k + 1), false); + if (full.compare(0, upto.size(), upto) != 0 || upto.compare(0, pre.size(), pre) != 0 || pre.size() >= upto.size()) { + why = at + ": the chat template does not render this conversation prefix-stably at message " + std::to_string(k) + + ", so the assistant turn cannot be located"; + return false; + } + spans.emplace_back(pre.size(), upto.size()); + } + } catch (const std::exception & e) { + why = at + " could not be rendered through the chat template: " + e.what(); + return false; + } + if (spans.empty()) { + why = at + " has no assistant turn to learn"; + return false; + } + // tokenize segment by segment so every token is wholly inside or outside a span + std::vector toks; + std::vector loss; + size_t pos = 0; + auto add = [&](size_t end, bool trainable) { + if (end <= pos) { + return; + } + const auto seg = common_tokenize(vocab, full.substr(pos, end - pos), /*add_special =*/ pos == 0, /*parse_special =*/ true); + toks.insert(toks.end(), seg.begin(), seg.end()); + loss.insert(loss.end(), seg.size(), trainable ? 1 : 0); + pos = end; + }; + for (const auto & [b, e] : spans) { + add(b, false); + add(e, true); + } + add(full.size(), false); + if ((int64_t) toks.size() > window + 1) { + why = at + " is " + std::to_string(toks.size()) + " tokens; one example is one window, and this window holds " + + std::to_string(window + 1) + " (raise \"window\" or split the example)"; + return false; + } + if (std::count(loss.begin() + 1, loss.end(), (uint8_t) 1) == 0) { + why = at + ": its assistant turns tokenize to nothing to learn"; + return false; + } + out.tokens.push_back(std::move(toks)); + out.loss.push_back(std::move(loss)); + } + return true; +} + +json server_trainer::start(const json & body_in) { + json body = body_in; + examples_data prepared; // OFF unless the server was started with --train-dir: the route writes files, so an engine // opts in explicitly and every write is confined to that directory (Cormac on #14). if (params_base.train_dir.empty()) { return json::object({{"ok", false}, {"error", "training is disabled on this server (start it with --train-dir DIR to enable /train)"}}); } - if (!body.contains("text") || !body.at("text").is_string() || body.at("text").get().empty()) { - return json::object({{"ok", false}, {"error", "\"text\" (the training corpus) is required"}}); + if (body.contains("examples")) { + if (body.contains("text")) { + return json::object({{"ok", false}, {"error", "give \"text\" or \"examples\", not both"}}); + } + if (!body.at("examples").is_array() || body.at("examples").empty()) { + return json::object({{"ok", false}, {"error", "\"examples\" must be a non-empty array"}}); + } + if (body.contains("parse_special") && !(body.at("parse_special").is_boolean() && body.at("parse_special").get())) { + return json::object({{"ok", false}, {"error", "\"examples\" are rendered through the chat template, so their control tokens are always parsed (drop \"parse_special\")"}}); + } + if (!render_chat) { + return json::object({{"ok", false}, {"error", "this server has no chat template to render \"examples\" with; send \"text\""}}); + } + if (!body.value("window", json(256)).is_number()) { + return json::object({{"ok", false}, {"error", "\"window\" must be a number"}}); + } + std::string why; + if (!prepare_examples(body.at("examples"), render_chat, llama_model_get_vocab(model), + (int64_t) body.value("window", 256.0), prepared, why)) { + return json::object({{"ok", false}, {"error", why}}); + } + body["n_examples"] = body.at("examples").size(); + body.erase("examples"); + } + if (prepared.tokens.empty() && (!body.contains("text") || !body.at("text").is_string() || body.at("text").get().empty())) { + return json::object({{"ok", false}, {"error", "\"text\" or \"examples\" (the training corpus) is required"}}); } if (!body.contains("out") || !body.at("out").is_string() || body.at("out").get().empty()) { return json::object({{"ok", false}, {"error", "\"out\" (the adapter file name to write) is required"}}); @@ -137,6 +252,9 @@ json server_trainer::start(const json & body) { { std::lock_guard lock(mu); state = json::object({{"state", "starting"}, {"out", body.at("out")}, {"epochs", json::array()}}); + if (body.contains("n_examples")) { + state["examples"] = body.at("n_examples"); + } } g_train_batch.store(0); g_train_batch_max.store(0); @@ -144,7 +262,7 @@ json server_trainer::start(const json & body) { cancel_requested.store(false); yield_to_turns.store(body.value("yield", true)); g_trainer.store(this); - worker = std::thread(&server_trainer::run, this, body); + worker = std::thread([this, body, ex = std::move(prepared)]() mutable { run(std::move(body), std::move(ex)); }); return json::object({{"ok", true}, {"status", status()}}); } @@ -193,7 +311,7 @@ json server_trainer::status() const { return s; } -void server_trainer::run(json req) { +void server_trainer::run(json req, examples_data ex) { auto fail = [&](const std::string & why) { LOG_ERR("%s: training run failed: %s\n", __func__, why.c_str()); std::lock_guard lock(mu); @@ -202,7 +320,8 @@ void server_trainer::run(json req) { running.store(false); }; - const std::string text = req.at("text").get(); + const bool by_example = !ex.tokens.empty(); + const std::string text = by_example ? std::string() : req.at("text").get(); const std::string out = confine_out(params_base.train_dir, req.at("out").get()); const int32_t rank = (int32_t) req.value("rank", (int64_t) 8); const float alpha = (float) req.value("alpha", 16.0); @@ -258,14 +377,31 @@ void server_trainer::run(json req) { return; } - std::vector tokens = common_tokenize(ctx, text, true, special); - if ((int64_t) tokens.size() < 2 * (int64_t) window) { - llama_adapter_lora_free(adapter); - llama_free(ctx); - fail("the corpus tokenizes to " + std::to_string(tokens.size()) + " tokens; at least two windows are needed"); - return; + // text: one stream, overlapping windows, every token carries loss. examples: one window per + // example, only the assistant turns carry loss (the rest of each window is label -1). + int64_t n_tokens = 0; + std::vector trainable_prefix; // examples: trainable targets in examples [0, i) + ggml_opt_dataset_t dataset = nullptr; + if (by_example) { + trainable_prefix.push_back(0); + for (size_t i = 0; i < ex.tokens.size(); ++i) { + n_tokens += (int64_t) ex.tokens[i].size(); + trainable_prefix.push_back(trainable_prefix.back() + std::count(ex.loss[i].begin() + 1, ex.loss[i].end(), (uint8_t) 1)); + } + const llama_vocab * vocab = llama_model_get_vocab(model); + const llama_token eos = llama_vocab_eos(vocab); + dataset = common_opt_dataset_init_masked(window, ex.tokens, ex.loss, eos >= 0 ? eos : 0); + } else { + std::vector tokens = common_tokenize(ctx, text, true, special); + if ((int64_t) tokens.size() < 2 * (int64_t) window) { + llama_adapter_lora_free(adapter); + llama_free(ctx); + fail("the corpus tokenizes to " + std::to_string(tokens.size()) + " tokens; at least two windows are needed"); + return; + } + n_tokens = (int64_t) tokens.size(); + dataset = common_opt_dataset_init(ctx, tokens, window / 2); } - ggml_opt_dataset_t dataset = common_opt_dataset_init(ctx, tokens, window / 2); lr_opt lr; lr.lr0 = lr0; @@ -291,19 +427,41 @@ void server_trainer::run(json req) { } } - const int64_t idata_split = (int64_t) (ggml_opt_dataset_ndata(dataset) * (1.0f - val)); - const int64_t train_tokens = idata_split * (int64_t) window; + const int64_t ndata = ggml_opt_dataset_ndata(dataset); + const int64_t idata_split = (int64_t) (ndata * (1.0f - val)); + const int64_t train_tokens = idata_split * (int64_t) window; // positions the forward runs (padding included) + if (idata_split < 1) { + ggml_opt_dataset_free(dataset); + { + std::lock_guard lock(mu); + ctx_live = nullptr; + } + llama_adapter_lora_free(adapter); + llama_free(ctx); + fail("no training windows after the validation split (" + std::to_string(ndata) + " in all); send more data or lower \"val_split\""); + return; + } + // ggml-opt averages the loss over every position of a window, masked ones counting zero; the + // reported loss is per TRAINABLE token so it reads the same as an unmasked run's + const int64_t trainable_train = by_example ? trainable_prefix[idata_split] : train_tokens; + const int64_t trainable_eval = by_example ? trainable_prefix.back() - trainable_prefix[idata_split] + : (ndata - idata_split) * (int64_t) window; + const double scale_train = trainable_train > 0 ? (double) train_tokens / trainable_train : 0.0; + const double scale_eval = trainable_eval > 0 ? (double) ((ndata - idata_split) * (int64_t) window) / trainable_eval : 0.0; ggml_opt_result_t result_train = ggml_opt_result_init(); ggml_opt_result_t result_eval = ggml_opt_result_init(); { std::lock_guard lock(mu); state["state"] = "running"; state["window"] = window; - state["tokens"] = (int64_t) tokens.size(); + state["tokens"] = n_tokens; state["train_tokens"] = train_tokens; + if (by_example) { + state["trainable_tokens"] = trainable_train; + } } LOG_INF("%s: training on the served model: %zu tokens, window %u, %u epochs, adapter -> %s\n", - __func__, tokens.size(), window, epochs, out.c_str()); + __func__, (size_t) n_tokens, window, epochs, out.c_str()); for (lr.epoch = 0; lr.epoch < lr.epochs && !cancel_requested.load(); ++lr.epoch) { const int64_t t0 = ggml_time_us(); @@ -315,6 +473,8 @@ void server_trainer::run(json req) { double loss_train = 0.0, unc_train = 0.0, loss_eval = 0.0, unc_eval = 0.0; ggml_opt_result_loss(result_train, &loss_train, &unc_train); ggml_opt_result_loss(result_eval, &loss_eval, &unc_eval); + loss_train *= scale_train; + loss_eval *= scale_eval; const double tok_s = seconds > 0 ? train_tokens / seconds : 0.0; LOG_INF("%s: epoch %u: train_loss=%.5f eval_loss=%.5f seconds=%.1f train_tok_s=%.1f\n", __func__, lr.epoch, loss_train, loss_eval, seconds, tok_s); diff --git a/tools/server/server-train.h b/tools/server/server-train.h index 732d842fe841..c310c96d7edb 100644 --- a/tools/server/server-train.h +++ b/tools/server/server-train.h @@ -14,6 +14,15 @@ // with per-epoch train/eval loss, train tok/s, and the adapter path // POST /train/cancel -> the run stops at the next training window and writes no adapter // +// POST /train {"examples": [{"prompt": "...", "completion": "..."} | {"messages": [...]}, ...], ...} +// -> each example is rendered through THIS model's chat template as a closed +// conversation (no generation prompt), control tokens parsed as tokens, and is +// ONE training window (padded; an example longer than the window is refused). +// Only the assistant turns carry loss (each turn's content and its end-of-turn +// token): the adapter learns her answers, not the prompts, the room or the +// system text. Losses are reported per trainable token. "text" and "examples" +// are exclusive; "text" trains on every token. +// // "parse_special": true tokenizes control tokens in the text (<|im_start|> ...) as the tokens // they name, which is what a corpus rendered through the model's chat template needs; the // default (false) trains on the text exactly as written. @@ -36,24 +45,36 @@ class server_trainer { public: // busy_slots: how many serving slots are working right now (read between training batches; // the trainer yields while it is non-zero, so a turn never waits behind more than one batch) - server_trainer(llama_model * model, const common_params & params_base, std::function busy_slots); + // render_chat(messages, add_generation_prompt): messages (OpenAI shape) through the model's + // chat template; throws on a template error. Empty = "examples" are refused. + using render_fn = std::function; + server_trainer(llama_model * model, const common_params & params_base, std::function busy_slots, + render_fn render_chat); + + // "examples", tokenized: one sequence per example with its loss mask (1 = the token is + // predicted with loss: the assistant's content and end-of-turn) + struct examples_data { + std::vector> tokens; + std::vector> loss; + }; ~server_trainer(); // Starts a run on a worker thread; refuses (ok=false) while one is running or on bad input. - common_json start(const common_json & body); + common_json start(const common_json & body_in); common_json status() const; // Stops the running job at its next training window (no adapter is written); ok=false when // nothing is running. common_json cancel(); private: - void run(common_json req); + void run(common_json req, examples_data ex); static void on_batch(bool train, ggml_opt_context_t, ggml_opt_dataset_t, ggml_opt_result_t, int64_t ibatch, int64_t ibatch_max, int64_t); llama_model * model; common_params params_base; std::function busy_slots; + render_fn render_chat; std::atomic yield_to_turns{true}; std::atomic yielded_ms{0};