From 1fd1918bdc94444eae42d58895d7037213178f79 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Fri, 15 Mar 2024 00:46:34 -0400 Subject: [PATCH 01/25] llama : greatly reduce logits memory usage --- examples/parallel/parallel.cpp | 1 - examples/speculative/speculative.cpp | 1 - llama.cpp | 450 ++++++++++++++++++++++----- llama.h | 4 +- 4 files changed, 378 insertions(+), 78 deletions(-) diff --git a/examples/parallel/parallel.cpp b/examples/parallel/parallel.cpp index a2ef0fb039c3f..f66c91013eaeb 100644 --- a/examples/parallel/parallel.cpp +++ b/examples/parallel/parallel.cpp @@ -132,7 +132,6 @@ int main(int argc, char ** argv) { llama_context * ctx = NULL; // load the target model - params.logits_all = true; std::tie(model, ctx) = llama_init_from_gpt_params(params); // load the prompts from an external file if there are any diff --git a/examples/speculative/speculative.cpp b/examples/speculative/speculative.cpp index e991b88460713..2a3d07bfd62ff 100644 --- a/examples/speculative/speculative.cpp +++ b/examples/speculative/speculative.cpp @@ -65,7 +65,6 @@ int main(int argc, char ** argv) { llama_context * ctx_dft = NULL; // load the target model - params.logits_all = true; std::tie(model_tgt, ctx_tgt) = llama_init_from_gpt_params(params); // load the draft model diff --git a/llama.cpp b/llama.cpp index e4db288ddd907..3e528d96b1148 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1737,6 +1737,7 @@ struct llama_cparams { uint32_t n_ctx; // context size used during inference uint32_t n_batch; uint32_t n_ubatch; + uint32_t n_seq_max; uint32_t n_threads; // number of threads to use for generation uint32_t n_threads_batch; // number of threads to use for batch processing @@ -2054,6 +2055,8 @@ struct llama_context { ggml_backend_free(backend); } + free(output_ids); + #ifdef GGML_USE_VULKAN ggml_vk_free_cpu_assist(); #endif @@ -2094,19 +2097,19 @@ struct llama_context { // host buffer for the model output (logits and embeddings) ggml_backend_buffer_t buf_output = nullptr; - // decode output (2-dimensional array: [n_tokens][n_vocab]) - size_t logits_size = 0; - float * logits = nullptr; + // decode output (2-dimensional array: [n_outputs][n_vocab]) + size_t logits_size = 0; // capacity (of floats) for logits + float * logits = nullptr; + + int32_t * output_ids = nullptr; // map token positions to ids of the logits and embd buffers + size_t output_size = 0; // capacity (of tokens positions) for the output buffer + int32_t n_outputs = 0; // number of actually-used outputs in the previous batch -#ifndef NDEBUG - // guard against access to unset logits - std::vector logits_valid; -#endif bool logits_all = false; - // embeddings output (2-dimensional array: [n_tokens][n_embd]) + // embeddings output (2-dimensional array: [n_outputs][n_embd]) // populated only when pooling_type == LLAMA_POOLING_TYPE_NONE - size_t embd_size = 0; + size_t embd_size = 0; // capacity (of floats) for embeddings float * embd = nullptr; // sequence embeddings output (map of [n_embd] vectors) @@ -2124,14 +2127,15 @@ struct llama_context { struct ggml_tensor * inp_tokens; // I32 [n_batch] struct ggml_tensor * inp_embd; // F32 [n_embd, n_batch] struct ggml_tensor * inp_pos; // I32 [n_batch] + struct ggml_tensor * inp_out_ids; // I32 [n_outputs] struct ggml_tensor * inp_KQ_mask; // F32 [kv_size, n_batch] - struct ggml_tensor * inp_KQ_pos; // F32 [kv_size] + struct ggml_tensor * inp_KQ_pos; // F32 [n_kv] struct ggml_tensor * inp_K_shift; // I32 [kv_size] struct ggml_tensor * inp_mean; // F32 [n_batch, n_batch] struct ggml_tensor * inp_cls; // I32 [n_batch] struct ggml_tensor * inp_s_copy; // I32 [kv_size] - struct ggml_tensor * inp_s_mask; // F32 [1, kv_size] - struct ggml_tensor * inp_s_seq; // I32 [kv_size, n_batch] + struct ggml_tensor * inp_s_mask; // F32 [1, n_kv] + struct ggml_tensor * inp_s_seq; // I32 [n_kv, n_batch] // control vectors struct llama_control_vector cvec; @@ -5562,7 +5566,8 @@ struct llm_build_context { const float norm_rms_eps; const int32_t n_tokens; - const int32_t n_kv; // size of KV cache to consider (n_kv <= n_ctx) + const int32_t n_kv; // size of KV cache to consider (n_kv <= kv_self.size) + const int32_t n_outputs; const int32_t kv_head; // index of where we store new KV data in the cache const int32_t n_orig_ctx; @@ -5609,6 +5614,7 @@ struct llm_build_context { norm_rms_eps (hparams.f_norm_rms_eps), n_tokens (batch.n_tokens), n_kv (worst_case ? kv_self.size : kv_self.n), + n_outputs (worst_case ? n_tokens : lctx.n_outputs), kv_head (worst_case ? (kv_self.recurrent ? 0 : kv_self.size - n_tokens) : kv_self.head), n_orig_ctx (cparams.n_yarn_orig_ctx), pooling_type (cparams.pooling_type), @@ -5753,6 +5759,13 @@ struct llm_build_context { return lctx.inp_pos; } + struct ggml_tensor * build_inp_out_ids() { + lctx.inp_out_ids = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_outputs); + cb(lctx.inp_out_ids, "inp_out_ids", -1); + ggml_set_input(lctx.inp_out_ids); + return lctx.inp_out_ids; + } + struct ggml_tensor * build_inp_KQ_mask(bool causal = true) { if (causal) { lctx.inp_KQ_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, n_tokens); @@ -5809,6 +5822,8 @@ struct llm_build_context { struct ggml_cgraph * build_llama() { struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + int32_t n_tokens = this->n_tokens; + const int64_t n_embd_head = hparams.n_embd_head_v; GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); GGML_ASSERT(n_embd_head == hparams.n_rot); @@ -5876,6 +5891,15 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + n_tokens = n_outputs; + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); cb(ffn_inp, "ffn_inp", il); @@ -6055,6 +6079,14 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); cb(ffn_inp, "ffn_inp", il); @@ -6170,6 +6202,15 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + attn_norm = ggml_get_rows(ctx0, attn_norm, inp_out_ids); + } + struct ggml_tensor * ffn_inp = cur; // feed forward @@ -6264,6 +6305,14 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + // add the input struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); cb(ffn_inp, "ffn_inp", il); @@ -6461,6 +6510,14 @@ struct llm_build_context { Kcur, Vcur, Q, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + residual = ggml_get_rows(ctx0, residual, inp_out_ids); + } + struct ggml_tensor * ffn_inp = ggml_add(ctx0, residual, cur); cb(ffn_inp, "ffn_inp", il); @@ -6550,6 +6607,14 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); cb(ffn_inp, "ffn_inp", il); @@ -6707,6 +6772,14 @@ struct llm_build_context { } cb(cur, "kqv_out", il); + if (il == n_layer - 1 && pooling_type == LLAMA_POOLING_TYPE_NONE) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + // re-add the layer input cur = ggml_add(ctx0, cur, inpL); @@ -6829,6 +6902,14 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + // Add the input struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); cb(ffn_inp, "ffn_inp", il); @@ -6927,6 +7008,14 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + // Add the input struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); cb(ffn_inp, "ffn_inp", il); @@ -7040,6 +7129,14 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); cb(ffn_inp, "ffn_inp", il); @@ -7146,6 +7243,14 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); cb(ffn_inp, "ffn_inp", il); @@ -7258,6 +7363,14 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); cb(ffn_inp, "ffn_inp", il); @@ -7376,6 +7489,15 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + attn_norm_output = ggml_get_rows(ctx0, attn_norm_output, inp_out_ids); + } + // FF { ffn_output = llm_build_ffn(ctx0, attn_norm_output, @@ -7473,6 +7595,15 @@ struct llm_build_context { cur = attention_norm; + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + sa_out = ggml_get_rows(ctx0, sa_out, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + // feed-forward network { cur = llm_build_ffn(ctx0, cur, @@ -7565,6 +7696,14 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + // add the input struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); cb(ffn_inp, "ffn_inp", il); @@ -7665,6 +7804,14 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + // add the input struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); cb(ffn_inp, "ffn_inp", il); @@ -7774,6 +7921,14 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); cb(ffn_inp, "ffn_inp", il); @@ -7884,6 +8039,14 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); cb(ffn_inp, "ffn_inp", il); @@ -8007,6 +8170,14 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + // scale_res - scale the hidden states for residual connection const float scale_res = scale_depth/sqrtf(float(n_layer)); cur = ggml_scale(ctx0, cur, scale_res); @@ -8121,6 +8292,14 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + struct ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL); cb(sa_out, "sa_out", il); @@ -8234,6 +8413,14 @@ struct llm_build_context { cb(cur, "kqv_out", il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); cb(ffn_inp, "ffn_inp", il); @@ -8381,6 +8568,16 @@ struct llm_build_context { struct ggml_tensor * y = ggml_view_2d(ctx0, y_ssm_states, d_inner, n_tokens, d_inner*ggml_element_size(y_ssm_states), 0); + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + x = ggml_get_rows(ctx0, x, inp_out_ids); + y = ggml_get_rows(ctx0, y, inp_out_ids); + z = ggml_get_rows(ctx0, z, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + // {d_inner, n_tokens} * {d_inner} => {d_inner, n_tokens} y = ggml_add(ctx0, y, ggml_mul(ctx0, x, model.layers[il].ssm_d)); y = ggml_mul(ctx0, y, ggml_silu(ctx0, z)); @@ -8483,6 +8680,14 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); } + if (il == n_layer - 1) { + // skip computing output for unused tokens + if (n_outputs == 0) { return gf; } + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + struct ggml_tensor * attn_out = cur; // feed-forward network @@ -8773,9 +8978,38 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { ggml_backend_tensor_set(lctx.inp_pos, batch.pos, 0, n_tokens*ggml_element_size(lctx.inp_pos)); } + { + GGML_ASSERT(lctx.inp_out_ids && "every model type must skip unused outputs"); + const int64_t n_tokens = batch.n_tokens; + + GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_out_ids->buffer)); + int32_t * data = (int32_t *) lctx.inp_out_ids->data; + + if (batch.logits) { + int32_t n_outputs = 0; + for (int i = 0; i < n_tokens; ++i) { + if (batch.logits[i]) { + data[n_outputs++] = i; + } + } + lctx.n_outputs = n_outputs; + } else if (lctx.logits_all || (cparams.embeddings && hparams.pooling_type != LLAMA_POOLING_TYPE_NONE)) { + for (int i = 0; i < n_tokens; ++i) { + data[i] = i; + } + lctx.n_outputs = n_tokens; + } else { + // only keep last output + data[0] = n_tokens - 1; + lctx.n_outputs = 1; + } + } + GGML_ASSERT( + // (!a || b) is a logical implication (a -> b) + // !hparams.causal_attn -> !cparams.causal_attn (hparams.causal_attn || !cparams.causal_attn) && - "non-causal attention with generative models is not supported" + "causal attention with embedding models is not supported" ); if (lctx.inp_KQ_mask) { @@ -8954,6 +9188,62 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { } } +// Only alloc when needed +static void llama_output_reserve(llama_context & lctx, int32_t n_outputs) { + GGML_ASSERT(0 <= n_outputs); + + const int32_t n_outputs_max = std::max((uint32_t) n_outputs, lctx.cparams.n_seq_max); + + const auto n_batch = lctx.cparams.n_batch; + const auto n_vocab = lctx.model.hparams.n_vocab; + const auto n_embd = lctx.model.hparams.n_embd; + const int64_t capacity = lctx.output_size; + + const bool has_logits = lctx.cparams.causal_attn; + const bool has_embd = lctx.cparams.embeddings; + + if (!lctx.output_ids) { + // never resized afterwards + lctx.output_ids = (int32_t *) malloc(n_batch*sizeof(int32_t)); + if (lctx.output_ids == nullptr) { + throw std::runtime_error("failed to allocate output_ids buffer"); + } + } + // alloc only when more than the current logits capacity is required + if (capacity < n_outputs_max) { + if (lctx.buf_output) { + ggml_backend_buffer_free(lctx.buf_output); + lctx.buf_output = nullptr; + lctx.logits = nullptr; + lctx.embd = nullptr; + } + { + lctx.output_size = n_outputs_max; + lctx.logits_size = has_logits ? n_vocab*n_outputs_max : 0; + lctx.embd_size = has_embd ? n_embd*n_outputs_max : 0; + + const size_t buf_output_size = (lctx.logits_size + lctx.embd_size)*sizeof(float); + + lctx.buf_output = ggml_backend_buft_alloc_buffer(llama_default_buffer_type_cpu(true), buf_output_size); + if (lctx.buf_output == nullptr) { + throw std::runtime_error(format("failed to allocate output buffer of size %.2f MiB", buf_output_size / (1024.0 * 1024.0))); + } + + float * output_base = (float *) ggml_backend_buffer_get_base(lctx.buf_output); + + lctx.logits = has_logits ? output_base : nullptr; + lctx.embd = has_embd ? output_base + lctx.logits_size : nullptr; + } + } + // set all ids as invalid (assume two's complement negative numbers) + memset(lctx.output_ids, -1, n_batch*sizeof(int32_t)); + + ggml_backend_buffer_clear(lctx.buf_output, 0); + + lctx.n_outputs = n_outputs; // also set in llama_set_inputs() before a batch +} + + static void llama_graph_compute( llama_context & lctx, ggml_cgraph * gf, @@ -9029,16 +9319,8 @@ static int llama_decode_internal( const int64_t n_embd = hparams.n_embd; const int64_t n_vocab = hparams.n_vocab; - - auto * logits_out = lctx.logits; - -#ifndef NDEBUG - auto & logits_valid = lctx.logits_valid; - logits_valid.clear(); - logits_valid.resize(n_tokens_all); - - memset(logits_out, 0, lctx.logits_size*sizeof(float)); -#endif + int32_t n_logits = 0; + int32_t n_logits_prev = 0; const auto n_ubatch = cparams.n_ubatch; @@ -9047,6 +9329,33 @@ static int llama_decode_internal( std::vector seq_id_arr; std::vector> seq_id; + // reserve output buffer + if (batch_all.logits) { + for (uint32_t i = 0; i < n_tokens_all; ++i) { + if (batch_all.logits[i]) { + n_logits++; + } + } + llama_output_reserve(lctx, n_logits); + int32_t i_logits = 0; + for (uint32_t i = 0; i < n_tokens_all; ++i) { + if (batch_all.logits[i]) { + lctx.output_ids[i] = i_logits++; + } + } + } else if (lctx.logits_all || (cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE)) { + n_logits = n_tokens_all; + llama_output_reserve(lctx, n_logits); + for (uint32_t i = 0; i < n_tokens_all; ++i) { + lctx.output_ids[i] = i; + } + } else { + // keep last logits only + n_logits = 1; + llama_output_reserve(lctx, n_logits); + lctx.output_ids[0] = 0; + } + for (uint32_t cur_token = 0; cur_token < n_tokens_all; cur_token += n_ubatch) { const uint32_t n_tokens = std::min(n_ubatch, n_tokens_all - cur_token); llama_batch u_batch = { @@ -9125,20 +9434,26 @@ static int llama_decode_internal( struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1]; struct ggml_tensor * embd = gf->nodes[gf->n_nodes - 2]; - if (!hparams.causal_attn) { + if (lctx.n_outputs == 0) { + // no output + res = nullptr; + embd = nullptr; + } else if (!hparams.causal_attn) { res = nullptr; // do not extract logits for embedding models such as BERT // token or sequence embeddings embd = gf->nodes[gf->n_nodes - 1]; GGML_ASSERT(strcmp(embd->name, "result_embd") == 0 || strcmp(embd->name, "result_embd_pooled") == 0); + // TODO: graph view to ignore the logits when not needed } else { if (strcmp(res->name, "result_output") == 0) { - // the token embeddings could be the second to last tensor, or the third to last tensor - if (strcmp(embd->name, "result_norm") != 0) { - embd = gf->nodes[gf->n_nodes - 3]; - GGML_ASSERT(strcmp(embd->name, "result_norm") == 0); + // the token embeddings could be the second to last tensor, or any of the previous tensors + // NOTE: see build_result_output() for an idea of up to how many tensors to skip + for (int i = 3; strcmp(embd->name, "result_norm") != 0 && i <= 10; ++i) { + embd = gf->nodes[gf->n_nodes - i]; } + GGML_ASSERT(strcmp(embd->name, "result_norm") == 0); } else { GGML_ASSERT(false && "missing result_output tensor"); } @@ -9189,41 +9504,29 @@ static int llama_decode_internal( if (res) { ggml_backend_t backend_res = ggml_backend_sched_get_tensor_backend(lctx.sched, res); GGML_ASSERT(backend_res != nullptr); + int32_t new_logits = 0; + if (u_batch.logits) { - int32_t i_first = -1; for (uint32_t i = 0; i < n_tokens; i++) { - if (u_batch.logits[i] && i_first == -1) { - i_first = (int32_t) i; + if (u_batch.logits[i]) { + new_logits++; } - if (u_batch.logits[i] == 0 || i == n_tokens - 1) { - if (i_first != -1) { - int i_last = u_batch.logits[i] == 0 ? i : i + 1; - // extract logits for the range [i_first, i_last) - // group the requests to minimize the number of calls to the backend - ggml_backend_tensor_get_async(backend_res, res, - logits_out + n_vocab*(cur_token + i_first), - i_first*n_vocab*sizeof(float), - (i_last - i_first)*n_vocab*sizeof(float)); - i_first = -1; - } - } -#ifndef NDEBUG - logits_valid[cur_token + i] = u_batch.logits[i] != 0;; -#endif } } else if (lctx.logits_all) { - ggml_backend_tensor_get_async(backend_res, res, logits_out + n_vocab*cur_token, 0, n_vocab*n_tokens*sizeof(float)); -#ifndef NDEBUG - std::fill(logits_valid.begin() + cur_token, logits_valid.begin() + cur_token + n_tokens, true); -#endif + new_logits += n_tokens; } else { + // keep last logits only if (cur_token + n_tokens >= n_tokens_all) { - ggml_backend_tensor_get_async(backend_res, res, logits_out, n_vocab*(n_tokens - 1)*sizeof(float), n_vocab*sizeof(float)); -#ifndef NDEBUG - logits_valid[0] = true; -#endif + new_logits += 1; } } + + if (new_logits) { + GGML_ASSERT(new_logits <= n_logits); + GGML_ASSERT((n_logits_prev+new_logits)*n_vocab <= (int64_t) lctx.logits_size); + ggml_backend_tensor_get_async(backend_res, res, lctx.logits, n_logits_prev*n_vocab*sizeof(float), new_logits*n_vocab*sizeof(float)); + n_logits_prev += new_logits; + } } // extract embeddings @@ -9243,6 +9546,7 @@ static int llama_decode_internal( if (u_batch.logits[i] == 0) { continue; } + // FIXME ggml_backend_tensor_get_async(backend_embd, embd, embd_out + n_embd*(i + cur_token), (n_embd*i)*sizeof(float), n_embd*sizeof(float)); } } @@ -13011,7 +13315,7 @@ struct llama_context * llama_new_context_with_model( const auto & hparams = model->hparams; auto & cparams = ctx->cparams; - // TODO: maybe add n_seq_max here too + cparams.n_seq_max = std::max(1u, params.n_seq_max); cparams.n_threads = params.n_threads; cparams.n_threads_batch = params.n_threads_batch; cparams.yarn_ext_factor = params.yarn_ext_factor; @@ -13214,25 +13518,14 @@ struct llama_context * llama_new_context_with_model( // graph outputs buffer { - // resized during inference, reserve maximum - ctx->logits_size = hparams.n_vocab*cparams.n_batch; - ctx->embd_size = params.embeddings ? hparams.n_embd*cparams.n_batch : 0; - - const size_t buf_output_size = (ctx->logits_size + ctx->embd_size)*sizeof(float); - - ctx->buf_output = ggml_backend_buft_alloc_buffer(llama_default_buffer_type_cpu(true), buf_output_size); - if (ctx->buf_output == nullptr) { - LLAMA_LOG_ERROR("%s: failed to allocate logits buffer\n", __func__); + // resized during inference when more than n_seq_max logits are requested in a batch + try { + llama_output_reserve(*ctx, 0); + } catch (const std::exception & err) { + LLAMA_LOG_ERROR("%s: error reserving logits buffer: %s\n", __func__, err.what()); llama_free(ctx); return nullptr; } - ggml_backend_buffer_clear(ctx->buf_output, 0); - - - ctx->logits = (float *) ggml_backend_buffer_get_base(ctx->buf_output); - if (params.embeddings) { - ctx->embd = ctx->logits + ctx->logits_size; - } LLAMA_LOG_INFO("%s: %10s output buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(ctx->buf_output), @@ -14269,17 +14562,20 @@ void llama_synchronize(struct llama_context * ctx) { } float * llama_get_logits(struct llama_context * ctx) { + // TODO: assert that really all logits are in the output llama_synchronize(ctx); return ctx->logits; } float * llama_get_logits_ith(struct llama_context * ctx, int32_t i) { - assert(ctx->logits_valid.at(i)); + const int32_t j = ctx->output_ids[i]; + GGML_ASSERT(0 <= j); llama_synchronize(ctx); - return ctx->logits + i*ctx->model.hparams.n_vocab; + // FIXME: check for nullptr + return ctx->logits + j*ctx->model.hparams.n_vocab; } float * llama_get_embeddings(struct llama_context * ctx) { @@ -14289,9 +14585,13 @@ float * llama_get_embeddings(struct llama_context * ctx) { } float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i) { + const int32_t j = ctx->output_ids[i]; + GGML_ASSERT(0 <= j); + llama_synchronize(ctx); - return ctx->embd + i*ctx->model.hparams.n_embd; + // FIXME: check for nullptr + return ctx->embd + j*ctx->model.hparams.n_embd; } float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id) { diff --git a/llama.h b/llama.h index 40dcf54e394f8..5eaf07e8a61dc 100644 --- a/llama.h +++ b/llama.h @@ -674,6 +674,7 @@ extern "C" { LLAMA_API void llama_synchronize(struct llama_context * ctx); // Token logits obtained from the last call to llama_decode() + // WARNING: the following layout is only valid when the batch outputs logits for all tokens // The logits for the last token are stored in the last row // Logits for which llama_batch.logits[i] == 0 are undefined // Rows: n_tokens provided with llama_batch @@ -681,10 +682,11 @@ extern "C" { LLAMA_API float * llama_get_logits(struct llama_context * ctx); // Logits for the ith token. Equivalent to: - // llama_get_logits(ctx) + i*n_vocab + // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i); // Get all output token embeddings + // WARNING: only use when all outputs are requested // shape: [n_tokens*n_embd] (1-dimensional) LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); From 98914c0ed02f1503762712bbe58bfacfcbf48b60 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Fri, 15 Mar 2024 12:21:24 -0400 Subject: [PATCH 02/25] llama : more compact state saving and reloading --- llama.cpp | 171 +++++++++++++++++++++++++++++++++++++++--------------- llama.h | 24 ++++---- 2 files changed, 138 insertions(+), 57 deletions(-) diff --git a/llama.cpp b/llama.cpp index 3e528d96b1148..47071700bfefb 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2102,8 +2102,8 @@ struct llama_context { float * logits = nullptr; int32_t * output_ids = nullptr; // map token positions to ids of the logits and embd buffers - size_t output_size = 0; // capacity (of tokens positions) for the output buffer - int32_t n_outputs = 0; // number of actually-used outputs in the previous batch + size_t output_size = 0; // capacity (of tokens positions) for the output buffers + int32_t n_outputs = 0; // number of actually-used outputs in the current or previous batch bool logits_all = false; @@ -9192,15 +9192,18 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { static void llama_output_reserve(llama_context & lctx, int32_t n_outputs) { GGML_ASSERT(0 <= n_outputs); - const int32_t n_outputs_max = std::max((uint32_t) n_outputs, lctx.cparams.n_seq_max); + const auto & cparams = lctx.cparams; + const auto & hparams = lctx.model.hparams; + + const int32_t n_outputs_max = std::max((uint32_t) n_outputs, cparams.n_seq_max); - const auto n_batch = lctx.cparams.n_batch; - const auto n_vocab = lctx.model.hparams.n_vocab; - const auto n_embd = lctx.model.hparams.n_embd; + const auto n_batch = cparams.n_batch; + const auto n_vocab = hparams.n_vocab; + const auto n_embd = hparams.n_embd; const int64_t capacity = lctx.output_size; - const bool has_logits = lctx.cparams.causal_attn; - const bool has_embd = lctx.cparams.embeddings; + const bool has_logits = cparams.causal_attn; + const bool has_embd = cparams.embeddings && (!hparams.causal_attn || cparams.pooling_type == LLAMA_POOLING_TYPE_NONE); if (!lctx.output_ids) { // never resized afterwards @@ -9211,29 +9214,32 @@ static void llama_output_reserve(llama_context & lctx, int32_t n_outputs) { } // alloc only when more than the current logits capacity is required if (capacity < n_outputs_max) { + lctx.output_size = n_outputs_max; + lctx.logits_size = has_logits ? n_vocab*n_outputs_max : 0; + lctx.embd_size = has_embd ? n_embd*n_outputs_max : 0; + + const size_t buf_output_size = (lctx.logits_size + lctx.embd_size)*sizeof(float); + if (lctx.buf_output) { +#ifndef NDEBUG + const size_t prev_size = ggml_backend_buffer_get_size(lctx.buf_output); + fprintf(stderr, "%s: reallocating output buffer from size %.02f MiB to %.02f MiB\n", __func__, prev_size / 1024.0 / 1024.0, buf_output_size/ 1024.0 / 1024.0); +#endif ggml_backend_buffer_free(lctx.buf_output); lctx.buf_output = nullptr; lctx.logits = nullptr; lctx.embd = nullptr; } - { - lctx.output_size = n_outputs_max; - lctx.logits_size = has_logits ? n_vocab*n_outputs_max : 0; - lctx.embd_size = has_embd ? n_embd*n_outputs_max : 0; - - const size_t buf_output_size = (lctx.logits_size + lctx.embd_size)*sizeof(float); - lctx.buf_output = ggml_backend_buft_alloc_buffer(llama_default_buffer_type_cpu(true), buf_output_size); - if (lctx.buf_output == nullptr) { - throw std::runtime_error(format("failed to allocate output buffer of size %.2f MiB", buf_output_size / (1024.0 * 1024.0))); - } + lctx.buf_output = ggml_backend_buft_alloc_buffer(llama_default_buffer_type_cpu(true), buf_output_size); + if (lctx.buf_output == nullptr) { + throw std::runtime_error(format("failed to allocate output buffer of size %.2f MiB", buf_output_size / (1024.0 * 1024.0))); + } - float * output_base = (float *) ggml_backend_buffer_get_base(lctx.buf_output); + float * output_base = (float *) ggml_backend_buffer_get_base(lctx.buf_output); - lctx.logits = has_logits ? output_base : nullptr; - lctx.embd = has_embd ? output_base + lctx.logits_size : nullptr; - } + lctx.logits = has_logits ? output_base : nullptr; + lctx.embd = has_embd ? output_base + lctx.logits_size : nullptr; } // set all ids as invalid (assume two's complement negative numbers) memset(lctx.output_ids, -1, n_batch*sizeof(int32_t)); @@ -14038,27 +14044,32 @@ void llama_kv_cache_update(struct llama_context * ctx) { // Returns the *maximum* size of the state size_t llama_get_state_size(const struct llama_context * ctx) { + const auto & cparams = ctx->cparams; + const auto & hparams = ctx->model.hparams; // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state. // for reference, std::mt19937(1337) serializes to 6701 bytes. const size_t s_rng_size = sizeof(size_t); const size_t s_rng = LLAMA_MAX_RNG_STATE; + const size_t s_n_outputs = sizeof(size_t); + // assume worst case for outputs although only currently set ones are serialized + const size_t s_output_pos = ctx->cparams.n_batch * sizeof(int32_t); const size_t s_logits_size = sizeof(size_t); - // assume worst case for logits although only currently set ones are serialized - const size_t s_logits = ctx->logits_size * sizeof(float); + const size_t s_logits = ctx->logits_size ? cparams.n_batch * hparams.n_vocab * sizeof(float) : 0; const size_t s_embedding_size = sizeof(size_t); - const size_t s_embedding = ctx->embd_size * sizeof(float); + const size_t s_embedding = ctx->embd_size ? cparams.n_batch * hparams.n_embd * sizeof(float) : 0; const size_t s_kv_buf_size = sizeof(size_t); const size_t s_kv_head = sizeof(uint32_t); const size_t s_kv_size = sizeof(uint32_t); const size_t s_kv_used = sizeof(uint32_t); const size_t s_kv = ctx->kv_self.total_size(); - // TODO: assume the max is more than 1 seq_id per KV cell - const size_t s_kv_cell = sizeof(llama_pos) + sizeof(size_t) + sizeof(llama_seq_id); + const size_t s_kv_cell = sizeof(llama_pos) + sizeof(size_t) + cparams.n_seq_max*sizeof(llama_seq_id); const size_t s_kv_cells = ctx->kv_self.size * s_kv_cell; const size_t s_total = ( + s_rng_size + s_rng + + s_n_outputs + + s_output_pos + s_logits_size + s_logits + s_embedding_size @@ -14142,25 +14153,60 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat data_ctx->write(rng_str.data(), rng_size); } - // copy logits + // copy outputs { - const size_t logits_size = ctx->logits_size; + size_t n_outputs = ctx->n_outputs; - data_ctx->write(&logits_size, sizeof(logits_size)); + // copy output ids + { + std::vector output_pos; + const size_t n_batch = ctx->cparams.n_batch; + const int32_t * output_ids = ctx->output_ids; + + output_pos.resize(n_outputs); + + // build a more compact representation of the output ids + for (size_t i = 0; i < n_batch; ++i) { + // map an output id to a position in the batch + int32_t pos = output_ids[i]; + if (pos >= 0) { + if ((size_t) pos >= output_pos.size()) { + // TODO: maybe fail here instead + LLAMA_LOG_WARN("%s: weird output buffer layout, possibly a bug\n", __func__); + n_outputs = pos + 1; + output_pos.resize(n_outputs); + } + output_pos[pos] = i; + } + } - if (logits_size) { - data_ctx->write(ctx->logits, logits_size * sizeof(float)); + data_ctx->write(&n_outputs, sizeof(n_outputs)); + + if (n_outputs) { + data_ctx->write(output_pos.data(), n_outputs * sizeof(int32_t)); + } } - } - // copy embeddings - { - const size_t embeddings_size = ctx->embd_size; + // copy logits + { + const size_t logits_size = std::min(ctx->logits_size, n_outputs * ctx->model.hparams.n_vocab); + + data_ctx->write(&logits_size, sizeof(logits_size)); - data_ctx->write(&embeddings_size, sizeof(embeddings_size)); + if (logits_size) { + data_ctx->write(ctx->logits, logits_size * sizeof(float)); + } + } - if (embeddings_size) { - data_ctx->write(ctx->embd, embeddings_size * sizeof(float)); + // copy embeddings + { + const size_t embeddings_size = std::min(ctx->embd_size, n_outputs * ctx->model.hparams.n_embd); + + data_ctx->write(&embeddings_size, sizeof(embeddings_size)); + + if (embeddings_size) { + data_ctx->write(ctx->embd, embeddings_size * sizeof(float)); + } } } @@ -14257,6 +14303,28 @@ size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { GGML_ASSERT(!rng_ss.fail()); } + // set output ids + { + size_t n_outputs; + std::vector output_pos; + + memcpy(&n_outputs, inp, sizeof(n_outputs)); inp += sizeof(n_outputs); + + llama_output_reserve(*ctx, n_outputs); + + if (n_outputs) { + output_pos.resize(n_outputs); + memcpy(output_pos.data(), inp, n_outputs * sizeof(int32_t)); + inp += n_outputs * sizeof(int32_t); + + for (int32_t i = 0; i < (int32_t) output_pos.size(); ++i) { + int32_t id = output_pos[i]; + GGML_ASSERT((uint32_t) id < ctx->cparams.n_batch); + ctx->output_ids[id] = i; + } + } + } + // set logits { size_t logits_size; @@ -14277,7 +14345,7 @@ size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { memcpy(&embeddings_size, inp, sizeof(embeddings_size)); inp += sizeof(embeddings_size); - GGML_ASSERT(ctx->embd_size == embeddings_size); + GGML_ASSERT(ctx->embd_size >= embeddings_size); if (embeddings_size) { memcpy(ctx->embd, inp, embeddings_size * sizeof(float)); @@ -14562,7 +14630,6 @@ void llama_synchronize(struct llama_context * ctx) { } float * llama_get_logits(struct llama_context * ctx) { - // TODO: assert that really all logits are in the output llama_synchronize(ctx); return ctx->logits; @@ -14570,12 +14637,17 @@ float * llama_get_logits(struct llama_context * ctx) { float * llama_get_logits_ith(struct llama_context * ctx, int32_t i) { const int32_t j = ctx->output_ids[i]; - GGML_ASSERT(0 <= j); llama_synchronize(ctx); - // FIXME: check for nullptr - return ctx->logits + j*ctx->model.hparams.n_vocab; + if (ctx->logits && 0 <= j && j < ctx->n_outputs) { + return ctx->logits + j*ctx->model.hparams.n_vocab; + } + LLAMA_LOG_ERROR("%s: invalid logits id %i\n", __func__, i); +#ifndef NDEBUG + GGML_ASSERT(false); +#endif + return nullptr; } float * llama_get_embeddings(struct llama_context * ctx) { @@ -14586,12 +14658,17 @@ float * llama_get_embeddings(struct llama_context * ctx) { float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i) { const int32_t j = ctx->output_ids[i]; - GGML_ASSERT(0 <= j); llama_synchronize(ctx); - // FIXME: check for nullptr - return ctx->embd + j*ctx->model.hparams.n_embd; + if (ctx->embd && 0 < j && j < ctx->n_outputs) { + return ctx->embd + j*ctx->model.hparams.n_embd; + } + LLAMA_LOG_ERROR("%s: invalid embeddings id %i\n", __func__, i); +#ifndef NDEBUG + GGML_ASSERT(false); +#endif + return nullptr; } float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id) { diff --git a/llama.h b/llama.h index 5eaf07e8a61dc..2e7de69b7049a 100644 --- a/llama.h +++ b/llama.h @@ -39,7 +39,7 @@ #define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn' #define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN -#define LLAMA_SESSION_VERSION 4 +#define LLAMA_SESSION_VERSION 5 #ifdef __cplusplus extern "C" { @@ -674,25 +674,29 @@ extern "C" { LLAMA_API void llama_synchronize(struct llama_context * ctx); // Token logits obtained from the last call to llama_decode() - // WARNING: the following layout is only valid when the batch outputs logits for all tokens - // The logits for the last token are stored in the last row - // Logits for which llama_batch.logits[i] == 0 are undefined - // Rows: n_tokens provided with llama_batch + // The logits for which llama_batch.logits[i] != 0 are stored contiguously + // in the order they have in the batch. + // Rows: number of tokens for which llama_batch.logits[i] != 0 // Cols: n_vocab LLAMA_API float * llama_get_logits(struct llama_context * ctx); // Logits for the ith token. Equivalent to: // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab + // returns NULL for invalid ids. LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i); - // Get all output token embeddings - // WARNING: only use when all outputs are requested - // shape: [n_tokens*n_embd] (1-dimensional) + // Get all output token embeddings. + // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model, + // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously + // in the order they have in the batch. + // shape: [n_outputs*n_embd] + // Otherwise, returns NULL. LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); - // Get the embeddings for the ith token - // llama_get_embeddings(ctx) + i*n_embd + // Get the embeddings for the ith token. Equivalent to: + // llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd // shape: [n_embd] (1-dimensional) + // returns NULL for invalid ids. LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i); // Get the embeddings for a sequence id From 705d3937eaa1f1f370fda188564405e358905d8c Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Sat, 16 Mar 2024 17:24:05 -0400 Subject: [PATCH 03/25] llama : fix lctx.n_outputs not being set before building graph --- llama.cpp | 149 +++++++++++++++++++++++++++++------------------------- 1 file changed, 80 insertions(+), 69 deletions(-) diff --git a/llama.cpp b/llama.cpp index 47071700bfefb..f397ffed4235a 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2103,7 +2103,7 @@ struct llama_context { int32_t * output_ids = nullptr; // map token positions to ids of the logits and embd buffers size_t output_size = 0; // capacity (of tokens positions) for the output buffers - int32_t n_outputs = 0; // number of actually-used outputs in the current or previous batch + int32_t n_outputs = 0; // number of actually-used outputs in the current or previous ubatch bool logits_all = false; @@ -8985,24 +8985,25 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_out_ids->buffer)); int32_t * data = (int32_t *) lctx.inp_out_ids->data; + int32_t n_outputs = 0; if (batch.logits) { - int32_t n_outputs = 0; for (int i = 0; i < n_tokens; ++i) { if (batch.logits[i]) { data[n_outputs++] = i; } } - lctx.n_outputs = n_outputs; } else if (lctx.logits_all || (cparams.embeddings && hparams.pooling_type != LLAMA_POOLING_TYPE_NONE)) { for (int i = 0; i < n_tokens; ++i) { data[i] = i; } - lctx.n_outputs = n_tokens; + n_outputs = n_tokens; } else { // only keep last output data[0] = n_tokens - 1; - lctx.n_outputs = 1; + n_outputs = 1; } + // the graph needs the have been passed the correct number of outputs + GGML_ASSERT(lctx.n_outputs == n_outputs); } GGML_ASSERT( @@ -9202,6 +9203,7 @@ static void llama_output_reserve(llama_context & lctx, int32_t n_outputs) { const auto n_embd = hparams.n_embd; const int64_t capacity = lctx.output_size; + // TODO: use a per-batch flag for logits presence instead const bool has_logits = cparams.causal_attn; const bool has_embd = cparams.embeddings && (!hparams.causal_attn || cparams.pooling_type == LLAMA_POOLING_TYPE_NONE); @@ -9221,10 +9223,11 @@ static void llama_output_reserve(llama_context & lctx, int32_t n_outputs) { const size_t buf_output_size = (lctx.logits_size + lctx.embd_size)*sizeof(float); if (lctx.buf_output) { -#ifndef NDEBUG + // This doesn't happen often +// #ifndef NDEBUG const size_t prev_size = ggml_backend_buffer_get_size(lctx.buf_output); - fprintf(stderr, "%s: reallocating output buffer from size %.02f MiB to %.02f MiB\n", __func__, prev_size / 1024.0 / 1024.0, buf_output_size/ 1024.0 / 1024.0); -#endif + LLAMA_LOG_INFO("%s: reallocating output buffer from size %.02f MiB to %.02f MiB\n", __func__, prev_size / 1024.0 / 1024.0, buf_output_size / 1024.0 / 1024.0); +// #endif ggml_backend_buffer_free(lctx.buf_output); lctx.buf_output = nullptr; lctx.logits = nullptr; @@ -9246,7 +9249,7 @@ static void llama_output_reserve(llama_context & lctx, int32_t n_outputs) { ggml_backend_buffer_clear(lctx.buf_output, 0); - lctx.n_outputs = n_outputs; // also set in llama_set_inputs() before a batch + lctx.n_outputs = 0; } @@ -9325,8 +9328,8 @@ static int llama_decode_internal( const int64_t n_embd = hparams.n_embd; const int64_t n_vocab = hparams.n_vocab; - int32_t n_logits = 0; - int32_t n_logits_prev = 0; + int32_t n_outputs = 0; + int32_t n_outputs_prev = 0; const auto n_ubatch = cparams.n_ubatch; @@ -9338,11 +9341,9 @@ static int llama_decode_internal( // reserve output buffer if (batch_all.logits) { for (uint32_t i = 0; i < n_tokens_all; ++i) { - if (batch_all.logits[i]) { - n_logits++; - } + n_outputs += batch_all.logits[i] != 0; } - llama_output_reserve(lctx, n_logits); + llama_output_reserve(lctx, n_outputs); int32_t i_logits = 0; for (uint32_t i = 0; i < n_tokens_all; ++i) { if (batch_all.logits[i]) { @@ -9350,15 +9351,15 @@ static int llama_decode_internal( } } } else if (lctx.logits_all || (cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE)) { - n_logits = n_tokens_all; - llama_output_reserve(lctx, n_logits); + n_outputs = n_tokens_all; + llama_output_reserve(lctx, n_outputs); for (uint32_t i = 0; i < n_tokens_all; ++i) { lctx.output_ids[i] = i; } } else { - // keep last logits only - n_logits = 1; - llama_output_reserve(lctx, n_logits); + // keep last output only + n_outputs = 1; + llama_output_reserve(lctx, n_outputs); lctx.output_ids[0] = 0; } @@ -9377,6 +9378,27 @@ static int llama_decode_internal( /* .all_seq_id = */ batch_all.all_seq_id, }; + // count the outputs in this u_batch + { + int32_t n_outputs_new = 0; + + if (u_batch.logits) { + for (uint32_t i = 0; i < n_tokens; i++) { + n_outputs_new += u_batch.logits[i] != 0; + } + } else if (lctx.logits_all) { + n_outputs_new = n_tokens; + } else { + // keep last output only + if (cur_token + n_tokens >= n_tokens_all) { + n_outputs_new = 1; + } + } + + // needs to happen before the graph is built + lctx.n_outputs = n_outputs_new; + } + int n_threads = n_tokens == 1 ? cparams.n_threads : cparams.n_threads_batch; GGML_ASSERT(n_threads > 0); @@ -9451,18 +9473,26 @@ static int llama_decode_internal( embd = gf->nodes[gf->n_nodes - 1]; GGML_ASSERT(strcmp(embd->name, "result_embd") == 0 || strcmp(embd->name, "result_embd_pooled") == 0); - // TODO: graph view to ignore the logits when not needed - } else { - if (strcmp(res->name, "result_output") == 0) { - // the token embeddings could be the second to last tensor, or any of the previous tensors - // NOTE: see build_result_output() for an idea of up to how many tensors to skip - for (int i = 3; strcmp(embd->name, "result_norm") != 0 && i <= 10; ++i) { - embd = gf->nodes[gf->n_nodes - i]; - } - GGML_ASSERT(strcmp(embd->name, "result_norm") == 0); - } else { - GGML_ASSERT(false && "missing result_output tensor"); + } else if (cparams.embeddings) { + // the embeddings could be in the second to last tensor, or any of the previous tensors + int i_embd = gf->n_nodes - 2; + for (int i = 3; strcmp(embd->name, "result_norm") != 0; ++i) { + i_embd = gf->n_nodes - i; + if (i_embd < 0) { break; } + embd = gf->nodes[i_embd]; + } + GGML_ASSERT(i_embd >= 0 && "missing result_norm tensor"); + + // TODO: use a per-batch flag to know when to skip logits while keeping embeddings + if (!cparams.causal_attn) { + res = nullptr; // do not extract logits when not needed + // skip computing logits + // TODO: is this safe? + gf->n_nodes = i_embd + 1; } + } else { + embd = nullptr; // do not extract embeddings when not needed + GGML_ASSERT(strcmp(res->name, "result_output") == 0 && "missing result_output tensor"); } // LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs); @@ -9505,38 +9535,22 @@ static int llama_decode_internal( //} // extract logits - // TODO: do not compute and extract logits if only embeddings are needed - // update the graphs to skip "result_output" if logits are not needed if (res) { ggml_backend_t backend_res = ggml_backend_sched_get_tensor_backend(lctx.sched, res); GGML_ASSERT(backend_res != nullptr); - int32_t new_logits = 0; - if (u_batch.logits) { - for (uint32_t i = 0; i < n_tokens; i++) { - if (u_batch.logits[i]) { - new_logits++; - } - } - } else if (lctx.logits_all) { - new_logits += n_tokens; - } else { - // keep last logits only - if (cur_token + n_tokens >= n_tokens_all) { - new_logits += 1; - } - } + float * logits_out = lctx.logits + n_outputs_prev*n_vocab; + const int32_t n_outputs_new = lctx.n_outputs; - if (new_logits) { - GGML_ASSERT(new_logits <= n_logits); - GGML_ASSERT((n_logits_prev+new_logits)*n_vocab <= (int64_t) lctx.logits_size); - ggml_backend_tensor_get_async(backend_res, res, lctx.logits, n_logits_prev*n_vocab*sizeof(float), new_logits*n_vocab*sizeof(float)); - n_logits_prev += new_logits; + if (n_outputs_new) { + GGML_ASSERT(n_outputs_prev+n_outputs_new <= n_outputs); + GGML_ASSERT((n_outputs_prev+n_outputs_new)*n_vocab <= (int64_t) lctx.logits_size); + ggml_backend_tensor_get_async(backend_res, res, logits_out, 0, n_outputs_new*n_vocab*sizeof(float)); } } // extract embeddings - if (cparams.embeddings && embd) { + if (embd) { ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(lctx.sched, embd); GGML_ASSERT(backend_embd != nullptr); @@ -9544,17 +9558,13 @@ static int llama_decode_internal( case LLAMA_POOLING_TYPE_NONE: { // extract token embeddings - auto & embd_out = lctx.embd; - - if (u_batch.logits) { - //embd_out.resize(n_embd * n_tokens); - for (uint32_t i = 0; i < n_tokens; i++) { - if (u_batch.logits[i] == 0) { - continue; - } - // FIXME - ggml_backend_tensor_get_async(backend_embd, embd, embd_out + n_embd*(i + cur_token), (n_embd*i)*sizeof(float), n_embd*sizeof(float)); - } + float * embd_out = lctx.embd + n_outputs_prev*n_embd; + const int32_t n_outputs_new = lctx.n_outputs; + + if (n_outputs_new) { + GGML_ASSERT(n_outputs_prev+n_outputs_new <= n_outputs); + GGML_ASSERT((n_outputs_prev+n_outputs_new)*n_embd <= (int64_t) lctx.embd_size); + ggml_backend_tensor_get_async(backend_embd, embd, embd_out, 0, n_outputs_new*n_embd*sizeof(float)); } } break; case LLAMA_POOLING_TYPE_CLS: @@ -9581,6 +9591,7 @@ static int llama_decode_internal( } break; } } + n_outputs_prev += lctx.n_outputs; } // wait for the computation to finish (automatically done when obtaining the model output) @@ -14639,11 +14650,11 @@ float * llama_get_logits_ith(struct llama_context * ctx, int32_t i) { const int32_t j = ctx->output_ids[i]; llama_synchronize(ctx); - - if (ctx->logits && 0 <= j && j < ctx->n_outputs) { + if (ctx->logits && 0 <= j && (size_t) j < ctx->output_size) { return ctx->logits + j*ctx->model.hparams.n_vocab; } - LLAMA_LOG_ERROR("%s: invalid logits id %i\n", __func__, i); + LLAMA_LOG_ERROR("%s: invalid logits id %i, reason: %s (j=%i, output_size=%li)\n", + __func__, i, !ctx->logits ? "no logits" : j < 0 ? "batch.logits[i] wasn't true" : "too big", j, ctx->output_size); #ifndef NDEBUG GGML_ASSERT(false); #endif @@ -14661,7 +14672,7 @@ float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i) { llama_synchronize(ctx); - if (ctx->embd && 0 < j && j < ctx->n_outputs) { + if (ctx->embd && 0 < j && (size_t) j < ctx->output_size) { return ctx->embd + j*ctx->model.hparams.n_embd; } LLAMA_LOG_ERROR("%s: invalid embeddings id %i\n", __func__, i); From 25981fca372c03d68c3c6ec5b72c24a4bbe02672 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Sat, 16 Mar 2024 21:36:48 -0400 Subject: [PATCH 04/25] perplexity : adapt to the logits API changes --- examples/imatrix/imatrix.cpp | 1 + examples/perplexity/perplexity.cpp | 120 +++++++++++++++++++---------- examples/server/server.cpp | 3 +- llama.cpp | 6 +- 4 files changed, 84 insertions(+), 46 deletions(-) diff --git a/examples/imatrix/imatrix.cpp b/examples/imatrix/imatrix.cpp index f21bc48f3b466..e1dbf24a3f8da 100644 --- a/examples/imatrix/imatrix.cpp +++ b/examples/imatrix/imatrix.cpp @@ -403,6 +403,7 @@ static bool compute_imatrix(llama_context * ctx, const gpt_params & params, bool tokens[batch_start] = llama_token_bos(llama_get_model(ctx)); } + // TODO: use batch.logits to save computations instead of relying on logits_all == true if (llama_decode(ctx, llama_batch_get_one(tokens.data() + batch_start, batch_size, j * n_batch, 0))) { fprintf(stderr, "%s : failed to eval\n", __func__); return false; diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index d766aef6ac1b1..72c1a55c4b9ea 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -380,6 +380,7 @@ static results_perplexity perplexity_v2(llama_context * ctx, const gpt_params & const int batch_size = std::min(end - batch_start, n_batch); //fprintf(stderr, " Batch %d: starts at %d, size is %d, n_past is %d\n",j,batch_start,batch_size,j * n_batch); + // TODO: use llama_batch.logits instead of relying on logits_all == true if (llama_decode(ctx, llama_batch_get_one(tokens.data() + batch_start, batch_size, j * n_batch, 0))) { //fprintf(stderr, "%s : failed to eval\n", __func__); return {tokens, -1, logit_history, prob_history}; @@ -551,6 +552,7 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par for (int j = 0; j < num_batches; ++j) { const int batch_start = start + j * n_batch; const int batch_size = std::min(end - batch_start, n_batch); + int n_outputs = 0; batch.n_tokens = 0; for (int seq = 0; seq < n_seq_batch; seq++) { @@ -571,6 +573,7 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par batch.n_seq_id[idx] = 1; batch.seq_id[idx][0] = seq; batch.logits[idx] = batch.pos[idx] >= first ? 1 : 0; + n_outputs += batch.logits[idx] != 0; } batch.n_tokens += batch_size; @@ -583,9 +586,9 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par return {tokens, -1, logit_history, prob_history}; } - if (num_batches > 1) { + if (num_batches > 1 && n_outputs > 0) { const auto * batch_logits = llama_get_logits(ctx); - logits.insert(logits.end(), batch_logits, batch_logits + batch_size * n_vocab); + logits.insert(logits.end(), batch_logits, batch_logits + n_outputs * n_vocab); } } @@ -604,14 +607,14 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par } for (int seq = 0; seq < n_seq_batch; seq++) { - const float * all_logits = num_batches > 1 ? logits.data() : llama_get_logits_ith(ctx, seq*n_ctx); + const float * all_logits = num_batches > 1 ? logits.data() : llama_get_logits_ith(ctx, seq*n_ctx + first); llama_token * tokens_data = tokens.data() + start + seq*n_ctx + first; if (!params.logits_file.empty()) { - process_logits(logits_stream, n_vocab, all_logits + first*n_vocab, + process_logits(logits_stream, n_vocab, all_logits, tokens_data, n_ctx - 1 - first, workers, log_probs, nll, nll2); } else { - process_logits(n_vocab, all_logits + first*n_vocab, + process_logits(n_vocab, all_logits, tokens_data, n_ctx - 1 - first, workers, nll, nll2, logit_history.data() + start + seq*n_ctx + first, @@ -652,6 +655,7 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par } static bool decode_helper(llama_context * ctx, llama_batch & batch, std::vector & batch_logits, int32_t n_batch, int32_t n_vocab) { + int prev_outputs = 0; for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch) { const int32_t n_tokens = std::min(n_batch, (int32_t) (batch.n_tokens - i)); @@ -672,7 +676,14 @@ static bool decode_helper(llama_context * ctx, llama_batch & batch, std::vector< return false; } - memcpy(batch_logits.data() + i*n_vocab, llama_get_logits(ctx), n_tokens*n_vocab*sizeof(float)); + int n_outputs = 0; + for (int i = 0; i < n_tokens; ++i) { + n_outputs += batch_view.logits[i] != 0; + } + + memcpy(batch_logits.data() + prev_outputs*n_vocab, llama_get_logits(ctx), n_outputs*n_vocab*sizeof(float)); + + prev_outputs += n_outputs; } return true; @@ -779,7 +790,7 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) { size_t ending_logprob_count[4]; double ending_logprob[4]; - size_t i_batch; // starting index in the llama_batch + size_t i_logits; // starting index of logits in the llama_batch size_t common_prefix; // max number of initial tokens that are the same in all sentences size_t required_tokens; // needed number of tokens to evaluate all 4 endings std::vector seq_tokens[4]; @@ -821,7 +832,9 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) { hs_cur.seq_tokens[0].size() - hs_cur.common_prefix + hs_cur.seq_tokens[1].size() - hs_cur.common_prefix + hs_cur.seq_tokens[2].size() - hs_cur.common_prefix + - hs_cur.seq_tokens[3].size() - hs_cur.common_prefix; + hs_cur.seq_tokens[3].size() - hs_cur.common_prefix + // the last tokens don't need to be evaluated + - 4; //GGML_ASSERT(hs_cur.common_prefix >= ::llama_tokenize(ctx, hs_cur.context, add_bos).size()); @@ -844,9 +857,10 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) { const int max_tasks_per_batch = 32; const int max_seq = std::min(4*max_tasks_per_batch, (int) llama_n_seq_max(ctx)); - llama_batch batch = llama_batch_init(n_ctx, 0, max_seq); + llama_batch batch = llama_batch_init(n_ctx, 0, 4); std::vector tok_logits(n_vocab); + // TODO: this could be made smaller; it's currently the worst-case size std::vector batch_logits(n_vocab*n_ctx); std::vector> eval_pairs; @@ -857,16 +871,17 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) { int n_cur = 0; size_t i1 = i0; - size_t i_batch = 0; // this tells us where in `llama_batch` we are currently + size_t i_logits = 0; // this tells us how many logits were needed before this point in the batch llama_batch_clear(batch); // batch as much tasks as possible into the available context - // each task has 4 unique seuqnce ids - one for each ending + // each task has 4 unique sequence ids - one for each ending // the common prefix is shared among the 4 sequences to save tokens // we extract logits only from the last common token and from all ending tokens of each sequence while (n_cur + (int) hs_data[i1].required_tokens <= n_ctx) { auto & hs_cur = hs_data[i1]; + int n_logits = 0; const int s0 = 4*(i1 - i0); if (s0 + 4 > max_seq) { @@ -874,18 +889,21 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) { } for (size_t i = 0; i < hs_cur.common_prefix; ++i) { - llama_batch_add(batch, hs_cur.seq_tokens[0][i], i, { s0 + 0, s0 + 1, s0 + 2, s0 + 3}, false); + llama_batch_add(batch, hs_cur.seq_tokens[0][i], i, { s0 + 0, s0 + 1, s0 + 2, s0 + 3 }, false); } batch.logits[batch.n_tokens - 1] = true; // we need logits for the last token of the common prefix + n_logits += 1; for (int s = 0; s < 4; ++s) { - for (size_t i = hs_cur.common_prefix; i < hs_cur.seq_tokens[s].size(); ++i) { + // end before the last token, no need to predict past the end of the sequences + for (size_t i = hs_cur.common_prefix; i < hs_cur.seq_tokens[s].size() - 1; ++i) { llama_batch_add(batch, hs_cur.seq_tokens[s][i], i, { s0 + s }, true); + n_logits += 1; } } - hs_cur.i_batch = i_batch; - i_batch += hs_cur.required_tokens; + hs_cur.i_logits = i_logits; + i_logits += n_logits; n_cur += hs_data[i1].required_tokens; if (++i1 == hs_task_count) { @@ -911,12 +929,11 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) { eval_pairs.clear(); for (size_t i = i0; i < i1; ++i) { auto & hs_cur = hs_data[i]; - size_t li = hs_cur.common_prefix; + size_t li = 1; // skip the last logit of the common prefix (computed separately below) for (int s = 0; s < 4; ++s) { for (size_t j = hs_cur.common_prefix; j < hs_cur.seq_tokens[s].size() - 1; j++) { - eval_pairs.emplace_back(hs_cur.i_batch + li++, hs_cur.seq_tokens[s][j + 1]); + eval_pairs.emplace_back(hs_cur.i_logits + li++, hs_cur.seq_tokens[s][j + 1]); } - ++li; } } // Then we do the actual calculation @@ -928,7 +945,8 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) { for (size_t i = i0; i < i1; ++i) { auto & hs_cur = hs_data[i]; - std::memcpy(tok_logits.data(), batch_logits.data() + n_vocab*(hs_cur.i_batch + hs_cur.common_prefix - 1), n_vocab*sizeof(float)); + // get the logits of the last token of the common prefix + std::memcpy(tok_logits.data(), batch_logits.data() + n_vocab*hs_cur.i_logits, n_vocab*sizeof(float)); const auto first_probs = softmax(tok_logits); @@ -978,7 +996,7 @@ struct winogrande_entry { std::array choices; int answer; - size_t i_batch; + size_t i_logits; size_t common_prefix; size_t required_tokens; size_t n_base1; // number of tokens for context + choice 1 @@ -1106,7 +1124,9 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) { task.required_tokens = task.common_prefix + task.seq_tokens[0].size() - task.common_prefix + - task.seq_tokens[1].size() - task.common_prefix; + task.seq_tokens[1].size() - task.common_prefix + // the last tokens don't need to be evaluated + - 2; task.n_base1 = ::llama_tokenize(ctx, task.first + task.choices[0], add_bos).size(); task.n_base2 = ::llama_tokenize(ctx, task.first + task.choices[1], add_bos).size(); @@ -1121,9 +1141,10 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) { const int max_tasks_per_batch = 128; const int max_seq = std::min(2*max_tasks_per_batch, (int) llama_n_seq_max(ctx)); - llama_batch batch = llama_batch_init(n_ctx, 0, max_seq); + llama_batch batch = llama_batch_init(n_ctx, 0, 2); std::vector tok_logits(n_vocab); + // TODO: this could be made smaller; it's currently the worst-case size std::vector batch_logits(n_vocab*n_ctx); std::vector> eval_pairs; @@ -1137,29 +1158,33 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) { int n_cur = 0; size_t i1 = i0; - size_t i_batch = 0; + size_t i_logits = 0; llama_batch_clear(batch); while (n_cur + (int) data[i1].required_tokens <= n_ctx) { + int n_logits = 0; const int s0 = 2*(i1 - i0); if (s0 + 2 > max_seq) { break; } for (size_t i = 0; i < data[i1].common_prefix; ++i) { - llama_batch_add(batch, data[i1].seq_tokens[0][i], i, { s0 + 0, s0 + 1}, false); + llama_batch_add(batch, data[i1].seq_tokens[0][i], i, { s0 + 0, s0 + 1 }, false); } batch.logits[batch.n_tokens - 1] = true; + n_logits += 1; for (int s = 0; s < 2; ++s) { - for (size_t i = data[i1].common_prefix; i < data[i1].seq_tokens[s].size(); ++i) { + // end before the last token, no need to predict past the end of the sequences + for (size_t i = data[i1].common_prefix; i < data[i1].seq_tokens[s].size() - 1; ++i) { llama_batch_add(batch, data[i1].seq_tokens[s][i], i, { s0 + s }, true); + n_logits += 1; } } - data[i1].i_batch = i_batch; - i_batch += data[i1].required_tokens; + data[i1].i_logits = i_logits; + i_logits += n_logits; n_cur += data[i1].required_tokens; if (++i1 == data.size()) { @@ -1184,21 +1209,25 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) { for (size_t i = i0; i < i1; ++i) { auto & task = data[i]; + // FIXME: this should not be needed. const bool skip_choice = task.seq_tokens[0].size() - task.common_prefix > k_min_trailing_ctx && task.seq_tokens[1].size() - task.common_prefix > k_min_trailing_ctx; const auto& n_base1 = skip_choice ? task.n_base1 : task.common_prefix; const int last_1st = task.seq_tokens[0].size() - n_base1 > 1 ? 1 : 0; - size_t li = n_base1 - 1; + // start from the end of the common prefix or the end token of the first choice + size_t li = n_base1 - task.common_prefix; for (size_t j = n_base1-1; j < task.seq_tokens[0].size()-1-last_1st; ++j) { - eval_pairs.emplace_back(task.i_batch + li++, task.seq_tokens[0][j+1]); + eval_pairs.emplace_back(task.i_logits + li++, task.seq_tokens[0][j+1]); } const auto& n_base2 = skip_choice ? task.n_base2 : task.common_prefix; const int last_2nd = task.seq_tokens[1].size() - n_base2 > 1 ? 1 : 0; - li = task.seq_tokens[0].size() - task.common_prefix + n_base2 - 1; + // TODO: consider fixing the following (maybe remove choice skipping too?) + // start from the end of the first version (!) or the end token of the second choice? + li = task.seq_tokens[0].size() - 1 - task.common_prefix + n_base2 - task.common_prefix; for (size_t j = n_base2-1; j < task.seq_tokens[1].size()-1-last_2nd; ++j) { - eval_pairs.emplace_back(task.i_batch + li++, task.seq_tokens[1][j+1]); + eval_pairs.emplace_back(task.i_logits + li++, task.seq_tokens[1][j+1]); } } compute_logprobs(batch_logits.data(), n_vocab, workers, eval_pairs, eval_results); @@ -1287,7 +1316,7 @@ struct multiple_choice_task { } // For evaluation - size_t i_batch; // starting index in the llama_batch + size_t i_logits; // starting index of logits in the llama_batch size_t common_prefix; // max number of initial tokens that are the same in all sentences size_t required_tokens; // needed number of tokens to evaluate all answers std::vector> seq_tokens; @@ -1334,6 +1363,8 @@ static bool multiple_choice_prepare_one_task(llama_context * ctx, bool add_bos, for (auto& seq : task.seq_tokens) { task.required_tokens += seq.size() - task.common_prefix; } + // the last tokens don't need to be evaluated + task.required_tokens -= task.seq_tokens.size(); return true; } @@ -1366,7 +1397,7 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & params std::vector task_pos(n_task); strstream.read((char *)task_pos.data(), task_pos.size()*sizeof(uint32_t)); if (strstream.fail()) { - printf("%s: failed to raad task positions from prompt\n", __func__); + printf("%s: failed to read task positions from prompt\n", __func__); return; } @@ -1491,17 +1522,18 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & params int n_cur = 0; size_t i1 = i0; - size_t i_batch = 0; // this tells us where in `llama_batch` we are currently + size_t i_logits = 0; // this tells us how many logits were needed before this point in the batch llama_batch_clear(batch); // batch as much tasks as possible into the available context - // each task has 4 unique seuqnce ids - one for each ending + // each task has 4 unique sequence ids - one for each ending // the common prefix is shared among the 4 sequences to save tokens // we extract logits only from the last common token and from all ending tokens of each sequence int s0 = 0; while (n_cur + (int) tasks[i1].required_tokens <= n_ctx) { auto& cur_task = tasks[i1]; + int n_logits = 0; int num_answers = cur_task.seq_tokens.size(); if (s0 + num_answers > max_seq) { @@ -1518,17 +1550,20 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & params llama_batch_add(batch, cur_task.seq_tokens[0][i], i, batch_indeces, false); } batch.logits[batch.n_tokens - 1] = true; // we need logits for the last token of the common prefix + n_logits += 1; for (int s = 0; s < int(cur_task.seq_tokens.size()); ++s) { - for (size_t i = cur_task.common_prefix; i < cur_task.seq_tokens[s].size(); ++i) { + // end before the last token, no need to predict past the end of the sequences + for (size_t i = cur_task.common_prefix; i < cur_task.seq_tokens[s].size() - 1; ++i) { llama_batch_add(batch, cur_task.seq_tokens[s][i], i, { s0 + s }, true); + n_logits += 1; } } s0 += num_answers; - cur_task.i_batch = i_batch; - i_batch += cur_task.required_tokens; + cur_task.i_logits = i_logits; + i_logits += n_logits; n_cur += cur_task.required_tokens; if (++i1 == tasks.size()) { @@ -1554,12 +1589,11 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & params eval_pairs.clear(); for (size_t i = i0; i < i1; ++i) { auto& cur_task = tasks[i]; - size_t li = cur_task.common_prefix; + size_t li = 1; // skip the last logit of the common prefix (computed separately below) for (int s = 0; s < int(cur_task.seq_tokens.size()); ++s) { for (size_t j = cur_task.common_prefix; j < cur_task.seq_tokens[s].size() - 1; j++) { - eval_pairs.emplace_back(cur_task.i_batch + li++, cur_task.seq_tokens[s][j + 1]); + eval_pairs.emplace_back(cur_task.i_logits + li++, cur_task.seq_tokens[s][j + 1]); } - ++li; } } // Then we do the actual calculation @@ -1578,7 +1612,8 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & params //} //printf("\n common_prefix: %zu\n", cur_task.common_prefix); - std::memcpy(tok_logits.data(), batch_logits.data() + n_vocab*(cur_task.i_batch + cur_task.common_prefix - 1), n_vocab*sizeof(float)); + // get the logits of the last token of the common prefix + std::memcpy(tok_logits.data(), batch_logits.data() + n_vocab*cur_task.i_logits, n_vocab*sizeof(float)); const auto first_probs = softmax(tok_logits); @@ -1730,6 +1765,7 @@ static void kl_divergence(llama_context * ctx, const gpt_params & params) { tokens[batch_start] = llama_token_bos(llama_get_model(ctx)); } + // TODO: use llama_batch.logits instead of relying on logits_all == true if (llama_decode(ctx, llama_batch_get_one(tokens.data() + batch_start, batch_size, j * n_batch, 0))) { fprintf(stderr, "%s : failed to eval\n", __func__); return; diff --git a/examples/server/server.cpp b/examples/server/server.cpp index d2a8e541d3305..504b78c21615a 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -744,7 +744,8 @@ struct server_context { { const int32_t n_batch = llama_n_batch(ctx); - batch = llama_batch_init(n_batch, 0, params.n_parallel); + // only a single seq_id per token is needed + batch = llama_batch_init(n_batch, 0, 1); } metrics.init(); diff --git a/llama.cpp b/llama.cpp index f397ffed4235a..577c09495fb22 100644 --- a/llama.cpp +++ b/llama.cpp @@ -9223,11 +9223,11 @@ static void llama_output_reserve(llama_context & lctx, int32_t n_outputs) { const size_t buf_output_size = (lctx.logits_size + lctx.embd_size)*sizeof(float); if (lctx.buf_output) { - // This doesn't happen often -// #ifndef NDEBUG +#ifndef NDEBUG + // This doesn't happen often, but may be annoying in some cases (like the HellaSwag benchmark) const size_t prev_size = ggml_backend_buffer_get_size(lctx.buf_output); LLAMA_LOG_INFO("%s: reallocating output buffer from size %.02f MiB to %.02f MiB\n", __func__, prev_size / 1024.0 / 1024.0, buf_output_size / 1024.0 / 1024.0); -// #endif +#endif ggml_backend_buffer_free(lctx.buf_output); lctx.buf_output = nullptr; lctx.logits = nullptr; From 17b45c96edc99a839f039ddd2bf154950aac1605 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Sat, 16 Mar 2024 22:05:44 -0400 Subject: [PATCH 05/25] perplexity : fix Winogrande, use correct logits for second choice start The first logits used to evaluate the second choice were not from the end of the common prefix; instead, they were the logits from the end of the first choice. This has been corrected. The previous implementation sometimes had outliers in the scores of choices for some tasks, and the logic to skip choices words in the log-likelihood evaluation probably was an attempt to reduce those, but it was complex and didn't quite seem to be the right thing. This is simpler now, and the outlier scores aren't there anymore. --- examples/perplexity/perplexity.cpp | 49 +++++++++--------------------- 1 file changed, 14 insertions(+), 35 deletions(-) diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index 72c1a55c4b9ea..766e240895633 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -999,8 +999,6 @@ struct winogrande_entry { size_t i_logits; size_t common_prefix; size_t required_tokens; - size_t n_base1; // number of tokens for context + choice 1 - size_t n_base2; // number of tokens for context + choice 2 std::vector seq_tokens[2]; }; @@ -1076,8 +1074,6 @@ static std::vector load_winogrande_from_csv(const std::string& */ static void winogrande_score(llama_context * ctx, const gpt_params & params) { - constexpr int k_min_trailing_ctx = 3; - auto data = load_winogrande_from_csv(params.prompt); if (data.empty()) { fprintf(stderr, "%s: no tasks\n", __func__); @@ -1127,9 +1123,6 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) { task.seq_tokens[1].size() - task.common_prefix // the last tokens don't need to be evaluated - 2; - - task.n_base1 = ::llama_tokenize(ctx, task.first + task.choices[0], add_bos).size(); - task.n_base2 = ::llama_tokenize(ctx, task.first + task.choices[1], add_bos).size(); } fprintf(stderr, "%s : calculating winogrande score over selected tasks.\n", __func__); @@ -1209,26 +1202,20 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) { for (size_t i = i0; i < i1; ++i) { auto & task = data[i]; - // FIXME: this should not be needed. - const bool skip_choice = - task.seq_tokens[0].size() - task.common_prefix > k_min_trailing_ctx && - task.seq_tokens[1].size() - task.common_prefix > k_min_trailing_ctx; - - const auto& n_base1 = skip_choice ? task.n_base1 : task.common_prefix; - const int last_1st = task.seq_tokens[0].size() - n_base1 > 1 ? 1 : 0; - // start from the end of the common prefix or the end token of the first choice - size_t li = n_base1 - task.common_prefix; - for (size_t j = n_base1-1; j < task.seq_tokens[0].size()-1-last_1st; ++j) { + // start from the end of the common prefix + size_t li = 0; + for (size_t j = task.common_prefix-1; j < task.seq_tokens[0].size()-1; ++j) { eval_pairs.emplace_back(task.i_logits + li++, task.seq_tokens[0][j+1]); } - const auto& n_base2 = skip_choice ? task.n_base2 : task.common_prefix; - const int last_2nd = task.seq_tokens[1].size() - n_base2 > 1 ? 1 : 0; - // TODO: consider fixing the following (maybe remove choice skipping too?) - // start from the end of the first version (!) or the end token of the second choice? - li = task.seq_tokens[0].size() - 1 - task.common_prefix + n_base2 - task.common_prefix; - for (size_t j = n_base2-1; j < task.seq_tokens[1].size()-1-last_2nd; ++j) { + // first token of the second choice is predicted by the end of the common prefix + eval_pairs.emplace_back(task.i_logits, task.seq_tokens[1][task.common_prefix]); + for (size_t j = task.common_prefix; j < task.seq_tokens[1].size()-1; ++j) { eval_pairs.emplace_back(task.i_logits + li++, task.seq_tokens[1][j+1]); } + if (i < i1 - 1) { + // make sure all logits have been processed as expected + GGML_ASSERT(task.i_logits + li == data[i+1].i_logits); + } } compute_logprobs(batch_logits.data(), n_vocab, workers, eval_pairs, eval_results); @@ -1236,25 +1223,17 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) { for (size_t i = i0; i < i1; ++i) { auto & task = data[i]; - const bool skip_choice = - task.seq_tokens[0].size() - task.common_prefix > k_min_trailing_ctx && - task.seq_tokens[1].size() - task.common_prefix > k_min_trailing_ctx; - float score_1st = 0; - const auto& n_base1 = skip_choice ? task.n_base1 : task.common_prefix; - const int last_1st = task.seq_tokens[0].size() - n_base1 > 1 ? 1 : 0; - for (size_t j = n_base1-1; j < task.seq_tokens[0].size()-1-last_1st; ++j) { + for (size_t j = task.common_prefix-1; j < task.seq_tokens[0].size()-1; ++j) { score_1st += eval_results[ir++]; } - score_1st /= (task.seq_tokens[0].size() - n_base1 - last_1st); + score_1st /= (task.seq_tokens[0].size() - task.common_prefix); float score_2nd = 0; - const auto& n_base2 = skip_choice ? task.n_base2 : task.common_prefix; - const int last_2nd = task.seq_tokens[1].size() - n_base2 > 1 ? 1 : 0; - for (size_t j = n_base2-1; j < task.seq_tokens[1].size()-1-last_2nd; ++j) { + for (size_t j = task.common_prefix-1; j < task.seq_tokens[1].size()-1; ++j) { score_2nd += eval_results[ir++]; } - score_2nd /= (task.seq_tokens[1].size() - n_base2 - last_2nd); + score_2nd /= (task.seq_tokens[1].size() - task.common_prefix); int result = score_1st > score_2nd ? 1 : 2; From d0129e8e2963c17dd9e17a7bec8f54b981fa949b Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Sun, 17 Mar 2024 14:54:09 -0400 Subject: [PATCH 06/25] perplexity : normalize spaces and punctuation in Winogrande sentences --- examples/perplexity/perplexity.cpp | 32 ++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index 766e240895633..8ee09ccfbedd1 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -1038,6 +1038,38 @@ static std::vector load_winogrande_from_csv(const std::string& auto choice2 = line.substr(comma_pos[2]+1, comma_pos[3] - comma_pos[2] - 1); auto answer = line.substr(comma_pos[3]+1, line.size() - comma_pos[3] - 1); auto index = line.substr(0, comma_pos[0]); + if ('a' <= sentence[0] && sentence[0] <= 'z') { + // make the first letter a capital letter + sentence[0] -= 'a' - 'A'; + } + for (int i = 0; i < (int) sentence.size() - 1; ++i) { + // trim repeated spaces and spaces before punctuation + if (sentence[i] == ' ') { + char next = sentence[i+1]; + if (next == ' ' || next == ',' || next == '.' || next == '\'') { + char r[2] = { next, 0 }; + sentence.replace(i, 2, r); + --i; // stay at the same index for repeated spaces + } + } else if (sentence[i] == ',' || sentence[i] == '.') { + if (sentence[i] == sentence[i+1]) { + // trim repeated punctuation (forward to work at the end of sentences) + char r[2] = { sentence[i], 0 }; + sentence.replace(i, 2, r); + --i; // same index to then run the other checks on that punctuation + } else if (0 < i && sentence[i-1] == sentence[i]) { + // trim repeated punctuation (looks back to work with the space trim) + char r[2] = { sentence[i], 0 }; + sentence.replace(i-1, 2, r); + i -= 2; // go back because content was shifted + } else if (sentence[i+1] != ' ') { + // add missing space after punctuation + // (since the loop stops before the end, this adds no trailing space) + char r[3] = { sentence[i], ' ', 0 }; + sentence.replace(i, 1, r); + } + } + } int where = 0; for ( ; where < int(sentence.size()); ++where) { if (sentence[where] == '_') break; From 487f89ec2e7856cc15ee5146c52d0ebfbad96fa2 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Sun, 17 Mar 2024 15:23:44 -0400 Subject: [PATCH 07/25] llama : fix embedding conditions --- llama.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index 577c09495fb22..538a3af32f7e0 100644 --- a/llama.cpp +++ b/llama.cpp @@ -8992,7 +8992,7 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { data[n_outputs++] = i; } } - } else if (lctx.logits_all || (cparams.embeddings && hparams.pooling_type != LLAMA_POOLING_TYPE_NONE)) { + } else if (lctx.logits_all || (cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE)) { for (int i = 0; i < n_tokens; ++i) { data[i] = i; } @@ -9205,7 +9205,7 @@ static void llama_output_reserve(llama_context & lctx, int32_t n_outputs) { // TODO: use a per-batch flag for logits presence instead const bool has_logits = cparams.causal_attn; - const bool has_embd = cparams.embeddings && (!hparams.causal_attn || cparams.pooling_type == LLAMA_POOLING_TYPE_NONE); + const bool has_embd = cparams.embeddings && (hparams.causal_attn || cparams.pooling_type == LLAMA_POOLING_TYPE_NONE); if (!lctx.output_ids) { // never resized afterwards From 408fcb0f91dba87386d9bf02a12eb0cf71c08559 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Sun, 17 Mar 2024 15:34:56 -0400 Subject: [PATCH 08/25] llama : fix llama_get_embeddings_ith when the resulting id is 0 --- llama.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/llama.cpp b/llama.cpp index 538a3af32f7e0..b82b5605947b2 100644 --- a/llama.cpp +++ b/llama.cpp @@ -14650,11 +14650,13 @@ float * llama_get_logits_ith(struct llama_context * ctx, int32_t i) { const int32_t j = ctx->output_ids[i]; llama_synchronize(ctx); + if (ctx->logits && 0 <= j && (size_t) j < ctx->output_size) { return ctx->logits + j*ctx->model.hparams.n_vocab; } LLAMA_LOG_ERROR("%s: invalid logits id %i, reason: %s (j=%i, output_size=%li)\n", - __func__, i, !ctx->logits ? "no logits" : j < 0 ? "batch.logits[i] wasn't true" : "too big", j, ctx->output_size); + __func__, i, !ctx->logits ? "no logits" : j < 0 ? "batch.logits[i] wasn't true" : "too big", + j, ctx->output_size); #ifndef NDEBUG GGML_ASSERT(false); #endif @@ -14672,10 +14674,12 @@ float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i) { llama_synchronize(ctx); - if (ctx->embd && 0 < j && (size_t) j < ctx->output_size) { + if (ctx->embd && 0 <= j && (size_t) j < ctx->output_size) { return ctx->embd + j*ctx->model.hparams.n_embd; } - LLAMA_LOG_ERROR("%s: invalid embeddings id %i\n", __func__, i); + LLAMA_LOG_ERROR("%s: invalid embeddings id %i, reason: %s (j=%i, output_size=%li)\n", + __func__, i, !ctx->embd ? "no embeddings" : j < 0 ? "batch.logits[i] wasn't true" : "too big", + j, ctx->output_size); #ifndef NDEBUG GGML_ASSERT(false); #endif From e19cb3aeb728987aa0a58a119d800c06fdd6aad7 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Sun, 17 Mar 2024 16:31:19 -0400 Subject: [PATCH 09/25] llama : fix wrong n_outputs in llama_set_inputs A mismatch happened when using a smaller n_ubatch than n_batch and then using llama_batch_get_one(). The decision of what n_outputs should be now almost fully depends on how lctx.n_outputs is set in llama_decode_internal. The conditions are simpler this way. * llama : when saving the state, recalculate n_outputs This ensures the correct number of outputs for the entire previous batch is stored in the session file, even when n_ubatch is smaller than n_batch. --- llama.cpp | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/llama.cpp b/llama.cpp index b82b5605947b2..e606bdda48c52 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2103,7 +2103,7 @@ struct llama_context { int32_t * output_ids = nullptr; // map token positions to ids of the logits and embd buffers size_t output_size = 0; // capacity (of tokens positions) for the output buffers - int32_t n_outputs = 0; // number of actually-used outputs in the current or previous ubatch + int32_t n_outputs = 0; // number of actually-used outputs in the current ubatch bool logits_all = false; @@ -8985,25 +8985,25 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_out_ids->buffer)); int32_t * data = (int32_t *) lctx.inp_out_ids->data; - int32_t n_outputs = 0; - if (batch.logits) { + if (lctx.n_outputs == n_tokens) { + for (int i = 0; i < n_tokens; ++i) { + data[i] = i; + } + } else if (batch.logits) { + int32_t n_outputs = 0; for (int i = 0; i < n_tokens; ++i) { if (batch.logits[i]) { data[n_outputs++] = i; } } - } else if (lctx.logits_all || (cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE)) { - for (int i = 0; i < n_tokens; ++i) { - data[i] = i; - } - n_outputs = n_tokens; - } else { + // the graph needs the have been passed the correct number of outputs + GGML_ASSERT(lctx.n_outputs == n_outputs); + } else if (lctx.n_outputs == 1) { // only keep last output data[0] = n_tokens - 1; - n_outputs = 1; + } else { + GGML_ASSERT(lctx.n_outputs == 0); } - // the graph needs the have been passed the correct number of outputs - GGML_ASSERT(lctx.n_outputs == n_outputs); } GGML_ASSERT( @@ -9386,7 +9386,7 @@ static int llama_decode_internal( for (uint32_t i = 0; i < n_tokens; i++) { n_outputs_new += u_batch.logits[i] != 0; } - } else if (lctx.logits_all) { + } else if ((uint32_t) n_outputs == n_tokens_all) { n_outputs_new = n_tokens; } else { // keep last output only @@ -14166,7 +14166,9 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat // copy outputs { - size_t n_outputs = ctx->n_outputs; + // Can't use ctx->n_outputs because it's not for the + // entire last batch when n_ubatch is smaller than n_batch + size_t n_outputs = 0; // copy output ids { @@ -14174,19 +14176,17 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat const size_t n_batch = ctx->cparams.n_batch; const int32_t * output_ids = ctx->output_ids; - output_pos.resize(n_outputs); + output_pos.resize(ctx->output_size); // build a more compact representation of the output ids for (size_t i = 0; i < n_batch; ++i) { // map an output id to a position in the batch int32_t pos = output_ids[i]; if (pos >= 0) { - if ((size_t) pos >= output_pos.size()) { - // TODO: maybe fail here instead - LLAMA_LOG_WARN("%s: weird output buffer layout, possibly a bug\n", __func__); + if ((size_t) pos >= n_outputs) { n_outputs = pos + 1; - output_pos.resize(n_outputs); } + GGML_ASSERT((size_t) pos < ctx->output_size); output_pos[pos] = i; } } @@ -14201,7 +14201,7 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat // copy logits { const size_t logits_size = std::min(ctx->logits_size, n_outputs * ctx->model.hparams.n_vocab); - + data_ctx->write(&logits_size, sizeof(logits_size)); if (logits_size) { From a57fa7faa4fc947f647b7974edd762bc6eed2790 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Sun, 17 Mar 2024 20:19:25 -0400 Subject: [PATCH 10/25] llama : fix not-skipping outputs of non-causal models --- llama.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index e606bdda48c52..b351779d9cbd7 100644 --- a/llama.cpp +++ b/llama.cpp @@ -5636,6 +5636,7 @@ struct llm_build_context { lctx.inp_tokens = nullptr; lctx.inp_embd = nullptr; lctx.inp_pos = nullptr; + lctx.inp_out_ids = nullptr; lctx.inp_KQ_mask = nullptr; lctx.inp_KQ_pos = nullptr; lctx.inp_K_shift = nullptr; @@ -8978,8 +8979,8 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { ggml_backend_tensor_set(lctx.inp_pos, batch.pos, 0, n_tokens*ggml_element_size(lctx.inp_pos)); } - { - GGML_ASSERT(lctx.inp_out_ids && "every model type must skip unused outputs"); + if (hparams.causal_attn || cparams.pooling_type == LLAMA_POOLING_TYPE_NONE) { + GGML_ASSERT(lctx.inp_out_ids && "every model that can must skip unused outputs"); const int64_t n_tokens = batch.n_tokens; GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_out_ids->buffer)); From 711b0bcb11a6b102fd848c62109956a2b49adb3a Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Sun, 17 Mar 2024 20:41:21 -0400 Subject: [PATCH 11/25] llama : fix running a batch with n_outputs == 0 It previously worked because lctx.inp_out_ids was not initialized, so it pointed to some garbage address which was somehow still valid when I ran my tests. --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index b351779d9cbd7..6650cfc6036be 100644 --- a/llama.cpp +++ b/llama.cpp @@ -8979,7 +8979,7 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { ggml_backend_tensor_set(lctx.inp_pos, batch.pos, 0, n_tokens*ggml_element_size(lctx.inp_pos)); } - if (hparams.causal_attn || cparams.pooling_type == LLAMA_POOLING_TYPE_NONE) { + if (lctx.n_outputs > 0 && (hparams.causal_attn || cparams.pooling_type == LLAMA_POOLING_TYPE_NONE)) { GGML_ASSERT(lctx.inp_out_ids && "every model that can must skip unused outputs"); const int64_t n_tokens = batch.n_tokens; From d100502251e4f197c008216d93fc9af67bc02694 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Sun, 17 Mar 2024 22:04:42 -0400 Subject: [PATCH 12/25] llama : keep same graph topology even when n_outputs == 0 --- ggml.c | 19 ++++++++++++------- llama.cpp | 23 ----------------------- 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/ggml.c b/ggml.c index fa23cb3c44f87..4f6f3c3967eb9 100644 --- a/ggml.c +++ b/ggml.c @@ -2542,6 +2542,11 @@ static inline bool ggml_is_padded_1d(const struct ggml_tensor * tensor) { tensor->nb[3] == tensor->nb[2]*tensor->ne[2]; } +static inline bool ggml_is_empty(const struct ggml_tensor * tensor) { + // nb[3] depends on the previous nb and ne + return tensor->nb[3] == 0 || tensor->ne[3] == 0; +} + bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); @@ -2556,11 +2561,11 @@ bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor static inline bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); - return - (t1->ne[0]%t0->ne[0] == 0) && - (t1->ne[1]%t0->ne[1] == 0) && - (t1->ne[2]%t0->ne[2] == 0) && - (t1->ne[3]%t0->ne[3] == 0); + return ggml_is_empty(t0) || + ((t1->ne[0]%t0->ne[0] == 0) && + (t1->ne[1]%t0->ne[1] == 0) && + (t1->ne[2]%t0->ne[2] == 0) && + (t1->ne[3]%t0->ne[3] == 0)); } static inline bool ggml_can_repeat_rows(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { @@ -16047,7 +16052,7 @@ static void ggml_compute_forward_cross_entropy_loss_back( static void ggml_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) { GGML_ASSERT(params); - if (tensor->op == GGML_OP_NONE) { + if (tensor->op == GGML_OP_NONE || ggml_is_empty(tensor)) { return; } @@ -18011,7 +18016,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads, int n_cur_ { // FIXME: the cost of launching additional threads decreases performance with GPU offloading //n_tasks = MIN(n_threads, ggml_nelements(node->src[1])); - n_tasks = MIN(n_cur_threads, ggml_nelements(node->src[1])); + n_tasks = MIN(n_cur_threads, MAX(ggml_nelements(node->src[1]), 1)); } break; case GGML_OP_SCALE: case GGML_OP_SET: diff --git a/llama.cpp b/llama.cpp index 6650cfc6036be..f8fa83f17712c 100644 --- a/llama.cpp +++ b/llama.cpp @@ -5894,7 +5894,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); n_tokens = n_outputs; cur = ggml_get_rows(ctx0, cur, inp_out_ids); @@ -6082,7 +6081,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); @@ -6205,7 +6203,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); @@ -6308,7 +6305,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); @@ -6513,7 +6509,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); residual = ggml_get_rows(ctx0, residual, inp_out_ids); @@ -6610,7 +6605,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); @@ -6775,7 +6769,6 @@ struct llm_build_context { if (il == n_layer - 1 && pooling_type == LLAMA_POOLING_TYPE_NONE) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); @@ -6905,7 +6898,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); @@ -7011,7 +7003,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); @@ -7132,7 +7123,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); @@ -7246,7 +7236,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); @@ -7366,7 +7355,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); @@ -7492,7 +7480,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); @@ -7598,7 +7585,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); sa_out = ggml_get_rows(ctx0, sa_out, inp_out_ids); @@ -7699,7 +7685,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); @@ -7807,7 +7792,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); @@ -7924,7 +7908,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); @@ -8042,7 +8025,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); @@ -8173,7 +8155,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); @@ -8295,7 +8276,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); @@ -8416,7 +8396,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); @@ -8571,7 +8550,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); x = ggml_get_rows(ctx0, x, inp_out_ids); y = ggml_get_rows(ctx0, y, inp_out_ids); @@ -8683,7 +8661,6 @@ struct llm_build_context { if (il == n_layer - 1) { // skip computing output for unused tokens - if (n_outputs == 0) { return gf; } struct ggml_tensor * inp_out_ids = build_inp_out_ids(); cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); From 99c37ccb6b7e3568d4bca4aebe2c5cb9cd7b5e01 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Sun, 17 Mar 2024 23:23:30 -0400 Subject: [PATCH 13/25] ggml : saner ggml_can_repeat with empty tensors * ggml : future-proof ggml_is_empty by using GGML_MAX_DIMS - 1 --- ggml.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ggml.c b/ggml.c index 4f6f3c3967eb9..1028f18b8fe17 100644 --- a/ggml.c +++ b/ggml.c @@ -2543,8 +2543,9 @@ static inline bool ggml_is_padded_1d(const struct ggml_tensor * tensor) { } static inline bool ggml_is_empty(const struct ggml_tensor * tensor) { - // nb[3] depends on the previous nb and ne - return tensor->nb[3] == 0 || tensor->ne[3] == 0; + // the stride of the last dimension is zero if any of the previous dimensions has no elements + // but to be sure, the number of elements in the last dimension must also be checked + return tensor->nb[GGML_MAX_DIMS - 1] == 0 || tensor->ne[GGML_MAX_DIMS - 1] == 0; } bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { @@ -2561,11 +2562,11 @@ bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor static inline bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); - return ggml_is_empty(t0) || - ((t1->ne[0]%t0->ne[0] == 0) && - (t1->ne[1]%t0->ne[1] == 0) && - (t1->ne[2]%t0->ne[2] == 0) && - (t1->ne[3]%t0->ne[3] == 0)); + return ggml_is_empty(t0) ? ggml_is_empty(t1) : + (t1->ne[0]%t0->ne[0] == 0) && + (t1->ne[1]%t0->ne[1] == 0) && + (t1->ne[2]%t0->ne[2] == 0) && + (t1->ne[3]%t0->ne[3] == 0); } static inline bool ggml_can_repeat_rows(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { From 6bf7f3f41ca9956210a604a29dcfc131410803f0 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Mon, 18 Mar 2024 00:35:03 -0400 Subject: [PATCH 14/25] ggml : do not multi-thread ops returning empty tensors --- ggml.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ggml.c b/ggml.c index 1028f18b8fe17..ed2cbe30f83fc 100644 --- a/ggml.c +++ b/ggml.c @@ -17935,6 +17935,12 @@ static void ggml_graph_compute_perf_stats_node(struct ggml_tensor * node, const static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads, int n_cur_threads) { int n_tasks = 0; + if (ggml_is_empty(node)) { + // no need to multi-thread a no-op + n_tasks = 1; + return n_tasks; + } + switch (node->op) { case GGML_OP_CPY: case GGML_OP_DUP: @@ -18017,7 +18023,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads, int n_cur_ { // FIXME: the cost of launching additional threads decreases performance with GPU offloading //n_tasks = MIN(n_threads, ggml_nelements(node->src[1])); - n_tasks = MIN(n_cur_threads, MAX(ggml_nelements(node->src[1]), 1)); + n_tasks = MIN(n_cur_threads, ggml_nelements(node->src[1])); } break; case GGML_OP_SCALE: case GGML_OP_SET: From 09bb15a66aff90dc7db1d386f9e738e1bbfc4e7f Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Mon, 18 Mar 2024 20:21:02 -0400 Subject: [PATCH 15/25] ggml : make ggml_is_empty public and work with views --- ggml.c | 12 ++++++++---- ggml.h | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ggml.c b/ggml.c index ed2cbe30f83fc..94ae9b9ef73e9 100644 --- a/ggml.c +++ b/ggml.c @@ -2542,10 +2542,14 @@ static inline bool ggml_is_padded_1d(const struct ggml_tensor * tensor) { tensor->nb[3] == tensor->nb[2]*tensor->ne[2]; } -static inline bool ggml_is_empty(const struct ggml_tensor * tensor) { - // the stride of the last dimension is zero if any of the previous dimensions has no elements - // but to be sure, the number of elements in the last dimension must also be checked - return tensor->nb[GGML_MAX_DIMS - 1] == 0 || tensor->ne[GGML_MAX_DIMS - 1] == 0; +GGML_CALL bool ggml_is_empty(const struct ggml_tensor * tensor) { + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + if (tensor->ne[i] == 0) { + // empty if any dimension has no elements + return true; + } + } + return false; } bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { diff --git a/ggml.h b/ggml.h index c937d4a535adb..7bb870f79b707 100644 --- a/ggml.h +++ b/ggml.h @@ -744,6 +744,7 @@ extern "C" { GGML_API GGML_CALL bool ggml_is_transposed(const struct ggml_tensor * tensor); GGML_API GGML_CALL bool ggml_is_contiguous(const struct ggml_tensor * tensor); GGML_API GGML_CALL bool ggml_is_permuted (const struct ggml_tensor * tensor); + GGML_API GGML_CALL bool ggml_is_empty (const struct ggml_tensor * tensor); GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor); GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor); GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor); From 4551e7eba805d2b5c9e4625e1095b8abc4f44bd9 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Mon, 18 Mar 2024 20:51:32 -0400 Subject: [PATCH 16/25] llama : use a vector for ctx->output_ids * llama : rework reallocation logic for llama_output_reserve Now comparing the actual size with the new total size of the output buffer to allow more efficient enabling and disabling of the embeddings and/or logits output in the future. --- llama.cpp | 66 +++++++++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/llama.cpp b/llama.cpp index f8fa83f17712c..075b1937ab7c6 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2055,8 +2055,6 @@ struct llama_context { ggml_backend_free(backend); } - free(output_ids); - #ifdef GGML_USE_VULKAN ggml_vk_free_cpu_assist(); #endif @@ -2098,19 +2096,19 @@ struct llama_context { ggml_backend_buffer_t buf_output = nullptr; // decode output (2-dimensional array: [n_outputs][n_vocab]) - size_t logits_size = 0; // capacity (of floats) for logits - float * logits = nullptr; + size_t logits_size = 0; // capacity (of floats) for logits + float * logits = nullptr; - int32_t * output_ids = nullptr; // map token positions to ids of the logits and embd buffers - size_t output_size = 0; // capacity (of tokens positions) for the output buffers - int32_t n_outputs = 0; // number of actually-used outputs in the current ubatch + std::vector output_ids; // map batch token positions to ids of the logits and embd buffers + size_t output_size = 0; // capacity (of tokens positions) for the output buffers + int32_t n_outputs = 0; // number of actually-used outputs in the current ubatch bool logits_all = false; // embeddings output (2-dimensional array: [n_outputs][n_embd]) // populated only when pooling_type == LLAMA_POOLING_TYPE_NONE - size_t embd_size = 0; // capacity (of floats) for embeddings - float * embd = nullptr; + size_t embd_size = 0; // capacity (of floats) for embeddings + float * embd = nullptr; // sequence embeddings output (map of [n_embd] vectors) // populated only when pooling_type != LLAMA_POOLING_TYPE_NONE @@ -9179,32 +9177,29 @@ static void llama_output_reserve(llama_context & lctx, int32_t n_outputs) { const auto n_batch = cparams.n_batch; const auto n_vocab = hparams.n_vocab; const auto n_embd = hparams.n_embd; - const int64_t capacity = lctx.output_size; // TODO: use a per-batch flag for logits presence instead const bool has_logits = cparams.causal_attn; const bool has_embd = cparams.embeddings && (hparams.causal_attn || cparams.pooling_type == LLAMA_POOLING_TYPE_NONE); - if (!lctx.output_ids) { - // never resized afterwards - lctx.output_ids = (int32_t *) malloc(n_batch*sizeof(int32_t)); - if (lctx.output_ids == nullptr) { - throw std::runtime_error("failed to allocate output_ids buffer"); - } + const size_t logits_size = has_logits ? n_vocab*n_outputs_max : 0; + const size_t embd_size = has_embd ? n_embd*n_outputs_max : 0; + + if (lctx.output_ids.empty()) { + // init, never resized afterwards + lctx.output_ids.resize(n_batch); } - // alloc only when more than the current logits capacity is required - if (capacity < n_outputs_max) { - lctx.output_size = n_outputs_max; - lctx.logits_size = has_logits ? n_vocab*n_outputs_max : 0; - lctx.embd_size = has_embd ? n_embd*n_outputs_max : 0; - const size_t buf_output_size = (lctx.logits_size + lctx.embd_size)*sizeof(float); + const size_t prev_size = lctx.buf_output ? ggml_backend_buffer_get_size(lctx.buf_output) : 0; + const size_t new_size = (logits_size + embd_size) * sizeof(float); + // alloc only when more than the current capacity is required + // TODO: also consider shrinking the buffer + if (prev_size < new_size) { if (lctx.buf_output) { #ifndef NDEBUG // This doesn't happen often, but may be annoying in some cases (like the HellaSwag benchmark) - const size_t prev_size = ggml_backend_buffer_get_size(lctx.buf_output); - LLAMA_LOG_INFO("%s: reallocating output buffer from size %.02f MiB to %.02f MiB\n", __func__, prev_size / 1024.0 / 1024.0, buf_output_size / 1024.0 / 1024.0); + LLAMA_LOG_INFO("%s: reallocating output buffer from size %.02f MiB to %.02f MiB\n", __func__, prev_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0); #endif ggml_backend_buffer_free(lctx.buf_output); lctx.buf_output = nullptr; @@ -9212,18 +9207,21 @@ static void llama_output_reserve(llama_context & lctx, int32_t n_outputs) { lctx.embd = nullptr; } - lctx.buf_output = ggml_backend_buft_alloc_buffer(llama_default_buffer_type_cpu(true), buf_output_size); + lctx.buf_output = ggml_backend_buft_alloc_buffer(llama_default_buffer_type_cpu(true), new_size); if (lctx.buf_output == nullptr) { - throw std::runtime_error(format("failed to allocate output buffer of size %.2f MiB", buf_output_size / (1024.0 * 1024.0))); + throw std::runtime_error(format("failed to allocate output buffer of size %.2f MiB", new_size / (1024.0 * 1024.0))); } + } + float * output_base = (float *) ggml_backend_buffer_get_base(lctx.buf_output); - float * output_base = (float *) ggml_backend_buffer_get_base(lctx.buf_output); + lctx.output_size = n_outputs_max; + lctx.logits = has_logits ? output_base : nullptr; + lctx.embd = has_embd ? output_base + logits_size : nullptr; + lctx.logits_size = logits_size; + lctx.embd_size = embd_size; - lctx.logits = has_logits ? output_base : nullptr; - lctx.embd = has_embd ? output_base + lctx.logits_size : nullptr; - } - // set all ids as invalid (assume two's complement negative numbers) - memset(lctx.output_ids, -1, n_batch*sizeof(int32_t)); + // set all ids as invalid (negative) + std::fill(lctx.output_ids.begin(), lctx.output_ids.end(), -1); ggml_backend_buffer_clear(lctx.buf_output, 0); @@ -14151,8 +14149,8 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat // copy output ids { std::vector output_pos; - const size_t n_batch = ctx->cparams.n_batch; - const int32_t * output_ids = ctx->output_ids; + const size_t n_batch = ctx->cparams.n_batch; + const auto & output_ids = ctx->output_ids; output_pos.resize(ctx->output_size); From 8b826c5b086c94aaf09977bf41cfbf7ae32786ed Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Mon, 18 Mar 2024 21:12:53 -0400 Subject: [PATCH 17/25] ggml : skip empty tensors in all backends --- ggml-cuda.cu | 2 +- ggml-kompute.cpp | 4 ++++ ggml-metal.m | 4 ++++ ggml-opencl.cpp | 5 +++++ ggml-sycl.cpp | 2 +- ggml-vulkan.cpp | 2 +- 6 files changed, 16 insertions(+), 3 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index db595409aa735..9ad1e16d6459a 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -11354,7 +11354,7 @@ GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; - if (node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { continue; } diff --git a/ggml-kompute.cpp b/ggml-kompute.cpp index 4caf2c9e78b02..b0bb4729370cf 100644 --- a/ggml-kompute.cpp +++ b/ggml-kompute.cpp @@ -1430,6 +1430,10 @@ static void ggml_vk_graph_compute(struct ggml_kompute_context * ctx, struct ggml struct ggml_tensor * dst = gf->nodes[i]; GGML_ASSERT(dst->data != nullptr); + if (ggml_is_empty(dst)) { + continue; + } + switch (dst->op) { case GGML_OP_NONE: case GGML_OP_RESHAPE: diff --git a/ggml-metal.m b/ggml-metal.m index c3451a79b103f..1e0136230b987 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -832,6 +832,10 @@ static enum ggml_status ggml_metal_graph_compute( struct ggml_tensor * src2 = gf->nodes[i]->src[2]; struct ggml_tensor * dst = gf->nodes[i]; + if (ggml_is_empty(dst)) { + continue; + } + switch (dst->op) { case GGML_OP_NONE: case GGML_OP_RESHAPE: diff --git a/ggml-opencl.cpp b/ggml-opencl.cpp index aa73d67df84b0..b3f8b7eaf0a3b 100644 --- a/ggml-opencl.cpp +++ b/ggml-opencl.cpp @@ -2234,6 +2234,11 @@ static ggml_backend_buffer_type_t ggml_backend_opencl_get_default_buffer_type(gg static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) { for (int i = 0; i < graph->n_nodes; ++i) { ggml_tensor * node = graph->nodes[i]; + + if (ggml_is_empty(node)) { + continue; + } + switch (node->op) { case GGML_OP_MUL_MAT: ggml_cl_mul_mat(node->src[0], node->src[1], node, nullptr, 0); diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 6dc5eb20c5a63..62fe74f66f38e 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -17229,7 +17229,7 @@ GGML_CALL static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t back params.ith = 0; for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; - if (node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { continue; } #ifndef NDEBUG diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 698b31496f417..db53eaa4756ff 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -5566,7 +5566,7 @@ GGML_CALL static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backen for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; - if (node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { continue; } From d04cfaf2f5844455c8e773edf0d7f02919d3249c Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Mon, 18 Mar 2024 21:26:08 -0400 Subject: [PATCH 18/25] llama : fix llama_output_reserve nullptr deref when new_size is 0 --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 075b1937ab7c6..faf65e3390261 100644 --- a/llama.cpp +++ b/llama.cpp @@ -9195,7 +9195,7 @@ static void llama_output_reserve(llama_context & lctx, int32_t n_outputs) { // alloc only when more than the current capacity is required // TODO: also consider shrinking the buffer - if (prev_size < new_size) { + if (!lctx.buf_output || prev_size < new_size) { if (lctx.buf_output) { #ifndef NDEBUG // This doesn't happen often, but may be annoying in some cases (like the HellaSwag benchmark) From 8f70dcb0f3b40cf333a1652e6ebab16640636edd Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Tue, 19 Mar 2024 14:07:48 -0400 Subject: [PATCH 19/25] perplexity : make Winogrande work as it does on master The problems with the Winogrande implementation will need to be fixed in a separate PR to ease review. --- examples/perplexity/perplexity.cpp | 87 +++++++++++++----------------- 1 file changed, 36 insertions(+), 51 deletions(-) diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index 8ee09ccfbedd1..d83777411ddab 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -999,6 +999,8 @@ struct winogrande_entry { size_t i_logits; size_t common_prefix; size_t required_tokens; + size_t n_base1; // number of tokens for context + choice 1 + size_t n_base2; // number of tokens for context + choice 2 std::vector seq_tokens[2]; }; @@ -1038,38 +1040,6 @@ static std::vector load_winogrande_from_csv(const std::string& auto choice2 = line.substr(comma_pos[2]+1, comma_pos[3] - comma_pos[2] - 1); auto answer = line.substr(comma_pos[3]+1, line.size() - comma_pos[3] - 1); auto index = line.substr(0, comma_pos[0]); - if ('a' <= sentence[0] && sentence[0] <= 'z') { - // make the first letter a capital letter - sentence[0] -= 'a' - 'A'; - } - for (int i = 0; i < (int) sentence.size() - 1; ++i) { - // trim repeated spaces and spaces before punctuation - if (sentence[i] == ' ') { - char next = sentence[i+1]; - if (next == ' ' || next == ',' || next == '.' || next == '\'') { - char r[2] = { next, 0 }; - sentence.replace(i, 2, r); - --i; // stay at the same index for repeated spaces - } - } else if (sentence[i] == ',' || sentence[i] == '.') { - if (sentence[i] == sentence[i+1]) { - // trim repeated punctuation (forward to work at the end of sentences) - char r[2] = { sentence[i], 0 }; - sentence.replace(i, 2, r); - --i; // same index to then run the other checks on that punctuation - } else if (0 < i && sentence[i-1] == sentence[i]) { - // trim repeated punctuation (looks back to work with the space trim) - char r[2] = { sentence[i], 0 }; - sentence.replace(i-1, 2, r); - i -= 2; // go back because content was shifted - } else if (sentence[i+1] != ' ') { - // add missing space after punctuation - // (since the loop stops before the end, this adds no trailing space) - char r[3] = { sentence[i], ' ', 0 }; - sentence.replace(i, 1, r); - } - } - } int where = 0; for ( ; where < int(sentence.size()); ++where) { if (sentence[where] == '_') break; @@ -1106,6 +1076,8 @@ static std::vector load_winogrande_from_csv(const std::string& */ static void winogrande_score(llama_context * ctx, const gpt_params & params) { + constexpr int k_min_trailing_ctx = 3; + auto data = load_winogrande_from_csv(params.prompt); if (data.empty()) { fprintf(stderr, "%s: no tasks\n", __func__); @@ -1150,11 +1122,13 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) { task.common_prefix++; } + // TODO: the last token of each of the sequences don't need to be evaluated task.required_tokens = task.common_prefix + task.seq_tokens[0].size() - task.common_prefix + - task.seq_tokens[1].size() - task.common_prefix - // the last tokens don't need to be evaluated - - 2; + task.seq_tokens[1].size() - task.common_prefix; + + task.n_base1 = ::llama_tokenize(ctx, task.first + task.choices[0], add_bos).size(); + task.n_base2 = ::llama_tokenize(ctx, task.first + task.choices[1], add_bos).size(); } fprintf(stderr, "%s : calculating winogrande score over selected tasks.\n", __func__); @@ -1201,8 +1175,8 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) { n_logits += 1; for (int s = 0; s < 2; ++s) { - // end before the last token, no need to predict past the end of the sequences - for (size_t i = data[i1].common_prefix; i < data[i1].seq_tokens[s].size() - 1; ++i) { + // TODO: end before the last token, no need to predict past the end of the sequences + for (size_t i = data[i1].common_prefix; i < data[i1].seq_tokens[s].size(); ++i) { llama_batch_add(batch, data[i1].seq_tokens[s][i], i, { s0 + s }, true); n_logits += 1; } @@ -1234,20 +1208,23 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) { for (size_t i = i0; i < i1; ++i) { auto & task = data[i]; - // start from the end of the common prefix - size_t li = 0; - for (size_t j = task.common_prefix-1; j < task.seq_tokens[0].size()-1; ++j) { + const bool skip_choice = + task.seq_tokens[0].size() - task.common_prefix > k_min_trailing_ctx && + task.seq_tokens[1].size() - task.common_prefix > k_min_trailing_ctx; + + const auto& n_base1 = skip_choice ? task.n_base1 : task.common_prefix; + const int last_1st = task.seq_tokens[0].size() - n_base1 > 1 ? 1 : 0; + size_t li = n_base1 - task.common_prefix; + for (size_t j = n_base1-1; j < task.seq_tokens[0].size()-1-last_1st; ++j) { eval_pairs.emplace_back(task.i_logits + li++, task.seq_tokens[0][j+1]); } - // first token of the second choice is predicted by the end of the common prefix - eval_pairs.emplace_back(task.i_logits, task.seq_tokens[1][task.common_prefix]); - for (size_t j = task.common_prefix; j < task.seq_tokens[1].size()-1; ++j) { + const auto& n_base2 = skip_choice ? task.n_base2 : task.common_prefix; + const int last_2nd = task.seq_tokens[1].size() - n_base2 > 1 ? 1 : 0; + // FIXME: this uses the wrong first logits when not skipping the choice word + li = task.seq_tokens[0].size() - task.common_prefix + n_base2 - task.common_prefix; + for (size_t j = n_base2-1; j < task.seq_tokens[1].size()-1-last_2nd; ++j) { eval_pairs.emplace_back(task.i_logits + li++, task.seq_tokens[1][j+1]); } - if (i < i1 - 1) { - // make sure all logits have been processed as expected - GGML_ASSERT(task.i_logits + li == data[i+1].i_logits); - } } compute_logprobs(batch_logits.data(), n_vocab, workers, eval_pairs, eval_results); @@ -1255,17 +1232,25 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) { for (size_t i = i0; i < i1; ++i) { auto & task = data[i]; + const bool skip_choice = + task.seq_tokens[0].size() - task.common_prefix > k_min_trailing_ctx && + task.seq_tokens[1].size() - task.common_prefix > k_min_trailing_ctx; + float score_1st = 0; - for (size_t j = task.common_prefix-1; j < task.seq_tokens[0].size()-1; ++j) { + const auto& n_base1 = skip_choice ? task.n_base1 : task.common_prefix; + const int last_1st = task.seq_tokens[0].size() - n_base1 > 1 ? 1 : 0; + for (size_t j = n_base1-1; j < task.seq_tokens[0].size()-1-last_1st; ++j) { score_1st += eval_results[ir++]; } - score_1st /= (task.seq_tokens[0].size() - task.common_prefix); + score_1st /= (task.seq_tokens[0].size() - n_base1 - last_1st); float score_2nd = 0; - for (size_t j = task.common_prefix-1; j < task.seq_tokens[1].size()-1; ++j) { + const auto& n_base2 = skip_choice ? task.n_base2 : task.common_prefix; + const int last_2nd = task.seq_tokens[1].size() - n_base2 > 1 ? 1 : 0; + for (size_t j = n_base2-1; j < task.seq_tokens[1].size()-1-last_2nd; ++j) { score_2nd += eval_results[ir++]; } - score_2nd /= (task.seq_tokens[1].size() - task.common_prefix); + score_2nd /= (task.seq_tokens[1].size() - n_base2 - last_2nd); int result = score_1st > score_2nd ? 1 : 2; From 615a3a4a50765d9fbb895ddd34f943c8bc92ce7e Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Tue, 19 Mar 2024 15:01:21 -0400 Subject: [PATCH 20/25] llama : clearer error messages for invalid logits or embeddings ids * llama : assert all models that can have inp_out_ids Since the graph topology is now constant, this presence check can be done even when there are no outputs. * llama : assert logits and embd buffers exist before writing to them --- llama.cpp | 66 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/llama.cpp b/llama.cpp index faf65e3390261..157ddadd8cc33 100644 --- a/llama.cpp +++ b/llama.cpp @@ -8954,7 +8954,7 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { ggml_backend_tensor_set(lctx.inp_pos, batch.pos, 0, n_tokens*ggml_element_size(lctx.inp_pos)); } - if (lctx.n_outputs > 0 && (hparams.causal_attn || cparams.pooling_type == LLAMA_POOLING_TYPE_NONE)) { + if (hparams.causal_attn || cparams.pooling_type == LLAMA_POOLING_TYPE_NONE) { GGML_ASSERT(lctx.inp_out_ids && "every model that can must skip unused outputs"); const int64_t n_tokens = batch.n_tokens; @@ -9514,6 +9514,7 @@ static int llama_decode_internal( if (res) { ggml_backend_t backend_res = ggml_backend_sched_get_tensor_backend(lctx.sched, res); GGML_ASSERT(backend_res != nullptr); + GGML_ASSERT(lctx.logits != nullptr); float * logits_out = lctx.logits + n_outputs_prev*n_vocab; const int32_t n_outputs_new = lctx.n_outputs; @@ -9534,6 +9535,7 @@ static int llama_decode_internal( case LLAMA_POOLING_TYPE_NONE: { // extract token embeddings + GGML_ASSERT(lctx.embd != nullptr); float * embd_out = lctx.embd + n_outputs_prev*n_embd; const int32_t n_outputs_new = lctx.n_outputs; @@ -14623,20 +14625,33 @@ float * llama_get_logits(struct llama_context * ctx) { } float * llama_get_logits_ith(struct llama_context * ctx, int32_t i) { - const int32_t j = ctx->output_ids[i]; - llama_synchronize(ctx); - if (ctx->logits && 0 <= j && (size_t) j < ctx->output_size) { + try { + if (ctx->logits == nullptr) { + throw std::runtime_error("no logits"); + } + if ((size_t) i >= ctx->output_ids.size()) { + throw std::runtime_error(format("out of range [0, %lu)", ctx->output_ids.size())); + } + const int32_t j = ctx->output_ids[i]; + + if (j < 0) { + throw std::runtime_error(format("batch.logits[%d] != true", i)); + } + if ((size_t) j >= ctx->output_size) { + // This should not happen + throw std::runtime_error(format("corrupt output buffer (j=%d, output_size=%lu)", j, ctx->output_size)); + } + return ctx->logits + j*ctx->model.hparams.n_vocab; - } - LLAMA_LOG_ERROR("%s: invalid logits id %i, reason: %s (j=%i, output_size=%li)\n", - __func__, i, !ctx->logits ? "no logits" : j < 0 ? "batch.logits[i] wasn't true" : "too big", - j, ctx->output_size); + } catch (const std::exception & err) { + LLAMA_LOG_ERROR("%s: invalid logits id %d, reason: %s\n", __func__, i, err.what()); #ifndef NDEBUG - GGML_ASSERT(false); + GGML_ASSERT(false); #endif - return nullptr; + return nullptr; + } } float * llama_get_embeddings(struct llama_context * ctx) { @@ -14646,20 +14661,33 @@ float * llama_get_embeddings(struct llama_context * ctx) { } float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i) { - const int32_t j = ctx->output_ids[i]; - llama_synchronize(ctx); - if (ctx->embd && 0 <= j && (size_t) j < ctx->output_size) { + try { + if (ctx->embd == nullptr) { + throw std::runtime_error("no embeddings"); + } + if ((size_t) i >= ctx->output_ids.size()) { + throw std::runtime_error(format("out of range [0, %lu)", ctx->output_ids.size())); + } + const int32_t j = ctx->output_ids[i]; + + if (j < 0) { + throw std::runtime_error(format("batch.logits[%d] != true", i)); + } + if ((size_t) j >= ctx->output_size) { + // This should not happen + throw std::runtime_error(format("corrupt output buffer (j=%d, output_size=%lu)", j, ctx->output_size)); + } + return ctx->embd + j*ctx->model.hparams.n_embd; - } - LLAMA_LOG_ERROR("%s: invalid embeddings id %i, reason: %s (j=%i, output_size=%li)\n", - __func__, i, !ctx->embd ? "no embeddings" : j < 0 ? "batch.logits[i] wasn't true" : "too big", - j, ctx->output_size); + } catch (const std::exception & err) { + LLAMA_LOG_ERROR("%s: invalid embeddings id %d, reason: %s\n", __func__, i, err.what()); #ifndef NDEBUG - GGML_ASSERT(false); + GGML_ASSERT(false); #endif - return nullptr; + return nullptr; + } } float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id) { From 7d8d6b589f4758930aa8b903d07bb67efb7b3cd8 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Wed, 20 Mar 2024 22:23:46 -0400 Subject: [PATCH 21/25] llama : handle errors from llama_output_reserve at call sites --- llama.cpp | 57 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/llama.cpp b/llama.cpp index 157ddadd8cc33..301f18c2f5b47 100644 --- a/llama.cpp +++ b/llama.cpp @@ -9165,14 +9165,13 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { } } -// Only alloc when needed -static void llama_output_reserve(llama_context & lctx, int32_t n_outputs) { - GGML_ASSERT(0 <= n_outputs); - +// Make sure enough space is available for outputs. +// Returns max number of outputs for which space was reserved. +static size_t llama_output_reserve(llama_context & lctx, size_t n_outputs) { const auto & cparams = lctx.cparams; const auto & hparams = lctx.model.hparams; - const int32_t n_outputs_max = std::max((uint32_t) n_outputs, cparams.n_seq_max); + const size_t n_outputs_max = std::max(n_outputs, (size_t) cparams.n_seq_max); const auto n_batch = cparams.n_batch; const auto n_vocab = hparams.n_vocab; @@ -9209,7 +9208,8 @@ static void llama_output_reserve(llama_context & lctx, int32_t n_outputs) { lctx.buf_output = ggml_backend_buft_alloc_buffer(llama_default_buffer_type_cpu(true), new_size); if (lctx.buf_output == nullptr) { - throw std::runtime_error(format("failed to allocate output buffer of size %.2f MiB", new_size / (1024.0 * 1024.0))); + LLAMA_LOG_ERROR("%s: failed to allocate output buffer of size %.2f MiB\n", __func__, new_size / (1024.0 * 1024.0)); + return 0; } } float * output_base = (float *) ggml_backend_buffer_get_base(lctx.buf_output); @@ -9226,6 +9226,8 @@ static void llama_output_reserve(llama_context & lctx, int32_t n_outputs) { ggml_backend_buffer_clear(lctx.buf_output, 0); lctx.n_outputs = 0; + + return n_outputs_max; } @@ -9304,8 +9306,8 @@ static int llama_decode_internal( const int64_t n_embd = hparams.n_embd; const int64_t n_vocab = hparams.n_vocab; - int32_t n_outputs = 0; - int32_t n_outputs_prev = 0; + uint32_t n_outputs = 0; + uint32_t n_outputs_prev = 0; const auto n_ubatch = cparams.n_ubatch; @@ -9314,29 +9316,34 @@ static int llama_decode_internal( std::vector seq_id_arr; std::vector> seq_id; - // reserve output buffer + // count outputs if (batch_all.logits) { for (uint32_t i = 0; i < n_tokens_all; ++i) { n_outputs += batch_all.logits[i] != 0; } - llama_output_reserve(lctx, n_outputs); + } else if (lctx.logits_all || (cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE)) { + n_outputs = n_tokens_all; + } else { + // keep last output only + n_outputs = 1; + } + // reserve output buffer + if (llama_output_reserve(lctx, n_outputs) < n_outputs) { + LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_outputs); + return -2; + }; + // set output mappings + if (batch_all.logits) { int32_t i_logits = 0; for (uint32_t i = 0; i < n_tokens_all; ++i) { if (batch_all.logits[i]) { lctx.output_ids[i] = i_logits++; } } - } else if (lctx.logits_all || (cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE)) { - n_outputs = n_tokens_all; - llama_output_reserve(lctx, n_outputs); - for (uint32_t i = 0; i < n_tokens_all; ++i) { + } else { + for (uint32_t i = 0; i < n_outputs; ++i) { lctx.output_ids[i] = i; } - } else { - // keep last output only - n_outputs = 1; - llama_output_reserve(lctx, n_outputs); - lctx.output_ids[0] = 0; } for (uint32_t cur_token = 0; cur_token < n_tokens_all; cur_token += n_ubatch) { @@ -9362,7 +9369,7 @@ static int llama_decode_internal( for (uint32_t i = 0; i < n_tokens; i++) { n_outputs_new += u_batch.logits[i] != 0; } - } else if ((uint32_t) n_outputs == n_tokens_all) { + } else if (n_outputs == n_tokens_all) { n_outputs_new = n_tokens; } else { // keep last output only @@ -13513,11 +13520,9 @@ struct llama_context * llama_new_context_with_model( // graph outputs buffer { - // resized during inference when more than n_seq_max logits are requested in a batch - try { - llama_output_reserve(*ctx, 0); - } catch (const std::exception & err) { - LLAMA_LOG_ERROR("%s: error reserving logits buffer: %s\n", __func__, err.what()); + // resized during inference when a batch uses more outputs + if (llama_output_reserve(*ctx, params.n_seq_max) < params.n_seq_max) { + LLAMA_LOG_ERROR("%s: failed to reserve initial output buffer\n", __func__); llama_free(ctx); return nullptr; } @@ -14299,7 +14304,7 @@ size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { memcpy(&n_outputs, inp, sizeof(n_outputs)); inp += sizeof(n_outputs); - llama_output_reserve(*ctx, n_outputs); + GGML_ASSERT(n_outputs <= llama_output_reserve(*ctx, n_outputs)); if (n_outputs) { output_pos.resize(n_outputs); From 5f33a675ca426c03adecad2f76099ee3217221d1 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Wed, 20 Mar 2024 22:48:19 -0400 Subject: [PATCH 22/25] perplexity : make hellaswag and multiple-choice outputs identical to master Due to how the KV cache is updated, the logprobs for tokens in a batch are very slightly affected by the other tokens present in the batch, so to make hellaswag and multiple-choice return exactly the same results as on master, the last token of each sequence needs to be evaluated even though its output is not used at all. This will probably be changed back in the future to make these benchmarks a tiny bit faster. * perplexity : fix division by zero when using less than 100 multiple-choice tasks --- examples/perplexity/perplexity.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index d83777411ddab..a65d9cd0bb140 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -832,9 +832,7 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) { hs_cur.seq_tokens[0].size() - hs_cur.common_prefix + hs_cur.seq_tokens[1].size() - hs_cur.common_prefix + hs_cur.seq_tokens[2].size() - hs_cur.common_prefix + - hs_cur.seq_tokens[3].size() - hs_cur.common_prefix - // the last tokens don't need to be evaluated - - 4; + hs_cur.seq_tokens[3].size() - hs_cur.common_prefix; //GGML_ASSERT(hs_cur.common_prefix >= ::llama_tokenize(ctx, hs_cur.context, add_bos).size()); @@ -895,10 +893,12 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) { n_logits += 1; for (int s = 0; s < 4; ++s) { - // end before the last token, no need to predict past the end of the sequences - for (size_t i = hs_cur.common_prefix; i < hs_cur.seq_tokens[s].size() - 1; ++i) { - llama_batch_add(batch, hs_cur.seq_tokens[s][i], i, { s0 + s }, true); - n_logits += 1; + const size_t seq_tokens_size = hs_cur.seq_tokens[s].size(); + // TODO: don't evaluate the last token of each sequence + for (size_t i = hs_cur.common_prefix; i < seq_tokens_size; ++i) { + const bool needs_logits = i < seq_tokens_size - 1; + llama_batch_add(batch, hs_cur.seq_tokens[s][i], i, { s0 + s }, needs_logits); + n_logits += needs_logits; } } @@ -1359,8 +1359,6 @@ static bool multiple_choice_prepare_one_task(llama_context * ctx, bool add_bos, for (auto& seq : task.seq_tokens) { task.required_tokens += seq.size() - task.common_prefix; } - // the last tokens don't need to be evaluated - task.required_tokens -= task.seq_tokens.size(); return true; } @@ -1474,7 +1472,7 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & params return; } } else { - int n_dot = n_task/100; + int n_dot = std::max((int) n_task/100, 1); int i_task = 0; for (auto& task : tasks) { ++i_task; @@ -1549,10 +1547,12 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & params n_logits += 1; for (int s = 0; s < int(cur_task.seq_tokens.size()); ++s) { - // end before the last token, no need to predict past the end of the sequences - for (size_t i = cur_task.common_prefix; i < cur_task.seq_tokens[s].size() - 1; ++i) { - llama_batch_add(batch, cur_task.seq_tokens[s][i], i, { s0 + s }, true); - n_logits += 1; + const size_t seq_tokens_size = cur_task.seq_tokens[s].size(); + // TODO: don't evaluate the last token of each sequence + for (size_t i = cur_task.common_prefix; i < seq_tokens_size; ++i) { + const bool needs_logits = i < seq_tokens_size - 1; + llama_batch_add(batch, cur_task.seq_tokens[s][i], i, { s0 + s }, needs_logits); + n_logits += needs_logits; } } From e9095aca209c968afe09654c428223c896abee65 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Mon, 25 Mar 2024 23:13:50 -0400 Subject: [PATCH 23/25] llama : allow loading state saved with a different ctx size When loading a session file, the context size is now only required to be at least enough to load the KV cells contained in that session file, instead of requiring to use exactly the same context size as when saving. Doing this enables the use-case of extending or shrinking the context size of a saved session. This breaks existing session files because the meaning of kv_buf_size is slightly changed (previously it was the size of the whole KV cache, now it's only the size of the saved part of it). This allows for finer-grained sanity checks when loading in an effort to keep kv_buf_size useful even when the kv_size is changed. --- llama.cpp | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/llama.cpp b/llama.cpp index 30b7788741ca7..18116b9628dad 100644 --- a/llama.cpp +++ b/llama.cpp @@ -14714,9 +14714,10 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa() + hparams.n_embd_k_s(); const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa() + hparams.n_embd_v_s(); - const size_t kv_buf_size = kv_self.total_size(); + // NOTE: kv_size and kv_buf_size are mostly used for sanity checks const uint32_t kv_head = llama_kv_cache_cell_max(kv_self); const uint32_t kv_size = kv_self.size; + const size_t kv_buf_size = kv_self.total_size() / (kv_size ? kv_size : 1) * kv_head; const uint32_t kv_used = kv_self.used; data_ctx->write(&kv_buf_size, sizeof(kv_buf_size)); @@ -14725,6 +14726,7 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat data_ctx->write(&kv_used, sizeof(kv_used)); if (kv_buf_size) { + const size_t pre_kv_buf_size = data_ctx->get_size_written(); std::vector tmp_buf; for (int il = 0; il < (int) n_layer; ++il) { const size_t k_size = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*kv_head); @@ -14754,6 +14756,7 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat data_ctx->write(tmp_buf.data(), tmp_buf.size()); } } + GGML_ASSERT(kv_buf_size == data_ctx->get_size_written() - pre_kv_buf_size); } for (uint32_t i = 0; i < kv_head; ++i) { @@ -14867,8 +14870,18 @@ size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { memcpy(&kv_size, inp, sizeof(kv_size)); inp += sizeof(kv_size); memcpy(&kv_used, inp, sizeof(kv_used)); inp += sizeof(kv_used); + if (kv_self.size != kv_size) { + // the KV cache needs to be big enough to load all the KV cells from the saved state + GGML_ASSERT(kv_self.size >= kv_head); + + LLAMA_LOG_INFO("%s: state contains %d KV cells, was saved with kv_size=%d, but is loaded with kv_size=%d (fine, but different)\n", + __func__, kv_head, kv_size, kv_self.size); + } + if (kv_buf_size) { - GGML_ASSERT(kv_self.total_size() == kv_buf_size); + const size_t pre_kv_buf_size = inp - src; + + GGML_ASSERT(kv_self.total_size() >= kv_buf_size); for (int il = 0; il < (int) n_layer; ++il) { const size_t k_size = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*kv_head); @@ -14888,23 +14901,21 @@ size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { // v is not contiguous, copy row by row const size_t v_row_size = ggml_row_size(kv_self.v_l[il]->type, kv_head); - const size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, kv_size); + const size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, kv_self.size); for (int ir = 0; ir < (int) n_embd_v_gqa; ++ir) { ggml_backend_tensor_set(kv_self.v_l[il], inp, ir*v_row_stride, v_row_size); inp += v_row_size; } } + GGML_ASSERT(kv_buf_size == inp - src - pre_kv_buf_size); } - GGML_ASSERT(kv_self.size == kv_size); + llama_kv_cache_clear(ctx); ctx->kv_self.head = kv_head; - ctx->kv_self.size = kv_size; ctx->kv_self.used = kv_used; - ctx->kv_self.cells.resize(kv_size); - for (uint32_t i = 0; i < kv_head; ++i) { llama_pos pos; size_t seq_id_size; @@ -14921,11 +14932,6 @@ size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { ctx->kv_self.cells[i].seq_id.insert(seq_id); } } - - for (uint32_t i = kv_head; i < kv_size; ++i) { - ctx->kv_self.cells[i].pos = -1; - ctx->kv_self.cells[i].seq_id.clear(); - } } const size_t nread = inp - src; From 5027d81f0a26fd72ceed7b02deb959c1c19ca943 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 26 Mar 2024 08:49:49 +0200 Subject: [PATCH 24/25] llama : minor ggml-ci --- examples/perplexity/perplexity.cpp | 13 ++++++++----- llama.cpp | 23 ++++++++++++++++------- llama.h | 4 ++-- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index a65d9cd0bb140..c70385c62bb07 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -552,6 +552,7 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par for (int j = 0; j < num_batches; ++j) { const int batch_start = start + j * n_batch; const int batch_size = std::min(end - batch_start, n_batch); + int n_outputs = 0; batch.n_tokens = 0; @@ -568,11 +569,12 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par for (int k = 0; k < batch_size; ++k) { const int idx = seq*n_ctx + k; - batch.token[idx] = tokens[seq_start + k]; - batch.pos[idx] = j*n_batch + k; - batch.n_seq_id[idx] = 1; - batch.seq_id[idx][0] = seq; - batch.logits[idx] = batch.pos[idx] >= first ? 1 : 0; + batch.token [idx] = tokens[seq_start + k]; + batch.pos [idx] = j*n_batch + k; + batch.n_seq_id[idx] = 1; + batch.seq_id [idx][0] = seq; + batch.logits [idx] = batch.pos[idx] >= first ? 1 : 0; + n_outputs += batch.logits[idx] != 0; } batch.n_tokens += batch_size; @@ -608,6 +610,7 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par for (int seq = 0; seq < n_seq_batch; seq++) { const float * all_logits = num_batches > 1 ? logits.data() : llama_get_logits_ith(ctx, seq*n_ctx + first); + llama_token * tokens_data = tokens.data() + start + seq*n_ctx + first; if (!params.logits_file.empty()) { process_logits(logits_stream, n_vocab, all_logits, diff --git a/llama.cpp b/llama.cpp index 18116b9628dad..5cf2dbf364281 100644 --- a/llama.cpp +++ b/llama.cpp @@ -6103,6 +6103,7 @@ struct llm_build_context { struct ggml_cgraph * build_llama() { struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + // mutable variable, needed during the last layer of the computation to skip unused tokens int32_t n_tokens = this->n_tokens; const int64_t n_embd_head = hparams.n_embd_head_v; @@ -6532,6 +6533,7 @@ struct llm_build_context { struct ggml_cgraph * build_grok() { struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + // mutable variable, needed during the last layer of the computation to skip unused tokens int32_t n_tokens = this->n_tokens; const int64_t n_embd_head = hparams.n_embd_head_v; @@ -9467,7 +9469,7 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { data[n_outputs++] = i; } } - // the graph needs the have been passed the correct number of outputs + // the graph needs to have been passed the correct number of outputs GGML_ASSERT(lctx.n_outputs == n_outputs); } else if (lctx.n_outputs == 1) { // only keep last output @@ -9707,11 +9709,13 @@ static size_t llama_output_reserve(llama_context & lctx, size_t n_outputs) { return 0; } } + float * output_base = (float *) ggml_backend_buffer_get_base(lctx.buf_output); - lctx.output_size = n_outputs_max; lctx.logits = has_logits ? output_base : nullptr; lctx.embd = has_embd ? output_base + logits_size : nullptr; + + lctx.output_size = n_outputs_max; lctx.logits_size = logits_size; lctx.embd_size = embd_size; @@ -9822,11 +9826,13 @@ static int llama_decode_internal( // keep last output only n_outputs = 1; } + // reserve output buffer if (llama_output_reserve(lctx, n_outputs) < n_outputs) { LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_outputs); return -2; }; + // set output mappings if (batch_all.logits) { int32_t i_logits = 0; @@ -10022,8 +10028,8 @@ static int llama_decode_internal( const int32_t n_outputs_new = lctx.n_outputs; if (n_outputs_new) { - GGML_ASSERT(n_outputs_prev+n_outputs_new <= n_outputs); - GGML_ASSERT((n_outputs_prev+n_outputs_new)*n_vocab <= (int64_t) lctx.logits_size); + GGML_ASSERT( n_outputs_prev + n_outputs_new <= n_outputs); + GGML_ASSERT((n_outputs_prev + n_outputs_new)*n_vocab <= (int64_t) lctx.logits_size); ggml_backend_tensor_get_async(backend_res, res, logits_out, 0, n_outputs_new*n_vocab*sizeof(float)); } } @@ -10042,8 +10048,8 @@ static int llama_decode_internal( const int32_t n_outputs_new = lctx.n_outputs; if (n_outputs_new) { - GGML_ASSERT(n_outputs_prev+n_outputs_new <= n_outputs); - GGML_ASSERT((n_outputs_prev+n_outputs_new)*n_embd <= (int64_t) lctx.embd_size); + GGML_ASSERT( n_outputs_prev + n_outputs_new <= n_outputs); + GGML_ASSERT((n_outputs_prev + n_outputs_new)*n_embd <= (int64_t) lctx.embd_size); ggml_backend_tensor_get_async(backend_embd, embd, embd_out, 0, n_outputs_new*n_embd*sizeof(float)); } } break; @@ -14541,6 +14547,7 @@ void llama_kv_cache_update(struct llama_context * ctx) { size_t llama_get_state_size(const struct llama_context * ctx) { const auto & cparams = ctx->cparams; const auto & hparams = ctx->model.hparams; + // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state. // for reference, std::mt19937(1337) serializes to 6701 bytes. const size_t s_rng_size = sizeof(size_t); @@ -14639,7 +14646,7 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat std::ostringstream rng_ss; rng_ss << ctx->rng; - const std::string & rng_str = rng_ss.str(); + const std::string & rng_str = rng_ss.str(); const size_t rng_size = rng_str.size(); GGML_ASSERT(rng_size <= LLAMA_MAX_RNG_STATE); @@ -14657,6 +14664,7 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat // copy output ids { std::vector output_pos; + const size_t n_batch = ctx->cparams.n_batch; const auto & output_ids = ctx->output_ids; @@ -14727,6 +14735,7 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat if (kv_buf_size) { const size_t pre_kv_buf_size = data_ctx->get_size_written(); + std::vector tmp_buf; for (int il = 0; il < (int) n_layer; ++il) { const size_t k_size = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*kv_head); diff --git a/llama.h b/llama.h index cd14c4311bd8d..c77d1af36ccb0 100644 --- a/llama.h +++ b/llama.h @@ -677,7 +677,7 @@ extern "C" { // Token logits obtained from the last call to llama_decode() // The logits for which llama_batch.logits[i] != 0 are stored contiguously - // in the order they have in the batch. + // in the order they have appeared in the batch. // Rows: number of tokens for which llama_batch.logits[i] != 0 // Cols: n_vocab LLAMA_API float * llama_get_logits(struct llama_context * ctx); @@ -690,7 +690,7 @@ extern "C" { // Get all output token embeddings. // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model, // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously - // in the order they have in the batch. + // in the order they have appeared in the batch. // shape: [n_outputs*n_embd] // Otherwise, returns NULL. LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); From 20248e80cd3eae1360d1ec8691b1ceb3282726c1 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Tue, 26 Mar 2024 10:28:19 -0400 Subject: [PATCH 25/25] readme : update recent API changes, and warn about Vulkan --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index f9cf1961629d0..06d6eeef5334c 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) ### Recent API changes +- [2024 Mar 26] Logits and embeddings API updated for compactness https://github.com/ggerganov/llama.cpp/pull/6122 - [2024 Mar 13] Add `llama_synchronize()` + `llama_context_params.n_ubatch` https://github.com/ggerganov/llama.cpp/pull/6017 - [2024 Mar 8] `llama_kv_cache_seq_rm()` returns a `bool` instead of `void`, and new `llama_n_seq_max()` returns the upper limit of acceptable `seq_id` in batches (relevant when dealing with multiple sequences) https://github.com/ggerganov/llama.cpp/pull/5328 - [2024 Mar 4] Embeddings API updated https://github.com/ggerganov/llama.cpp/pull/5796 @@ -633,6 +634,15 @@ Building the program with BLAS support may lead to some performance improvements - #### Vulkan +> [!WARNING] +> +> Vulkan support has been broken in https://github.com/ggerganov/llama.cpp/pull/6122 +> due to relying on `GGML_OP_GET_ROWS` which is not yet properly supported by the Vulkan backend, +> but should be fixed relatively soon (possibly in https://github.com/ggerganov/llama.cpp/pull/6155 +> (ref: https://github.com/ggerganov/llama.cpp/pull/6122#issuecomment-2015327635)). +> +> Meanwhile, if you want to use the Vulkan backend, you should use the commit right before the breaking change, https://github.com/ggerganov/llama.cpp/commit/55c1b2a3bbd470e9e2a3a0618b92cf64a885f806 + **With docker**: You don't need to install Vulkan SDK. It will be installed inside the container.