Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fixes #7999 (adds control vectors to all build_XXX() functions in llama.cpp [needs testing] #8060

Merged
merged 8 commits into from
Jun 25, 2024
Merged

fixes #7999 (adds control vectors to all build_XXX() functions in llama.cpp [needs testing] #8060

Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8e09261
fixes #7999
jukofyork Jun 22, 2024
af2e033
Fixes qwen2 too
jukofyork Jun 22, 2024
2c42c51
Merge branch 'ggerganov:master' into jukofyork-command_r-control-vect…
jukofyork Jun 22, 2024
f57ebf2
Fixed all models' control vectors
jukofyork Jun 22, 2024
aac1e66
Merge branch 'ggerganov:master' into jukofyork-command_r-control-vect…
jukofyork Jun 22, 2024
f393795
Removed double calls to `cb(cur, "l_out", il)`
jukofyork Jun 22, 2024
ed90e43
Merge branch 'ggerganov:master' into jukofyork-command_r-control-vect…
jukofyork Jun 25, 2024
a5a5319
Moved control vector logic to llama_control_vector:apply_to()
jukofyork Jun 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 73 additions & 39 deletions llama.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2295,13 +2295,21 @@ struct llama_control_vector {
int32_t layer_start = -1;
int32_t layer_end = -1;

ggml_tensor * tensor_for(int il) const {
struct ggml_tensor * tensor_for(int il) const {
if (il < 0 || il < layer_start || il > layer_end || (size_t) il >= tensors.size()) {
return nullptr;
}
return tensors[il];
}

struct ggml_tensor * apply_to(struct ggml_context * ctx, struct ggml_tensor * cur, int il) const {
ggml_tensor * layer_dir = tensor_for(il);
if (layer_dir != nullptr) {
cur = ggml_add(ctx, cur, layer_dir);
}
return cur;
}

~llama_control_vector() {
for (struct ggml_context * ctx : ctxs) {
ggml_free(ctx);
Expand Down Expand Up @@ -7901,10 +7909,7 @@ struct llm_build_context {
cur = ggml_add(ctx0, cur, ffn_inp);
cb(cur, "ffn_out", il);

ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
if (layer_dir != nullptr) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -8019,6 +8024,7 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -8123,6 +8129,7 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -8238,9 +8245,8 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
cb(cur, "l_out", il);

cur = ggml_add(ctx0, cur, inpL);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -8392,10 +8398,7 @@ struct llm_build_context {
cur = ggml_add(ctx0, cur, ffn_inp);
cb(cur, "ffn_out", il);

ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
if (layer_dir != nullptr) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -8526,10 +8529,7 @@ struct llm_build_context {
cur = ggml_add(ctx0, cur, ffn_inp);
cb(cur, "ffn_out", il);

ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
if (layer_dir != nullptr) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -8635,8 +8635,12 @@ struct llm_build_context {
cb(cur, "ffn_out", il);
}

inpL = ggml_add(ctx0, cur, ffn_inp);
cb(inpL, "l_out", il);
cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
inpL = cur;
}

cur = llm_build_norm(ctx0, inpL, hparams,
Expand Down Expand Up @@ -8724,6 +8728,7 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -9019,8 +9024,12 @@ struct llm_build_context {
cb(cur, "ffn_out", il);
}

inpL = ggml_add(ctx0, cur, ffn_inp);
cb(inpL, "l_out", il);
cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
inpL = cur;
}

cur = llm_build_norm(ctx0, inpL, hparams,
Expand Down Expand Up @@ -9154,6 +9163,7 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -9302,6 +9312,7 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -9414,6 +9425,7 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -9525,6 +9537,7 @@ struct llm_build_context {
cb(cur, "ffn_out", il);

cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -9670,6 +9683,7 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -9790,11 +9804,11 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_output);
cb(cur, "l_out", il);

cur = ggml_add(ctx0, cur, inpL);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
inpL = cur;
}

Expand Down Expand Up @@ -9926,8 +9940,10 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, residual, cur);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
inpL = cur;
}

Expand Down Expand Up @@ -10026,9 +10042,8 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, sa_out);
cb(cur, "l_out", il);

cur = ggml_add(ctx0, cur, inpL);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -10134,8 +10149,12 @@ struct llm_build_context {
cb(cur, "ffn_out", il);
}

inpL = ggml_add(ctx0, cur, ffn_inp);
cb(inpL, "l_out", il);
cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
inpL = cur;
}

cur = llm_build_norm(ctx0, inpL, hparams,
Expand Down Expand Up @@ -10241,8 +10260,12 @@ struct llm_build_context {
cb(cur, "ffn_out", il);
}

inpL = ggml_add(ctx0, cur, ffn_inp);
cb(inpL, "l_out", il);
cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
inpL = cur;
}

cur = llm_build_norm(ctx0, inpL, hparams,
Expand Down Expand Up @@ -10354,6 +10377,7 @@ struct llm_build_context {
cb(cur, "ffn_out", il);

cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -10471,6 +10495,7 @@ struct llm_build_context {
cb(cur, "ffn_out", il);

cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -10612,6 +10637,7 @@ struct llm_build_context {
cb(cur, "hidden_scaled_ffn", -1);

cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -10724,6 +10750,7 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, sa_out);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -10840,7 +10867,9 @@ struct llm_build_context {
NULL,
LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);
cb(cur, "ffn_out", il);

cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -10989,6 +11018,7 @@ struct llm_build_context {

// residual
cur = ggml_add(ctx0, cur, inpL);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -11130,6 +11160,7 @@ struct llm_build_context {
// add together residual + FFN + self-attention
cur = ggml_add(ctx0, cur, inpL);
cur = ggml_add(ctx0, cur, attn_out);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -11265,10 +11296,7 @@ struct llm_build_context {
cur = ggml_add(ctx0, cur, ffn_inp);
cb(cur, "ffn_out", il);

ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
if (layer_dir != nullptr) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -11382,8 +11410,12 @@ struct llm_build_context {
cur = ggml_add(ctx0, cur, inpL);
cb(cur, "ffn_out", il);

inpL = ggml_add(ctx0, cur, attn_out);
cb(inpL, "l_out", il);
cur = ggml_add(ctx0, cur, attn_out);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
inpL = cur;
} else {
// attention and ffn are computed sequentially
// x = x + attn(ln1(x))
Expand All @@ -11406,8 +11438,12 @@ struct llm_build_context {
LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);
cb(cur, "ffn_out", il);

inpL = ggml_add(ctx0, cur, ffn_inp);
cb(inpL, "l_out", il);
cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
inpL = cur;
}
}

Expand Down Expand Up @@ -11534,10 +11570,7 @@ struct llm_build_context {
cur = ggml_add(ctx0, cur, ffn_out);
cb(cur, "ffn_out", il);

ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
if (layer_dir != nullptr) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -11770,6 +11803,7 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);

// input for next layer
Expand Down
Loading