Skip adding special tokens to targets

jncraton · Nov 18, 2024 · 54c7257 · 54c7257
1 parent 3f5a7fd
commit 54c7257
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/changelog.md b/changelog.md
@@ -5,6 +5,7 @@
 ### Fixed
 
 - Properly apply prompt format when providing `choices`
+- Do not add special tokens before `choices`
 
 ## 0.22 - 2024-11-02
 

diff --git a/languagemodels/inference.py b/languagemodels/inference.py
@@ -284,7 +284,7 @@ def rank_instruct(inputs, targets):
     fmt = model_info.get("prompt_fmt", "{instruction}")
     inputs = [fmt.replace("{instruction}", inst) for inst in inputs]
 
-    targ_tok = [tokenizer.encode(t).tokens for t in targets]
+    targ_tok = [tokenizer.encode(t, add_special_tokens=False).tokens for t in targets]
     targ_tok *= len(inputs)
 
     in_tok = []