Fix issue causing choices to be scored improperly

changelog.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.18.0
+
+### Fixed
+
+- Correct issue causing `choices` to be scored improperly
+
 ## 0.17.0 - 2024-02-15
 
 ### Added

languagemodels/__init__.py

@@ -83,6 +83,14 @@ def do(prompt, choices=None):
 
     >>> do(["Say red", "Say blue"], choices=["red", "blue"])
     ['red', 'blue']
+
+    >>> do("Classify as positive or negative: LLMs are bad",
+    ... choices=["Positive", "Negative"])
+    'Negative'
+
+    >>> do("Classify as positive or negative: LLMs are great",
+    ... choices=["Positive", "Negative"])
+    'Positive'
     """
 
     prompts = [prompt] if isinstance(prompt, str) else prompt

languagemodels/inference.py

@@ -196,12 +196,12 @@ def rank_instruct(inputs, targets):
     """
     tokenizer, model = get_model("instruct")
 
-    targ_tok = [tokenizer.encode(t, add_special_tokens=False).tokens for t in targets]
+    targ_tok = [tokenizer.encode(t).tokens for t in targets]
     targ_tok *= len(inputs)
 
     in_tok = []
     for input in inputs:
-        toks = [tokenizer.encode(input, add_special_tokens=False).tokens]
+        toks = [tokenizer.encode(input).tokens]
         in_tok += toks * len(targets)
 
     if "Generator" in str(type(model)):