undo commit for fastembed

michaelfeil · Feb 27, 2024 · 91b1edf · 91b1edf
1 parent e7905c2
commit 91b1edf
Showing 1 changed file with 0 additions and 1 deletion.
diff --git a/libs/infinity_emb/infinity_emb/transformer/embedder/fastembed.py b/libs/infinity_emb/infinity_emb/transformer/embedder/fastembed.py
@@ -72,6 +72,5 @@ def encode_post(self, embedding: np.ndarray) -> EmbeddingReturnType:
     def tokenize_lengths(self, sentences: List[str]) -> List[int]:
         tks = self._infinity_tokenizer.encode_batch(
             sentences,
-            truncation=True,
         )
         return [len(t.tokens) for t in tks]