Fix cohere tokenizer (#307)

pinecone-io · Feb 26, 2024 · 580dc2c · 580dc2c
1 parent 50414f5
commit 580dc2c
Showing 1 changed file with 23 additions and 31 deletions.
diff --git a/tests/unit/tokenizer/test_cohere_hf_tokenizer.py b/tests/unit/tokenizer/test_cohere_hf_tokenizer.py
@@ -20,37 +20,29 @@ def tokenizer():
     @staticmethod
     @pytest.fixture
     def expected_tokens(text):
-        return [
-            'string',
-            'Ġwith',
-            'Ġspecial',
-            'Ġcharacters',
-            'Ġlike',
-            'Ġ!',
-            '@',
-            '#',
-            '$',
-            '%',
-            '^',
-            '&',
-            '*',
-            '()',
-            '_',
-            '+',
-            'Ġæ',
-            'Ĺ',
-            '¥',
-            'æľ¬',
-            'Ġspaces',
-            'ĠĠĠ',
-            'ĊĠ',
-            'ĊĊ',
-            'ĠCASE',
-            'Ġc',
-            'A',
-            'se',
-            'Ġ',
-        ]
+        return ['string',
+                'Ġwith',
+                'Ġspecial',
+                'Ġcharacters',
+                'Ġlike',
+                'Ġ!',
+                '@',
+                '#$',
+                '%^',
+                '&',
+                '*',
+                '()',
+                '_',
+                '+',
+                'ĠæĹ¥æľ¬',
+                'Ġspaces',
+                'ĠĠĠ',
+                'ĊĠĊĊ',
+                'ĠCASE',
+                'Ġc',
+                'A',
+                'se',
+                'Ġ']
 
     @staticmethod
     def test_messages_token_count(tokenizer):