bugfix for IS_ALPHA

huggingface · Jan 30, 2025 · b9fb72a · b9fb72a
1 parent b3daef2
commit b9fb72a
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/src/datatrove/utils/word_tokenizers.py b/src/datatrove/utils/word_tokenizers.py
@@ -136,9 +136,9 @@ def _do_tokenize(self, text: str):
         self.tokenizer.max_length = len(text)
         try:
             return [self.tokenizer(t, disable=["parser", "tagger", "ner"]) for t in texts]
-        except KeyError as e:
+        except Exception as e:
             # this dumb string breaks the tokenizer completely
-            if "IS_ALPHA" in str(e):
+            if "IS_ALPHA" in text:
                 return [self.tokenizer(t.replace("IS_ALPHA", ""), disable=["parser", "tagger", "ner"]) for t in texts]
             else:
                 raise e