fix make_dummy_tokenizer

okdshin · Nov 19, 2023 · 4a0ef75 · 4a0ef75
1 parent 2fac127
commit 4a0ef75
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/make_dummy_tokenizer.py b/make_dummy_tokenizer.py
@@ -1,2 +1,4 @@
 import sentencepiece as spm
+from pathlib import Path
+Path("./dummy_tokenizer").mkdir(exist_ok=True)
 spm.SentencePieceTrainer.train(input="dummy_file", model_prefix='dummy_tokenizer/tokenizer', vocab_size=51200, byte_fallback=True)