diff --git a/make_dummy_tokenizer.py b/make_dummy_tokenizer.py index 241b6fd..bca16de 100644 --- a/make_dummy_tokenizer.py +++ b/make_dummy_tokenizer.py @@ -1,2 +1,4 @@ import sentencepiece as spm +from pathlib import Path +Path("./dummy_tokenizer").mkdir(exist_ok=True) spm.SentencePieceTrainer.train(input="dummy_file", model_prefix='dummy_tokenizer/tokenizer', vocab_size=51200, byte_fallback=True)