diff --git a/scripts/train_tokenizer.py b/scripts/train_tokenizer.py index 9faf3fc8..335de573 100644 --- a/scripts/train_tokenizer.py +++ b/scripts/train_tokenizer.py @@ -2,10 +2,9 @@ import os from datasets import load_dataset -from tqdm.auto import tqdm +from tqdm.auto import tqdm, trange from transformers import LlamaTokenizerFast from tokenizers import SentencePieceBPETokenizer -from tqdm import trange from delphi.train.tokenizer import train_vocab, get_tokenizer_model_path