Skip to content

Commit

Permalink
Update convert-hf-to-gguf.py
Browse files Browse the repository at this point in the history
Replacing with a less risky alternative to the 'assert' statement
  • Loading branch information
0xspringtime authored Jun 19, 2024
1 parent 9c77ec1 commit 620633e
Showing 1 changed file with 3 additions and 1 deletion.
4 changes: 3 additions & 1 deletion convert-hf-to-gguf.py
Original file line number Diff line number Diff line change
Expand Up @@ -967,7 +967,9 @@ def set_vocab(self):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(dir_model)
vocab_size = hparams.get("vocab_size", len(tokenizer.vocab))
assert max(tokenizer.vocab.values()) < vocab_size
if max(tokenizer.get_vocab().values()) >= vocab_size:
raise ValueError("Vocabulary size exceeds expected maximum size.")


reverse_vocab: dict[int, str] = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}
added_vocab = tokenizer.get_added_vocab()
Expand Down

0 comments on commit 620633e

Please sign in to comment.