Skip to content

Commit

Permalink
Update convert-hf-to-gguf.py
Browse files Browse the repository at this point in the history
  • Loading branch information
0xspringtime authored Jun 19, 2024
1 parent 620633e commit a05faca
Showing 1 changed file with 5 additions and 1 deletion.
6 changes: 5 additions & 1 deletion convert-hf-to-gguf.py
Original file line number Diff line number Diff line change
Expand Up @@ -967,10 +967,14 @@ def set_vocab(self):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(dir_model)
vocab_size = hparams.get("vocab_size", len(tokenizer.vocab))
if max(tokenizer.get_vocab().values()) >= vocab_size:
# Since we are checking the maximum index, we need to ensure it's strictly less than vocab_size,
# because vocab_size is the count of items, and indexes start at 0.
max_vocab_index = max(tokenizer.get_vocab().values())
if max_vocab_index >= vocab_size:
raise ValueError("Vocabulary size exceeds expected maximum size.")



reverse_vocab: dict[int, str] = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}
added_vocab = tokenizer.get_added_vocab()

Expand Down

0 comments on commit a05faca

Please sign in to comment.