Update convert_hf_to_gguf.py

Co-authored-by: compilade <[email protected]>
ggml-org · Aug 12, 2024 · e6f08bc · e6f08bc
1 parent 786152a
commit e6f08bc
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
@@ -2728,8 +2728,8 @@ def set_vocab(self):
                 toktypes.append(gguf.TokenType.NORMAL)
         remainder = vocab_size - len(tokens)
         assert remainder >= 0
-        for i in range(remainder):
-            tokens.append(f"<unused {i}>".encode("utf-8"))
+        for i in range(len(tokens), vocab_size):
+            tokens.append(f"[PAD{i}]".encode("utf-8"))
             toktypes.append(gguf.TokenType.UNUSED)
 
         self.gguf_writer.add_tokenizer_model("rwkv")