From 6edbe8190c3407459a93eef67d7620577ec617e6 Mon Sep 17 00:00:00 2001 From: Molly Sophia Date: Mon, 12 Aug 2024 09:12:16 +0800 Subject: [PATCH] Update convert_hf_to_gguf.py Co-authored-by: compilade --- convert_hf_to_gguf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index a1fac81046ca79..f72400b5aa6c3b 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -2719,7 +2719,7 @@ def set_vocab(self): for line in lines: parts = line.split(' ') assert len(parts) >= 3 - _, token, token_len = int(parts[0]), ast.literal_eval(' '.join(parts[1:-1])), int(parts[-1]) + token, token_len = ast.literal_eval(' '.join(parts[1:-1])), int(parts[-1]) token = token.encode("utf-8") if isinstance(token, str) else token assert isinstance(token, bytes) assert len(token) == token_len