From a2e03b826fe29f93c3c49168a55667f2456900af Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 13 Dec 2024 16:41:40 -0700 Subject: [PATCH 1/2] fix: Use gpt2 tokenizer for roberta and add eos/bos tokens Branch: RobertaTokenizer Signed-off-by: Gabe Goodhart --- convert_hf_to_gguf.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index c63d929c187a8..7cd60c0a2bec3 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -663,7 +663,11 @@ def get_vocab_base_pre(self, tokenizer) -> str: res = "minerva-7b" if chkhsh == "8b5a93ed704057481f240da0be7e7dca721d7f8f4755263b6807227a2cbeae65": # ref: https://huggingface.co/sentence-transformers/stsb-roberta-base - res = "roberta-bpe" + # NOTE: The Roberta tokenizer is the same as GPT-2, but it always + # adds the cls/sep tokens as bos/eos. This is handled as a + # post-processor in tokenizers, so the chkhsh is different, but + # it still maps to gpt-2 internally. + res = "gpt-2" if res is None: logger.warning("\n") @@ -2544,7 +2548,7 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter return [(self.map_tensor_name(name), data_torch)] -@Model.register("BertModel", "CamembertModel", "RobertaModel") +@Model.register("BertModel", "CamembertModel") class BertModel(Model): model_arch = gguf.MODEL_ARCH.BERT @@ -2617,6 +2621,27 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter return [(self.map_tensor_name(name), data_torch)] +@Model.register("RobertaModel") +class RobertaModel(BertModel): + model_arch = gguf.MODEL_ARCH.BERT + + def set_vocab(self): + """Support BPE tokenizers for roberta models""" + bpe_tok_path = self.dir_model / "tokenizer.json" + if bpe_tok_path.exists(): + self._set_vocab_gpt2() + self.gguf_writer.add_add_bos_token(True) + self.gguf_writer.add_add_eos_token(True) + + # we need this to validate the size of the token_type embeddings + # though currently we are passing all zeros to the token_type embeddings + # "Sequence A" or "Sequence B" + self.gguf_writer.add_token_type_count(self.hparams.get("type_vocab_size", 1)) + + else: + return super().set_vocab() + + @Model.register("NomicBertModel") class NomicBertModel(BertModel): model_arch = gguf.MODEL_ARCH.NOMIC_BERT From d5f69e8a437ed0cc09785e596b707669c87d1d1a Mon Sep 17 00:00:00 2001 From: Sukriti-Sharma4 Date: Mon, 16 Dec 2024 15:28:09 -0700 Subject: [PATCH 2/2] fixes to position embeddings Signed-off-by: Sukriti-Sharma4 --- convert_hf_to_gguf.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 7cd60c0a2bec3..482ba264dd3c5 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -2624,6 +2624,16 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter @Model.register("RobertaModel") class RobertaModel(BertModel): model_arch = gguf.MODEL_ARCH.BERT + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # we need the pad_token_id to know how to chop down position_embd matrix + if (pad_token_id := self.hparams.get("pad_token_id")) is not None: + self._position_offset = 1 + pad_token_id + if "max_position_embeddings" in self.hparams: + self.hparams["max_position_embeddings"] -= self._position_offset + else: + self._position_offset = None def set_vocab(self): """Support BPE tokenizers for roberta models""" @@ -2641,6 +2651,19 @@ def set_vocab(self): else: return super().set_vocab() + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # if name starts with "roberta.", remove the prefix + # e.g. https://huggingface.co/BAAI/bge-reranker-v2-m3/tree/main + if name.startswith("roberta."): + name = name[8:] + + # position embeddings start at pad_token_id + 1, so just chop down the weight tensor + if name == "embeddings.position_embeddings.weight": + if self._position_offset is not None: + data_torch = data_torch[self._position_offset:,:] + + return super().modify_tensors(data_torch, name, bid) + @Model.register("NomicBertModel") class NomicBertModel(BertModel):