Skip to content

Commit

Permalink
*.py: fix flake8 warnings
Browse files Browse the repository at this point in the history
  • Loading branch information
@mofosyne
mofosyne committed Apr 30, 2024
1 parent 5e5e74e commit fcc5a5e
Show file tree
Hide file tree
Showing 6 changed files with 103 additions and 95 deletions.
2 changes: 1 addition & 1 deletion .flake8
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
[flake8]
max-line-length = 125
ignore = E203,E211,E221,E225,E231,E241,E251,E261,E266,E501,E701,E704,W503
exclude = examples/*,examples/*/**,*/**/__init__.py,convert-hf-to-gguf-update.py
exclude = examples/*,examples/*/**,*/**/__init__.py
156 changes: 81 additions & 75 deletions convert-hf-to-gguf-update.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,56 +21,64 @@
# TODO: automate the update of convert-hf-to-gguf.py
#

import logging
import os
import requests
import sys
import json

from hashlib import sha256
from enum import IntEnum, auto
from transformers import AutoTokenizer

logger = logging.getLogger("convert-hf-to-gguf-update")


class TOKENIZER_TYPE(IntEnum):
SPM = auto()
BPE = auto()
WPM = auto()


# TODO: this string has to exercise as much pre-tokenizer functionality as possible
# will be updated with time - contributions welcome
chktxt = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български \'\'\'\'\'\'```````\"\"\"\"......!!!!!!?????? I\'ve been \'told he\'s there, \'RE you sure? \'M not sure I\'ll make it, \'D you like some tea? We\'Ve a\'lL'

if len(sys.argv) == 2:
token = sys.argv[1]
else:
print("Usage: python convert-hf-to-gguf-update.py <huggingface_token>")
logger.info("Usage: python convert-hf-to-gguf-update.py <huggingface_token>")
sys.exit(1)

# TODO: add models here, base models preferred
models = [
{ "name": "llama-spm", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/meta-llama/Llama-2-7b-hf", },
{ "name": "llama-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", },
{ "name": "phi-3", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct", },
{ "name": "deepseek-llm", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-llm-7b-base", },
{ "name": "deepseek-coder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base", },
{ "name": "falcon", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/falcon-7b", },
{ "name": "bert-bge", "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/BAAI/bge-small-en-v1.5", },
{ "name": "mpt", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mosaicml/mpt-7b", },
{ "name": "starcoder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/bigcode/starcoder2-3b", },
{ "name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openai-community/gpt2", },
]
{"name": "llama-spm", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/meta-llama/Llama-2-7b-hf", },
{"name": "llama-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", },
{"name": "phi-3", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct", },
{"name": "deepseek-llm", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-llm-7b-base", },
{"name": "deepseek-coder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base", },
{"name": "falcon", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/falcon-7b", },
{"name": "bert-bge", "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/BAAI/bge-small-en-v1.5", },
{"name": "mpt", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mosaicml/mpt-7b", },
{"name": "starcoder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/bigcode/starcoder2-3b", },
{"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openai-community/gpt2", },
]

# make directory "models/tokenizers" if it doesn't exist
if not os.path.exists("models/tokenizers"):
os.makedirs("models/tokenizers")


def download_file_with_auth(url, token, save_path):
headers = {"Authorization": f"Bearer {token}"}
response = requests.get(url, headers=headers)
if response.status_code == 200:
with open(save_path, 'wb') as f:
f.write(response.content)
print(f"File {save_path} downloaded successfully")
logger.info(f"File {save_path} downloaded successfully")
else:
print(f"Failed to download file. Status code: {response.status_code}")
logger.info(f"Failed to download file. Status code: {response.status_code}")


# download the tokenizer models
for model in models:
Expand All @@ -81,10 +89,10 @@ def download_file_with_auth(url, token, save_path):
if not os.path.exists(f"models/tokenizers/{name}"):
os.makedirs(f"models/tokenizers/{name}")
else:
print(f"Directory models/tokenizers/{name} already exists - skipping")
logger.info(f"Directory models/tokenizers/{name} already exists - skipping")
continue

print(f"Downloading {name} to models/tokenizers/{name}")
logger.info(f"Downloading {name} to models/tokenizers/{name}")

url = f"{repo}/raw/main/config.json"
save_path = f"models/tokenizers/{name}/config.json"
Expand Down Expand Up @@ -115,76 +123,75 @@ def download_file_with_auth(url, token, save_path):
continue

# create the tokenizer
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")

chktok = tokenizer.encode(chktxt)
chkhsh = sha256(str(chktok).encode()).hexdigest()

print(f"model: {name}")
print(f"tokt: {tokt}")
print(f"repo: {model['repo']}")
print(f"chktok: {chktok}")
print(f"chkhsh: {chkhsh}")
logger.info(f"model: {name}")
logger.info(f"tokt: {tokt}")
logger.info(f"repo: {model['repo']}")
logger.info(f"chktok: {chktok}")
logger.info(f"chkhsh: {chkhsh}")

# print the "pre_tokenizer" content from the tokenizer.json
with open(f"models/tokenizers/{name}/tokenizer.json", "r", encoding="utf-8") as f:
cfg = json.load(f)
pre_tokenizer = cfg["pre_tokenizer"]
print("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4))
logger.info("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4))

print(f"\n")
logger.info("")

src_ifs += f" if chkhsh == \"{chkhsh}\":\n"
src_ifs += f" # ref: {model['repo']}\n"
src_ifs += f" res = \"{name}\"\n"

src_func = ""
src_func += " def get_vocab_base_pre(self, tokenizer) -> str:\n"
src_func += " # encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that\n"
src_func += " # is specific for the BPE pre-tokenizer used by the model\n"
src_func += " # we will use this unique identifier to write a \"tokenizer.ggml.pre\" entry in the GGUF file which we can\n"
src_func += " # use in llama.cpp to implement the same pre-tokenizer\n"
src_func += "\n"
src_func += f" chktxt = {repr(chktxt)}\n"
src_func += "\n"
src_func += " chktok = tokenizer.encode(chktxt)\n"
src_func += " chkhsh = sha256(str(chktok).encode()).hexdigest()\n"
src_func += "\n"
src_func += " print(f\"chktok: {chktok}\")\n"
src_func += " print(f\"chkhsh: {chkhsh}\")\n"
src_func += "\n"
src_func += " res = None\n"
src_func += "\n"
src_func += " # NOTE: if you get an error here, you need to update the convert-hf-to-gguf-update.py script\n"
src_func += " # or pull the latest version of the model from Huggingface\n"
src_func += " # don't edit the hashes manually!\n"
src_func += f"{src_ifs}\n"
src_func += " if res is None:\n"
src_func += " print(\"\\n\")\n"
src_func += " print(\"**************************************************************************************\")\n"
src_func += " print(\"** WARNING: The BPE pre-tokenizer was not recognized!\")\n"
src_func += " print(\"** There are 2 possible reasons for this:\")\n"
src_func += " print(\"** - the model has not been added to convert-hf-to-gguf-update.py yet\")\n"
src_func += " print(\"** - the pre-tokenization config has changed upstream\")\n"
src_func += " print(\"** Check your model files and convert-hf-to-gguf-update.py and update them accordingly.\")\n"
src_func += " print(\"** ref: https://github.com/ggerganov/llama.cpp/pull/6920\")\n"
src_func += " print(\"**\")\n"
src_func += " print(f\"** chkhsh: {chkhsh}\")\n"
src_func += " print(\"**************************************************************************************\")\n"
src_func += " print(\"\\n\")\n"
src_func += " raise NotImplementedError(\"BPE pre-tokenizer was not recognized - update get_vocab_base_pre()\")\n"
src_func += "\n"
src_func += " print(f\"tokenizer.ggml.pre: {res}\")\n"
src_func += " print(f\"chkhsh: {chkhsh}\")\n"
src_func += "\n"
src_func += " return res\n"

print(src_func)

print("\n")
print("!!! Copy-paste the function above into convert-hf-to-gguf.py !!!")
print("\n")
src_func = "" # noqa: E222
src_func += " def get_vocab_base_pre(self, tokenizer) -> str:\n" # noqa: E222
src_func += " # encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that\n" # noqa: E222
src_func += " # is specific for the BPE pre-tokenizer used by the model\n" # noqa: E222
src_func += " # we will use this unique identifier to write a \"tokenizer.ggml.pre\" entry in the GGUF file which we can\n" # noqa: E222
src_func += " # use in llama.cpp to implement the same pre-tokenizer\n" # noqa: E222
src_func += "\n" # noqa: E222
src_func += f" chktxt = {repr(chktxt)}\n" # noqa: E222
src_func += "\n" # noqa: E222
src_func += " chktok = tokenizer.encode(chktxt)\n" # noqa: E222
src_func += " chkhsh = sha256(str(chktok).encode()).hexdigest()\n" # noqa: E222
src_func += "\n" # noqa: E222
src_func += " print(f\"chktok: {chktok}\")\n" # noqa: E222
src_func += " print(f\"chkhsh: {chkhsh}\")\n" # noqa: E222
src_func += "\n" # noqa: E222
src_func += " res = None\n" # noqa: E222
src_func += "\n" # noqa: E222
src_func += " # NOTE: if you get an error here, you need to update the convert-hf-to-gguf-update.py script\n" # noqa: E222
src_func += " # or pull the latest version of the model from Huggingface\n" # noqa: E222
src_func += " # don't edit the hashes manually!\n" # noqa: E222
src_func += f"{src_ifs}\n" # noqa: E222
src_func += " if res is None:\n" # noqa: E222
src_func += " print(\"\\n\")\n" # noqa: E222
src_func += " print(\"**************************************************************************************\")\n" # noqa: E222
src_func += " print(\"** WARNING: The BPE pre-tokenizer was not recognized!\")\n" # noqa: E222
src_func += " print(\"** There are 2 possible reasons for this:\")\n" # noqa: E222
src_func += " print(\"** - the model has not been added to convert-hf-to-gguf-update.py yet\")\n" # noqa: E222
src_func += " print(\"** - the pre-tokenization config has changed upstream\")\n" # noqa: E222
src_func += " print(\"** Check your model files and convert-hf-to-gguf-update.py and update them accordingly.\")\n" # noqa: E222
src_func += " print(\"** ref: https://github.com/ggerganov/llama.cpp/pull/6920\")\n" # noqa: E222
src_func += " print(\"**\")\n" # noqa: E222
src_func += " print(f\"** chkhsh: {chkhsh}\")\n" # noqa: E222
src_func += " print(\"**************************************************************************************\")\n" # noqa: E222
src_func += " print(\"\\n\")\n" # noqa: E222
src_func += " raise NotImplementedError(\"BPE pre-tokenizer was not recognized - update get_vocab_base_pre()\")\n" # noqa: E222
src_func += "\n" # noqa: E222
src_func += " print(f\"tokenizer.ggml.pre: {res}\")\n" # noqa: E222
src_func += " print(f\"chkhsh: {chkhsh}\")\n" # noqa: E222
src_func += "\n" # noqa: E222
src_func += " return res\n" # noqa: E222

print(src_func) # noqa: NP100

logger.info("\n")
logger.info("!!! Copy-paste the function above into convert-hf-to-gguf.py !!!")
logger.info("\n")

# generate tests for each tokenizer model

Expand Down Expand Up @@ -250,7 +257,6 @@ def download_file_with_auth(url, token, save_path):
tokt = model["tokt"]

# create the tokenizer
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")

with open(f"models/ggml-vocab-{name}.gguf.inp", "w", encoding="utf-8") as f:
Expand All @@ -265,15 +271,15 @@ def download_file_with_auth(url, token, save_path):
f.write(f" {r}")
f.write("\n")

print(f"Tests for {name} written in ./models/ggml-vocab-{name}.gguf.*")
logger.info(f"Tests for {name} written in ./models/ggml-vocab-{name}.gguf.*")

# generate commands for creating vocab files

print("\nRun the following commands to generate the vocab files for testing:\n")
logger.info("\nRun the following commands to generate the vocab files for testing:\n")

for model in models:
name = model["name"]

print(f"python3 convert-hf-to-gguf.py models/tokenizers/{name}/ --outfile models/ggml-vocab-{name}.gguf --vocab-only")
logger.info(f"python3 convert-hf-to-gguf.py models/tokenizers/{name}/ --outfile models/ggml-vocab-{name}.gguf --vocab-only")

print("\n")
logger.info("\n")
32 changes: 16 additions & 16 deletions convert-hf-to-gguf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,8 +276,8 @@ def get_vocab_base_pre(self, tokenizer) -> str:
chktok = tokenizer.encode(chktxt)
chkhsh = sha256(str(chktok).encode()).hexdigest()

print(f"chktok: {chktok}")
print(f"chkhsh: {chkhsh}")
logger.debug(f"chktok: {chktok}")
logger.debug(f"chkhsh: {chkhsh}")

res = None

Expand Down Expand Up @@ -310,22 +310,22 @@ def get_vocab_base_pre(self, tokenizer) -> str:
res = "gpt-2"

if res is None:
print("\n")
print("**************************************************************************************")
print("** WARNING: The BPE pre-tokenizer was not recognized!")
print("** There are 2 possible reasons for this:")
print("** - the model has not been added to convert-hf-to-gguf-update.py yet")
print("** - the pre-tokenization config has changed upstream")
print("** Check your model files and convert-hf-to-gguf-update.py and update them accordingly.")
print("** ref: https://github.com/ggerganov/llama.cpp/pull/6920")
print("**")
print(f"** chkhsh: {chkhsh}")
print("**************************************************************************************")
print("\n")
logger.warning("\n")
logger.warning("**************************************************************************************")
logger.warning("** WARNING: The BPE pre-tokenizer was not recognized!")
logger.warning("** There are 2 possible reasons for this:")
logger.warning("** - the model has not been added to convert-hf-to-gguf-update.py yet")
logger.warning("** - the pre-tokenization config has changed upstream")
logger.warning("** Check your model files and convert-hf-to-gguf-update.py and update them accordingly.")
logger.warning("** ref: https://github.com/ggerganov/llama.cpp/pull/6920")
logger.warning("**")
logger.warning(f"** chkhsh: {chkhsh}")
logger.warning("**************************************************************************************")
logger.warning("\n")
raise NotImplementedError("BPE pre-tokenizer was not recognized - update get_vocab_base_pre()")

print(f"tokenizer.ggml.pre: {res}")
print(f"chkhsh: {chkhsh}")
logger.debug(f"tokenizer.ggml.pre: {res}")
logger.debug(f"chkhsh: {chkhsh}")

return res

Expand Down
4 changes: 3 additions & 1 deletion gguf-py/gguf/gguf_reader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#
from __future__ import annotations

import logging
import os
from collections import OrderedDict
from typing import Any, Literal, NamedTuple, TypeVar, Union
Expand All @@ -27,6 +28,7 @@
GGUFValueType,
)

logger = logging.getLogger(__name__)

READER_SUPPORTED_VERSIONS = [2, GGUF_VERSION]

Expand Down Expand Up @@ -142,7 +144,7 @@ def _push_field(self, field: ReaderField, skip_sum: bool = False) -> int:
# TODO: add option to generate error on duplicate keys
# raise KeyError(f'Duplicate {field.name} already in list at offset {field.offset}')

print(f'Warning: Duplicate key {field.name} at offset {field.offset}')
logger.warning(f'Duplicate key {field.name} at offset {field.offset}')
self.fields[field.name + '_{}'.format(field.offset)] = field
else:
self.fields[field.name] = field
Expand Down
2 changes: 1 addition & 1 deletion tests/test-tokenizer-0-bpe.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

from transformers import AutoTokenizer

logger = logging.getLogger("convert")
logger = logging.getLogger("test-tokenizer-0-bpe")

parser = argparse.ArgumentParser()
parser.add_argument("dir_tokenizer", help="directory containing 'tokenizer.model' file")
Expand Down
2 changes: 1 addition & 1 deletion tests/test-tokenizer-0-spm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

from sentencepiece import SentencePieceProcessor

logger = logging.getLogger("test-tokenizer-0-llama")
logger = logging.getLogger("test-tokenizer-0-spm")

parser = argparse.ArgumentParser()
parser.add_argument("dir_tokenizer", help="directory containing 'tokenizer.model' file")
Expand Down

0 comments on commit fcc5a5e

Please sign in to comment.