llama : improve BPE pre-processing + LLaMA 3 and Deepseek support

.github/workflows/python-lint.yml

@@ -21,4 +21,4 @@ jobs:
         uses: py-actions/flake8@v2
         with:
             ignore: "E203,E211,E221,E225,E231,E241,E251,E261,E266,E501,E701,E704,W503"
-            exclude: "examples/*,examples/*/**,*/**/__init__.py"
+            exclude: "examples/*,examples/*/**,*/**/__init__.py,convert-hf-to-gguf-update.py"

.gitignore

@@ -108,3 +108,20 @@ examples/server/*.mjs.hpp
 poetry.lock
 poetry.toml
 nppBackup
+
+# Test binaries
+/tests/test-grammar-parser
+/tests/test-llama-grammar
+/tests/test-double-float
+/tests/test-grad0
+/tests/test-opt
+/tests/test-quantize-fns
+/tests/test-quantize-perf
+/tests/test-sampling
+/tests/test-tokenizer-0-llama
+/tests/test-tokenizer-0-falcon
+/tests/test-tokenizer-0-deepseek-coder
+/tests/test-tokenizer-1-llama
+/tests/test-tokenizer-1-bpe
+/tests/test-rope
+/tests/test-backend-ops

Makefile

@@ -6,11 +6,27 @@ BUILD_TARGETS = \
 
 # Binaries only useful for tests
 TEST_TARGETS = \
-	tests/test-llama-grammar tests/test-grammar-parser tests/test-double-float tests/test-grad0 tests/test-opt \
-	tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0-llama          \
-	tests/test-tokenizer-0-falcon tests/test-tokenizer-1-llama tests/test-tokenizer-1-bpe tests/test-rope      \
-	tests/test-backend-ops tests/test-model-load-cancel tests/test-autorelease                                 \
-	tests/test-json-schema-to-grammar tests/test-grammar-integration
+	tests/test-autorelease \
+	tests/test-backend-ops \
+	tests/test-double-float \
+	tests/test-grad0 \
+	tests/test-grammar-integration \
+	tests/test-grammar-parser \
+	tests/test-json-schema-to-grammar \
+	tests/test-llama-grammar \
+	tests/test-model-load-cancel \
+	tests/test-opt \
+	tests/test-quantize-fns \
+	tests/test-quantize-perf \
+	tests/test-rope \
+	tests/test-sampling \
+	tests/test-tokenizer-0-deepseek-coder \
+	tests/test-tokenizer-0-deepseek-llm \
+	tests/test-tokenizer-0-falcon \
+	tests/test-tokenizer-0-llama \
+	tests/test-tokenizer-0-llama-v3 \
+	tests/test-tokenizer-1-bpe \
+	tests/test-tokenizer-1-llama
 
 # Code coverage output files
 COV_TARGETS = *.gcno tests/*.gcno *.gcda tests/*.gcda *.gcov tests/*.gcov lcov-report gcovr-report
@@ -51,8 +67,14 @@ test: $(TEST_TARGETS)
 	for test_target in $(TEST_TARGETS); do \
 		if [ "$$test_target" = "tests/test-tokenizer-0-llama" ]; then \
 			./$$test_target $(CURDIR)/models/ggml-vocab-llama.gguf; \
+		elif [ "$$test_target" = "tests/test-tokenizer-0-llama-v3" ]; then \
+			./$$test_target $(CURDIR)/models/ggml-vocab-llama-v3.gguf; \
 		elif [ "$$test_target" = "tests/test-tokenizer-0-falcon" ]; then \
 			./$$test_target $(CURDIR)/models/ggml-vocab-falcon.gguf; \
+		elif [ "$$test_target" = "tests/test-tokenizer-0-deepseek-coder" ]; then \
+			./$$test_target $(CURDIR)/models/ggml-vocab-deepseek-coder.gguf; \
+		elif [ "$$test_target" = "tests/test-tokenizer-0-deepseek-llm" ]; then \
+			./$$test_target $(CURDIR)/models/ggml-vocab-deepseek-llm.gguf; \
 		elif [ "$$test_target" = "tests/test-tokenizer-1-llama" ]; then \
 			continue; \
 		elif [ "$$test_target" = "tests/test-tokenizer-1-bpe" ]; then \
@@ -979,6 +1001,16 @@ tests/test-tokenizer-0-llama: tests/test-tokenizer-0-llama.cpp ggml.o llama.o $(
 	$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
 
+tests/test-tokenizer-0-llama-v3: tests/test-tokenizer-0-llama-v3.cpp ggml.o llama.o $(COMMON_DEPS) console.o $(OBJS)
+	$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
+	$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
+
+tests/test-tokenizer-0-deepseek-coder: tests/test-tokenizer-0-deepseek-coder.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS)
+	$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
+
+tests/test-tokenizer-0-deepseek-llm: tests/test-tokenizer-0-deepseek-llm.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS)
+	$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
+
 tests/test-tokenizer-1-bpe: tests/test-tokenizer-1-bpe.cpp ggml.o llama.o $(COMMON_DEPS) console.o $(OBJS)
 	$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)

common/common.cpp

@@ -1693,6 +1693,18 @@ std::vector<std::string> string_split(std::string input, char separator) {
     return parts;
 }
 
+std::string string_strip(const std::string & str) {
+    size_t start = 0;
+    size_t end = str.size();
+    while (start < end && std::isspace(str[start])) {
+        start++;
+    }
+    while (end > start && std::isspace(str[end - 1])) {
+        end--;
+    }
+    return str.substr(start, end - start);
+}
+
 std::vector<llama_sampler_type> sampler_types_from_names(const std::vector<std::string> & names, bool allow_alt_names) {
     std::unordered_map<std::string, llama_sampler_type> sampler_canonical_name_map {
         {"top_k",       llama_sampler_type::TOP_K},

common/common.h

@@ -196,6 +196,7 @@ bool validate_file_name(const std::string & filename);
 std::vector<llama_sampler_type> sampler_types_from_names(const std::vector<std::string> & names, bool allow_alt_names);
 std::vector<llama_sampler_type> sampler_types_from_chars(const std::string & names_string);
 std::vector<std::string> string_split(std::string input, char separator);
+std::string string_strip(const std::string & str);
 std::string sampler_type_to_name_string(llama_sampler_type sampler_type);
 
 //

convert-hf-to-gguf-update.py

@@ -0,0 +1,164 @@
+# Instructions:
+#
+# - Add a new model to the "models" list
+# - Run the script with your huggingface token:
+#
+#   python3 convert-hf-to-gguf-update.py <huggingface_token>
+#
+# - Copy-paste the generated get_vocab_base_pre() function into convert-hf-to-gguf.py
+# - Update llama.cpp with the new pre-tokenizer if necessary
+#
+# TODO: generate tokenizer tests for llama.cpp
+#
+
+import os
+import requests
+import sys
+import json
+
+from hashlib import sha256
+from enum import IntEnum, auto
+
+class TOKENIZER_TYPE(IntEnum):
+    SPM = auto()
+    BPE = auto()
+    WPM = auto()
+
+# TODO: this string has to exercise as much pre-tokenizer functionality as possible
+#       will be updated with time - contributions welcome
+chktxt = '\n \n\n \n\n\n \t \t\t \t\n  \n   \n    \n     \n🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български what\'s \'\'\'\'\'\'```````\"\"\"\"......!!!!!!??????'
+
+if len(sys.argv) == 2:
+    token = sys.argv[1]
+else:
+    print("Usage: python convert-hf-to-gguf-update.py <huggingface_token>")
+    sys.exit(1)
+
+# TODO: add models here, base models preferred
+models = [
+        { "name": "llama-v2",       "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/meta-llama/Llama-2-7b-hf",                },
+        { "name": "llama-v3",       "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Meta-Llama-3-8B",              },
+        { "name": "deepseek-llm",   "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-llm-7b-base",        },
+        { "name": "deepseek-coder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base",    },
+        { "name": "falcon",         "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/falcon-7b",                        },
+        { "name": "bert-bge",       "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/BAAI/bge-small-en-v1.5",                  },
+        ]
+
+# make directory "models/tokenizers" if it doesn't exist
+if not os.path.exists("models/tokenizers"):
+    os.makedirs("models/tokenizers")
+
+def download_file_with_auth(url, token, save_path):
+    headers = {"Authorization": f"Bearer {token}"}
+    response = requests.get(url, headers=headers)
+    if response.status_code == 200:
+        with open(save_path, 'wb') as f:
+            f.write(response.content)
+        print("File downloaded successfully.")
+    else:
+        print(f"Failed to download file. Status code: {response.status_code}")
+
+for model in models:
+    name = model["name"]
+    repo = model["repo"]
+    tokt = model["tokt"]
+
+    if not os.path.exists(f"models/tokenizers/{name}"):
+        os.makedirs(f"models/tokenizers/{name}")
+    else:
+        print(f"Directory models/tokenizers/{name} already exists - skipping")
+        continue
+
+    print(f"Downloading {name} to models/tokenizers/{name}")
+
+    url = f"{repo}/raw/main/tokenizer.json"
+    save_path = f"models/tokenizers/{name}/tokenizer.json"
+    download_file_with_auth(url, token, save_path)
+
+    if tokt == TOKENIZER_TYPE.SPM:
+        url = f"{repo}/resolve/main/tokenizer.model"
+        save_path = f"models/tokenizers/{name}/tokenizer.model"
+        download_file_with_auth(url, token, save_path)
+
+    url = f"{repo}/raw/main/tokenizer_config.json"
+    save_path = f"models/tokenizers/{name}/tokenizer_config.json"
+    download_file_with_auth(url, token, save_path)
+
+# generate the source code for the convert-hf-to-gguf.py:get_vocab_base_pre() function:
+# TODO: auto-update convert-hf-to-gguf.py with the generated function
+
+src_ifs = ""
+for model in models:
+    name = model["name"]
+    tokt = model["tokt"]
+
+    if tokt == TOKENIZER_TYPE.SPM:
+        continue
+
+    # create the tokenizer
+    from transformers import AutoTokenizer
+    tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")
+
+    chktok = tokenizer.encode(chktxt)
+    chkhsh = sha256(str(chktok).encode()).hexdigest()
+
+    print(f"model: {name}")
+    print(f"tokt: {tokt}")
+    print(f"repo: {model['repo']}")
+    print(f"chktok: {chktok}")
+    print(f"chkhsh: {chkhsh}")
+
+    # print the "pre_tokenizer" content from the tokenizer.json
+    with open(f"models/tokenizers/{name}/tokenizer.json", "r") as f:
+        cfg = json.load(f)
+        pre_tokenizer = cfg["pre_tokenizer"]
+        print("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4))
+
+    print(f"\n")
+
+    src_ifs += f"        if chkhsh == \"{chkhsh}\":\n"
+    src_ifs += f"            # ref: {model['repo']}\n"
+    src_ifs += f"            res = \"{name}\"\n"
+
+src_func = ""
+src_func +=  "    def get_vocab_base_pre(self, tokenizer) -> str:\n"
+src_func +=  "        # encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that\n"
+src_func +=  "        # is specific for the BPE pre-tokenizer used by the model\n"
+src_func +=  "        # we will use this unique identifier to write a \"tokenizer.ggml.pre\" entry in the GGUF file which we can\n"
+src_func +=  "        # use in llama.cpp to implement the same pre-tokenizer\n"
+src_func +=  "\n"
+src_func += f"        chktxt = {repr(chktxt)}\n"
+src_func +=  "\n"
+src_func +=  "        chktok = tokenizer.encode(chktxt)\n"
+src_func +=  "        chkhsh = sha256(str(chktok).encode()).hexdigest()\n"
+src_func +=  "\n"
+src_func +=  "        print(f\"chktok: {chktok}\")\n"
+src_func +=  "        print(f\"chkhsh: {chkhsh}\")\n"
+src_func +=  "\n"
+src_func +=  "        res = None\n"
+src_func +=  "\n"
+src_func +=  "        # NOTE: if you get an error here, you need to add the model to the if-elif chain below\n"
+src_func += f"{src_ifs}\n"
+src_func +=  "        if res is None:\n"
+src_func +=  "            print( \"\\n\")\n"
+src_func +=  "            print( \"**************************************************************************************\")\n"
+src_func +=  "            print( \"** WARNING: The BPE pre-tokenizer was not recognized!\")\n"
+src_func +=  "            print( \"**          This means that it was not added yet or you are using an older version.\")\n"
+src_func +=  "            print( \"**          Check convert-hf-to-gguf-update.py and update it accordingly.\")\n"
+src_func +=  "            print( \"**\")\n"
+src_func +=  "            print(f\"** chkhsh:  {chkhsh}\")\n"
+src_func +=  "            print( \"**************************************************************************************\")\n"
+src_func +=  "            print( \"\\n\")\n"
+src_func +=  "            raise NotImplementedError(\"BPE pre-tokenizer was not recognized - update get_vocab_base_pre()\")\n"
+src_func +=  "\n"
+src_func +=  "        print(f\"tokenizer.ggml.pre: {res}\")\n"
+src_func +=  "        print(f\"chkhsh: {chkhsh}\")\n"
+src_func +=  "\n"
+src_func +=  "        return res\n"
+
+print(src_func)
+
+print("\n")
+print("!!! Copy-paste the function above into convert-hf-to-gguf.py !!!")
+print("\n")
+