handle regex

ggerganov · Jul 21, 2024 · f4600e6 · f4600e6
1 parent 689e38c
commit f4600e6
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/llama.cpp b/src/llama.cpp
@@ -15558,6 +15558,7 @@ struct llm_tokenizer_bpe {
             case LLAMA_VOCAB_PRE_TYPE_STARCODER:
             case LLAMA_VOCAB_PRE_TYPE_REFACT:
             case LLAMA_VOCAB_PRE_TYPE_COMMAND_R:
+            case LLAMA_VOCAB_PRE_TYPE_SMOLLM:
                 regex_exprs = {
                     "\\p{N}",
                     "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",