grammar : hide decode_utf8 overload

ggml-ci
ggerganov · Aug 20, 2024 · 267f138 · 267f138
1 parent 05ce8ce
commit 267f138
Show file tree

Hide file tree

Showing 6 changed files with 179 additions and 191 deletions.
diff --git a/common/sampling.cpp b/common/sampling.cpp
@@ -157,25 +157,21 @@ std::vector<llama_sampler_type> llama_sampling_types_from_names(const std::vecto
 
     std::vector<llama_sampler_type> sampler_types;
     sampler_types.reserve(names.size());
-    for (const auto & name : names)
-    {
+
+    for (const auto & name : names) {
         auto sampler_item = sampler_canonical_name_map.find(name);
-        if (sampler_item != sampler_canonical_name_map.end())
-        {
+        if (sampler_item != sampler_canonical_name_map.end()) {
             sampler_types.push_back(sampler_item->second);
-        }
-        else
-        {
-            if (allow_alt_names)
-            {
+        } else {
+            if (allow_alt_names) {
                 sampler_item = sampler_alt_name_map.find(name);
-                if (sampler_item != sampler_alt_name_map.end())
-                {
+                if (sampler_item != sampler_alt_name_map.end()) {
                     sampler_types.push_back(sampler_item->second);
                 }
             }
         }
     }
+
     return sampler_types;
 }
 

diff --git a/examples/gbnf-validator/gbnf-validator.cpp b/examples/gbnf-validator/gbnf-validator.cpp
@@ -1,7 +1,5 @@
-#include "ggml.h"
-#include "llama.h"
-#include "llama-grammar.h"
 #include "unicode.h"
+#include "llama-grammar.h"
 
 #include <cstdio>
 #include <cstdlib>
@@ -11,21 +9,20 @@
 #include <vector>
 
 static bool llama_grammar_validate(struct llama_grammar * grammar, const std::string & input_str, size_t & error_pos, std::string & error_msg) {
-    auto decoded = decode_utf8(input_str, {});
-    const auto & code_points = decoded.first;
+    const auto cpts = unicode_cpts_from_utf8(input_str);
 
     const llama_grammar_rules  & rules      = llama_grammar_get_rules (grammar);
           llama_grammar_stacks & cur_stacks = llama_grammar_get_stacks(grammar);
 
     size_t pos = 0;
-    for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) {
+    for (const auto & cpt : cpts) {
         const llama_grammar_stacks prev_stacks = llama_grammar_get_stacks(grammar); // copy
 
-        cur_stacks = llama_grammar_accept(rules, prev_stacks, *it);
+        cur_stacks = llama_grammar_accept(rules, prev_stacks, cpt);
 
         if (cur_stacks.empty()) {
             error_pos = pos;
-            error_msg = "Unexpected character '" + unicode_cpt_to_utf8(*it) + "'";
+            error_msg = "Unexpected character '" + unicode_cpt_to_utf8(cpt) + "'";
             cur_stacks = prev_stacks;
             return false;
         }