fix absurdly fast benchmarks: cleanup lm-format-enforcer cache in tea…

…rdown step, cleanup json generator
dottxt-ai · Oct 16, 2024 · 0d73c1c · 0d73c1c
1 parent 5bc6379
commit 0d73c1c
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 4 deletions.
diff --git a/src/benchmark_lfe.py b/src/benchmark_lfe.py
@@ -37,6 +37,9 @@ def time_lfe(self, _, regex_name):
             for i in range(len(regex_sample_tokens)):
                 _ = token_enforcer.get_allowed_tokens(regex_sample_tokens[: i + 1])
 
+    def teardown(self, *args):
+        del self.tokenizer_data
+
 
 class LMFormatEnforcerJsonSchema:
     params = [models, json_cases.keys()]
@@ -66,3 +69,6 @@ def time_lfe(self, _, json_schema_name):
             json_sample_tokens = self.tokenizer.encode(json_sample)
             for i in range(len(json_sample_tokens)):
                 _ = token_enforcer.get_allowed_tokens(json_sample_tokens[: i + 1])
+
+    def teardown(self, *args):
+        del self.tokenizer_data
diff --git a/src/data.py b/src/data.py
@@ -57,8 +57,8 @@
             "title": "Character",
             "type": "object",
         },
-        "samples": map(
-            json.dumps, json.load(open(SAMPLES_PATH / "rpg_characters.json"))
+        "samples": list(
+            map(json.dumps, json.load(open(SAMPLES_PATH / "rpg_characters.json")))
         ),
     },
     "Simple nested schema": {
@@ -94,8 +94,8 @@
             },
             "required": ["id", "work", "recording_artists"],
         },
-        "samples": map(
-            json.dumps, json.load(open(SAMPLES_PATH / "recording_schema.json"))
+        "samples": list(
+            map(json.dumps, json.load(open(SAMPLES_PATH / "recording_schema.json")))
         ),
     },
 }