diff --git a/src/benchmark_lfe.py b/src/benchmark_lfe.py index bd3fa31..7379de8 100644 --- a/src/benchmark_lfe.py +++ b/src/benchmark_lfe.py @@ -37,6 +37,9 @@ def time_lfe(self, _, regex_name): for i in range(len(regex_sample_tokens)): _ = token_enforcer.get_allowed_tokens(regex_sample_tokens[: i + 1]) + def teardown(self, *args): + del self.tokenizer_data + class LMFormatEnforcerJsonSchema: params = [models, json_cases.keys()] @@ -66,3 +69,6 @@ def time_lfe(self, _, json_schema_name): json_sample_tokens = self.tokenizer.encode(json_sample) for i in range(len(json_sample_tokens)): _ = token_enforcer.get_allowed_tokens(json_sample_tokens[: i + 1]) + + def teardown(self, *args): + del self.tokenizer_data diff --git a/src/data.py b/src/data.py index 3c5144a..2159be1 100644 --- a/src/data.py +++ b/src/data.py @@ -57,8 +57,8 @@ "title": "Character", "type": "object", }, - "samples": map( - json.dumps, json.load(open(SAMPLES_PATH / "rpg_characters.json")) + "samples": list( + map(json.dumps, json.load(open(SAMPLES_PATH / "rpg_characters.json"))) ), }, "Simple nested schema": { @@ -94,8 +94,8 @@ }, "required": ["id", "work", "recording_artists"], }, - "samples": map( - json.dumps, json.load(open(SAMPLES_PATH / "recording_schema.json")) + "samples": list( + map(json.dumps, json.load(open(SAMPLES_PATH / "recording_schema.json"))) ), }, }