fix: Update count_tokens function in utils.py

arc53 · May 4, 2024 · 4317551 · 4317551
1 parent de0193f
commit 4317551
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/application/utils.py b/application/utils.py
@@ -1,6 +1,6 @@
 from transformers import GPT2TokenizerFast
 
-
+tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
 def count_tokens(string):
-    tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
+
     return len(tokenizer(string)['input_ids'])