Update tokenizer.py

ryao · May 31, 2024 · afa919b · afa919b
1 parent 857df09
commit afa919b
Showing 1 changed file with 1 addition and 2 deletions.
diff --git a/tokenizer.py b/tokenizer.py
@@ -108,8 +108,7 @@ def export(self):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("-t", "--tokenizer-model", type=str, help="optional path to custom tokenizer ")
-        "-t", "--tokenizer-model", type=str, help="optional path to custom tokenizer "
-    )
+
     args = parser.parse_args()
 
     t = Tokenizer(args.tokenizer_model)