revert

huggingface · Jul 24, 2024 · 08a9ecf · 08a9ecf
1 parent 16af28b
commit 08a9ecf
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 6 deletions.
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
@@ -29,7 +29,3 @@ pyo3 = { version = "0.21", features = ["auto-initialize"] }
 
 [features]
 defaut = ["pyo3/extension-module"]
-
-[profile.release]
-debug = true
-strip = false
diff --git a/tokenizers/benches/bpe_benchmark.rs b/tokenizers/benches/bpe_benchmark.rs
@@ -24,8 +24,8 @@ fn create_gpt2_tokenizer(bpe: BPE) -> Tokenizer {
     let mut tokenizer = Tokenizer::new(bpe);
     tokenizer.with_pre_tokenizer(ByteLevel::default());
     tokenizer.with_decoder(ByteLevel::default());
-    // tokenizer.add_tokens(&[AddedToken::from("ing", false).single_word(false)]);
-    // tokenizer.add_special_tokens(&[AddedToken::from("[ENT]", true).single_word(true)]);
+    tokenizer.add_tokens(&[AddedToken::from("ing", false).single_word(false)]);
+    tokenizer.add_special_tokens(&[AddedToken::from("[ENT]", true).single_word(true)]);
     tokenizer
 }