just testing some sutff

huggingface · Jun 20, 2024 · 1e027dc · 1e027dc
1 parent e5976a6
commit 1e027dc
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/tokenizers/benches/bpe_benchmark.rs b/tokenizers/benches/bpe_benchmark.rs
@@ -24,8 +24,8 @@ fn create_gpt2_tokenizer(bpe: BPE) -> Tokenizer {
     let mut tokenizer = Tokenizer::new(bpe);
     tokenizer.with_pre_tokenizer(ByteLevel::default());
     tokenizer.with_decoder(ByteLevel::default());
-    tokenizer.add_tokens(&[AddedToken::from("ing", false).single_word(false)]);
-    tokenizer.add_special_tokens(&[AddedToken::from("[ENT]", true).single_word(true)]);
+    // tokenizer.add_tokens(&[AddedToken::from("ing", false).single_word(false)]);
+    // tokenizer.add_special_tokens(&[AddedToken::from("[ENT]", true).single_word(true)]);
     tokenizer
 }
 

diff --git a/tokenizers/src/pre_tokenizers/byte_level.rs b/tokenizers/src/pre_tokenizers/byte_level.rs
@@ -169,7 +169,8 @@ impl PreTokenizer for ByteLevel {
                         .map(|(i, b)| (BYTES_CHAR[b], isize::from(i > 0))),
                 );
             }
-            normalized.transform(transformations, 0);
+            // normalized.transform(transformations, 0); // TODO here what whould happen if we ignore
+            // aligments?
             Ok(())
         })
     }
@@ -199,6 +200,7 @@ impl Decoder for ByteLevel {
     }
 }
 
+// TODO this is also somewhere we want to just skip if we are fast
 /// As a `PostProcessor`, `ByteLevel` is in charge of trimming the offsets if necessary.
 impl PostProcessor for ByteLevel {
     fn added_tokens(&self, _is_pair: bool) -> usize {