Synchronize README.md and src/lib.rs

huggingface · Jan 2, 2025 · 4eee3f2 · 4eee3f2
1 parent ad4d404
commit 4eee3f2
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/tokenizers/README.md b/tokenizers/README.md
@@ -137,3 +137,5 @@ fn main() -> Result<()> {
 **progressbar**: The progress bar visualization is enabled by default. It might be disabled if
   compilation for certain targets is not supported by the [termios](https://crates.io/crates/termios)
   dependency of the [indicatif](https://crates.io/crates/indicatif) progress bar.
+**http**: This feature enables downloading the tokenizer via HTTP. It is disabled by default.
+  With this feature enabled, `Tokenizer::from_pretrained` becomes accessible.
diff --git a/tokenizers/src/lib.rs b/tokenizers/src/lib.rs
@@ -30,10 +30,11 @@
 //! fn main() -> Result<()> {
 //!     # #[cfg(feature = "http")]
 //!     # {
-//!         let tokenizer = Tokenizer::from_pretrained("bert-base-cased", None)?;
+//!     // needs http feature enabled
+//!     let tokenizer = Tokenizer::from_pretrained("bert-base-cased", None)?;
 //!
-//!         let encoding = tokenizer.encode("Hey there!", false)?;
-//!         println!("{:?}", encoding.get_tokens());
+//!     let encoding = tokenizer.encode("Hey there!", false)?;
+//!     println!("{:?}", encoding.get_tokens());
 //!     # }
 //!     Ok(())
 //! }