wasm tokenizer

santiagomed · Nov 8, 2023 · 07ae2b3 · 07ae2b3
1 parent 478f6eb
commit 07ae2b3
Showing 1 changed file with 8 additions and 6 deletions.
diff --git a/models/Cargo.toml b/models/Cargo.toml
@@ -9,12 +9,10 @@ edition = "2021"
 candle = { git = "https://github.com/huggingface/candle", package = "candle-core" }
 candle-transformers = { git = "https://github.com/huggingface/candle" }
 candle-nn = { git = "https://github.com/huggingface/candle" }
-tokenizers = {version = "0.14.0", features = ["http"]}
-hf-hub = {version = "0.3.0", features = ["tokio"], optional = true}
-serde_json = "^1.0"
-anyhow = "1.0.75"
-serde = { version = "^1.0.130", features = ["derive"] }
-tokio = {version = "1.33.0", optional = true}
+tokenizers = { version = "0.13.4", features = ["unstable_wasm"] }
+serde_json = "1.0.99"
+anyhow = "1"
+serde = { version = "1.0.171", features = ["derive"] }
 
 # Wasm specific crates.
 console_error_panic_hook = "0.1.7"
@@ -24,5 +22,9 @@ js-sys = "0.3.64"
 wasm-bindgen = "0.2.87"
 serde-wasm-bindgen = "0.6.0"
 
+# Optional dependencies (for async support).
+tokio = {version = "1.33.0", optional = true}
+hf-hub = {version = "0.3.0", features = ["tokio"], optional = true}
+
 [features]
 async = ["dep:tokio", "dep:hf-hub"]