Allow tokenizer_py to run in oss

Summary: Changes to let tokenizer run in OSS env Reviewed By: larryliu0820 Differential Revision: D53676371 fbshipit-source-id: 96178ff52c74633a570b6fa45f45874d98cd07c3
kirklandsign · Feb 13, 2024 · 3b623f9 · 3b623f9
1 parent 084abda
commit 3b623f9
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 5 deletions.
diff --git a/examples/models/llama2/install_requirements.sh b/examples/models/llama2/install_requirements.sh
@@ -6,4 +6,5 @@
 # LICENSE file in the root directory of this source tree.
 
 # Install snakeviz for cProfile flamegraph
-pip install snakeviz
+# Install sentencepiece for llama tokenizer
+pip install snakeviz sentencepiece
diff --git a/examples/models/llama2/tokenizer/targets.bzl b/examples/models/llama2/tokenizer/targets.bzl
@@ -30,10 +30,7 @@ def define_common_targets():
             "//bento/...",
         ],
         _is_external_target = True,
-        deps = [
-            "fbsource//third-party/pypi/sentencepiece:sentencepiece",
-            "//caffe2:torch",
-        ],
+        deps = [] if runtime.is_oss else ["fbsource//third-party/pypi/sentencepiece:sentencepiece"],
     )
 
     runtime.python_binary(