Use external_deps for sentencepiece (pytorch#4269)

Summary: Pull Request resolved: pytorch#4269 as title ghstack-source-id: 234090425 Reviewed By: dbort Differential Revision: D59770172 fbshipit-source-id: 4bb63b2497cd3ddb04726da6fb8cefb0a2add391
dijopaul · Jul 17, 2024 · 8950d90 · 8950d90
1 parent 037cfcf
commit 8950d90
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/extension/llm/tokenizer/targets.bzl b/extension/llm/tokenizer/targets.bzl
@@ -20,8 +20,9 @@ def define_common_targets():
             "//bento_kernels/...",
         ],
         _is_external_target = True,
-        # TODO: Define an external_deps entry for sentencepiece instead of pointing to an fbsource path.
-        deps = [] if runtime.is_oss else ["fbsource//third-party/pypi/sentencepiece:sentencepiece"],
+        external_deps = [
+            "sentencepiece-py",
+        ],
     )
 
     runtime.python_binary(

diff --git a/shim/xplat/executorch/build/env_interface.bzl b/shim/xplat/executorch/build/env_interface.bzl
@@ -42,6 +42,7 @@ _EXTERNAL_DEPS = {
     "prettytable": "//third-party:prettytable",
     "pybind11": "//third-party:pybind11",
     "re2": [],  # TODO(larryliu0820): Add support
+    "sentencepiece-py": [],
     # Core C++ PyTorch functionality like Tensor and ScalarType.
     "torch-core-cpp": "//third-party:libtorch",
     "torchgen": "//third-party:torchgen",