Remove dependency from pytorch core as git submodule (pytorch#3530)

Summary: Pull Request resolved: pytorch#3530 Reviewed By: larryliu0820 Differential Revision: D57078775 Pulled By: mergennachin fbshipit-source-id: 5a6047bd779e59e7833ca01453ca8efb4657cdab
dijopaul · May 8, 2024 · 0492eeb · 0492eeb
1 parent c86128e
commit 0492eeb
Show file tree

Hide file tree

Showing 17 changed files with 40 additions and 69 deletions.
diff --git a/.buckconfig b/.buckconfig
@@ -31,6 +31,5 @@
   ignore = \
       .git, \
       **/.git, \
-      third-party/pytorch/third_party, \
       cmake-out, \
       pip-out
diff --git a/.gitmodules b/.gitmodules
@@ -4,9 +4,6 @@
 [submodule "third-party/flatbuffers"]
 	path = third-party/flatbuffers
 	url = https://github.com/google/flatbuffers.git
-[submodule "third-party/pytorch"]
-	path = third-party/pytorch
-	url = https://github.com/pytorch/pytorch.git
 [submodule "third-party/gflags"]
 	path = third-party/gflags
 	url = https://github.com/gflags/gflags.git

diff --git a/backends/cadence/hifi/operators/CMakeLists.txt b/backends/cadence/hifi/operators/CMakeLists.txt
@@ -11,11 +11,6 @@ if(NOT CMAKE_CXX_STANDARD)
   set(CMAKE_CXX_STANDARD 17)
 endif()
 
-# Source root directory for pytorch.
-if(NOT TORCH_ROOT)
-  set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch)
-endif()
-
 include(${EXECUTORCH_ROOT}/build/Utils.cmake)
 include(${EXECUTORCH_ROOT}/build/Codegen.cmake)
 

diff --git a/build/Codegen.cmake b/build/Codegen.cmake
@@ -69,19 +69,28 @@ function(generate_bindings_for_kernels)
 
   # Command to generate selected_operators.yaml from custom_ops.yaml.
   file(GLOB_RECURSE _codegen_templates "${EXECUTORCH_ROOT}/codegen/templates/*")
-  file(GLOB_RECURSE _torchgen_srcs "${TORCH_ROOT}/torchgen/*.py")
 
   set(_out_dir ${CMAKE_CURRENT_BINARY_DIR}/${GEN_LIB_NAME})
   # By default selective build output is selected_operators.yaml
   set(_oplist_yaml ${_out_dir}/selected_operators.yaml)
 
   # Command to codegen C++ wrappers to register custom ops to both PyTorch and
   # Executorch runtime.
+  execute_process(
+    COMMAND
+      "${PYTHON_EXECUTABLE}" -c "from distutils.sysconfig import get_python_lib;print(get_python_lib())"
+    OUTPUT_VARIABLE site-packages-out
+    ERROR_VARIABLE site-packages-out-error
+    RESULT_VARIABLE site-packages-result
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+  file(GLOB_RECURSE _torchgen_srcs "${site-packages-out}/torchgen/*.py")
   set(_gen_command
       "${PYTHON_EXECUTABLE}" -m torchgen.gen_executorch
       --source-path=${EXECUTORCH_ROOT}/codegen --install-dir=${_out_dir}
-      --tags-path=${TORCH_ROOT}/aten/src/ATen/native/tags.yaml
-      --aten-yaml-path=${TORCH_ROOT}/aten/src/ATen/native/native_functions.yaml
+      --tags-path=${site-packages-out}/torchgen/packaged/ATen/native/tags.yaml
+      --aten-yaml-path=${site-packages-out}/torchgen/packaged/ATen/native/native_functions.yaml
       --op-selection-yaml-path=${_oplist_yaml}
   )
 

diff --git a/configurations/CMakeLists.txt b/configurations/CMakeLists.txt
@@ -15,10 +15,6 @@ endif()
 if(NOT EXECUTORCH_ROOT)
   set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/..)
 endif()
-# Source root directory for pytorch. This is needed for kernel binding.
-if(NOT TORCH_ROOT)
-  set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch)
-endif()
 
 include(${EXECUTORCH_ROOT}/build/Utils.cmake)
 

diff --git a/examples/apple/mps/CMakeLists.txt b/examples/apple/mps/CMakeLists.txt
@@ -33,11 +33,6 @@ if(NOT PYTHON_EXECUTABLE)
   resolve_python_executable()
 endif()
 
-# Source root directory for pytorch.
-if(NOT TORCH_ROOT)
-  set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch)
-endif()
-
 add_compile_options("-Wall" "-Werror")
 
 include(${EXECUTORCH_ROOT}/build/Utils.cmake)

diff --git a/examples/arm/CMakeLists.txt b/examples/arm/CMakeLists.txt
@@ -27,11 +27,6 @@ if(NOT PYTHON_EXECUTABLE)
   resolve_python_executable()
 endif()
 
-# Source root directory for pytorch.
-if(NOT TORCH_ROOT)
-  set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch)
-endif()
-
 set(_common_compile_options -Wno-deprecated-declarations -fPIC)
 
 # Let files say "include <executorch/path/to/header.h>".

diff --git a/examples/models/llama2/custom_ops/CMakeLists.txt b/examples/models/llama2/custom_ops/CMakeLists.txt
@@ -20,11 +20,6 @@ if(NOT EXECUTORCH_ROOT)
   set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../..)
 endif()
 
-# Source root directory for pytorch.
-if(NOT TORCH_ROOT)
-  set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch)
-endif()
-
 set(_common_compile_options -Wno-deprecated-declarations -fPIC)
 
 include(${EXECUTORCH_ROOT}/build/Utils.cmake)

diff --git a/examples/portable/custom_ops/CMakeLists.txt b/examples/portable/custom_ops/CMakeLists.txt
@@ -26,10 +26,6 @@ endif()
 if(NOT EXECUTORCH_ROOT)
   set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
 endif()
-# Source root directory for pytorch.
-if(NOT TORCH_ROOT)
-  set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch)
-endif()
 
 include(${EXECUTORCH_ROOT}/build/Utils.cmake)
 include(${EXECUTORCH_ROOT}/build/Codegen.cmake)

diff --git a/examples/qualcomm/CMakeLists.txt b/examples/qualcomm/CMakeLists.txt
@@ -15,10 +15,6 @@ project(qualcomm_runner_example)
 if(NOT EXECUTORCH_ROOT)
   set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..)
 endif()
-# Source root directory for pytorch.
-if(NOT TORCH_ROOT)
-  set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch)
-endif()
 
 include(${EXECUTORCH_ROOT}/build/Utils.cmake)
 include(${EXECUTORCH_ROOT}/build/Codegen.cmake)

diff --git a/examples/sdk/CMakeLists.txt b/examples/sdk/CMakeLists.txt
@@ -18,10 +18,6 @@ endif()
 if(NOT EXECUTORCH_ROOT)
   set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..)
 endif()
-# Source root directory for pytorch.
-if(NOT TORCH_ROOT)
-  set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch)
-endif()
 
 include(${EXECUTORCH_ROOT}/build/Utils.cmake)
 include(${EXECUTORCH_ROOT}/build/Codegen.cmake)

diff --git a/install_requirements.sh b/install_requirements.sh
@@ -54,8 +54,7 @@ done
 #
 
 # Since ExecuTorch often uses main-branch features of pytorch, only the nightly
-# pip versions will have the required features. The NIGHTLY_VERSION value should
-# agree with the third-party/pytorch pinned submodule commit.
+# pip versions will have the required features.
 #
 # NOTE: If a newly-fetched version of the executorch repo changes the value of
 # NIGHTLY_VERSION, you should re-run this script to install the necessary

diff --git a/kernels/optimized/CMakeLists.txt b/kernels/optimized/CMakeLists.txt
@@ -20,10 +20,6 @@ endif()
 if(NOT EXECUTORCH_ROOT)
   set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..)
 endif()
-# Source root directory for pytorch. This is needed for kernel binding.
-if(NOT TORCH_ROOT)
-  set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch)
-endif()
 
 set(_common_compile_options -Wno-deprecated-declarations)
 

diff --git a/kernels/portable/CMakeLists.txt b/kernels/portable/CMakeLists.txt
@@ -20,10 +20,6 @@ endif()
 if(NOT EXECUTORCH_ROOT)
   set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..)
 endif()
-# Source root directory for pytorch.
-if(NOT TORCH_ROOT)
-  set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch)
-endif()
 
 set(_common_compile_options -Wno-deprecated-declarations)
 

diff --git a/kernels/quantized/CMakeLists.txt b/kernels/quantized/CMakeLists.txt
@@ -23,10 +23,6 @@ endif()
 if(NOT EXECUTORCH_ROOT)
   set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..)
 endif()
-# Source root directory for pytorch.
-if(NOT TORCH_ROOT)
-  set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch)
-endif()
 
 set(_common_compile_options -Wno-deprecated-declarations)
 

diff --git a/third-party/TARGETS b/third-party/TARGETS
@@ -36,14 +36,31 @@ prebuilt_python_library_defs = {
 
 add_prebuilt_python_library_targets(prebuilt_python_library_defs)
 
+runtime.genrule(
+    name = "torchgen_files",
+    outs = {
+        "torchgen": ["torchgen"],
+    },
+    default_outs = ["."],
+    srcs = ["link_torch.sh"],
+    bash = "bash $SRCS -f torchgen -o ${OUT}",
+)
+
+runtime.genrule(
+    name = "yaml_files",
+    outs = {
+        "tags": ["aten/src/ATen/native/tags.yaml"],
+        "native_functions": ["aten/src/ATen/native/native_functions.yaml"],
+    },
+    default_outs = ["."],
+    srcs = ["link_torch.sh"],
+    bash = "mkdir -p ${OUT}/aten/src/ATen/native/ && bash $SRCS -f torchgen/packaged/ATen/native/tags.yaml,torchgen/packaged/ATen/native/native_functions.yaml -o ${OUT}/aten/src/ATen/native/",
+)
+
 runtime.python_library(
     name = "torchgen",
-    srcs = subdir_glob(
-        [
-            ("pytorch/torchgen", "**/*.py"),
-        ],
-    ),
-    base_module = "torchgen",
+    srcs = [":torchgen_files[torchgen]"] + glob(["**/*.py"]),
+    base_module = "torchgen,torchgen.executorch",
     visibility = ["PUBLIC"],
     deps = [
         "//third-party:pyyaml",
@@ -78,11 +95,10 @@ runtime.python_binary(
 
 runtime.filegroup(
     name = "aten_src_path",
-    srcs = subdir_glob([
-        ("pytorch", "aten/src/ATen/templates/*"),
-        ("pytorch", "aten/src/ATen/native/native_functions.yaml"),
-        ("pytorch", "aten/src/ATen/native/tags.yaml"),
-    ]),
+    srcs = [
+        ":yaml_files[tags]",
+        ":yaml_files[native_functions]",
+    ],
     visibility = [
         "PUBLIC",
     ],

diff --git a/third-party/pytorch b/third-party/pytorch