refactor: build everything in build.rs

.github/workflows/ci.yml

@@ -19,6 +19,8 @@ jobs:
       with:
         toolchain: nightly
         target: x86_64-unknown-linux-gnu
+    - name: Rust cache
+      uses: Swatinem/rust-cache@v2
     - name: Use Node.js 20.x
       uses: actions/setup-node@v4
       with:
@@ -37,33 +39,16 @@ jobs:
       with:
         path: .venv
         key: ${{ runner.os }}-venv-3_10
-    - name: Cached build
-      uses: actions/cache@v2
-      with:
-        path: executorch/cmake-out
-        key: ${{ runner.os }}-cmake-${{ hashFiles('build.rs', 'executorch/version.txt') }}
-    - name: Build executorch
+    - name: Install python dependencies
       run: |
-        python -m venv .venv
+        python3 -m venv .venv
         source .venv/bin/activate
-        cd executorch
-        pip install tomli zstd setuptools wheel
-        pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu
-        if [ ! -d cmake-out ]; then
-          ./install_requirements.sh
-          cmake \
-            -S . \
-            -B cmake-out \
-            -DCMAKE_INSTALL_PREFIX=cmake-out \
-            -DEXECUTORCH_BUILD_XNNPACK=ON \
-            -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
-            -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
-            -DEXECUTORCH_ENABLE_LOGGING=ON
-          cmake --build cmake-out --target install --config Release
-        fi
+        ./executorch/install_requirements.sh
+        echo "$VIRTUAL_ENV/bin" >> $GITHUB_PATH
+        echo "VIRTUAL_ENV=$VIRTUAL_ENV" >> $GITHUB_ENV
     - name: Install dependencies
       run: yarn install
-    - name: Build
-      run: yarn build
+    - name: Minimal Build
+      run: yarn debug
     - name: Run tests
       run: yarn test-all

.github/workflows/cmake-format.yml

@@ -0,0 +1,23 @@
+name: Run cmake-format linter
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Format CMake files
+      id: cmake-format
+      uses: PuneetMatharu/[email protected]
+      with:
+        args: --in-place
+
+    - name: Commit changes
+      uses: stefanzweifel/git-auto-commit-action@v4
+      with:
+        commit_user_name: cmake-format-bot
+        commit_message: 'chore: automated commit of cmake-format changes.'

.github/workflows/release.yml

@@ -105,59 +105,16 @@ jobs:
         rm llvm-mingw.tar.xz
         MINGW_PATH=$(realpath llvm-mingw-*)
         echo "PATH=$MINGW_PATH/bin:$PATH" >> $GITHUB_ENV
-    - name: Build executorch
-      shell: bash
-      env:
-        PLATFORM: ${{ matrix.plat }}
-        ARCH: ${{ matrix.arch }}
+    - name: Install python dependencies
       run: |
         python3 -m venv .venv
         source .venv/bin/activate
-        cd executorch
-        cd backends/xnnpack/third-party/cpuinfo/
-        patch -p1 -i ${{ github.workspace }}/scripts/cpuinfo.patch
-        cd ${{ github.workspace }}/executorch
-        pip install tomli zstd setuptools wheel
-        pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu
-        if [ ! -d cmake-out ]; then
-          ./install_requirements.sh
-          EXTRA_CMAKE_ARGS=""
-          if [[ "$PLATFORM" == "windows" ]]; then
-            EXTRA_CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=../cmake/mingw-w64-${ARCH}.clang.toolchain.cmake"
-            if [[ "$ARCH" == "aarch64" ]]; then
-              EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DEXECUTORCH_BUILD_QNN=ON -DQNN_SDK_ROOT=$QNN_SDK_ROOT"
-            fi
-          elif [[ "$PLATFORM" == "macos" ]]; then
-            EXTRA_CMAKE_ARGS="-DEXECUTORCH_BUILD_COREML=ON -DCMAKE_OSX_ARCHITECTURES=${ARCH}"
-          elif [[ "$PLATFORM" == "linux" ]] && [[ "$ARCH" == "aarch64" ]]; then
-            EXTRA_CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=../cmake/aarch64-linux-gnu.gcc.toolchain.cmake"
-          fi
-          if [[ "$ARCH" == "aarch64" ]]; then
-            EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DEXECUTORCH_XNNPACK_ENABLE_KLEIDI=ON"
-          fi
-          cmake \
-            -S . \
-            -B cmake-out \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DCMAKE_INSTALL_PREFIX=cmake-out \
-            -DEXECUTORCH_ENABLE_LOGGING=ON \
-            -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
-            -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
-            -DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON \
-            -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
-            -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
-            -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
-            -DEXECUTORCH_BUILD_PTHREADPOOL=ON \
-            -DEXECUTORCH_BUILD_CPUINFO=ON \
-            -DEXECUTORCH_BUILD_XNNPACK=ON \
-            -DEXECUTORCH_XNNPACK_SHARED_WORKSPACE=ON \
-            $EXTRA_CMAKE_ARGS
-          cmake --build cmake-out --target install --config Release -j$(nproc)
-        fi
+        ./executorch/install_requirements.sh
+        echo "$VIRTUAL_ENV/bin" >> $GITHUB_PATH
+        echo "VIRTUAL_ENV=$VIRTUAL_ENV" >> $GITHUB_ENV
     - name: Install dependencies
       run: yarn install
     - name: Build
-      shell: bash
       run: yarn build --target ${{ matrix.target }}
     - name: Upload build artifacts
       uses: actions/upload-artifact@v4
@@ -192,11 +149,3 @@ jobs:
       uses: JS-DevTools/npm-publish@v3
       with:
         token: ${{ secrets.NPM_TOKEN }}
-    - name: Draft release
-      uses: actions/create-release@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN  }}
-      with:
-        tag_name: ${{ github.ref  }}
-        release_name: ${{ github.ref  }}
-        draft: true

CMakeLists.txt

@@ -0,0 +1,143 @@
+cmake_minimum_required(VERSION 3.18)
+project(executorch_all)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+include(ExternalProject)
+
+if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/executorch)
+  ExternalProject_Add(
+    executorch_src
+    PREFIX executorch
+    GIT_REPOSITORY https://github.com/mybigday/node-executorch.git
+    GIT_TAG dev
+    GIT_SHALLOW TRUE
+    GIT_SUBMODULES_RECURSE TRUE
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+    INSTALL_COMMAND "")
+  set(EXECUTORCH_DIR ${CMAKE_CURRENT_SOURCE_DIR}/executorch)
+else()
+  set(EXECUTORCH_DIR ${CMAKE_CURRENT_SOURCE_DIR}/executorch)
+endif()
+
+# Apply patch for cpuinfo
+
+find_program(GIT git REQUIRED)
+
+cmake_language(DEFER DIRECTORY ${CMAKE_SOURCE_DIR} CALL _reverse_patch_cpuinfo
+               ())
+function(_reverse_patch_cpuinfo)
+  # reverse patch
+  execute_process(
+    COMMAND ${GIT} apply ${CMAKE_CURRENT_SOURCE_DIR}/scripts/cpuinfo.patch -R
+    WORKING_DIRECTORY ${EXECUTORCH_DIR}/backends/xnnpack/third-party/cpuinfo)
+endfunction(_reverse_patch_cpuinfo)
+
+add_custom_target(
+  patch_cpuinfo ALL
+  COMMAND ${GIT} apply ${CMAKE_CURRENT_SOURCE_DIR}/scripts/cpuinfo.patch || true
+  WORKING_DIRECTORY ${EXECUTORCH_DIR}/backends/xnnpack/third-party/cpuinfo)
+
+# ExecuTorch main
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Windows" AND CMAKE_SYSTEM_PROCESSOR STREQUAL
+                                            "ARM64")
+  get_filename_component(
+    CMAKE_TOOLCHAIN_FILE
+    ${CMAKE_CURRENT_SOURCE_DIR}/cmake/mingw-w64-aarch64.clang.toolchain.cmake
+    ABSOLUTE)
+  if(QNN_SDK_ROOT AND CMAKE_BUILD_TYPE STREQUAL "Release")
+    list(APPEND EXECUTORCH_CMAKE_ARGS "-DEXECUTORCH_BUILD_QNN=ON")
+  endif()
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Windows" AND CMAKE_SYSTEM_PROCESSOR STREQUAL
+                                            "x86_64")
+  get_filename_component(
+    CMAKE_TOOLCHAIN_FILE
+    ${CMAKE_CURRENT_SOURCE_DIR}/cmake/mingw-w64-x86_64.clang.toolchain.cmake
+    ABSOLUTE)
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_BUILD_TYPE STREQUAL "Release")
+  list(APPEND EXECUTORCH_CMAKE_ARGS -DEXECUTORCH_BUILD_COREML=ON)
+endif()
+
+if(CMAKE_BUILD_TYPE STREQUAL "Release")
+  list(APPEND EXECUTORCH_CMAKE_ARGS "-DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON")
+  list(APPEND EXECUTORCH_CMAKE_ARGS "-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON")
+  list(APPEND EXECUTORCH_CMAKE_ARGS "-DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON")
+  list(APPEND EXECUTORCH_CMAKE_ARGS "-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON")
+  list(APPEND EXECUTORCH_CMAKE_ARGS "-DEXECUTORCH_BUILD_CPUINFO=ON")
+  list(APPEND EXECUTORCH_CMAKE_ARGS "-DEXECUTORCH_XNNPACK_SHARED_WORKSPACE=ON")
+  if(CMAKE_SYSTEM_PROCESSOR STREQUAL "ARM64" AND NOT CMAKE_SYSTEM_NAME STREQUAL
+                                                 "Darwin")
+    list(APPEND EXECUTORCH_CMAKE_ARGS "-DEXECUTORCH_XNNPACK_ENABLE_KLEIDI=ON")
+  endif()
+endif()
+
+ExternalProject_Add(
+  executorch
+  PREFIX ${CMAKE_CURRENT_BINARY_DIR}/executorch
+  SOURCE_DIR ${EXECUTORCH_DIR}
+  DEPENDS patch_cpuinfo
+  CMAKE_ARGS -DCMAKE_TOOLCHAIN_FILE:PATH=${CMAKE_TOOLCHAIN_FILE}
+             -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+             -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+             -DCMAKE_ASM_COMPILER=${CMAKE_ASM_COMPILER}
+             -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}
+             -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+             -DCMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME}
+             -DCMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}
+             -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_INSTALL_PREFIX}
+             -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}
+             -DEXECUTORCH_ENABLE_LOGGING=ON
+             -DEXECUTORCH_BUILD_XNNPACK=ON
+             -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON
+             -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON
+             -DQNN_SDK_ROOT:PATH=$ENV{QNN_SDK_ROOT}
+             ${EXECUTORCH_CMAKE_ARGS})
+
+ExternalProject_Get_Property(executorch SOURCE_DIR)
+
+# TOKENIZERS
+
+set(ABSL_ENABLE_INSTALL ON)
+set(ABSL_PROPAGATE_CXX_STD ON)
+add_subdirectory(executorch/extension/llm/third-party/abseil-cpp)
+add_subdirectory(executorch/extension/llm/third-party/re2)
+add_subdirectory(executorch/extension/llm/third-party/sentencepiece)
+
+file(GLOB TOKENIZER_SRCS executorch/extension/llm/tokenizer/bpe_tokenizer.cpp
+     executorch/extension/llm/tokenizer/tiktoken.cpp)
+file(GLOB TOKENIZER_HDRS executorch/extension/llm/tokenizer/*.h)
+add_library(tokenizer STATIC ${TOKENIZER_SRCS} ${TOKENIZER_HDRS})
+target_include_directories(
+  tokenizer
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${CMAKE_CURRENT_SOURCE_DIR}/executorch/extension/llm/third-party/abseil-cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/executorch/extension/llm/third-party/re2)
+target_link_libraries(tokenizer PUBLIC re2::re2 sentencepiece-static)
+
+install(
+  TARGETS tokenizer
+  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
+
+# SAMPLER
+
+file(GLOB SAMPLER_SRCS executorch/extension/llm/sampler/sampler.cpp)
+file(GLOB SAMPLER_HDRS executorch/extension/llm/sampler/sampler.h)
+add_library(sampler STATIC ${SAMPLER_SRCS} ${SAMPLER_HDRS})
+target_include_directories(sampler PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+target_link_libraries(sampler PUBLIC tokenizer)
+
+install(
+  TARGETS sampler
+  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})

Cargo.toml

@@ -24,4 +24,4 @@ libc = "0.2"
 [build-dependencies]
 cc = "1.1.21"
 cpp_build = "0.5"
-build-target = "0.4"
+cmake = "0.1"