Initial commit

fbshipit-source-id: 38182c9de097e3893c12e841e89c1b809c806b20

.buckconfig

@@ -0,0 +1,25 @@
+[executorch]
+  is_oss = 1
+
+[buildfile]
+  name = TARGETS
+
+[repositories]
+  root = .
+  prelude = third-party/prelude
+  shim = shim
+
+[repository_aliases]
+  config = prelude
+  ovr_config = prelude
+  toolchains = shim
+  fbcode = shim
+  fbcode_macros = shim
+  fbsource = shim
+  buck = shim
+
+[cxx]
+  cxxflags = -g -std=c++17
+
+[parser]
+  target_platform_detector_spec = target:root//...->prelude//platforms:default target:shim//...->prelude//platforms:default

.ci/docker/README.md

@@ -0,0 +1,25 @@
+# Docker images for Executorch CI
+
+This directory contains everything needed to build the Docker images
+that are used in Executorch CI. The content of this directory are copied
+from PyTorch CI https://github.com/pytorch/pytorch/tree/main/.ci/docker.
+It also uses the same directory structure as PyTorch.
+
+## Contents
+
+* `build.sh` -- dispatch script to launch all builds
+* `common` -- scripts used to execute individual Docker build stages
+* `ubuntu` -- Dockerfile for Ubuntu image for CPU build and test jobs
+
+## Usage
+
+```bash
+# Generic usage
+./build.sh "${IMAGE_NAME}" "${DOCKER_BUILD_PARAMETERS}"
+
+# Build a specific image
+./build.sh executorch-ubuntu-22.04-clang12 -t myimage:latest
+
+# Set CLANG version (see build.sh) and build image
+CLANG_VERSION=11 ./build.sh executorch-ubuntu-22.04-clang11 -t myimage:latest
+```

.ci/docker/build.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+IMAGE_NAME="$1"
+shift
+
+echo "Building ${IMAGE_NAME} Docker image"
+
+OS=ubuntu
+OS_VERSION=22.04
+CLANG_VERSION=12
+PYTHON_VERSION=3.10
+MINICONDA_VERSION='23.5.1-0'
+
+docker build \
+  --no-cache \
+  --progress=plain \
+  --build-arg "OS_VERSION=${OS_VERSION}" \
+  --build-arg "CLANG_VERSION=${CLANG_VERSION}" \
+  --build-arg "PYTHON_VERSION=${PYTHON_VERSION}" \
+  --build-arg "MINICONDA_VERSION=${MINICONDA_VERSION}" \
+  -f "${OS}"/Dockerfile \
+  "$@" \
+  .

.ci/docker/common/install_base.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+install_ubuntu() {
+  apt-get update
+
+  apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    curl \
+    git \
+    wget \
+    sudo \
+    vim \
+    jq \
+    vim \
+    unzip \
+    gdb \
+    rsync
+
+  # Cleanup package manager
+  apt-get autoclean && apt-get clean
+  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+  ubuntu)
+    install_ubuntu
+    ;;
+  *)
+    echo "Unable to determine OS..."
+    exit 1
+    ;;
+esac

.ci/docker/common/install_buck.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+install_ubuntu() {
+  apt-get update
+  apt-get install -y zstd
+
+  wget -q https://github.com/facebook/buck2/releases/download/2023-07-18/buck2-x86_64-unknown-linux-gnu.zst
+  zstd -d buck2-x86_64-unknown-linux-gnu.zst -o buck2
+
+  chmod +x buck2
+  mv buck2 /usr/bin/
+
+  rm buck2-x86_64-unknown-linux-gnu.zst
+  # Cleanup package manager
+  apt-get autoclean && apt-get clean
+  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+  ubuntu)
+    install_ubuntu
+    ;;
+  *)
+    echo "Unable to determine OS..."
+    exit 1
+    ;;
+esac

.ci/docker/common/install_clang.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+install_ubuntu() {
+  apt-get update
+
+  apt-get install -y --no-install-recommends clang-"$CLANG_VERSION"
+  apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION"
+  # Also require LLD linker from llvm
+  apt-get install -y lld
+
+  # Use update-alternatives to make this version the default
+  update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50
+  update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-"$CLANG_VERSION" 50
+  # Override cc/c++ to clang as well
+  update-alternatives --install /usr/bin/cc cc /usr/bin/clang 50
+  update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 50
+
+  # Cleanup package manager
+  apt-get autoclean && apt-get clean
+  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+if [ -n "$CLANG_VERSION" ]; then
+  # Install base packages depending on the base OS
+  ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+  case "$ID" in
+    ubuntu)
+      install_ubuntu
+      ;;
+    *)
+      echo "Unable to determine OS..."
+      exit 1
+      ;;
+  esac
+fi

.ci/docker/common/install_conda.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+# shellcheck source=/dev/null
+source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+
+install_miniconda() {
+  BASE_URL="https://repo.anaconda.com/miniconda"
+  CONDA_FILE="Miniconda3-py${PYTHON_VERSION//./}_${MINICONDA_VERSION}-Linux-x86_64.sh"
+
+  mkdir -p /opt/conda
+  chown ci-user:ci-user /opt/conda
+
+  pushd /tmp
+  wget -q "${BASE_URL}/${CONDA_FILE}"
+  # Install miniconda
+  as_ci_user bash "${CONDA_FILE}" -b -f -p "/opt/conda"
+  # Clean up the download file
+  rm "${CONDA_FILE}"
+  popd
+
+  sed -e 's|PATH="\(.*\)"|PATH="/opt/conda/bin:\1"|g' -i /etc/environment
+  export PATH="/opt/conda/bin:$PATH"
+}
+
+install_python() {
+  pushd /opt/conda
+  # Install the correct Python version
+  as_ci_user conda create -n "py_${PYTHON_VERSION}" -y --file /opt/conda/conda-env-ci.txt python="${PYTHON_VERSION}"
+  popd
+}
+
+install_pip_dependencies() {
+  pushd /opt/conda
+  # Install all Python dependencies, including PyTorch
+  pip_install -r /opt/conda/requirements-ci.txt
+  pip_install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
+  popd
+}
+
+install_miniconda
+install_python
+install_pip_dependencies

.ci/docker/common/install_user.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+# Same as ec2-user
+echo "ci-user:x:1000:1000::/var/lib/ci-user:" >> /etc/passwd
+echo "ci-user:x:1000:" >> /etc/group
+# Needed on Focal or newer
+echo "ci-user:*:19110:0:99999:7:::" >> /etc/shadow
+
+# Create $HOME
+mkdir -p /var/lib/ci-user
+chown ci-user:ci-user /var/lib/ci-user
+
+# Allow sudo
+echo 'ci-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ci-user
+
+# Test that sudo works
+sudo -u ci-user sudo -v

.ci/docker/common/utils.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+as_ci_user() {
+  # NB: unsetting the environment variables works around a conda bug
+  #     https://github.com/conda/conda/issues/6576
+  # NB: Pass on PATH and LD_LIBRARY_PATH to sudo invocation
+  # NB: This must be run from a directory that the user has access to
+  sudo -E -H -u ci-user env -u SUDO_UID -u SUDO_GID -u SUDO_COMMAND -u SUDO_USER env "PATH=${PATH}" "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" "$@"
+}
+
+conda_install() {
+  # Ensure that the install command don't upgrade/downgrade Python
+  # This should be called as
+  #   conda_install pkg1 pkg2 ... [-c channel]
+  as_ci_user conda install -q -n "py_${PYTHON_VERSION}" -y python="${PYTHON_VERSION}" "$@"
+}
+
+conda_run() {
+  as_ci_user conda run -n "py_${PYTHON_VERSION}" --no-capture-output "$@"
+}
+
+pip_install() {
+  as_ci_user conda run -n "py_${PYTHON_VERSION}" pip install --progress-bar off "$@"
+}

.ci/docker/conda-env-ci.txt

@@ -0,0 +1,3 @@
+# TODO: We might need to update this to install flatbuffers from the pinned commit
+# in fbcode/executorch/third-party/flatbuffers.submodule.txt
+flatbuffers=2.0.0

.ci/docker/requirements-ci.txt

@@ -0,0 +1,4 @@
+mpmath==1.3.0
+PyYAML==6.0.1
+ruamel.yaml==0.17.32
+sympy==1.12

.ci/docker/ubuntu/Dockerfile

@@ -0,0 +1,38 @@
+ARG OS_VERSION
+
+FROM ubuntu:${OS_VERSION}
+
+ARG OS_VERSION
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install common dependencies
+COPY ./common/install_base.sh install_base.sh
+RUN bash ./install_base.sh && rm install_base.sh
+
+# Install clang
+ARG CLANG_VERSION
+COPY ./common/install_clang.sh install_clang.sh
+RUN bash ./install_clang.sh && rm install_clang.sh
+
+# Setup buck
+COPY ./common/install_buck.sh install_buck.sh
+RUN bash ./install_buck.sh && rm install_buck.sh
+
+# Setup user
+COPY ./common/install_user.sh install_user.sh
+RUN bash ./install_user.sh && rm install_user.sh
+
+# Install conda and other dependencies
+ARG MINICONDA_VERSION
+ARG PYTHON_VERSION
+ENV PYTHON_VERSION=$PYTHON_VERSION
+ENV PATH /opt/conda/envs/py_$PYTHON_VERSION/bin:/opt/conda/bin:$PATH
+COPY requirements-ci.txt /opt/conda/
+COPY conda-env-ci.txt /opt/conda/
+COPY ./common/install_conda.sh install_conda.sh
+COPY ./common/utils.sh utils.sh
+RUN bash ./install_conda.sh && rm install_conda.sh utils.sh /opt/conda/requirements-ci.txt /opt/conda/conda-env-ci.txt
+
+USER ci-user
+CMD ["bash"]

.clang-tidy

@@ -0,0 +1,10 @@
+---
+# NOTE there must be no spaces before the '-', so put the comma last.
+InheritParentConfig: true
+Checks: '
+-facebook-hte-BadMemberName,
+-facebook-hte-NullableReturn,
+'
+AnalyzeTemporaryDtors: false
+CheckOptions:
+...