Dev

.github/workflows/build.yml

@@ -1,73 +1,106 @@
 name: Build
 
 on:
+  workflow_dispatch:
   push:
     branches-ignore:
       - "update-dependencies-pr"
     paths:
       - ".github/workflows/build.yml"
       - "docker/**"
       - "*.sh"
-  pull_request:
-    paths:
-      - ".github/workflows/build.yml"
-      - "docker/**"
-      - "build.sh"
+  # pull_request:
+  #   paths:
+  #     - ".github/workflows/build.yml"
+  #     - "docker/**"
+  #     - "build.sh"
 
 concurrency:
   group: test-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  REGION_ID: cn-beijing
+  ACR_REGISTRY: registry.cn-beijing.aliyuncs.com
+  ACR_NAMESPACE: oneflow
+  DOCKER_HUB_NAMESPACE: oneflowinc
+
 jobs:
   build_manylinux:
-    name: ${{ matrix.policy }}_${{ matrix.platform }}
+    name: ${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
-        policy: ["manylinux2014", "manylinux_2_24", "musllinux_1_1"]
-        platform: ["i686", "x86_64", "ppc64le"]
         include:
-          - policy: "manylinux2010"
-            platform: "i686"
-          - policy: "manylinux2010"
+          - tag-suffix: "cuda12.2"
+            policy: "manylinux2014"
+            platform: "x86_64"
+            CUDA_BASE_IMAGE: "nvidia/cuda:12.2.2-cudnn8-devel-centos7"
+          - tag-suffix: "cuda12.1"
+            policy: "manylinux2014"
+            platform: "x86_64"
+            CUDA_BASE_IMAGE: "nvidia/cuda:12.1.1-cudnn8-devel-centos7"
+          - tag-suffix: "cuda11.8"
+            policy: "manylinux2014"
+            platform: "x86_64"
+            CUDA_BASE_IMAGE: "nvidia/cuda:11.8.0-cudnn8-devel-centos7"
+          - tag-suffix: "cpu"
+            policy: "manylinux2014"
             platform: "x86_64"
+            CUDA_BASE_IMAGE: ""
 
     env:
       POLICY: ${{ matrix.policy }}
       PLATFORM: ${{ matrix.platform }}
       COMMIT_SHA: ${{ github.sha }}
+      DOCKER_REPO: "${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}"
+      TEST_TAG: ${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }}
+      CUDA_BASE_IMAGE: ${{ matrix.CUDA_BASE_IMAGE }}
 
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+      # 1.1 Login to ACR
+      - name: Login to ACR with the AccessKey pair
+        uses: aliyun/acr-login@v1
+        with:
+          login-server: https://registry.${{env.REGION_ID}}.aliyuncs.com
+          username: "${{ secrets.ACR_USERNAME }}"
+          password: "${{ secrets.ACR_PASSWORD }}"
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Set up emulation
         if: matrix.platform == 'ppc64le'
         uses: docker/setup-qemu-action@v1
         with:
           platforms: ${{ matrix.platform }}
 
-      # - name: Set BuildKit for ppc64le
-      #   if: matrix.platform == 'ppc64le'
-      #   run: echo 'MANYLINUX_BUILD_FRONTEND=buildkit' >> $GITHUB_ENV 
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
-      - name: Setup cache
-        uses: actions/cache@v2
-        with:
-          path: .buildx-cache-${{ matrix.policy }}_${{ matrix.platform }}/*
-          key: buildx-cache-${{ matrix.policy }}-${{ matrix.platform }}-${{ hashFiles('docker/**') }}
-          restore-keys: buildx-cache-${{ matrix.policy }}-${{ matrix.platform }}-
-
       - name: Build
         run: ./build.sh
-
-      - name: Deploy
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'pypa/manylinux'
-        run: ./deploy.sh
-        env:
-          QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
-          QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: |
+            ${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest
+            ${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ env.COMMIT_SHA }}
+            ${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest
+            ${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ env.COMMIT_SHA }}
+          cache-from: type=registry,ref=${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest
+          cache-to: type=inline
+          context: ./docker/
+          build-args: |
+            POLICY
+            PLATFORM
+            BASEIMAGE
+            DEVTOOLSET_ROOTPATH
+            PREPEND_PATH
+            LD_LIBRARY_PATH_ARG

.gitignore

@@ -61,3 +61,4 @@ docker/sources
 
 # buildx cache
 .buildx-cache-*/
+.vscode/

build.sh

@@ -48,7 +48,8 @@ elif [ "${POLICY}" == "manylinux2014" ]; then
 	if [ "${PLATFORM}" == "s390x" ]; then
 		BASEIMAGE="s390x/clefos:7"
 	else
-		BASEIMAGE="${MULTIARCH_PREFIX}centos:7"
+		DEFAULT_BASEIMAGE="${MULTIARCH_PREFIX}centos:7"
+		BASEIMAGE="${CUDA_BASE_IMAGE:-${DEFAULT_BASEIMAGE}}"
 	fi
 	DEVTOOLSET_ROOTPATH="/opt/rh/devtoolset-10/root"
 	PREPEND_PATH="${DEVTOOLSET_ROOTPATH}/usr/bin:"
@@ -91,31 +92,18 @@ fi
 if [ "${MANYLINUX_BUILD_FRONTEND}" == "docker" ]; then
 	docker build ${BUILD_ARGS_COMMON}
 elif [ "${MANYLINUX_BUILD_FRONTEND}" == "docker-buildx" ]; then
-	docker buildx build \
-		--load \
-		--cache-from=type=local,src=$(pwd)/.buildx-cache-${POLICY}_${PLATFORM} \
-		--cache-to=type=local,dest=$(pwd)/.buildx-cache-staging-${POLICY}_${PLATFORM} \
-		${BUILD_ARGS_COMMON}
+	env
 elif [ "${MANYLINUX_BUILD_FRONTEND}" == "buildkit" ]; then
-	buildctl build \
-		--frontend=dockerfile.v0 \
-		--local context=./docker/ \
-		--local dockerfile=./docker/ \
-		--import-cache type=local,src=$(pwd)/.buildx-cache-${POLICY}_${PLATFORM} \
-		--export-cache type=local,dest=$(pwd)/.buildx-cache-staging-${POLICY}_${PLATFORM} \
-		--opt build-arg:POLICY=${POLICY} --opt build-arg:PLATFORM=${PLATFORM} --opt build-arg:BASEIMAGE=${BASEIMAGE} \
-		--opt "build-arg:DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}" --opt "build-arg:PREPEND_PATH=${PREPEND_PATH}" --opt "build-arg:LD_LIBRARY_PATH_ARG=${LD_LIBRARY_PATH_ARG}" \
-		--output type=docker,name=quay.io/pypa/${POLICY}_${PLATFORM}:${COMMIT_SHA} | docker load
+	echo "Unsupported build frontend: buildkit"
+	exit 1
 else
 	echo "Unsupported build frontend: '${MANYLINUX_BUILD_FRONTEND}'"
 	exit 1
 fi
 
-docker run --rm -v $(pwd)/tests:/tests:ro quay.io/pypa/${POLICY}_${PLATFORM}:${COMMIT_SHA} /tests/run_tests.sh
-
-if [ "${MANYLINUX_BUILD_FRONTEND}" != "docker" ]; then
-	if [ -d $(pwd)/.buildx-cache-${POLICY}_${PLATFORM} ]; then
-		rm -rf $(pwd)/.buildx-cache-${POLICY}_${PLATFORM}
-	fi
-	mv $(pwd)/.buildx-cache-staging-${POLICY}_${PLATFORM} $(pwd)/.buildx-cache-${POLICY}_${PLATFORM}
-fi
+echo "POLICY=${POLICY}" >>  $GITHUB_ENV
+echo "PLATFORM=${PLATFORM}" >>  $GITHUB_ENV
+echo "BASEIMAGE=${BASEIMAGE}" >>  $GITHUB_ENV
+echo "DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}" >>  $GITHUB_ENV
+echo "PREPEND_PATH=${PREPEND_PATH}" >>  $GITHUB_ENV
+echo "LD_LIBRARY_PATH_ARG=${LD_LIBRARY_PATH_ARG}" >>  $GITHUB_ENV

docker/Dockerfile

@@ -26,18 +26,18 @@ COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors
 
 # setup entrypoint, this will wrap commands with `linux32` with i686 images
 COPY build_scripts/install-entrypoint.sh \
-     build_scripts/update-system-packages.sh \
-     build_scripts/build_utils.sh \
-     /build_scripts/
+    build_scripts/update-system-packages.sh \
+    build_scripts/build_utils.sh \
+    /build_scripts/
 
 RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts
 COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint
 ENTRYPOINT ["manylinux-entrypoint"]
 
 COPY build_scripts/install-runtime-packages.sh \
-     build_scripts/update-system-packages.sh \
-     build_scripts/build_utils.sh \
-     /build_scripts/
+    build_scripts/update-system-packages.sh \
+    build_scripts/build_utils.sh \
+    /build_scripts/
 RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/
 
 COPY build_scripts/build_utils.sh /build_scripts/
@@ -136,9 +136,9 @@ RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.2
 
 FROM build_cpython AS all_python
 COPY build_scripts/install-pypy.sh \
-     build_scripts/pypy.sha256 \
-     build_scripts/finalize-python.sh \
-     /build_scripts/
+    build_scripts/pypy.sha256 \
+    build_scripts/finalize-python.sh \
+    /build_scripts/
 RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.7
 RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.7
 COPY --from=build_cpython36 /opt/_internal /opt/_internal/
@@ -155,18 +155,28 @@ COPY --from=build_swig /manylinux-rootfs /
 COPY --from=build_cpython /manylinux-rootfs /
 COPY --from=all_python /opt/_internal /opt/_internal/
 COPY build_scripts/finalize.sh \
-     build_scripts/update-system-packages.sh \
-     build_scripts/python-tag-abi-tag.py \
-     build_scripts/requirements3.6.txt \
-     build_scripts/requirements3.7.txt \
-     build_scripts/requirements3.8.txt \
-     build_scripts/requirements3.9.txt \
-     build_scripts/requirements3.10.txt \
-     build_scripts/requirements-base-tools.txt \
-     /build_scripts/
+    build_scripts/update-system-packages.sh \
+    build_scripts/python-tag-abi-tag.py \
+    build_scripts/requirements3.6.txt \
+    build_scripts/requirements3.7.txt \
+    build_scripts/requirements3.8.txt \
+    build_scripts/requirements3.9.txt \
+    build_scripts/requirements3.10.txt \
+    build_scripts/requirements-base-tools.txt \
+    /build_scripts/
 COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/
 RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
 
 ENV SSL_CERT_FILE=/opt/_internal/certs.pem
 
+ARG BAZEL_URL="https://github.com/bazelbuild/bazel/releases/download/3.4.1/bazel-3.4.1-linux-x86_64"
+RUN curl -L $BAZEL_URL -o /usr/local/bin/bazel \
+    && chmod +x /usr/local/bin/bazel
+RUN yum install -y wget nasm rdma-core-devel rsync gdb ninja-build openblas-static devtoolset-7-gcc* vim ccache htop fuse-devel
+RUN yum install -y devtoolset-9
+RUN yum install -y devtoolset-10-libasan-devel devtoolset-10-libubsan-devel devtoolset-10-libtsan-devel
+
+RUN wget https://github.com/Oneflow-Inc/llvm-project/releases/download/maybe-14.0.4/clang-tidy-14.AppImage -O /usr/local/bin/clangd && \
+    chmod +x /usr/local/bin/clangd
+
 CMD ["/bin/bash"]

docker/build_scripts/install-runtime-packages.sh

@@ -34,6 +34,14 @@ source $MY_DIR/build_utils.sh
 # MANYLINUX_DEPS: Install development packages (except for libgcc which is provided by gcc install)
 if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
 	MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel zlib-devel expat-devel"
+	yum -y install yum-versionlock
+	yum versionlock add libcudnn8-devel
+	yum versionlock add libcudnn8
+	yum versionlock add cuda-*
+	yum versionlock add libnccl
+	yum versionlock add libnccl-devel
+	yum versionlock list libcudnn8-devel
+	yum versionlock list libcudnn8
 elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
 	MANYLINUX_DEPS="libc6-dev libstdc++-6-dev libglib2.0-dev libx11-dev libxext-dev libxrender-dev libgl1-mesa-dev libice-dev libsm-dev libz-dev libexpat1-dev"
 elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then