From 8a256f0fdbdf0a733a8a440eabb96193802e33f5 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Sat, 21 Dec 2024 10:21:05 +0800 Subject: [PATCH] [GLUTEN-8266][VL] Use pre-installed resources for Spark/Celeborn (#8294) * Initial * Change docker for celeborn test --- .github/workflows/velox_backend.yml | 46 +++++------------------------ 1 file changed, 8 insertions(+), 38 deletions(-) diff --git a/.github/workflows/velox_backend.yml b/.github/workflows/velox_backend.yml index 3ec7e36ca08b..55cb42176f94 100644 --- a/.github/workflows/velox_backend.yml +++ b/.github/workflows/velox_backend.yml @@ -546,7 +546,7 @@ jobs: spark: [ "spark-3.2" ] celeborn: [ "celeborn-0.5.2", "celeborn-0.4.2", "celeborn-0.3.2-incubating" ] runs-on: ubuntu-20.04 - container: ubuntu:22.04 + container: apache/gluten:centos-8 steps: - uses: actions/checkout@v2 - name: Download All Native Artifacts @@ -559,15 +559,6 @@ jobs: with: name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - - name: Setup tzdata - run: | - apt-get update - TZ="Etc/GMT" DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata - - name: Setup java and maven - run: | - apt-get update && apt-get install -y openjdk-8-jdk maven wget - apt remove openjdk-11* -y - echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV - name: Build for Spark ${{ matrix.spark }} run: | cd $GITHUB_WORKSPACE/ @@ -582,7 +573,6 @@ jobs: fi echo "EXTRA_PROFILE: ${EXTRA_PROFILE}" cd /opt && mkdir -p celeborn && \ - ${WGET_CMD} https://archive.apache.org/dist/celeborn/${{ matrix.celeborn }}/apache-${{ matrix.celeborn }}-bin.tgz && \ tar xzf apache-${{ matrix.celeborn }}-bin.tgz -C /opt/celeborn --strip-components=1 && cd celeborn && \ mv ./conf/celeborn-env.sh.template ./conf/celeborn-env.sh && \ bash -c "echo -e 'CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g' > ./conf/celeborn-env.sh" && \ @@ -608,9 +598,8 @@ jobs: with: name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - - name: Prepare spark.test.home for Spark 3.2.2 (other tests) + - name: Prepare run: | - bash .github/workflows/util/install_spark_resources.sh 3.2 dnf module -y install python39 && \ alternatives --set python3 /usr/bin/python3.9 && \ pip3 install setuptools && \ @@ -652,9 +641,6 @@ jobs: with: name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - - name: Prepare spark.test.home for Spark 3.2.2 (slow tests) - run: | - bash .github/workflows/util/install_spark_resources.sh 3.2 - name: Build and run unit test for Spark 3.2.2 (slow tests) run: | cd $GITHUB_WORKSPACE/ @@ -683,9 +669,8 @@ jobs: with: name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - - name: Prepare spark.test.home for Spark 3.3.1 (other tests) + - name: Prepare run: | - bash .github/workflows/util/install_spark_resources.sh 3.3 dnf module -y install python39 && \ alternatives --set python3 /usr/bin/python3.9 && \ pip3 install setuptools && \ @@ -728,9 +713,7 @@ jobs: with: name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - - name: Prepare spark.test.home for Spark 3.3.1 (slow tests) - run: | - bash .github/workflows/util/install_spark_resources.sh 3.3 + - name: Build and Run unit test for Spark 3.3.1 (slow tests) run: | cd $GITHUB_WORKSPACE/ @@ -760,9 +743,8 @@ jobs: with: name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - - name: Prepare spark.test.home for Spark 3.4.3 (other tests) + - name: Prepare run: | - bash .github/workflows/util/install_spark_resources.sh 3.4 dnf module -y install python39 && \ alternatives --set python3 /usr/bin/python3.9 && \ pip3 install setuptools && \ @@ -805,9 +787,6 @@ jobs: with: name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - - name: Prepare spark.test.home for Spark 3.4.3 (slow tests) - run: | - bash .github/workflows/util/install_spark_resources.sh 3.4 - name: Build and Run unit test for Spark 3.4.3 (slow tests) run: | cd $GITHUB_WORKSPACE/ @@ -837,9 +816,8 @@ jobs: with: name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - - name: Prepare spark.test.home for Spark 3.5.2 (other tests) + - name: Prepare run: | - bash .github/workflows/util/install_spark_resources.sh 3.5 dnf module -y install python39 && \ alternatives --set python3 /usr/bin/python3.9 && \ pip3 install setuptools && \ @@ -881,9 +859,8 @@ jobs: with: name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - - name: Prepare spark.test.home for Spark 3.5.2 (other tests) + - name: Prepare run: | - bash .github/workflows/util/install_spark_resources.sh 3.5-scala2.13 dnf module -y install python39 && \ alternatives --set python3 /usr/bin/python3.9 && \ pip3 install setuptools && \ @@ -919,9 +896,6 @@ jobs: with: name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - - name: Prepare spark.test.home for Spark 3.5.2 (slow tests) - run: | - bash .github/workflows/util/install_spark_resources.sh 3.5 - name: Build and Run unit test for Spark 3.5.2 (slow tests) run: | cd $GITHUB_WORKSPACE/ @@ -951,9 +925,8 @@ jobs: with: name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - - name: Prepare spark.test.home for Spark 3.5.2 (other tests) + - name: Prepare run: | - bash .github/workflows/util/install_spark_resources.sh 3.5 dnf module -y install python39 && \ alternatives --set python3 /usr/bin/python3.9 && \ pip3 install setuptools && \ @@ -988,9 +961,6 @@ jobs: with: name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - - name: Prepare spark.test.home for Spark 3.5.2 (slow tests) - run: | - bash .github/workflows/util/install_spark_resources.sh 3.5 - name: Build and Run unit test for Spark 3.5.2 (slow tests) run: | cd $GITHUB_WORKSPACE/