diff --git a/.github/workflows/util/install_spark_resources.sh b/.github/workflows/util/install_spark_resources.sh index 0709b5f04a479..17fa454efc9bb 100755 --- a/.github/workflows/util/install_spark_resources.sh +++ b/.github/workflows/util/install_spark_resources.sh @@ -21,12 +21,12 @@ case "$1" in 3.2) # Spark-3.2 cd ${INSTALL_DIR} && \ - wget https://archive.apache.org/dist/spark/spark-3.2.2/spark-3.2.2-bin-hadoop3.2.tgz && \ + wget -nv https://archive.apache.org/dist/spark/spark-3.2.2/spark-3.2.2-bin-hadoop3.2.tgz && \ tar --strip-components=1 -xf spark-3.2.2-bin-hadoop3.2.tgz spark-3.2.2-bin-hadoop3.2/jars/ && \ rm -rf spark-3.2.2-bin-hadoop3.2.tgz && \ mkdir -p ${INSTALL_DIR}//shims/spark32/spark_home/assembly/target/scala-2.12 && \ mv jars ${INSTALL_DIR}//shims/spark32/spark_home/assembly/target/scala-2.12 && \ - wget https://github.com/apache/spark/archive/refs/tags/v3.2.2.tar.gz && \ + wget -nv https://github.com/apache/spark/archive/refs/tags/v3.2.2.tar.gz && \ tar --strip-components=1 -xf v3.2.2.tar.gz spark-3.2.2/sql/core/src/test/resources/ && \ mkdir -p shims/spark32/spark_home/ && \ mv sql shims/spark32/spark_home/ @@ -34,12 +34,12 @@ case "$1" in 3.3) # Spark-3.3 cd ${INSTALL_DIR} && \ - wget https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz && \ + wget -nv https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz && \ tar --strip-components=1 -xf spark-3.3.1-bin-hadoop3.tgz spark-3.3.1-bin-hadoop3/jars/ && \ rm -rf spark-3.3.1-bin-hadoop3.tgz && \ mkdir -p ${INSTALL_DIR}//shims/spark33/spark_home/assembly/target/scala-2.12 && \ mv jars ${INSTALL_DIR}//shims/spark33/spark_home/assembly/target/scala-2.12 && \ - wget https://github.com/apache/spark/archive/refs/tags/v3.3.1.tar.gz && \ + wget -nv https://github.com/apache/spark/archive/refs/tags/v3.3.1.tar.gz && \ tar --strip-components=1 -xf v3.3.1.tar.gz spark-3.3.1/sql/core/src/test/resources/ && \ mkdir -p shims/spark33/spark_home/ && \ mv sql shims/spark33/spark_home/ @@ -47,12 +47,12 @@ case "$1" in 3.4) # Spark-3.4 cd ${INSTALL_DIR} && \ - wget https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3.tgz && \ + wget -nv https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3.tgz && \ tar --strip-components=1 -xf spark-3.4.2-bin-hadoop3.tgz spark-3.4.2-bin-hadoop3/jars/ && \ rm -rf spark-3.4.2-bin-hadoop3.tgz && \ mkdir -p ${INSTALL_DIR}//shims/spark34/spark_home/assembly/target/scala-2.12 && \ mv jars ${INSTALL_DIR}//shims/spark34/spark_home/assembly/target/scala-2.12 && \ - wget https://github.com/apache/spark/archive/refs/tags/v3.4.2.tar.gz && \ + wget -nv https://github.com/apache/spark/archive/refs/tags/v3.4.2.tar.gz && \ tar --strip-components=1 -xf v3.4.2.tar.gz spark-3.4.2/sql/core/src/test/resources/ && \ mkdir -p shims/spark34/spark_home/ && \ mv sql shims/spark34/spark_home/ @@ -60,12 +60,12 @@ case "$1" in 3.5) # Spark-3.5 cd ${INSTALL_DIR} && \ - wget https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz && \ + wget -nv https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz && \ tar --strip-components=1 -xf spark-3.5.1-bin-hadoop3.tgz spark-3.5.1-bin-hadoop3/jars/ && \ rm -rf spark-3.5.1-bin-hadoop3.tgz && \ mkdir -p ${INSTALL_DIR}//shims/spark35/spark_home/assembly/target/scala-2.12 && \ mv jars ${INSTALL_DIR}//shims/spark35/spark_home/assembly/target/scala-2.12 && \ - wget https://github.com/apache/spark/archive/refs/tags/v3.5.1.tar.gz && \ + wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.1.tar.gz && \ tar --strip-components=1 -xf v3.5.1.tar.gz spark-3.5.1/sql/core/src/test/resources/ && \ mkdir -p shims/spark35/spark_home/ && \ mv sql shims/spark35/spark_home/ diff --git a/.github/workflows/velox_backend.yml b/.github/workflows/velox_backend.yml index b2da38532d220..fb6161aed3921 100644 --- a/.github/workflows/velox_backend.yml +++ b/.github/workflows/velox_backend.yml @@ -43,6 +43,7 @@ on: env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true MVN_CMD: 'mvn -ntp' + WGET_CMD: 'wget -nv' concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} @@ -205,7 +206,7 @@ jobs: else yum update -y && yum install -y java-1.8.0-openjdk-devel wget fi - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven - name: Set environment variables @@ -435,7 +436,7 @@ jobs: - name: Setup java and maven run: | yum update -y && yum install -y java-1.8.0-openjdk-devel wget git - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven - name: Build for Uniffle 0.9.0 @@ -448,9 +449,9 @@ jobs: cd incubator-uniffle && \ $MVN_CMD clean install -Phadoop2.8,spark3 -DskipTests cd /opt && \ - wget -nv https://archive.apache.org/dist/incubator/uniffle/0.9.0/apache-uniffle-0.9.0-incubating-bin.tar.gz && \ + ${WGET_CMD} https://archive.apache.org/dist/incubator/uniffle/0.9.0/apache-uniffle-0.9.0-incubating-bin.tar.gz && \ tar xzf apache-uniffle-0.9.0-incubating-bin.tar.gz -C /opt/ && mv /opt/rss-0.9.0-hadoop2.8 /opt/uniffle && \ - wget -nv https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz && \ + ${WGET_CMD} https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz && \ tar xzf hadoop-2.8.5.tar.gz -C /opt/ rm -rf /opt/incubator-uniffle cd /opt/uniffle && mkdir shuffle_data && \ @@ -518,7 +519,7 @@ jobs: fi echo "EXTRA_PROFILE: ${EXTRA_PROFILE}" cd /opt && mkdir -p celeborn && \ - wget https://archive.apache.org/dist/celeborn/${{ matrix.celeborn }}/apache-${{ matrix.celeborn }}-bin.tgz && \ + ${WGET_CMD} https://archive.apache.org/dist/celeborn/${{ matrix.celeborn }}/apache-${{ matrix.celeborn }}-bin.tgz && \ tar xzf apache-${{ matrix.celeborn }}-bin.tgz -C /opt/celeborn --strip-components=1 && cd celeborn && \ mv ./conf/celeborn-env.sh.template ./conf/celeborn-env.sh && \ bash -c "echo -e 'CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g' > ./conf/celeborn-env.sh" && \ @@ -553,7 +554,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV @@ -613,7 +614,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV @@ -659,7 +660,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV @@ -720,7 +721,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV @@ -768,7 +769,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV @@ -829,7 +830,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV @@ -877,7 +878,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV @@ -937,7 +938,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV @@ -991,7 +992,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV