Add -nv for wget

apache · Aug 21, 2024 · dec18c6 · dec18c6
1 parent 26a608e
commit dec18c6
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 22 deletions.
diff --git a/.github/workflows/util/install_spark_resources.sh b/.github/workflows/util/install_spark_resources.sh
@@ -21,51 +21,51 @@ case "$1" in
 3.2)
     # Spark-3.2
     cd ${INSTALL_DIR} && \
-    wget https://archive.apache.org/dist/spark/spark-3.2.2/spark-3.2.2-bin-hadoop3.2.tgz && \
+    wget -nv https://archive.apache.org/dist/spark/spark-3.2.2/spark-3.2.2-bin-hadoop3.2.tgz && \
     tar --strip-components=1 -xf spark-3.2.2-bin-hadoop3.2.tgz spark-3.2.2-bin-hadoop3.2/jars/ && \
     rm -rf spark-3.2.2-bin-hadoop3.2.tgz && \
     mkdir -p ${INSTALL_DIR}//shims/spark32/spark_home/assembly/target/scala-2.12 && \
     mv jars ${INSTALL_DIR}//shims/spark32/spark_home/assembly/target/scala-2.12 && \
-    wget https://github.com/apache/spark/archive/refs/tags/v3.2.2.tar.gz && \
+    wget -nv https://github.com/apache/spark/archive/refs/tags/v3.2.2.tar.gz && \
     tar --strip-components=1 -xf v3.2.2.tar.gz spark-3.2.2/sql/core/src/test/resources/  && \
     mkdir -p shims/spark32/spark_home/ && \
     mv sql shims/spark32/spark_home/
     ;;
 3.3)
     # Spark-3.3
     cd ${INSTALL_DIR} && \
-    wget https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz && \
+    wget -nv https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz && \
     tar --strip-components=1 -xf spark-3.3.1-bin-hadoop3.tgz spark-3.3.1-bin-hadoop3/jars/ && \
     rm -rf spark-3.3.1-bin-hadoop3.tgz && \
     mkdir -p ${INSTALL_DIR}//shims/spark33/spark_home/assembly/target/scala-2.12 && \
     mv jars ${INSTALL_DIR}//shims/spark33/spark_home/assembly/target/scala-2.12 && \
-    wget https://github.com/apache/spark/archive/refs/tags/v3.3.1.tar.gz && \
+    wget -nv https://github.com/apache/spark/archive/refs/tags/v3.3.1.tar.gz && \
     tar --strip-components=1 -xf v3.3.1.tar.gz spark-3.3.1/sql/core/src/test/resources/  && \
     mkdir -p shims/spark33/spark_home/ && \
     mv sql shims/spark33/spark_home/
     ;;
 3.4)
     # Spark-3.4
     cd ${INSTALL_DIR} && \
-    wget https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3.tgz && \
+    wget -nv https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3.tgz && \
     tar --strip-components=1 -xf spark-3.4.2-bin-hadoop3.tgz spark-3.4.2-bin-hadoop3/jars/ && \
     rm -rf spark-3.4.2-bin-hadoop3.tgz && \
     mkdir -p ${INSTALL_DIR}//shims/spark34/spark_home/assembly/target/scala-2.12 && \
     mv jars ${INSTALL_DIR}//shims/spark34/spark_home/assembly/target/scala-2.12 && \
-    wget https://github.com/apache/spark/archive/refs/tags/v3.4.2.tar.gz && \
+    wget -nv https://github.com/apache/spark/archive/refs/tags/v3.4.2.tar.gz && \
     tar --strip-components=1 -xf v3.4.2.tar.gz spark-3.4.2/sql/core/src/test/resources/  && \
     mkdir -p shims/spark34/spark_home/ && \
     mv sql shims/spark34/spark_home/
     ;;
 3.5)
     # Spark-3.5
     cd ${INSTALL_DIR} && \
-    wget https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz && \
+    wget -nv https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz && \
     tar --strip-components=1 -xf spark-3.5.1-bin-hadoop3.tgz spark-3.5.1-bin-hadoop3/jars/ && \
     rm -rf spark-3.5.1-bin-hadoop3.tgz && \
     mkdir -p ${INSTALL_DIR}//shims/spark35/spark_home/assembly/target/scala-2.12 && \
     mv jars ${INSTALL_DIR}//shims/spark35/spark_home/assembly/target/scala-2.12 && \
-    wget https://github.com/apache/spark/archive/refs/tags/v3.5.1.tar.gz && \
+    wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.1.tar.gz && \
     tar --strip-components=1 -xf v3.5.1.tar.gz spark-3.5.1/sql/core/src/test/resources/  && \
     mkdir -p shims/spark35/spark_home/ && \
     mv sql shims/spark35/spark_home/

diff --git a/.github/workflows/velox_backend.yml b/.github/workflows/velox_backend.yml
@@ -43,6 +43,7 @@ on:
 env:
   ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
   MVN_CMD: 'mvn -ntp'
+  WGET_CMD: 'wget -nv'
 
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
@@ -205,7 +206,7 @@ jobs:
           else
             yum update -y && yum install -y java-1.8.0-openjdk-devel wget
           fi
-          wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+          ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
           tar -xvf apache-maven-3.8.8-bin.tar.gz
           mv apache-maven-3.8.8 /usr/lib/maven
       - name: Set environment variables
@@ -435,7 +436,7 @@ jobs:
       - name: Setup java and maven
         run: |
           yum update -y && yum install -y java-1.8.0-openjdk-devel wget git
-          wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+          ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
           tar -xvf apache-maven-3.8.8-bin.tar.gz
           mv apache-maven-3.8.8 /usr/lib/maven
       - name: Build for Uniffle 0.9.0
@@ -448,9 +449,9 @@ jobs:
           cd incubator-uniffle && \
           $MVN_CMD clean install -Phadoop2.8,spark3 -DskipTests
           cd /opt && \
-          wget -nv https://archive.apache.org/dist/incubator/uniffle/0.9.0/apache-uniffle-0.9.0-incubating-bin.tar.gz && \
+          ${WGET_CMD} https://archive.apache.org/dist/incubator/uniffle/0.9.0/apache-uniffle-0.9.0-incubating-bin.tar.gz && \
           tar xzf apache-uniffle-0.9.0-incubating-bin.tar.gz -C /opt/ && mv /opt/rss-0.9.0-hadoop2.8 /opt/uniffle && \
-          wget -nv https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz && \
+          ${WGET_CMD} https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz && \
           tar xzf hadoop-2.8.5.tar.gz -C /opt/
           rm -rf /opt/incubator-uniffle
           cd /opt/uniffle && mkdir shuffle_data && \
@@ -518,7 +519,7 @@ jobs:
           fi
           echo "EXTRA_PROFILE: ${EXTRA_PROFILE}"
           cd /opt && mkdir -p celeborn && \
-          wget https://archive.apache.org/dist/celeborn/${{ matrix.celeborn }}/apache-${{ matrix.celeborn }}-bin.tgz && \
+          ${WGET_CMD} https://archive.apache.org/dist/celeborn/${{ matrix.celeborn }}/apache-${{ matrix.celeborn }}-bin.tgz && \
           tar xzf apache-${{ matrix.celeborn }}-bin.tgz -C /opt/celeborn --strip-components=1 && cd celeborn && \
           mv ./conf/celeborn-env.sh.template ./conf/celeborn-env.sh && \
           bash -c "echo -e 'CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g' > ./conf/celeborn-env.sh" && \
@@ -553,7 +554,7 @@ jobs:
       - name: Setup build dependency
         run: |
           yum install sudo patch java-1.8.0-openjdk-devel wget -y
-          wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+          ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
           tar -xvf apache-maven-3.8.8-bin.tar.gz
           mv apache-maven-3.8.8 /usr/lib/maven
           echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
@@ -613,7 +614,7 @@ jobs:
       - name: Setup build dependency
         run: |
           yum install sudo patch java-1.8.0-openjdk-devel wget -y
-          wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+          ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
           tar -xvf apache-maven-3.8.8-bin.tar.gz
           mv apache-maven-3.8.8 /usr/lib/maven
           echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
@@ -659,7 +660,7 @@ jobs:
       - name: Setup build dependency
         run: |
           yum install sudo patch java-1.8.0-openjdk-devel wget -y
-          wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+          ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
           tar -xvf apache-maven-3.8.8-bin.tar.gz
           mv apache-maven-3.8.8 /usr/lib/maven
           echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
@@ -720,7 +721,7 @@ jobs:
       - name: Setup build dependency
         run: |
           yum install sudo patch java-1.8.0-openjdk-devel wget -y
-          wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+          ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
           tar -xvf apache-maven-3.8.8-bin.tar.gz
           mv apache-maven-3.8.8 /usr/lib/maven
           echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
@@ -768,7 +769,7 @@ jobs:
       - name: Setup build dependency
         run: |
           yum install sudo patch java-1.8.0-openjdk-devel wget -y
-          wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+          ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
           tar -xvf apache-maven-3.8.8-bin.tar.gz
           mv apache-maven-3.8.8 /usr/lib/maven
           echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
@@ -829,7 +830,7 @@ jobs:
       - name: Setup build dependency
         run: |
           yum install sudo patch java-1.8.0-openjdk-devel wget -y
-          wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+          ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
           tar -xvf apache-maven-3.8.8-bin.tar.gz
           mv apache-maven-3.8.8 /usr/lib/maven
           echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
@@ -877,7 +878,7 @@ jobs:
       - name: Setup build dependency
         run: |
           yum install sudo patch java-1.8.0-openjdk-devel wget -y
-          wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+          ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
           tar -xvf apache-maven-3.8.8-bin.tar.gz
           mv apache-maven-3.8.8 /usr/lib/maven
           echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
@@ -937,7 +938,7 @@ jobs:
       - name: Setup build dependency
         run: |
           yum install sudo patch java-1.8.0-openjdk-devel wget -y
-          wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+          ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
           tar -xvf apache-maven-3.8.8-bin.tar.gz
           mv apache-maven-3.8.8 /usr/lib/maven
           echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
@@ -991,7 +992,7 @@ jobs:
       - name: Setup build dependency
         run: |
           yum install sudo patch java-1.8.0-openjdk-devel wget -y
-          wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+          ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
           tar -xvf apache-maven-3.8.8-bin.tar.gz
           mv apache-maven-3.8.8 /usr/lib/maven
           echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV