Have presto-agg-fuzzer build velox

facebookincubator · Jan 12, 2024 · 5411aa6 · 5411aa6
1 parent ee80668
commit 5411aa6
Showing 1 changed file with 8 additions and 133 deletions.
diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml
@@ -46,13 +46,15 @@ permissions:
   contents: read
 
 jobs:
-  compile:
+
+  presto-java-aggregation-fuzzer-run:
     runs-on: 8-core
+    container: ghcr.io/facebookincubator/velox-dev:presto-java
     timeout-minutes: 120
     env:
       CCACHE_DIR: "${{ github.workspace }}/.ccache/"
       CCACHE_BASEDIR: "${{ github.workspace }}"
-      LINUX_DISTRO: "ubuntu"
+      LINUX_DISTRO: "centos"
     steps:
 
       - name: "Restore ccache"
@@ -61,71 +63,25 @@ jobs:
           path: "${{ env.CCACHE_DIR }}"
           # We are using the benchmark ccache as it has all
           # required features enabled, so no need to create a new one
-          key: ccache-benchmark-${{ github.sha }}
+          key: ccache-presto-${{ github.sha }}
           restore-keys: |
-            ccache-benchmark-
+            ccache-presto-
+
       - name: "Checkout Repo"
         uses: actions/checkout@v3
         with:
           path: velox
           submodules: 'recursive'
           ref: "${{ inputs.ref || 'main' }}"
 
-      - name: "Install dependencies"
-        run: cd  velox && source ./scripts/setup-ubuntu.sh
 
       - name: "Build"
         run: |
           cd velox
+          source /opt/rh/gcc-toolset-9/enable
           make debug NUM_THREADS="${{ inputs.numThreads || 8 }}" MAX_HIGH_MEM_JOBS="${{ inputs.maxHighMemJobs || 8 }}" MAX_LINK_JOBS="${{ inputs.maxLinkJobs || 4 }}" EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_ARROW=ON ${{ inputs.extraCMakeFlags }}"
           ccache -s
 
-      - name: Upload aggregation fuzzer
-        uses: actions/upload-artifact@v3
-        with:
-          name: aggregation
-          path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_aggregation_fuzzer_test
-
-      - name: Upload spark aggregation fuzzer
-        uses: actions/upload-artifact@v3
-        with:
-          name: spark_aggregation_fuzzer
-          path: velox/_build/debug/velox/functions/sparksql/fuzzer/spark_aggregation_fuzzer_test
-
-      - name: Upload aggregation fuzzer
-        uses: actions/upload-artifact@v3
-        with:
-          name: aggregation
-          path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_aggregation_fuzzer_test
-
-      - name: Upload join fuzzer
-        uses: actions/upload-artifact@v3
-        with:
-          name: join
-          path: velox/_build/debug/velox/exec/tests/velox_join_fuzzer_test
-
-
-  presto-java-aggregation-fuzzer-run:
-    runs-on: 8-core
-    container: ghcr.io/facebookincubator/velox-dev:presto-java
-    needs: compile
-    timeout-minutes: 120
-    steps:
-
-      - name: "Checkout Repo"
-        uses: actions/checkout@v3
-        with:
-          path: velox
-          ref: "${{ inputs.ref || 'main' }}"
-
-      - name: "Install dependencies"
-        run: cd  velox && source ./scripts/setup-centos8.sh
-
-      - name: Download aggregation fuzzer
-        uses: actions/download-artifact@v3
-        with:
-          name: aggregation
-
       - name: "Run Aggregate Fuzzer"
         run: |
           cd  velox
@@ -157,84 +113,3 @@ jobs:
             /tmp/aggregate_fuzzer_repro
             /tmp/server.log
 
-  linux-spark-fuzzer-run:
-    runs-on: ubuntu-latest
-    needs: compile
-    timeout-minutes: 120
-    steps:
-
-      - name: "Checkout Repo"
-        uses: actions/checkout@v3
-        with:
-          ref: "${{ inputs.ref || 'main' }}"
-
-      - name: "Install dependencies"
-        run: source ./scripts/setup-ubuntu.sh
-
-      - name: Download spark aggregation fuzzer
-        uses: actions/download-artifact@v3
-        with:
-          name: spark_aggregation_fuzzer
-
-      - name: "Run Spark Aggregate Fuzzer"
-        run: |
-          mkdir -p /tmp/spark_aggregate_fuzzer_repro/
-          chmod -R 777 /tmp/spark_aggregate_fuzzer_repro
-          chmod +x spark_aggregation_fuzzer_test
-          ./spark_aggregation_fuzzer_test \
-                --seed ${RANDOM} \
-                --duration_sec 1800 \
-                --logtostderr=1 \
-                --minloglevel=0 \
-                --repro_persist_path=/tmp/spark_aggregate_fuzzer_repro \
-                --enable_sorted_aggregations=true \
-          && echo -e "\n\nSpark Aggregation Fuzzer run finished successfully."
-
-      - name: Archive Spark aggregate production artifacts
-        if: always()
-        uses: actions/upload-artifact@v3
-        with:
-          name: spark-agg-fuzzer-failure-artifacts
-          path: |
-            /tmp/spark_aggregate_fuzzer_repro
-
-  linux-join-fuzzer-run:
-    runs-on: ubuntu-latest
-    needs: compile
-    timeout-minutes: 120
-    steps:
-
-      - name: "Checkout Repo"
-        uses: actions/checkout@v3
-        with:
-          ref: "${{ inputs.ref || 'main' }}"
-
-      - name: "Install dependencies"
-        run: source ./scripts/setup-ubuntu.sh
-
-      - name: Download join fuzzer
-        uses: actions/download-artifact@v3
-        with:
-          name: join
-
-      - name: "Run Join Fuzzer"
-        run: |
-          ls /lib64
-          mkdir -p /tmp/join_fuzzer_repro/
-          rm -rfv /tmp/join_fuzzer_repro/*
-          chmod -R 777 /tmp/join_fuzzer_repro
-          chmod +x velox_join_fuzzer_test
-          ./velox_join_fuzzer_test \
-                --seed ${RANDOM} \
-                --duration_sec 1800 \
-                --logtostderr=1 \
-                --minloglevel=0 \
-            && echo -e "\n\nAggregation fuzzer run finished successfully."
-
-      - name: Archive aggregate production artifacts
-        if: always()
-        uses: actions/upload-artifact@v3
-        with:
-          name: join-fuzzer-failure-artifacts
-          path: |
-            /tmp/join_fuzzer_repro