From e43acc2dd78b0540fe92e0ba237f4387f2565d6e Mon Sep 17 00:00:00 2001 From: Krishna Pai Date: Thu, 11 Jan 2024 10:20:23 -0800 Subject: [PATCH] Ensure that server logs are archived. --- .github/workflows/experimental.yml | 42 ++++++++++++++++++------- scripts/etc/jvm.config.example | 1 + scripts/prestojava-container.dockerfile | 6 ++-- scripts/start-prestojava.sh | 7 ++--- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index 0c2a7f0b6b6ea..a1705ecfba42c 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -96,33 +96,52 @@ jobs: presto-java-aggregation-fuzzer-run: - runs-on: ubuntu-latest + runs-on: 8-core container: ghcr.io/facebookincubator/velox-dev:presto-java - needs: compile timeout-minutes: 120 + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache/" + CCACHE_BASEDIR: "${{ github.workspace }}" + LINUX_DISTRO: "centos" steps: + - name: "Restore ccache" + uses: actions/cache@v3 + with: + path: "${{ env.CCACHE_DIR }}" + # We are using the benchmark ccache as it has all + # required features enabled, so no need to create a new one + key: ccache-presto-${{ github.sha }} + restore-keys: | + ccache-presto- + - name: "Checkout Repo" uses: actions/checkout@v3 with: + path: velox + submodules: 'recursive' ref: "${{ inputs.ref || 'main' }}" - - name: "Install dependencies" - run: source ./scripts/setup-ubuntu.sh - - name: Download aggregation fuzzer - uses: actions/download-artifact@v3 - with: - name: aggregation + - name: "Build" + run: | + cd velox + source /opt/rh/gcc-toolset-9/enable + make debug NUM_THREADS="${{ inputs.numThreads || 8 }}" MAX_HIGH_MEM_JOBS="${{ inputs.maxHighMemJobs || 8 }}" MAX_LINK_JOBS="${{ inputs.maxLinkJobs || 4 }}" EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_ARROW=ON ${{ inputs.extraCMakeFlags }}" + ccache -s - name: "Run Aggregate Fuzzer" run: | - /opt/start-prestojava.sh 2>&1 >> /tmp/server.log & + cd velox + cp ./scripts/etc/hive.properties $PRESTO_HOME/etc/catalog + ls -lR $PRESTO_HOME/etc + /opt/start-prestojava.sh > /tmp/server.log 2>&1 & + sleep 60 + /opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;' mkdir -p /tmp/aggregate_fuzzer_repro/ rm -rfv /tmp/aggregate_fuzzer_repro/* chmod -R 777 /tmp/aggregate_fuzzer_repro - chmod +x velox_aggregation_fuzzer_test - ./velox_aggregation_fuzzer_test \ + _build/debug/velox/functions/prestosql/fuzzer/velox_aggregation_fuzzer_test \ --seed ${RANDOM} \ --duration_sec 3600 \ --logtostderr=1 \ @@ -139,6 +158,7 @@ jobs: name: aggregate-fuzzer-failure-artifacts path: | /tmp/aggregate_fuzzer_repro + /tmp/server.log linux-spark-fuzzer-run: runs-on: ubuntu-latest diff --git a/scripts/etc/jvm.config.example b/scripts/etc/jvm.config.example index 8436145168343..74f19da617f9f 100644 --- a/scripts/etc/jvm.config.example +++ b/scripts/etc/jvm.config.example @@ -7,3 +7,4 @@ -XX:+HeapDumpOnOutOfMemoryError -XX:+ExitOnOutOfMemoryError -Djdk.attach.allowAttachSelf=true +-Duser.timezone=America/Los_Angeles diff --git a/scripts/prestojava-container.dockerfile b/scripts/prestojava-container.dockerfile index fe627d93c62f3..f5dca495c526c 100644 --- a/scripts/prestojava-container.dockerfile +++ b/scripts/prestojava-container.dockerfile @@ -39,9 +39,9 @@ RUN dnf install -y java-11-openjdk less procps python3 tzdata \ && mkdir -p $PRESTO_HOME/etc/data \ && mkdir -p /usr/lib/presto/utils - -# Setting timezone to deal with Velox timezone conversion problem -ENV TZ=America/Bahia_Banderas +# We set the timezone to America/Los_Angeles due to issue +# detailed here : https://github.com/facebookincubator/velox/issues/8127 +ENV TZ=America/Los_Angeles COPY scripts/etc/config.properties.example $PRESTO_HOME/etc/config.properties COPY scripts/etc/jvm.config.example $PRESTO_HOME/etc/jvm.config diff --git a/scripts/start-prestojava.sh b/scripts/start-prestojava.sh index 7057131a1e5db..416af62185fe8 100755 --- a/scripts/start-prestojava.sh +++ b/scripts/start-prestojava.sh @@ -15,8 +15,5 @@ set -e -nohup $PRESTO_HOME/bin/launcher --pid-file=/tmp/pidfile run > /tmp/server.log -#wait a few seconds for presto to start -sleep 60 -echo 'CREATE SCHEMA hive.tpch;' > /tmp/hive_create.sql -/opt/presto-cli --server 127.0.0.1:8080 --file /tmp/hive_create.sql +$PRESTO_HOME/bin/launcher --pid-file=/tmp/pidfile run +