From db5aa54191fa55fed0d26fe65d9c4248c6b0457f Mon Sep 17 00:00:00 2001 From: Krishna Pai Date: Tue, 9 Jan 2024 15:12:36 -0800 Subject: [PATCH 1/3] Setup AggregationFuzzer against Presto in experimental jobs. --- .github/workflows/experimental.yml | 5 +++++ scripts/etc/hive.properties | 4 ++++ scripts/etc/node.properties | 5 +++-- scripts/prestojava-container.dockerfile | 3 ++- scripts/start-prestojava.sh | 1 - 5 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 scripts/etc/hive.properties diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index a16b9d0f522ec..c66050fbbef40 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -115,6 +115,10 @@ jobs: with: name: aggregation + - name: "Start Presto Java" + - run: | + - /opt/start-prestojava.sh + - name: "Run Aggregate Fuzzer" run: | mkdir -p /tmp/aggregate_fuzzer_repro/ @@ -128,6 +132,7 @@ jobs: --minloglevel=0 \ --repro_persist_path=/tmp/aggregate_fuzzer_repro \ --enable_sorted_aggregations=true \ + --presto_url=http://127.0.0.1:8080 \ && echo -e "\n\nAggregation fuzzer run finished successfully." - name: Archive aggregate production artifacts if: always() diff --git a/scripts/etc/hive.properties b/scripts/etc/hive.properties new file mode 100644 index 0000000000000..e9a0d05c76a77 --- /dev/null +++ b/scripts/etc/hive.properties @@ -0,0 +1,4 @@ +connector.name=hive-hadoop2 +hive.metastore=file +hive.metastore.catalog.dir=file:/opt/presto-server/etc/data +hive.allow-drop-table=true \ No newline at end of file diff --git a/scripts/etc/node.properties b/scripts/etc/node.properties index 78c7942c91453..9aeda0213159b 100644 --- a/scripts/etc/node.properties +++ b/scripts/etc/node.properties @@ -1,2 +1,3 @@ -node.environment=test -node.data-dir=/var/lib/presto/data +node.environment=production +node.id=ffffffff-ffff-ffff-ffff-ffffffffffff +node.data-dir=/tmp/presto/data diff --git a/scripts/prestojava-container.dockerfile b/scripts/prestojava-container.dockerfile index a19def4b2a3e8..64f6c0f7a6fd7 100644 --- a/scripts/prestojava-container.dockerfile +++ b/scripts/prestojava-container.dockerfile @@ -36,12 +36,13 @@ RUN dnf install -y java-11-openjdk less procps python3 \ && ln -s /opt/presto-cli /usr/local/bin/ \ && mkdir -p $PRESTO_HOME/etc \ && mkdir -p $PRESTO_HOME/etc/catalog \ - && mkdir -p /var/lib/presto/data \ + && mkdir -p $PRESTO_HOME/etc/data \ && mkdir -p /usr/lib/presto/utils COPY scripts/etc/config.properties.example $PRESTO_HOME/etc/config.properties COPY scripts/etc/jvm.config.example $PRESTO_HOME/etc/jvm.config COPY scripts/etc/node.properties $PRESTO_HOME/etc/node.properties +COPY scripts/etc/hive.properties $PRESTO_HOME/etc/catalog COPY scripts/start-prestojava.sh /opt WORKDIR /velox diff --git a/scripts/start-prestojava.sh b/scripts/start-prestojava.sh index 147b845c2fec2..d94afc0099cae 100755 --- a/scripts/start-prestojava.sh +++ b/scripts/start-prestojava.sh @@ -14,6 +14,5 @@ # limitations under the License. set -e -echo "node.id=$HOSTNAME" >> $PRESTO_HOME/etc/node.properties $PRESTO_HOME/bin/launcher run From 57d8781fc5ce624d6b58fdce38784aa5db463a1c Mon Sep 17 00:00:00 2001 From: Krishna Pai Date: Wed, 10 Jan 2024 08:52:24 -0800 Subject: [PATCH 2/3] Add support for another repo to take reference from. --- .github/workflows/experimental.yml | 8 +++----- scripts/prestojava-container.dockerfile | 6 +++++- scripts/start-prestojava.sh | 6 +++++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index c66050fbbef40..0c2a7f0b6b6ea 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -84,7 +84,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: aggregation - path: velox/_build/debug/velox/exec/tests/velox_aggregation_fuzzer_test + path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_aggregation_fuzzer_test - name: Upload spark fuzzer uses: actions/upload-artifact@v3 @@ -115,12 +115,9 @@ jobs: with: name: aggregation - - name: "Start Presto Java" - - run: | - - /opt/start-prestojava.sh - - name: "Run Aggregate Fuzzer" run: | + /opt/start-prestojava.sh 2>&1 >> /tmp/server.log & mkdir -p /tmp/aggregate_fuzzer_repro/ rm -rfv /tmp/aggregate_fuzzer_repro/* chmod -R 777 /tmp/aggregate_fuzzer_repro @@ -134,6 +131,7 @@ jobs: --enable_sorted_aggregations=true \ --presto_url=http://127.0.0.1:8080 \ && echo -e "\n\nAggregation fuzzer run finished successfully." + - name: Archive aggregate production artifacts if: always() uses: actions/upload-artifact@v3 diff --git a/scripts/prestojava-container.dockerfile b/scripts/prestojava-container.dockerfile index 64f6c0f7a6fd7..fe627d93c62f3 100644 --- a/scripts/prestojava-container.dockerfile +++ b/scripts/prestojava-container.dockerfile @@ -28,7 +28,7 @@ ARG PRESTO_CLI_JAR=presto-cli-$PRESTO_VERSION-executable.jar ENV PRESTO_HOME="/opt/presto-server" RUN cp $PRESTO_CLI_JAR /opt/presto-cli -RUN dnf install -y java-11-openjdk less procps python3 \ +RUN dnf install -y java-11-openjdk less procps python3 tzdata \ && ln -s $(which python3) /usr/bin/python \ && tar -zxf $PRESTO_PKG \ && mv ./presto-server-$PRESTO_VERSION $PRESTO_HOME \ @@ -39,6 +39,10 @@ RUN dnf install -y java-11-openjdk less procps python3 \ && mkdir -p $PRESTO_HOME/etc/data \ && mkdir -p /usr/lib/presto/utils + +# Setting timezone to deal with Velox timezone conversion problem +ENV TZ=America/Bahia_Banderas + COPY scripts/etc/config.properties.example $PRESTO_HOME/etc/config.properties COPY scripts/etc/jvm.config.example $PRESTO_HOME/etc/jvm.config COPY scripts/etc/node.properties $PRESTO_HOME/etc/node.properties diff --git a/scripts/start-prestojava.sh b/scripts/start-prestojava.sh index d94afc0099cae..7057131a1e5db 100755 --- a/scripts/start-prestojava.sh +++ b/scripts/start-prestojava.sh @@ -15,4 +15,8 @@ set -e -$PRESTO_HOME/bin/launcher run +nohup $PRESTO_HOME/bin/launcher --pid-file=/tmp/pidfile run > /tmp/server.log +#wait a few seconds for presto to start +sleep 60 +echo 'CREATE SCHEMA hive.tpch;' > /tmp/hive_create.sql +/opt/presto-cli --server 127.0.0.1:8080 --file /tmp/hive_create.sql From 5193e7fca0ccc85f9e9518b4b0ecec4ce71dd8ea Mon Sep 17 00:00:00 2001 From: Krishna Pai Date: Thu, 11 Jan 2024 10:20:23 -0800 Subject: [PATCH 3/3] Ensure that server logs are archived. --- .github/workflows/experimental.yml | 7 ++++++- scripts/prestojava-container.dockerfile | 1 - scripts/start-prestojava.sh | 7 ++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index 0c2a7f0b6b6ea..1727b9ac3e486 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -117,7 +117,11 @@ jobs: - name: "Run Aggregate Fuzzer" run: | - /opt/start-prestojava.sh 2>&1 >> /tmp/server.log & + ls -lR $PRESTO_HOME/etc + /opt/start-prestojava.sh > /tmp/server.log 2>&1 & + sleep 60 + echo 'CREATE SCHEMA hive.tpch;' > /tmp/hive_create.sql + /opt/presto-cli --server 127.0.0.1:8080 --file /tmp/hive_create.sql mkdir -p /tmp/aggregate_fuzzer_repro/ rm -rfv /tmp/aggregate_fuzzer_repro/* chmod -R 777 /tmp/aggregate_fuzzer_repro @@ -139,6 +143,7 @@ jobs: name: aggregate-fuzzer-failure-artifacts path: | /tmp/aggregate_fuzzer_repro + /tmp/server.log linux-spark-fuzzer-run: runs-on: ubuntu-latest diff --git a/scripts/prestojava-container.dockerfile b/scripts/prestojava-container.dockerfile index fe627d93c62f3..44930042adfcc 100644 --- a/scripts/prestojava-container.dockerfile +++ b/scripts/prestojava-container.dockerfile @@ -40,7 +40,6 @@ RUN dnf install -y java-11-openjdk less procps python3 tzdata \ && mkdir -p /usr/lib/presto/utils -# Setting timezone to deal with Velox timezone conversion problem ENV TZ=America/Bahia_Banderas COPY scripts/etc/config.properties.example $PRESTO_HOME/etc/config.properties diff --git a/scripts/start-prestojava.sh b/scripts/start-prestojava.sh index 7057131a1e5db..416af62185fe8 100755 --- a/scripts/start-prestojava.sh +++ b/scripts/start-prestojava.sh @@ -15,8 +15,5 @@ set -e -nohup $PRESTO_HOME/bin/launcher --pid-file=/tmp/pidfile run > /tmp/server.log -#wait a few seconds for presto to start -sleep 60 -echo 'CREATE SCHEMA hive.tpch;' > /tmp/hive_create.sql -/opt/presto-cli --server 127.0.0.1:8080 --file /tmp/hive_create.sql +$PRESTO_HOME/bin/launcher --pid-file=/tmp/pidfile run +