Merge branch 'presto_java_hive_metaserver' into kgpai-experimental

facebookincubator · Jan 12, 2024 · 78bbf95 · 78bbf95
2 parents 265cd94 + 5193e7f
commit 78bbf95
Show file tree

Hide file tree

Showing 5 changed files with 31 additions and 10 deletions.
diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml
@@ -84,7 +84,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: aggregation
-          path: velox/_build/debug/velox/exec/tests/velox_aggregation_fuzzer_test
+          path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_aggregation_fuzzer_test
 
       - name: Upload spark aggregation fuzzer
         uses: actions/upload-artifact@v3
@@ -115,10 +115,11 @@ jobs:
       - name: "Checkout Repo"
         uses: actions/checkout@v3
         with:
+          path: velox
           ref: "${{ inputs.ref || 'main' }}"
 
       - name: "Install dependencies"
-        run: source ./scripts/setup-ubuntu.sh
+        run: cd  velox && source ./scripts/setup-ubuntu.sh
 
       - name: Download aggregation fuzzer
         uses: actions/download-artifact@v3
@@ -127,25 +128,36 @@ jobs:
 
       - name: "Run Aggregate Fuzzer"
         run: |
+          pwd
+          ls -lR .
+          cd  velox
+          cp ./scripts/etc/hive.properties $PRESTO_HOME/etc/catalog
+          ls -lR $PRESTO_HOME/etc
+          /opt/start-prestojava.sh > /tmp/server.log 2>&1 &
+          sleep 60
+          /opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;'
           mkdir -p /tmp/aggregate_fuzzer_repro/
           rm -rfv /tmp/aggregate_fuzzer_repro/*
           chmod -R 777 /tmp/aggregate_fuzzer_repro
-          chmod +x velox_aggregation_fuzzer_test
-          ./velox_aggregation_fuzzer_test \
+          chmod +x ../velox_aggregation_fuzzer_test
+          ../velox_aggregation_fuzzer_test \
                 --seed ${RANDOM} \
                 --duration_sec 3600 \
                 --logtostderr=1 \
                 --minloglevel=0 \
                 --repro_persist_path=/tmp/aggregate_fuzzer_repro \
                 --enable_sorted_aggregations=true \
+                --presto_url=http://127.0.0.1:8080 \
           && echo -e "\n\nAggregation fuzzer run finished successfully."
+
       - name: Archive aggregate production artifacts
         if: always()
         uses: actions/upload-artifact@v3
         with:
           name: aggregate-fuzzer-failure-artifacts
           path: |
             /tmp/aggregate_fuzzer_repro
+            /tmp/server.log
 
   linux-spark-fuzzer-run:
     runs-on: ubuntu-latest

diff --git a/scripts/etc/hive.properties b/scripts/etc/hive.properties
@@ -0,0 +1,4 @@
+connector.name=hive-hadoop2
+hive.metastore=file
+hive.metastore.catalog.dir=file:/opt/presto-server/etc/data
+hive.allow-drop-table=true
diff --git a/scripts/etc/node.properties b/scripts/etc/node.properties
@@ -1,2 +1,3 @@
-node.environment=test
-node.data-dir=/var/lib/presto/data
+node.environment=production
+node.id=ffffffff-ffff-ffff-ffff-ffffffffffff
+node.data-dir=/tmp/presto/data
diff --git a/scripts/prestojava-container.dockerfile b/scripts/prestojava-container.dockerfile
@@ -28,20 +28,24 @@ ARG PRESTO_CLI_JAR=presto-cli-$PRESTO_VERSION-executable.jar
 ENV PRESTO_HOME="/opt/presto-server"
 RUN cp $PRESTO_CLI_JAR /opt/presto-cli
 
-RUN dnf install -y java-11-openjdk less procps python3 \
+RUN dnf install -y java-11-openjdk less procps python3 tzdata \
     && ln -s $(which python3) /usr/bin/python \
     && tar -zxf $PRESTO_PKG \
     && mv ./presto-server-$PRESTO_VERSION $PRESTO_HOME \
     && chmod +x /opt/presto-cli \
     && ln -s /opt/presto-cli /usr/local/bin/ \
     && mkdir -p $PRESTO_HOME/etc \
     && mkdir -p $PRESTO_HOME/etc/catalog \
-    && mkdir -p /var/lib/presto/data \
+    && mkdir -p $PRESTO_HOME/etc/data \
     && mkdir -p /usr/lib/presto/utils
 
+
+ENV TZ=America/Bahia_Banderas
+
 COPY scripts/etc/config.properties.example $PRESTO_HOME/etc/config.properties
 COPY scripts/etc/jvm.config.example $PRESTO_HOME/etc/jvm.config
 COPY scripts/etc/node.properties $PRESTO_HOME/etc/node.properties
+COPY scripts/etc/hive.properties $PRESTO_HOME/etc/catalog
 COPY scripts/start-prestojava.sh /opt
 
 WORKDIR /velox
diff --git a/scripts/start-prestojava.sh b/scripts/start-prestojava.sh
@@ -14,6 +14,6 @@
 # limitations under the License.
 
 set -e
-echo "node.id=$HOSTNAME" >> $PRESTO_HOME/etc/node.properties
 
-$PRESTO_HOME/bin/launcher run
+$PRESTO_HOME/bin/launcher --pid-file=/tmp/pidfile  run
+