Update to use koku-presto (#2520)

* Fix grafana dockerfile; Update jvm.config for jre11; Use koku-presto image

* update to presto 347

* use cloudservices/ubi-presto image

* Testing the hive image build

* testing with ubi images for hive, hadoop

* Change presto DROP to DELETE then DROP

Makefile

@@ -363,10 +363,11 @@ docker-iqe-vortex-tests: docker-reinitdb _set-test-dir-permissions clear-testing
 
 docker-metastore-setup:
 	@cp -fr deploy/metastore/ testing/metastore/
-	@chmod -R o+rwx ./testing/metastore
+	find ./testing/metastore -type d -exec chmod a+rwx {} \;
 	@[[ ! -d ./testing/metastore/db-data ]] && mkdir -p -m a+rwx ./testing/metastore/db-data || chmod a+rwx ./testing/metastore/db-data
 	@cp -fr deploy/hadoop/ testing/hadoop/
-	@chmod o+rwx ./testing/hadoop
+#	@[[ ! -d ./testing/hadoop/hadoop-logs ]] && mkdir -p -m a+rwx ./hadoop/hadoop-logs || chmod a+rwx ./hadoop/hadoop-logs
+	find ./testing/hadoop -type d -exec chmod a+rwx {} \;
 	@$(SED_IN_PLACE) -e 's/s3path/$(shell echo $(or $(S3_BUCKET_NAME),metastore))/g' testing/hadoop/hadoop-config/core-site.xml
 	@$(SED_IN_PLACE) -e 's/s3path/$(shell echo $(or $(S3_BUCKET_NAME),metastore))/g' testing/metastore/hive-config/hive-site.xml
 	@$(SED_IN_PLACE) -e 's%s3endpoint%$(shell echo $(or $(S3_ENDPOINT),localhost))%g' testing/metastore/hive-config/hive-site.xml
@@ -375,9 +376,9 @@ docker-metastore-setup:
 
 docker-presto-setup:
 	@cp -fr deploy/presto/ testing/presto/
-	@chmod o+rwx ./testing/presto
+	find ./testing/presto -type d -exec chmod a+rwx {} \;
 	@cp -fr deploy/hadoop/ testing/hadoop/
-	@chmod o+rwx ./testing/hadoop
+	find ./testing/hadoop -type d -exec chmod a+rwx {} \;
 	@[[ ! -d ./testing/parquet_data ]] && mkdir -p -m a+rwx ./testing/parquet_data || chmod a+rwx ./testing/parquet_data
 	@$(SED_IN_PLACE) -e 's/s3path/$(shell echo $(or $(S3_BUCKET_NAME),metastore))/g' testing/hadoop/hadoop-config/core-site.xml
 	@$(SED_IN_PLACE) -e 's/DATABASE_NAME/$(shell echo $(or $(DATABASE_NAME),postgres))/g' testing/presto/presto-catalog-config/postgres.properties
@@ -391,6 +392,9 @@ docker-presto-cleanup:
 docker-presto-up: docker-metastore-setup docker-presto-setup
 	docker-compose -f ./testing/compose_files/docker-compose-presto.yml up -d
 
+docker-presto-ps:
+	docker-compose -f ./testing/compose_files/docker-compose-presto.yml ps
+
 docker-presto-down:
 	docker-compose -f ./testing/compose_files/docker-compose-presto.yml down -v
 	make docker-presto-cleanup

deploy/metastore/hive-scripts/entrypoint.sh

@@ -65,9 +65,9 @@ ln -s -f /hive-config/hive-exec-log4j2.properties $HIVE_HOME/conf/hive-exec-log4
 
 export HADOOP_LOG_DIR="${HADOOP_HOME}/logs"
 # Set garbage collection settings
-export GC_SETTINGS="-XX:+UseG1GC -XX:G1HeapRegionSize=32M -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${HADOOP_LOG_DIR}/heap_dump.bin -XX:+ExitOnOutOfMemoryError -XX:ErrorFile=${HADOOP_LOG_DIR}/java_error%p.log"
+#export GC_SETTINGS="-XX:+UseG1GC -XX:G1HeapRegionSize=32M -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${HADOOP_LOG_DIR}/heap_dump.bin -XX:+ExitOnOutOfMemoryError -XX:ErrorFile=${HADOOP_LOG_DIR}/java_error%p.log"
 
-export VM_OPTIONS="$VM_OPTIONS -XX:+UseContainerSupport"
+export VM_OPTIONS="$VM_OPTIONS -XX:+UseContainerSupport -XX:ErrorFile=${HADOOP_LOG_DIR}/java_error%p.log"
 
 if [ -n "$JVM_INITIAL_RAM_PERCENTAGE" ]; then
 VM_OPTIONS="$VM_OPTIONS -XX:InitialRAMPercentage=$JVM_INITIAL_RAM_PERCENTAGE"
@@ -83,8 +83,7 @@ fi
 export JMX_OPTIONS="-javaagent:/opt/jmx_exporter/jmx_exporter.jar=8082:/opt/jmx_exporter/config/config.yml -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=8081 -Dcom.sun.management.jmxremote.rmi.port=8081 -Djava.rmi.server.hostname=127.0.0.1"
 
 # Set garbage collection logs
-GC_SETTINGS="${GC_SETTINGS} -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc.log"
-GC_SETTINGS="${GC_SETTINGS} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=3M"
+export GC_SETTINGS="${GC_SETTINGS} -verbose:gc -Xlog:gc*:${HADOOP_LOG_DIR}/gc.log"
 
 export HIVE_LOGLEVEL="${HIVE_LOGLEVEL:-INFO}"
 export HADOOP_OPTS="${HADOOP_OPTS} ${VM_OPTIONS} ${GC_SETTINGS} ${JMX_OPTIONS}"

deploy/presto/presto-coordinator-config/jvm.config

@@ -22,19 +22,7 @@
 
 -verbose:gc
 
--Xloggc:/var/presto/logs/gc.log
-
--XX:+PrintGCDetails
-
--XX:+PrintGCTimeStamps
-
--XX:+PrintGCDateStamps
-
--XX:+UseGCLogFileRotation
-
--XX:NumberOfGCLogFiles=5
-
--XX:GCLogFileSize=3M
+-Xlog:gc*:/var/presto/logs/gc.log
 
 -javaagent:/opt/jmx_exporter/jmx_exporter.jar=8082:/opt/jmx_exporter/config/config.yml
 

grafana/Dockerfile-grafana

@@ -2,5 +2,5 @@ FROM grafana/grafana:latest
 USER root
 RUN apk add sqlite
 USER grafana
-COPY --chown=grafana:grafana grafana.db.sql /var/lib/grafana/grafana.db.sql
+COPY --chown=grafana:root grafana.db.sql /var/lib/grafana/grafana.db.sql
 RUN cat /var/lib/grafana/grafana.db.sql | sqlite3 /var/lib/grafana/grafana.db && rm /var/lib/grafana/grafana.db.sql

koku/masu/database/presto_sql/reporting_ocp_lineitem_daily_summary.sql

@@ -396,6 +396,9 @@ SELECT uuid() as "uuid",
  * ====================================
  */
 
+DELETE FROM hive.{{schema | sqlsafe}}.__ocp_node_label_line_item_daily_{{uuid | sqlsafe}};
 DROP TABLE IF EXISTS hive.{{schema | sqlsafe}}.__ocp_node_label_line_item_daily_{{uuid | sqlsafe}};
+DELETE FROM hive.{{schema | sqlsafe}}.__ocp_cluster_capacity_{{uuid | sqlsafe}};
 DROP TABLE IF EXISTS hive.{{schema | sqlsafe}}.__ocp_cluster_capacity_{{uuid | sqlsafe}};
+DELETE FROM hive.{{schema | sqlsafe}}.__volume_nodes_{{uuid | sqlsafe}};
 DROP TABLE IF EXISTS hive.{{schema | sqlsafe}}.__volume_nodes_{{uuid | sqlsafe}};

koku/masu/database/presto_sql/reporting_ocp_usage_label_summary.sql

@@ -189,6 +189,7 @@ SELECT DISTINCT
 ;
 
 -- DROP gather table as it is no longer needed
+DELETE from hive.{{schema | sqlsafe}}.__label_summary_gather_{{uuid | sqlsafe}};
 DROP TABLE IF EXISTS hive.{{schema | sqlsafe}}.__label_summary_gather_{{uuid | sqlsafe}};
 
 
@@ -436,4 +437,5 @@ VALUES (
  * ====================================
  */
 
+DELETE FROM hive.{{schema | sqlsafe}}.__label_summary_{{uuid | sqlsafe}};
 DROP TABLE IF EXISTS hive.{{schema | sqlsafe}}.__label_summary_{{uuid | sqlsafe}};

koku/masu/database/presto_sql/reporting_ocpawscostlineitem_daily_summary.sql

@@ -241,6 +241,7 @@ CREATE TABLE hive.{{schema | sqlsafe}}.__reporting_ocp_pod_tags_{{uuid | sqlsafe
 )
 ;
 
+DELETE FROM hive.{{schema | sqlsafe}}.__matched_tags_{{uuid | sqlsafe}};
 DROP TABLE IF EXISTS hive.{{schema | sqlsafe}}.__matched_tags_{{uuid | sqlsafe}};
 
 
@@ -562,7 +563,9 @@ INSERT INTO hive.{{schema | sqlsafe}}.__reporting_ocpawsusagelineitem_daily_{{uu
         ON tm.aws_id = shared.aws_id
 ;
 
+DELETE FROM hive.{{schema | sqlsafe}}.__reporting_aws_daily_{{uuid | sqlsafe}};
 DROP TABLE IF EXISTS hive.{{schema | sqlsafe}}.__reporting_aws_daily_{{uuid | sqlsafe}};
+DELETE FROM hive.{{schema | sqlsafe}}.__reporting_ocp_pod_tags_{{uuid | sqlsafe}};
 DROP TABLE IF EXISTS hive.{{schema | sqlsafe}}.__reporting_ocp_pod_tags_{{uuid | sqlsafe}};
 
 
@@ -747,6 +750,8 @@ INSERT INTO hive.{{schema | sqlsafe}}.__reporting_ocpawsstoragelineitem_daily_{{
 
 ;
 
+DELETE FROM hive.{{schema | sqlsafe}}.__reporting_aws_special_case_tags_{{uuid | sqlsafe}}
+;
 DROP TABLE IF EXISTS hive.{{schema | sqlsafe}}.__reporting_aws_special_case_tags_{{uuid | sqlsafe}}
 ;
 
@@ -810,9 +815,13 @@ DROP TABLE IF EXISTS hive.{{schema | sqlsafe}}.__reporting_aws_special_case_tags
 
 ;
 
+DELETE FROM hive.{{schema | sqlsafe}}.__reporting_ocp_storage_tags_{{uuid | sqlsafe}}
+;
 DROP TABLE IF EXISTS hive.{{schema | sqlsafe}}.__reporting_ocp_storage_tags_{{uuid | sqlsafe}}
 ;
 
+DELETE FROM hive.{{schema | sqlsafe}}.__reporting_aws_tags_{{uuid | sqlsafe}}
+;
 DROP TABLE IF EXISTS hive.{{schema | sqlsafe}}.__reporting_aws_tags_{{uuid | sqlsafe}}
 ;
 
@@ -1020,8 +1029,10 @@ CREATE TABLE hive.{{schema | sqlsafe}}.__reporting_ocpawscostlineitem_project_da
 )
 ;
 
+DELETE FROM hive.{{schema | sqlsafe}}.__reporting_ocpawsusagelineitem_daily_{{uuid | sqlsafe}};
 DROP TABLE IF EXISTS hive.{{schema | sqlsafe}}.__reporting_ocpawsusagelineitem_daily_{{uuid | sqlsafe}};
 
+DELETE FROM hive.{{schema | sqlsafe}}.__reporting_ocpawsstoragelineitem_daily_{{uuid | sqlsafe}};
 DROP TABLE IF EXISTS hive.{{schema | sqlsafe}}.__reporting_ocpawsstoragelineitem_daily_{{uuid | sqlsafe}};
 
 
@@ -1107,6 +1118,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_daily_sum
     FROM hive.{{schema | sqlsafe}}.__reporting_ocpawscostlineitem_daily_summary_{{uuid | sqlsafe}}
 ;
 
+DELETE FROM hive.{{schema | sqlsafe}}.__reporting_ocpawscostlineitem_daily_summary_{{uuid | sqlsafe}};
 DROP TABLE IF EXISTS hive.{{schema | sqlsafe}}.__reporting_ocpawscostlineitem_daily_summary_{{uuid | sqlsafe}};
 
 -- Clear out old entries first
@@ -1192,4 +1204,5 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_d
     FROM hive.{{schema | sqlsafe}}.__reporting_ocpawscostlineitem_project_daily_summary_{{uuid | sqlsafe}}
 ;
 
+DELETE FROM hive.{{schema | sqlsafe}}.__reporting_ocpawscostlineitem_project_daily_summary_{{uuid | sqlsafe}};
 DROP TABLE IF EXISTS hive.{{schema | sqlsafe}}.__reporting_ocpawscostlineitem_project_daily_summary_{{uuid | sqlsafe}};

testing/compose_files/docker-compose-presto.yml

@@ -32,7 +32,7 @@ services:
 
   hive-metastore:
     container_name: hive-metastore
-    image: quay.io/coreos/hive:release-4.6
+    image: quay.io/cloudservices/ubi-hive:3.1.2
     entrypoint:  ["/hive-scripts/entrypoint.sh", "/opt/hive/bin/hive", "--service", "metastore"]
     ports:
       - "9083:9083"
@@ -47,13 +47,13 @@ services:
       - ./../../testing/metastore/db-data:/var/lib/hive
       - ./../../testing/hadoop/hadoop-dfs-name:/hadoop/dfs/name
       - ./../../testing/hadoop/hadoop-dfs-data:/hadoop/dfs/data
-      - ./../../testing/hadoop/hadoop-logs:/opt/hadoop/logs
+#      - ./../../testing/hadoop/hadoop-logs:/opt/hadoop/logs
       - ./../../testing/hadoop/hadoop-config:/hadoop-config
       - ./../../testing/hadoop/hadoop-starting-config:/hadoop-starting-config
 
   presto:
     container_name: presto
-    image: quay.io/coreos/presto:release-4.6
+    image: quay.io/cloudservices/ubi-presto:348
     user: root
     command: /presto-common/initialize_presto.sh
     ports: