Merge pull request #38 from arenadata/4.0.0-develop

Shading fixes

druid-handler/src/main/resources/META-INF/services/java.sql.Driver

@@ -0,0 +1 @@
+org.apache.hive.org.apache.calcite.jdbc.Driver

iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInternalRecordWrapper.java

@@ -142,7 +142,9 @@ private Map<String, Integer> buildFieldPositionMap(StructType schema) {
   private static Function<Object, Object> converter(Type type) {
     switch (type.typeId()) {
       case TIMESTAMP:
-        return timestamp -> DateTimeUtil.timestamptzFromMicros((Long) timestamp);
+        return timestamp -> ((Types.TimestampType) type).shouldAdjustToUTC() ?
+            DateTimeUtil.timestamptzFromMicros((Long) timestamp) :
+            DateTimeUtil.timestampFromMicros((Long) timestamp);
       case DATE:
         return date -> DateTimeUtil.dateFromDays((Integer) date);
       case STRUCT:

iceberg/iceberg-handler/src/test/queries/positive/iceberg_metadata_table.q

@@ -0,0 +1,18 @@
+-- SORT_QUERY_RESULTS
+-- MASK_TIMESTAMP
+create table ice_ts_4 (id int, ts timestamp ) stored by iceberg stored as parquet tblproperties ('format-version'='2');
+insert into ice_ts_4 values (1, cast('2023-07-20 00:00:00' as timestamp)), (2, cast('2023-07-20 00:00:00' as timestamp));
+select * from ice_ts_4;
+delete from ice_ts_4 where id = 2;
+select * from ice_ts_4;
+select readable_metrics from default.ice_ts_4.FILES;
+select readable_metrics from default.ice_ts_4.ALL_FILES;
+select readable_metrics from default.ice_ts_4.DATA_FILES;
+select readable_metrics from default.ice_ts_4.ALL_DATA_FILES;
+select readable_metrics from default.ice_ts_4.DELETE_FILES;
+
+-- Test partitions table
+CREATE EXTERNAL TABLE ice_part  (`col1` int, `decimalA` decimal(5,2), `decimalC` decimal(5,2)) PARTITIONED BY SPEC
+(decimalC) stored by iceberg tblproperties('format-version'='2');
+insert into ice_part values(1, 122.91, 102.21), (1, 12.32, 200.12);
+select last_updated_at from default.ice_part.PARTITIONS;

iceberg/iceberg-handler/src/test/results/positive/iceberg_metadata_table.q.out

@@ -0,0 +1,118 @@
+PREHOOK: query: create table ice_ts_4 (id int, ts timestamp ) stored by iceberg stored as parquet tblproperties ('format-version'='2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_ts_4
+POSTHOOK: query: create table ice_ts_4 (id int, ts timestamp ) stored by iceberg stored as parquet tblproperties ('format-version'='2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_ts_4
+PREHOOK: query: insert into ice_ts_4 values (1, cast('2023-07-20 00:00:00' as timestamp)), (2, cast('2023-07-20 00:00:00' as timestamp))
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_ts_4
+POSTHOOK: query: insert into ice_ts_4 values (1, cast('2023-07-20 00:00:00' as timestamp)), (2, cast('2023-07-20 00:00:00' as timestamp))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_ts_4
+PREHOOK: query: select * from ice_ts_4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_ts_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from ice_ts_4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_ts_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1	2023-07-20 00:00:00
+2	2023-07-20 00:00:00
+PREHOOK: query: delete from ice_ts_4 where id = 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_ts_4
+PREHOOK: Output: default@ice_ts_4
+POSTHOOK: query: delete from ice_ts_4 where id = 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_ts_4
+POSTHOOK: Output: default@ice_ts_4
+PREHOOK: query: select * from ice_ts_4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_ts_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from ice_ts_4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_ts_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1	2023-07-20 00:00:00
+PREHOOK: query: select readable_metrics from default.ice_ts_4.FILES
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_ts_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select readable_metrics from default.ice_ts_4.FILES
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_ts_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+{"id":{"column_size":46,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":1,"upper_bound":2},"ts":{"column_size":76,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"2023-07-20 00:00:00","upper_bound":"2023-07-20 00:00:00"}}
+{"id":{"column_size":null,"value_count":null,"null_value_count":null,"nan_value_count":null,"lower_bound":null,"upper_bound":null},"ts":{"column_size":null,"value_count":null,"null_value_count":null,"nan_value_count":null,"lower_bound":null,"upper_bound":null}}
+PREHOOK: query: select readable_metrics from default.ice_ts_4.ALL_FILES
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_ts_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select readable_metrics from default.ice_ts_4.ALL_FILES
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_ts_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+{"id":{"column_size":46,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":1,"upper_bound":2},"ts":{"column_size":76,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"2023-07-20 00:00:00","upper_bound":"2023-07-20 00:00:00"}}
+{"id":{"column_size":null,"value_count":null,"null_value_count":null,"nan_value_count":null,"lower_bound":null,"upper_bound":null},"ts":{"column_size":null,"value_count":null,"null_value_count":null,"nan_value_count":null,"lower_bound":null,"upper_bound":null}}
+PREHOOK: query: select readable_metrics from default.ice_ts_4.DATA_FILES
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_ts_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select readable_metrics from default.ice_ts_4.DATA_FILES
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_ts_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+{"id":{"column_size":46,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":1,"upper_bound":2},"ts":{"column_size":76,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"2023-07-20 00:00:00","upper_bound":"2023-07-20 00:00:00"}}
+PREHOOK: query: select readable_metrics from default.ice_ts_4.ALL_DATA_FILES
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_ts_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select readable_metrics from default.ice_ts_4.ALL_DATA_FILES
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_ts_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+{"id":{"column_size":46,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":1,"upper_bound":2},"ts":{"column_size":76,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"2023-07-20 00:00:00","upper_bound":"2023-07-20 00:00:00"}}
+PREHOOK: query: select readable_metrics from default.ice_ts_4.DELETE_FILES
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_ts_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select readable_metrics from default.ice_ts_4.DELETE_FILES
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_ts_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+{"id":{"column_size":null,"value_count":null,"null_value_count":null,"nan_value_count":null,"lower_bound":null,"upper_bound":null},"ts":{"column_size":null,"value_count":null,"null_value_count":null,"nan_value_count":null,"lower_bound":null,"upper_bound":null}}
+PREHOOK: query: CREATE EXTERNAL TABLE ice_part  (`col1` int, `decimalA` decimal(5,2), `decimalC` decimal(5,2)) PARTITIONED BY SPEC
+(decimalC) stored by iceberg tblproperties('format-version'='2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_part
+POSTHOOK: query: CREATE EXTERNAL TABLE ice_part  (`col1` int, `decimalA` decimal(5,2), `decimalC` decimal(5,2)) PARTITIONED BY SPEC
+(decimalC) stored by iceberg tblproperties('format-version'='2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_part
+PREHOOK: query: insert into ice_part values(1, 122.91, 102.21), (1, 12.32, 200.12)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_part
+POSTHOOK: query: insert into ice_part values(1, 122.91, 102.21), (1, 12.32, 200.12)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_part
+PREHOOK: query: select last_updated_at from default.ice_part.PARTITIONS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_part
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select last_updated_at from default.ice_part.PARTITIONS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_part
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+  ###MaskedTimeStamp### 
+  ###MaskedTimeStamp### 

itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java

@@ -60,6 +60,10 @@ public class QOutProcessor {
   private static final PatternReplacementPair MASK_DATA_SIZE = new PatternReplacementPair(
       Pattern.compile(" Data size: [1-9][0-9]*"),
       " Data size: ###Masked###");
+    private static final PatternReplacementPair MASK_TIMESTAMP = new PatternReplacementPair(
+      Pattern.compile(
+          "[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1]) (2[0-3]|[01][0-9]):[0-5][0-9]:[0-5][0-9].[0-9]{1,3} [a-zA-Z/]*"),
+        "  ###MaskedTimeStamp### ");
   private static final PatternReplacementPair MASK_LINEAGE = new PatternReplacementPair(
       Pattern.compile("POSTHOOK: Lineage: .*"),
       "POSTHOOK: Lineage: ###Masked###");
@@ -143,7 +147,7 @@ public String get() {
   };
 
   private enum Mask {
-    STATS("-- MASK_STATS"), DATASIZE("-- MASK_DATA_SIZE"), LINEAGE("-- MASK_LINEAGE");
+    STATS("-- MASK_STATS"), DATASIZE("-- MASK_DATA_SIZE"), LINEAGE("-- MASK_LINEAGE"), TIMESTAMP("-- MASK_TIMESTAMP");
     private Pattern pattern;
 
     Mask(String pattern) {