From 9a7aa70063659da5a15ffc604ae172799ec4d1a8 Mon Sep 17 00:00:00 2001 From: yan ma Date: Tue, 13 Aug 2024 16:46:10 +0800 Subject: [PATCH] disable parquet files reading as velox not supported yet --- .../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 5 +++++ .../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 5 +++++ .../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 5 +++++ .../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index f8a075ec0c41f..71d8073012f55 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -901,6 +901,10 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenParquetV1PartitionDiscoverySuite] enableSuite[GlutenParquetV2PartitionDiscoverySuite] enableSuite[GlutenParquetProtobufCompatibilitySuite] + // Velox issue + .exclude("unannotated array of struct with unannotated array") + .exclude("struct with unannotated array") + .exclude("unannotated array of struct") enableSuite[GlutenParquetV1QuerySuite] // Unsupport spark.sql.files.ignoreCorruptFiles. .exclude("Enabling/disabling ignoreCorruptFiles") @@ -944,6 +948,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenParquetThriftCompatibilitySuite] // Rewrite for file locating. .exclude("Read Parquet file generated by parquet-thrift") + .exclude("SPARK-10136 list of primitive list") enableSuite[GlutenFileTableSuite] enableSuite[GlutenBucketingUtilsSuite] enableSuite[GlutenDataSourceStrategySuite] diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 6c21fc7cc2ba1..e0dd1a7b4f63f 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -717,6 +717,10 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenParquetV1PartitionDiscoverySuite] enableSuite[GlutenParquetV2PartitionDiscoverySuite] enableSuite[GlutenParquetProtobufCompatibilitySuite] + // Velox issue + .exclude("unannotated array of struct with unannotated array") + .exclude("struct with unannotated array") + .exclude("unannotated array of struct") enableSuite[GlutenParquetV1QuerySuite] // Unsupport spark.sql.files.ignoreCorruptFiles. .exclude("Enabling/disabling ignoreCorruptFiles") @@ -762,6 +766,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenParquetThriftCompatibilitySuite] // Rewrite for file locating. .exclude("Read Parquet file generated by parquet-thrift") + .exclude("SPARK-10136 list of primitive list") enableSuite[GlutenParquetVectorizedSuite] enableSuite[GlutenTextV1Suite] enableSuite[GlutenTextV2Suite] diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 95a5fe107f474..c8dbecc97b229 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -701,6 +701,10 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenParquetV1PartitionDiscoverySuite] enableSuite[GlutenParquetV2PartitionDiscoverySuite] enableSuite[GlutenParquetProtobufCompatibilitySuite] + // Velox issue + .exclude("unannotated array of struct with unannotated array") + .exclude("struct with unannotated array") + .exclude("unannotated array of struct") enableSuite[GlutenParquetV1QuerySuite] // Unsupport spark.sql.files.ignoreCorruptFiles. .exclude("Enabling/disabling ignoreCorruptFiles") @@ -756,6 +760,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenParquetThriftCompatibilitySuite] // Rewrite for file locating. .exclude("Read Parquet file generated by parquet-thrift") + .exclude("SPARK-10136 list of primitive list") enableSuite[GlutenParquetVectorizedSuite] enableSuite[GlutenTextV1Suite] enableSuite[GlutenTextV2Suite] diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index e8d8730e93663..672f186022b73 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -706,6 +706,10 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenParquetV1PartitionDiscoverySuite] enableSuite[GlutenParquetV2PartitionDiscoverySuite] enableSuite[GlutenParquetProtobufCompatibilitySuite] + // Velox issue + .exclude("unannotated array of struct with unannotated array") + .exclude("struct with unannotated array") + .exclude("unannotated array of struct") enableSuite[GlutenParquetV1QuerySuite] .exclude("row group skipping doesn't overflow when reading into larger type") // Unsupport spark.sql.files.ignoreCorruptFiles. @@ -763,6 +767,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenParquetThriftCompatibilitySuite] // Rewrite for file locating. .exclude("Read Parquet file generated by parquet-thrift") + .exclude("SPARK-10136 list of primitive list") enableSuite[GlutenParquetVectorizedSuite] enableSuite[GlutenTextV1Suite] enableSuite[GlutenTextV2Suite]