From cc86a9a622476342ea0e6c239344c1d9cab41711 Mon Sep 17 00:00:00 2001 From: yan ma Date: Tue, 13 Aug 2024 16:46:10 +0800 Subject: [PATCH] disable parquet files reading as velox not supported yet --- .../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 5 +++++ .../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 5 +++++ .../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 5 +++++ .../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index e064f2afc9d7..78ea77129ae9 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -899,6 +899,10 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenParquetV1PartitionDiscoverySuite] enableSuite[GlutenParquetV2PartitionDiscoverySuite] enableSuite[GlutenParquetProtobufCompatibilitySuite] + // Velox issue + .exclude("unannotated array of struct with unannotated array") + .exclude("struct with unannotated array") + .exclude("unannotated array of struct") enableSuite[GlutenParquetV1QuerySuite] // Unsupport spark.sql.files.ignoreCorruptFiles. .exclude("Enabling/disabling ignoreCorruptFiles") @@ -942,6 +946,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenParquetThriftCompatibilitySuite] // Rewrite for file locating. .exclude("Read Parquet file generated by parquet-thrift") + .exclude("SPARK-10136 list of primitive list") enableSuite[GlutenFileTableSuite] enableSuite[GlutenBucketingUtilsSuite] enableSuite[GlutenDataSourceStrategySuite] diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 6c21fc7cc2ba..e0dd1a7b4f63 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -717,6 +717,10 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenParquetV1PartitionDiscoverySuite] enableSuite[GlutenParquetV2PartitionDiscoverySuite] enableSuite[GlutenParquetProtobufCompatibilitySuite] + // Velox issue + .exclude("unannotated array of struct with unannotated array") + .exclude("struct with unannotated array") + .exclude("unannotated array of struct") enableSuite[GlutenParquetV1QuerySuite] // Unsupport spark.sql.files.ignoreCorruptFiles. .exclude("Enabling/disabling ignoreCorruptFiles") @@ -762,6 +766,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenParquetThriftCompatibilitySuite] // Rewrite for file locating. .exclude("Read Parquet file generated by parquet-thrift") + .exclude("SPARK-10136 list of primitive list") enableSuite[GlutenParquetVectorizedSuite] enableSuite[GlutenTextV1Suite] enableSuite[GlutenTextV2Suite] diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 95a5fe107f47..c8dbecc97b22 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -701,6 +701,10 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenParquetV1PartitionDiscoverySuite] enableSuite[GlutenParquetV2PartitionDiscoverySuite] enableSuite[GlutenParquetProtobufCompatibilitySuite] + // Velox issue + .exclude("unannotated array of struct with unannotated array") + .exclude("struct with unannotated array") + .exclude("unannotated array of struct") enableSuite[GlutenParquetV1QuerySuite] // Unsupport spark.sql.files.ignoreCorruptFiles. .exclude("Enabling/disabling ignoreCorruptFiles") @@ -756,6 +760,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenParquetThriftCompatibilitySuite] // Rewrite for file locating. .exclude("Read Parquet file generated by parquet-thrift") + .exclude("SPARK-10136 list of primitive list") enableSuite[GlutenParquetVectorizedSuite] enableSuite[GlutenTextV1Suite] enableSuite[GlutenTextV2Suite] diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index e8d8730e9366..672f186022b7 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -706,6 +706,10 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenParquetV1PartitionDiscoverySuite] enableSuite[GlutenParquetV2PartitionDiscoverySuite] enableSuite[GlutenParquetProtobufCompatibilitySuite] + // Velox issue + .exclude("unannotated array of struct with unannotated array") + .exclude("struct with unannotated array") + .exclude("unannotated array of struct") enableSuite[GlutenParquetV1QuerySuite] .exclude("row group skipping doesn't overflow when reading into larger type") // Unsupport spark.sql.files.ignoreCorruptFiles. @@ -763,6 +767,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenParquetThriftCompatibilitySuite] // Rewrite for file locating. .exclude("Read Parquet file generated by parquet-thrift") + .exclude("SPARK-10136 list of primitive list") enableSuite[GlutenParquetVectorizedSuite] enableSuite[GlutenTextV1Suite] enableSuite[GlutenTextV2Suite]