enable complex type in parquet write

apache · Jun 25, 2024 · f7aaed8 · f7aaed8
1 parent 9cceba6
commit f7aaed8
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 7 deletions.
diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
@@ -224,9 +224,6 @@ object VeloxBackendSettings extends BackendSettingsApi {
         field =>
           field.dataType match {
             case _: TimestampType => Some("TimestampType")
-            case _: StructType => Some("StructType")
-            case _: ArrayType => Some("ArrayType")
-            case _: MapType => Some("MapType")
             case _: YearMonthIntervalType => Some("YearMonthIntervalType")
             case _ => None
           }
@@ -292,7 +289,7 @@ object VeloxBackendSettings extends BackendSettingsApi {
     fields.map {
       field =>
         field.dataType match {
-          case _: TimestampType | _: StructType | _: ArrayType | _: MapType => return false
+          case _: TimestampType => return false
           case _ =>
         }
     }

diff --git a/...elox/src/test/scala/org/apache/gluten/execution/VeloxParquetDataTypeValidationSuite.scala b/...elox/src/test/scala/org/apache/gluten/execution/VeloxParquetDataTypeValidationSuite.scala
@@ -485,9 +485,6 @@ class VeloxParquetDataTypeValidationSuite extends VeloxWholeStageTransformerSuit
             .format("parquet")
             .load(data_path)
             .drop("timestamp")
-            .drop("array")
-            .drop("struct")
-            .drop("map")
           df.write.mode("append").format("parquet").save(write_path)
           val parquetDf = spark.read
             .format("parquet")