From 87ae908f6563ee9f1c0b9d91cebeadc8f6c8421d Mon Sep 17 00:00:00 2001
From: yan ma <yan.ma@intel.com>
Date: Wed, 14 Aug 2024 20:02:55 +0800
Subject: [PATCH] fallback timestamp scan for parquet if necessary

---
 .../gluten/backendsapi/velox/VeloxBackend.scala       | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
index ccc924a58670a..d1e9dc46c59b0 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
@@ -25,7 +25,6 @@ import org.apache.gluten.extension.ValidationResult
 import org.apache.gluten.sql.shims.SparkShimLoader
 import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat
 import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat.{DwrfReadFormat, OrcReadFormat, ParquetReadFormat}
-import org.apache.gluten.utils._
 
 import org.apache.spark.sql.catalyst.catalog.BucketSpec
 import org.apache.spark.sql.catalyst.expressions.{Alias, CumeDist, DenseRank, Descending, EulerNumber, Expression, Lag, Lead, Literal, MakeYMInterval, NamedExpression, NthValue, NTile, PercentRank, Pi, Rand, RangeFrame, Rank, RowNumber, SortOrder, SparkPartitionID, SparkVersion, SpecialFrameBoundary, SpecifiedWindowFrame, Uuid}
@@ -100,7 +99,15 @@ object VeloxBackendSettings extends BackendSettingsApi {
     }
 
     format match {
-      case ParquetReadFormat | DwrfReadFormat => ValidationResult.succeeded
+      case ParquetReadFormat =>
+        val typeValidator: PartialFunction[StructField, String] = {
+          // Parquet timestamp is not fully supported yet
+          case StructField(_, TimestampType, _, _)
+              if GlutenConfig.getConf.forceParquetTimestampTypeScanFallbackEnabled =>
+            "TimestampType"
+        }
+        validateTypes(typeValidator)
+      case DwrfReadFormat => ValidationResult.succeeded
       case OrcReadFormat =>
         if (!GlutenConfig.getConf.veloxOrcScanEnabled) {
           ValidationResult.failed(s"Velox ORC scan is turned off.")