From 37ea5c11a3f3d067369c3530b1a92cd22e505387 Mon Sep 17 00:00:00 2001 From: gaoyangxiaozhu Date: Thu, 27 Jun 2024 14:15:12 +0800 Subject: [PATCH] remove isVeloxBackendLoaded usage of file metadata ut --- .../datasources/GlutenFileMetadataStructSuite.scala | 8 ++++---- .../datasources/GlutenFileMetadataStructSuite.scala | 8 ++++---- .../datasources/GlutenFileMetadataStructSuite.scala | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala index efa0fbae062b..ed347d024c1c 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.execution.datasources +import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.{FileSourceScanExecTransformer, FilterExecTransformer} -import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{Column, DataFrame, Row} import org.apache.spark.sql.GlutenSQLTestsBaseTrait @@ -108,7 +108,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS METADATA_FILE_MODIFICATION_TIME, "age") dfWithMetadata.collect - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](dfWithMetadata) } else { checkOperatorMatch[FileSourceScanExec](dfWithMetadata) @@ -133,7 +133,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS .where(Column(METADATA_FILE_NAME) === f0((METADATA_FILE_NAME))) val ret = filterDF.collect assert(ret.size == 1) - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](filterDF) } else { checkOperatorMatch[FileSourceScanExec](filterDF) @@ -149,7 +149,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS Row(f1(METADATA_FILE_PATH)) ) ) - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](filterDF) } else { checkOperatorMatch[FileSourceScanExec](filterDF) diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala index b3b9ea7393c3..6e47a94e3c13 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.execution.datasources +import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.{FileSourceScanExecTransformer, FilterExecTransformer} -import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{Column, DataFrame, Row} import org.apache.spark.sql.GlutenSQLTestsBaseTrait @@ -109,7 +109,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS METADATA_FILE_MODIFICATION_TIME, "age") dfWithMetadata.collect - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](dfWithMetadata) } else { checkOperatorMatch[FileSourceScanExec](dfWithMetadata) @@ -134,7 +134,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS .where(Column(METADATA_FILE_NAME) === f0((METADATA_FILE_NAME))) val ret = filterDF.collect assert(ret.size == 1) - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](filterDF) } else { checkOperatorMatch[FileSourceScanExec](filterDF) @@ -150,7 +150,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS Row(f1(METADATA_FILE_PATH)) ) ) - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](filterDF) } else { checkOperatorMatch[FileSourceScanExec](filterDF) diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala index efa0fbae062b..ed347d024c1c 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.execution.datasources +import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.{FileSourceScanExecTransformer, FilterExecTransformer} -import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{Column, DataFrame, Row} import org.apache.spark.sql.GlutenSQLTestsBaseTrait @@ -108,7 +108,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS METADATA_FILE_MODIFICATION_TIME, "age") dfWithMetadata.collect - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](dfWithMetadata) } else { checkOperatorMatch[FileSourceScanExec](dfWithMetadata) @@ -133,7 +133,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS .where(Column(METADATA_FILE_NAME) === f0((METADATA_FILE_NAME))) val ret = filterDF.collect assert(ret.size == 1) - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](filterDF) } else { checkOperatorMatch[FileSourceScanExec](filterDF) @@ -149,7 +149,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS Row(f1(METADATA_FILE_PATH)) ) ) - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](filterDF) } else { checkOperatorMatch[FileSourceScanExec](filterDF)