fixup

.github/labeler.yml

@@ -47,6 +47,7 @@ DOCS:
 CORE:
   - changed-files:
     - any-glob-to-any-file: [
+      'gluten-core/**/*',
       'gluten-substrait/**/*',
       'shims/**/*',
       'gluten-ras/**/*',

.github/workflows/clickhouse_be_trigger.yml

@@ -25,6 +25,7 @@ on:
       - 'gluten-celeborn/common/**'
       - 'gluten-celeborn/package/**'
       - 'gluten-celeborn/clickhouse/**'
+      - 'gluten-core/**'
       - 'gluten-substrait/**'
       - 'gluten-ut/**'
       - 'shims/**'

.github/workflows/velox_backend.yml

@@ -26,6 +26,7 @@ on:
       - 'gluten-celeborn/package/**'
       - 'gluten-celeborn/velox/**'
       - 'gluten-ras/**'
+      - 'gluten-core/**'
       - 'gluten-substrait/**'
       - 'gluten-data/**'
       - 'gluten-delta/**'

backends-clickhouse/src/test/scala/org/apache/gluten/RunTPCHTest.scala

@@ -63,7 +63,7 @@ object RunTPCHTest {
     FileUtils.forceMkdir(new File(warehouse))
     FileUtils.forceMkdir(new File(metaStorePathAbsolute))
 
-    val resourcePath = rootPath + "../../../../gluten-substrait/src/test/resources/"
+    val resourcePath = rootPath + "../../../../gluten-core/src/test/resources/"
     val queryPath = resourcePath + "/tpch-queries/"
     // which sql to execute
     val sqlFilePath = queryPath + "q01.sql"

backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala

@@ -40,7 +40,7 @@ class GlutenClickHouseDecimalSuite
 
   override protected val tablesPath: String = basePath + "/tpch-data"
   override protected val tpchQueries: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpch-queries"
+    rootPath + "../../../../gluten-core/src/test/resources/tpch-queries"
   override protected val queriesResults: String = rootPath + "queries-output"
   override protected val createNullableTables = true
   override protected def createTPCHNotNullTables(): Unit = {}

backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseFileFormatSuite.scala

@@ -53,7 +53,7 @@ class GlutenClickHouseFileFormatSuite
 
   override protected val tablesPath: String = basePath + "/tpch-data"
   override protected val tpchQueries: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpch-queries"
+    rootPath + "../../../../gluten-core/src/test/resources/tpch-queries"
   override protected val queriesResults: String = rootPath + "queries-output"
 
   protected val orcDataPath: String = rootPath + "orc-data"

backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala

@@ -25,7 +25,7 @@ class GlutenClickHouseJoinSuite extends GlutenClickHouseWholeStageTransformerSui
 
   protected val tablesPath: String = basePath + "/tpch-data"
   protected val tpchQueries: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpch-queries"
+    rootPath + "../../../../gluten-core/src/test/resources/tpch-queries"
   protected val queriesResults: String = rootPath + "queries-output"
 
   private val joinAlgorithm = "spark.gluten.sql.columnar.backend.ch.runtime_settings.join_algorithm"

backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseS3SourceSuite.scala

@@ -27,7 +27,7 @@ class GlutenClickHouseS3SourceSuite extends GlutenClickHouseTPCHAbstractSuite {
 
   override protected val tablesPath: String = basePath + "/tpch-data"
   override protected val tpchQueries: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpch-queries"
+    rootPath + "../../../../gluten-core/src/test/resources/tpch-queries"
   override protected val queriesResults: String = rootPath + "queries-output"
 
   override protected def sparkConf: SparkConf = {

backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala

@@ -43,7 +43,7 @@ abstract class GlutenClickHouseTPCDSAbstractSuite
   protected val tablesPath: String = UTSystemParameters.tpcdsDecimalDataPath + "/"
   protected val db_name: String = "tpcdsdb"
   protected val tpcdsQueries: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpcds-queries/tpcds.queries.original"
+    rootPath + "../../../../gluten-core/src/test/resources/tpcds-queries/tpcds.queries.original"
   protected val queriesResults: String = rootPath + "tpcds-decimal-queries-output"
 
   /** Return values: (sql num, is fall back) */

backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala

@@ -38,7 +38,7 @@ abstract class GlutenClickHouseTPCHAbstractSuite
   protected val needCopyParquetToTablePath = false
 
   protected val parquetTableDataPath: String =
-    "../../../../gluten-substrait/src/test/resources/tpch-data"
+    "../../../../gluten-core/src/test/resources/tpch-data"
 
   protected val tablesPath: String
   protected val tpchQueries: String

backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHBucketSuite.scala

@@ -61,7 +61,7 @@ class GlutenClickHouseTPCHBucketSuite
 
     val parquetTablePath = basePath + "/tpch-data"
     val parquetTableDataPath: String =
-      "../../../../gluten-substrait/src/test/resources/tpch-data"
+      "../../../../gluten-core/src/test/resources/tpch-data"
     FileUtils.copyDirectory(new File(rootPath + parquetTableDataPath), new File(parquetTablePath))
 
     createNotNullTPCHTablesInParquet(parquetTablePath)

backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala

@@ -34,7 +34,7 @@ class GlutenFunctionValidateSuite extends GlutenClickHouseWholeStageTransformerS
 
   protected val tablesPath: String = basePath + "/tpch-data"
   protected val tpchQueries: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpch-queries"
+    rootPath + "../../../../gluten-core/src/test/resources/tpch-queries"
   protected val queriesResults: String = rootPath + "queries-output"
 
   private var parquetPath: String = _

backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala

@@ -27,7 +27,7 @@ class GlutenClickhouseFunctionSuite extends GlutenClickHouseTPCHAbstractSuite {
 
   override protected val tablesPath: String = basePath + "/tpch-data"
   override protected val tpchQueries: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpch-queries"
+    rootPath + "../../../../gluten-core/src/test/resources/tpch-queries"
   override protected val queriesResults: String = rootPath + "queries-output"
 
   override protected def createTPCHNotNullTables(): Unit = {

backends-clickhouse/src/test/scala/org/apache/gluten/execution/extension/GlutenCustomAggExpressionSuite.scala

@@ -32,7 +32,7 @@ class GlutenCustomAggExpressionSuite extends GlutenClickHouseTPCHAbstractSuite {
 
   override protected val tablesPath: String = basePath + "/tpch-data"
   override protected val tpchQueries: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpch-queries"
+    rootPath + "../../../../gluten-core/src/test/resources/tpch-queries"
   override protected val queriesResults: String = rootPath + "queries-output"
 
   override protected def sparkConf: SparkConf = {

backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala

@@ -33,7 +33,7 @@ class GlutenClickHouseTPCHMetricsSuite extends GlutenClickHouseTPCHAbstractSuite
 
   override protected val tablesPath: String = basePath + "/tpch-data"
   override protected val tpchQueries: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpch-queries"
+    rootPath + "../../../../gluten-core/src/test/resources/tpch-queries"
   override protected val queriesResults: String = rootPath + "queries-output"
 
   protected val metricsJsonFilePath: String = rootPath + "metrics-json"

backends-clickhouse/src/test/scala/org/apache/gluten/execution/parquet/GlutenParquetFilterSuite.scala

@@ -35,7 +35,7 @@ class GlutenParquetFilterSuite
   with Logging {
 
   private val tpchQueriesResourceFolder: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpch-queries"
+    rootPath + "../../../../gluten-core/src/test/resources/tpch-queries"
 
   override protected def sparkConf: SparkConf =
     super.sparkConf

backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseDatetimeExpressionSuite.scala

@@ -32,7 +32,7 @@ class GlutenClickHouseDatetimeExpressionSuite
 
   override protected val tablesPath: String = basePath + "/tpch-data"
   override protected val tpchQueries: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpch-queries"
+    rootPath + "../../../../gluten-core/src/test/resources/tpch-queries"
   override protected val queriesResults: String = rootPath + "queries-output"
 
   /** Run Gluten + ClickHouse Backend with SortShuffleManager */

backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseHDFSSuite.scala

@@ -29,7 +29,7 @@ class GlutenClickHouseHDFSSuite
 
   override protected val tablesPath: String = HDFS_URL_ENDPOINT + "/tpch-data"
   override protected val tpchQueries: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpch-queries"
+    rootPath + "../../../../gluten-core/src/test/resources/tpch-queries"
   override protected val queriesResults: String = rootPath + "queries-output"
 
   private val hdfsCachePath = "/tmp/gluten_hdfs_cache/"

backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala

@@ -33,7 +33,7 @@ class GlutenClickHouseTPCHColumnarShuffleParquetAQESuite
 
   override protected val tablesPath: String = basePath + "/tpch-data"
   override protected val tpchQueries: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpch-queries"
+    rootPath + "../../../../gluten-core/src/test/resources/tpch-queries"
   override protected val queriesResults: String = rootPath + "queries-output"
 
   /** Run Gluten + ClickHouse Backend with SortShuffleManager */

backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHParquetAQEConcurrentSuite.scala

@@ -36,7 +36,7 @@ class GlutenClickHouseTPCHParquetAQEConcurrentSuite
 
   override protected val tablesPath: String = basePath + "/tpch-data"
   override protected val tpchQueries: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpch-queries"
+    rootPath + "../../../../gluten-core/src/test/resources/tpch-queries"
   override protected val queriesResults: String = rootPath + "queries-output"
 
   /** Run Gluten + ClickHouse Backend with SortShuffleManager */

backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHParquetAQESuite.scala

@@ -34,7 +34,7 @@ class GlutenClickHouseTPCHParquetAQESuite
 
   override protected val tablesPath: String = basePath + "/tpch-data"
   override protected val tpchQueries: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpch-queries"
+    rootPath + "../../../../gluten-core/src/test/resources/tpch-queries"
   override protected val queriesResults: String = rootPath + "queries-output"
 
   /** Run Gluten + ClickHouse Backend with SortShuffleManager */

backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHParquetBucketSuite.scala

@@ -39,7 +39,7 @@ class GlutenClickHouseTPCHParquetBucketSuite
 
   override protected val tablesPath: String = basePath + "/tpch-data-ch"
   override protected val tpchQueries: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpch-queries"
+    rootPath + "../../../../gluten-core/src/test/resources/tpch-queries"
   override protected val queriesResults: String = rootPath + "queries-output"
 
   protected val bucketTableResourcePath: String = rootPath + "tpch-data-bucket/parquet_bucket"

backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala

@@ -38,7 +38,7 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr
 
   override protected val tablesPath: String = basePath + "/tpch-data"
   override protected val tpchQueries: String =
-    rootPath + "../../../../gluten-substrait/src/test/resources/tpch-queries"
+    rootPath + "../../../../gluten-core/src/test/resources/tpch-queries"
   override protected val queriesResults: String = rootPath + "queries-output"
 
   protected val BACKEND_CONF_KEY = "spark.gluten.sql.columnar.backend.ch."

backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCDSSuite.scala

@@ -35,7 +35,7 @@ class VeloxTPCDSSuite extends VeloxWholeStageTransformerSuite {
   override protected val fileFormat: String = "parquet"
 
   private val queryPath = System.getProperty("user.dir") +
-    "/gluten-substrait/src/test/resources/tpcds-queries/tpcds.queries.original/"
+    "/gluten-core/src/test/resources/tpcds-queries/tpcds.queries.original/"
 
   protected var queryTables: Map[String, DataFrame] = _
 

docs/get-started/Velox.md

@@ -662,12 +662,12 @@ All TPC-H and TPC-DS queries are supported in Gluten Velox backend.
 The data generation scripts are [TPC-H dategen script](../../tools/workload/tpch/gen_data/parquet_dataset/tpch_datagen_parquet.sh) and
 [TPC-DS dategen script](../../tools/workload/tpcds/gen_data/parquet_dataset/tpcds_datagen_parquet.sh).
 
-The used TPC-H and TPC-DS queries are the original ones, and can be accessed from [TPC-DS queries](../../gluten-substrait/src/test/resources/tpcds-queries/tpcds.queries.original)
-and [TPC-H queries](../../gluten-substrait/src/test/resources/tpch-queries).
+The used TPC-H and TPC-DS queries are the original ones, and can be accessed from [TPC-DS queries](../../gluten-core/src/test/resources/tpcds-queries/tpcds.queries.original)
+and [TPC-H queries](../../gluten-core/src/test/resources/tpch-queries).
 
 Some other versions of TPC-DS queries are also provided, but are **not** recommended for testing, including:
 
-- the modified TPC-DS queries with "Decimal-to-Double": [TPC-DS non-decimal queries](../../gluten-substrait/src/test/resources/tpcds-queries/tpcds.queries.no-decimal) (outdated).
+- the modified TPC-DS queries with "Decimal-to-Double": [TPC-DS non-decimal queries](../../gluten-core/src/test/resources/tpcds-queries/tpcds.queries.no-decimal) (outdated).
 
 ## Submit the Spark SQL job
 

gluten-celeborn/clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseRSSColumnarMemorySortShuffleSuite.scala

@@ -29,7 +29,7 @@ class GlutenClickHouseRSSColumnarMemorySortShuffleSuite
     rootPath + "../../../../../backends-clickhouse/src/test/resources/mergetree-queries-output"
 
   override protected val parquetTableDataPath: String =
-    "../../../../../gluten-substrait/src/test/resources/tpch-data"
+    "../../../../../gluten-core/src/test/resources/tpch-data"
 
   /** Run Gluten + ClickHouse Backend with ColumnarShuffleManager */
   override protected def sparkConf: SparkConf = {

gluten-celeborn/clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseRSSColumnarShuffleAQESuite.scala

@@ -33,7 +33,7 @@ class GlutenClickHouseRSSColumnarShuffleAQESuite
     rootPath + "../../../../../backends-clickhouse/src/test/resources/mergetree-queries-output"
 
   override protected val parquetTableDataPath: String =
-    "../../../../../gluten-substrait/src/test/resources/tpch-data"
+    "../../../../../gluten-core/src/test/resources/tpch-data"
 
   /** Run Gluten + ClickHouse Backend with ColumnarShuffleManager */
   override protected def sparkConf: SparkConf = {

gluten-substrait/pom.xml

@@ -25,6 +25,13 @@
       <artifactId>gluten-core</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.gluten</groupId>
+      <artifactId>gluten-core</artifactId>
+      <version>${project.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.gluten</groupId>
       <artifactId>gluten-ui</artifactId>

tools/workload/tpcds/README.md

@@ -29,5 +29,5 @@ val dbgenDir = "/PATH/TO/TPCDS_DBGEN" // location of dbgen
 Currently, Gluten with Velox can support Parquet file format and three compression codec including snappy, gzip, zstd.
 
 ## Test Queries
-We provide the test queries in [TPC-DS Queries](../../../gluten-substrait/src/test/resources/tpcds-queries).
+We provide the test queries in [TPC-DS Queries](../../../gluten-core/src/test/resources/tpcds-queries).
 We also provide a Scala script in [Run TPC-DS](./run_tpcds) directory about how to run TPC-DS queries.

tools/workload/tpcds/run_tpcds/tpcds_parquet.scala

@@ -28,7 +28,7 @@ var gluten_root = "/PATH/TO/GLUTEN"
 // e.g. hdfs://hostname:8020
 var paq_file_root = "/ROOT_PATH"
 
-var tpcds_queries_path = "/gluten-substrait/src/test/resources/tpcds-queries/"
+var tpcds_queries_path = "/gluten-core/src/test/resources/tpcds-queries/"
 var queries_no_decimal = "tpcds.queries.no-decimal"
 var queries_original = "tpcds.queries.original"
 

tools/workload/tpch/README.md

@@ -48,6 +48,6 @@ val dwrf_file_path = "/PATH/TO/TPCH_DWRF_PATH"
 ```
 
 ## Test Queries
-We provide the test queries in [TPC-H queries](../../../gluten-substrait/src/test/resources/tpch-queries).
+We provide the test queries in [TPC-H queries](../../../gluten-core/src/test/resources/tpch-queries).
 We also provide a scala script in [Run TPC-H](./run_tpch/) directory about how to run TPC-H queries.
 Please note if you are using DWRF test, please remember to set the file format to DWRF in the code.

tools/workload/tpch/run_tpch/tpch_parquet.scala

@@ -63,7 +63,7 @@ def getListOfFiles(dir: String):List[File] = {
          List[File]()
      }
 }
-val fileLists = getListOfFiles(gluten_root + "/gluten-substrait/src/test/resources/tpch-queries/")
+val fileLists = getListOfFiles(gluten_root + "/gluten-core/src/test/resources/tpch-queries/")
 val sorted = fileLists.sortBy {
        f => f.getName match {
        case name =>