From 50430ebdd32a427ad6e185d8a4ba8c477823fd19 Mon Sep 17 00:00:00 2001 From: Chang chen Date: Thu, 26 Sep 2024 14:47:26 +0800 Subject: [PATCH] [GLUTEN-7356][CORE] Make GlutenConfig.GLUTEN_CONFIG_PREFIX private(#7357) --- .../apache/gluten/backendsapi/clickhouse/CHBackend.scala | 6 +----- .../shuffle/CHCelebornColumnarShuffleWriterFactory.scala | 4 ++-- .../src/main/scala/org/apache/gluten/GlutenConfig.scala | 2 +- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala index ff0cd39109e8..d8966519da1c 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala @@ -48,7 +48,7 @@ import java.util.Locale import scala.util.control.Breaks.{break, breakable} class CHBackend extends SubstraitBackend { - override def name(): String = CHBackend.BACKEND_NAME + override def name(): String = CHConf.BACKEND_NAME override def batchType: Convention.BatchType = CHBatch override def buildInfo(): Backend.BuildInfo = Backend.BuildInfo("ClickHouse", CH_BRANCH, CH_COMMIT, "UNKNOWN") @@ -62,10 +62,6 @@ class CHBackend extends SubstraitBackend { override def settings(): BackendSettingsApi = CHBackendSettings } -object CHBackend { - val BACKEND_NAME: String = CHConf.BACKEND_NAME -} - object CHBackendSettings extends BackendSettingsApi with Logging { private val GLUTEN_CLICKHOUSE_SEP_SCAN_RDD = "spark.gluten.sql.columnar.separate.scan.rdd.for.ch" diff --git a/gluten-celeborn/clickhouse/src/main/scala/org/apache/spark/shuffle/CHCelebornColumnarShuffleWriterFactory.scala b/gluten-celeborn/clickhouse/src/main/scala/org/apache/spark/shuffle/CHCelebornColumnarShuffleWriterFactory.scala index 7c105933e2f9..42dcd9bed567 100644 --- a/gluten-celeborn/clickhouse/src/main/scala/org/apache/spark/shuffle/CHCelebornColumnarShuffleWriterFactory.scala +++ b/gluten-celeborn/clickhouse/src/main/scala/org/apache/spark/shuffle/CHCelebornColumnarShuffleWriterFactory.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.shuffle -import org.apache.gluten.backendsapi.clickhouse.CHBackend +import org.apache.gluten.backend.Backend import org.apache.spark.TaskContext import org.apache.spark.shuffle.celeborn.CelebornShuffleHandle @@ -26,7 +26,7 @@ import org.apache.celeborn.client.ShuffleClient import org.apache.celeborn.common.CelebornConf class CHCelebornColumnarShuffleWriterFactory extends CelebornShuffleWriterFactory { - override def backendName(): String = CHBackend.BACKEND_NAME + override def backendName(): String = Backend.get().name() override def createShuffleWriterInstance[K, V]( shuffleId: Int, diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 4e27e6330a62..7a384f762074 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -533,7 +533,7 @@ object GlutenConfig { val GLUTEN_IAA_BACKEND_NAME = "iaa" val GLUTEN_IAA_SUPPORTED_CODEC: Set[String] = Set("gzip") - val GLUTEN_CONFIG_PREFIX = "spark.gluten.sql.columnar.backend." + private val GLUTEN_CONFIG_PREFIX = "spark.gluten.sql.columnar.backend." // Private Spark configs. val SPARK_ONHEAP_SIZE_KEY = "spark.executor.memory"