diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseHiveTableSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseHiveTableSuite.scala index 57fda77144dc..74e8683389fe 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseHiveTableSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseHiveTableSuite.scala @@ -1286,6 +1286,7 @@ class GlutenClickHouseHiveTableSuite sql(s"drop table if exists $tbl") } +<<<<<<< HEAD test("test mergetree write with column case sensitive on hive") { val dataPath = s"$basePath/lineitem_mergetree_bucket" val sourceDF = spark.sql(s""" @@ -1314,4 +1315,30 @@ class GlutenClickHouseHiveTableSuite compareResultsAgainstVanillaSpark(select_sql, true, _ => {}) spark.sql("drop table test_tbl_6506") } + + test("GLUTEN-6879: Fix partition value diff when it contains blanks") { + val tableName = "test_tbl_6879" + sql(s"drop table if exists $tableName") + + val createSql = + s""" + |CREATE TABLE $tableName ( + | id INT, + | name STRING + |) PARTITIONED BY (part STRING) + |STORED AS PARQUET; + |""".stripMargin + sql(createSql) + + val insertSql = + s""" + |INSERT INTO $tableName PARTITION (part='part with spaces') + |VALUES (1, 'John Doe'); + |""".stripMargin + sql(insertSql) + + val selectSql = s"SELECT * FROM $tableName" + compareResultsAgainstVanillaSpark(selectSql, true, _ => {}) + sql(s"drop table if exists $tableName") + } } diff --git a/cpp-ch/local-engine/Common/GlutenStringUtils.cpp b/cpp-ch/local-engine/Common/GlutenStringUtils.cpp index b6d11ac1b267..c699efb1f588 100644 --- a/cpp-ch/local-engine/Common/GlutenStringUtils.cpp +++ b/cpp-ch/local-engine/Common/GlutenStringUtils.cpp @@ -14,10 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "GlutenStringUtils.h" -#include #include #include +#include + +#include "GlutenStringUtils.h" namespace local_engine { @@ -27,14 +28,22 @@ PartitionValues GlutenStringUtils::parsePartitionTablePath(const std::string & f Poco::StringTokenizer path(file, "/"); for (const auto & item : path) { - auto position = item.find('='); - if (position != std::string::npos) + auto pos = item.find('='); + if (pos != std::string::npos) { - result.emplace_back(PartitionValue(boost::algorithm::to_lower_copy(item.substr(0, position)), item.substr(position + 1))); + auto key = boost::to_lower_copy(item.substr(0, pos)); + auto value = item.substr(pos + 1); + + std::string unescaped_key; + std::string unescaped_value; + Poco::URI::decode(key, unescaped_key); + Poco::URI::decode(value, unescaped_value); + result.emplace_back(std::move(unescaped_key), std::move(unescaped_value)); } } return result; } + bool GlutenStringUtils::isNullPartitionValue(const std::string & value) { return value == "__HIVE_DEFAULT_PARTITION__"; diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.cpp b/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.cpp index e449ede988ee..59ef10a5e047 100644 --- a/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.cpp +++ b/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.cpp @@ -55,25 +55,20 @@ FormatFile::FormatFile( : context(context_), file_info(file_info_), read_buffer_builder(read_buffer_builder_) { PartitionValues part_vals = GlutenStringUtils::parsePartitionTablePath(file_info.uri_file()); - String partition_values_str = "["; for (size_t i = 0; i < part_vals.size(); ++i) { const auto & part = part_vals[i]; partition_keys.push_back(part.first); partition_values[part.first] = part.second; - if (i > 0) - partition_values_str += ", "; - partition_values_str += part.first + "=" + part.second; } - partition_values_str += "]"; + LOG_INFO( &Poco::Logger::get("FormatFile"), - "Reading File path: {}, format: {}, range: {}, partition_index: {}, partition_values: {}", + "Reading File path: {}, format: {}, range: {}, partition_index: {}", file_info.uri_file(), file_info.file_format_case(), std::to_string(file_info.start()) + "-" + std::to_string(file_info.start() + file_info.length()), - file_info.partition_index(), - partition_values_str); + file_info.partition_index()); } FormatFilePtr FormatFileUtil::createFile(