Skip to content

Commit

Permalink
fix partition values diff
Browse files Browse the repository at this point in the history
  • Loading branch information
taiyang-li committed Aug 16, 2024
1 parent 39480ad commit c22b230
Show file tree
Hide file tree
Showing 3 changed files with 44 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -1286,6 +1286,7 @@ class GlutenClickHouseHiveTableSuite
sql(s"drop table if exists $tbl")
}

<<<<<<< HEAD
test("test mergetree write with column case sensitive on hive") {
val dataPath = s"$basePath/lineitem_mergetree_bucket"
val sourceDF = spark.sql(s"""
Expand Down Expand Up @@ -1314,4 +1315,30 @@ class GlutenClickHouseHiveTableSuite
compareResultsAgainstVanillaSpark(select_sql, true, _ => {})
spark.sql("drop table test_tbl_6506")
}

test("GLUTEN-6879: Fix partition value diff when it contains blanks") {
val tableName = "test_tbl_6879"
sql(s"drop table if exists $tableName")

val createSql =
s"""
|CREATE TABLE $tableName (
| id INT,
| name STRING
|) PARTITIONED BY (part STRING)
|STORED AS PARQUET;
|""".stripMargin
sql(createSql)

val insertSql =
s"""
|INSERT INTO $tableName PARTITION (part='part with spaces')
|VALUES (1, 'John Doe');
|""".stripMargin
sql(insertSql)

val selectSql = s"SELECT * FROM $tableName"
compareResultsAgainstVanillaSpark(selectSql, true, _ => {})
sql(s"drop table if exists $tableName")
}
}
19 changes: 14 additions & 5 deletions cpp-ch/local-engine/Common/GlutenStringUtils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "GlutenStringUtils.h"
#include <filesystem>
#include <boost/algorithm/string.hpp>
#include <Poco/StringTokenizer.h>
#include <Poco/URI.h>

#include "GlutenStringUtils.h"

namespace local_engine
{
Expand All @@ -27,14 +28,22 @@ PartitionValues GlutenStringUtils::parsePartitionTablePath(const std::string & f
Poco::StringTokenizer path(file, "/");
for (const auto & item : path)
{
auto position = item.find('=');
if (position != std::string::npos)
auto pos = item.find('=');
if (pos != std::string::npos)
{
result.emplace_back(PartitionValue(boost::algorithm::to_lower_copy(item.substr(0, position)), item.substr(position + 1)));
auto key = boost::to_lower_copy(item.substr(0, pos));
auto value = item.substr(pos + 1);

std::string unescaped_key;
std::string unescaped_value;
Poco::URI::decode(key, unescaped_key);
Poco::URI::decode(value, unescaped_value);
result.emplace_back(std::move(unescaped_key), std::move(unescaped_value));
}
}
return result;
}

bool GlutenStringUtils::isNullPartitionValue(const std::string & value)
{
return value == "__HIVE_DEFAULT_PARTITION__";
Expand Down
11 changes: 3 additions & 8 deletions cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -55,25 +55,20 @@ FormatFile::FormatFile(
: context(context_), file_info(file_info_), read_buffer_builder(read_buffer_builder_)
{
PartitionValues part_vals = GlutenStringUtils::parsePartitionTablePath(file_info.uri_file());
String partition_values_str = "[";
for (size_t i = 0; i < part_vals.size(); ++i)
{
const auto & part = part_vals[i];
partition_keys.push_back(part.first);
partition_values[part.first] = part.second;
if (i > 0)
partition_values_str += ", ";
partition_values_str += part.first + "=" + part.second;
}
partition_values_str += "]";

LOG_INFO(
&Poco::Logger::get("FormatFile"),
"Reading File path: {}, format: {}, range: {}, partition_index: {}, partition_values: {}",
"Reading File path: {}, format: {}, range: {}, partition_index: {}",
file_info.uri_file(),
file_info.file_format_case(),
std::to_string(file_info.start()) + "-" + std::to_string(file_info.start() + file_info.length()),
file_info.partition_index(),
partition_values_str);
file_info.partition_index());
}

FormatFilePtr FormatFileUtil::createFile(
Expand Down

0 comments on commit c22b230

Please sign in to comment.