Skip to content

Commit

Permalink
[GLUTEN-2942][CH] fix inconsistency of native write and non-native re…
Browse files Browse the repository at this point in the history
…ad of orc format table (apache#2948)

* fix inconsistency of orc native write

* fix bugs
  • Loading branch information
taiyang-li authored Aug 30, 2023
1 parent c22bd77 commit 4a4d995
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -951,4 +951,22 @@ class GlutenClickHouseNativeWriteTableSuite
}
}
}

test("test native write and non-native read consistency") {
withSQLConf(("spark.gluten.sql.native.writer.enabled", "true")) {
for (format <- formats) {
val table_name = "t_" + format
spark.sql(s"drop table IF EXISTS $table_name")
spark.sql(s"create table $table_name (id int, name string, info char(4)) stored as $format")
spark.sql(
s"insert overwrite table $table_name " +
"select id, cast(id as string), concat('aaa', cast(id as string)) from range(10)")
compareResultsAgainstVanillaSpark(
s"select * from $table_name",
compareResult = true,
_ => {})
}
}
}

}
1 change: 1 addition & 0 deletions cpp-ch/local-engine/Common/CHUtil.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -584,6 +584,7 @@ void BackendInitializerUtil::initSettings(std::map<std::string, std::string> & b
settings.set("input_format_parquet_allow_missing_columns", true);
settings.set("input_format_parquet_case_insensitive_column_matching", true);
settings.set("input_format_parquet_import_nested", true);
settings.set("output_format_orc_string_as_string", true);
settings.set("output_format_parquet_version", "1.0");
settings.set("output_format_parquet_compression_method", "snappy");
settings.set("output_format_parquet_string_as_string", true);
Expand Down

0 comments on commit 4a4d995

Please sign in to comment.