Skip to content

Commit

Permalink
Add ut for #2584
Browse files Browse the repository at this point in the history
- Rebase failed with ClickHouse/ClickHouse#67879, and hence we can remove Kyligence/ClickHouse#454

(cherry picked from commit 583aa8d6566a9e1c0924c1a3ab1d315fcc229fa6)
  • Loading branch information
baibaichen committed Aug 15, 2024
1 parent 5844401 commit 8897a44
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -937,4 +937,37 @@ class GlutenClickHouseNativeWriteTableSuite
_ => {})
)
}

test("GLUTEN-2584: fix native write and read mismatch about complex types") {
def table(format: String): String = s"t_$format"
def create(format: String, table_name: Option[String] = None): String =
s"""CREATE TABLE ${table_name.getOrElse(table(format))}(
| id INT,
| info STRUCT<name:STRING, age:INT>,
| data MAP<STRING, INT>,
| values ARRAY<INT>
|) stored as $format""".stripMargin
def insert(format: String, table_name: Option[String] = None): String =
s"""INSERT overwrite ${table_name.getOrElse(table(format))} VALUES
| (6, null, null, null);
""".stripMargin

nativeWrite2(
format => (table(format), create(format), insert(format)),
(table_name, format) => {
val vanilla_table = s"${table_name}_v"
val vanilla_create = create(format, Some(vanilla_table))
vanillaWrite {
withDestinationTable(vanilla_table, Option(vanilla_create)) {
checkInsertQuery(insert(format, Some(vanilla_table)), checkNative = false)
}
}
val rowsFromOriginTable =
spark.sql(s"select * from $vanilla_table").collect()
val dfFromWriteTable =
spark.sql(s"select * from $table_name")
checkAnswer(dfFromWriteTable, rowsFromOriginTable)
}
)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@ trait NativeWriteChecker
}
}

def vanillaWrite(block: => Unit): Unit = {
withSQLConf(("spark.gluten.sql.native.writer.enabled", "false")) {
block
}
}

def withSource(df: Dataset[Row], viewName: String, pairs: (String, String)*)(
block: => Unit): Unit = {
withSQLConf(pairs: _*) {
Expand Down

0 comments on commit 8897a44

Please sign in to comment.