From 8bf55a7887bcd86333127458bff1ea369c5e5b73 Mon Sep 17 00:00:00 2001 From: "joey.ljy" Date: Tue, 12 Dec 2023 12:10:00 +0800 Subject: [PATCH] use iceberg FANOUT_ENABLED option --- .../execution/VeloxTPCHIcebergSuite.scala | 29 +++++-------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/gluten-iceberg/src/test/scala/io/glutenproject/execution/VeloxTPCHIcebergSuite.scala b/gluten-iceberg/src/test/scala/io/glutenproject/execution/VeloxTPCHIcebergSuite.scala index 52d0c3989875d..7f76fee112702 100644 --- a/gluten-iceberg/src/test/scala/io/glutenproject/execution/VeloxTPCHIcebergSuite.scala +++ b/gluten-iceberg/src/test/scala/io/glutenproject/execution/VeloxTPCHIcebergSuite.scala @@ -18,6 +18,8 @@ package io.glutenproject.execution import org.apache.spark.SparkConf +import org.apache.iceberg.spark.SparkWriteOptions + import java.io.File class VeloxTPCHIcebergSuite extends VeloxTPCHSuite { @@ -83,30 +85,13 @@ class VeloxTPCHIcebergSuite extends VeloxTPCHSuite { test("iceberg partition table") { withTable("lineitem_p") { - spark.sql(""" - |CREATE TABLE lineitem_p ( - | l_orderkey BIGINT, - | l_partkey BIGINT, - | l_suppkey BIGINT, - | l_linenumber INT, - | l_quantity DECIMAL(12,2), - | l_extendedprice DECIMAL(12,2), - | l_discount DECIMAL(12,2), - | l_tax DECIMAL(12,2), - | l_returnflag STRING, - | l_linestatus STRING, - | l_commitdate DATE, - | l_receiptdate DATE, - | l_shipinstruct STRING, - | l_shipmode STRING, - | l_comment STRING, - | l_shipdate DATE) - |USING iceberg - |PARTITIONED BY (l_shipdate); - |""".stripMargin) val tablePath = new File(resourcePath, "lineitem").getAbsolutePath val tableDF = spark.read.format(fileFormat).load(tablePath) - tableDF.write.format("iceberg").mode("append").saveAsTable("lineitem_p") + tableDF.write + .format("iceberg") + .option(SparkWriteOptions.FANOUT_ENABLED, "true") + .mode("append") + .saveAsTable("lineitem_p") runQueryAndCompare(""" |SELECT | sum(l_extendedprice * l_discount) AS revenue