From ca29456dac485492fdf5e01188faed7f42b507b2 Mon Sep 17 00:00:00 2001 From: liuneng1994 Date: Wed, 29 May 2024 15:00:14 +0800 Subject: [PATCH] fix bug --- ...utenClickHouseMergeTreeOptimizeSuite.scala | 20 +++++++++---------- .../Shuffle/CachedShuffleWriter.cpp | 1 + 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala index d4302193f5b8c..f016f9dc5d144 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala @@ -157,9 +157,9 @@ class GlutenClickHouseMergeTreeOptimizeSuite assert(ret.apply(0).get(0) == 600572) spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 812) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 372) spark.sql("VACUUM lineitem_mergetree_optimize_p2 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 232) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 239) spark.sql("VACUUM lineitem_mergetree_optimize_p2 RETAIN 0 HOURS") // the second VACUUM will remove some empty folders assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 220) @@ -188,11 +188,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite assert(ret.apply(0).get(0) == 600572) spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 398) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 516) spark.sql("VACUUM lineitem_mergetree_optimize_p3 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 286) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 306) spark.sql("VACUUM lineitem_mergetree_optimize_p3 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 270) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 276) spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p3").collect() @@ -219,11 +219,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite assert(ret.apply(0).get(0) == 600572) spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 398) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 516) spark.sql("VACUUM lineitem_mergetree_optimize_p4 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 286) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 306) spark.sql("VACUUM lineitem_mergetree_optimize_p4 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 270) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 276) spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p4").collect() @@ -313,12 +313,12 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p6")) == { - if (sparkVersion.equals("3.2")) 931 else 1014 + if (sparkVersion.equals("3.2")) 499 else 528 }) spark.sql("VACUUM lineitem_mergetree_optimize_p6 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p6 RETAIN 0 HOURS") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p6")) == { - if (sparkVersion.equals("3.2")) 439 else 445 + if (sparkVersion.equals("3.2")) 315 else 321 }) spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") diff --git a/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp b/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp index 0b28af5c62476..559d9031862e8 100644 --- a/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp +++ b/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp @@ -171,6 +171,7 @@ SplitResult CachedShuffleWriter::stop() size_t CachedShuffleWriter::evictPartitions() { + if (!partition_writer) return 0; return partition_writer->evictPartitions(true, options.flush_block_buffer_before_evict); } } \ No newline at end of file