From a211e23aebe75b1a96aab9cdf73f487a133f8dce Mon Sep 17 00:00:00 2001 From: Zhichao Zhang Date: Mon, 16 Sep 2024 17:04:15 +0800 Subject: [PATCH] profile bhj --- cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp | 3 +++ .../gluten/execution/GlutenWholeStageColumnarRDD.scala | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp b/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp index a5ed605ed0a20..69b511e8d7711 100644 --- a/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp +++ b/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp @@ -207,6 +207,7 @@ void JoinRelParser::renamePlanColumns(DB::QueryPlan & left, DB::QueryPlan & righ DB::QueryPlanPtr JoinRelParser::parseJoin(const substrait::JoinRel & join, DB::QueryPlanPtr left, DB::QueryPlanPtr right) { + Stopwatch watch(CLOCK_MONOTONIC); auto join_config = JoinConfig::loadFromContext(getContext()); google::protobuf::StringValue optimization_info; optimization_info.ParseFromString(join.advanced_extension().optimization().value()); @@ -315,6 +316,8 @@ DB::QueryPlanPtr JoinRelParser::parseJoin(const substrait::JoinRel & join, DB::Q query_plan = std::move(left); /// hold right plan for profile extra_plan_holder.emplace_back(std::move(right)); + std::cout << "parse bhj time: " << join_opt_info.storage_join_key << "---" << watch.elapsedMicroseconds() / 1000.0 << std::endl; + watch.stop(); } else if (join_opt_info.is_smj) { diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/GlutenWholeStageColumnarRDD.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/GlutenWholeStageColumnarRDD.scala index de5fc85f356d0..70846efc3058e 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/GlutenWholeStageColumnarRDD.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/GlutenWholeStageColumnarRDD.scala @@ -77,8 +77,11 @@ class GlutenWholeStageColumnarRDD( _ => ExecutorManager.tryTaskSet(numaBindingInfo) val (inputPartition, inputColumnarRDDPartitions) = castNativePartition(split) + var st = System.nanoTime() val inputIterators = rdds.getIterators(inputColumnarRDDPartitions, context) - BackendsApiManager.getIteratorApiInstance.genFirstStageIterator( + logWarning(s"${split.index} rdds get iterators: ${System.nanoTime() - st}") + st = System.nanoTime() + val gen = BackendsApiManager.getIteratorApiInstance.genFirstStageIterator( inputPartition, context, pipelineTime, @@ -87,6 +90,8 @@ class GlutenWholeStageColumnarRDD( split.index, inputIterators ) + logWarning(s"${split.index} gen iterators: ${System.nanoTime() - st}") + gen } }