fix: Various metrics bug fixes and improvements

common/src/main/scala/org/apache/comet/vector/NativeUtil.scala

@@ -26,6 +26,7 @@ import org.apache.arrow.vector.VectorSchemaRoot
 import org.apache.arrow.vector.dictionary.DictionaryProvider
 import org.apache.spark.SparkException
 import org.apache.spark.sql.comet.util.Utils
+import org.apache.spark.sql.execution.metric.SQLMetric
 import org.apache.spark.sql.vectorized.ColumnarBatch
 
 import org.apache.comet.CometArrowAllocator
@@ -148,20 +149,28 @@ class NativeUtil {
    */
   def getNextBatch(
       numOutputCols: Int,
+      arrowFfiMetric: Option[SQLMetric],
       func: (Array[Long], Array[Long]) => Long): Option[ColumnarBatch] = {
+
+    val start = System.nanoTime()
     val (arrays, schemas) = allocateArrowStructs(numOutputCols)
 
     val arrayAddrs = arrays.map(_.memoryAddress())
     val schemaAddrs = schemas.map(_.memoryAddress())
 
+    var arrowFfiTime = System.nanoTime() - start
+
     val result = func(arrayAddrs, schemaAddrs)
 
     result match {
       case -1 =>
         // EOF
         None
       case numRows =>
+        val start = System.nanoTime()
         val cometVectors = importVector(arrays, schemas)
+        arrowFfiTime += System.nanoTime() - start
+        arrowFfiMetric.foreach(_.add(arrowFfiTime))
         Some(new ColumnarBatch(cometVectors.toArray, numRows.toInt))
       case flag =>
         throw new IllegalStateException(s"Invalid native flag: $flag")

docs/source/user-guide/tuning.md

@@ -113,8 +113,14 @@ Some Comet metrics are not directly comparable to Spark metrics in some cases:
 
 Comet also adds some custom metrics:
 
-### ShuffleWriterExec
+### CometScanExec
 
-| Metric           | Description                                                                                                                                                                       |
-| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `jvm_fetch_time` | Measure the time it takes for `ShuffleWriterExec` to fetch batches from the JVM. Note that this does not include the execution time of the query that produced the input batches. |
+| Metric     | Description                                                                                                                                     |
+| ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
+| `scanTime` | Total time to read Parquet batches into JVM. This does not include the Arrow FFI cost of exporting these batches to native code for processing. |
+
+### Common to all Comet Operators
+
+| Metric           | Description                                                                                      |
+| ---------------- | ------------------------------------------------------------------------------------------------ |
+| `arrow_ffi_time` | Measure the time it takes to transfer Arrow batches between JVM and native code using Arrow FFI. |

native/core/src/execution/datafusion/mod.rs

@@ -21,4 +21,5 @@ pub mod expressions;
 mod operators;
 pub mod planner;
 pub mod shuffle_writer;
+pub(crate) mod spark_plan;
 mod util;