+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Parameters¶
+
+
+
+
+
+
+
+
+start analysis cluster and run¶
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Setting default log level to "WARN". +To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel). ++
+
+
+
+24/12/06 05:53:36 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable ++
+
+
+
+24/12/06 05:53:37 WARN DomainSocketFactory: The short-circuit local reads feature cannot be used because libhadoop cannot be loaded. +24/12/06 05:53:37 WARN Client: Neither spark.yarn.jars nor spark.yarn.archive is set, falling back to uploading libraries under SPARK_HOME. ++
+
+
+
+/home/sparkuser/spark/python/pyspark/sql/context.py:112: FutureWarning: Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead. + warnings.warn( ++
+
+
+
+
+
+
+
+
+
+
+
+
+
+Sparklog¶
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Content¶
+
+
+
+
+
+
+
+
+
+Self app info¶
+
+
+
+
+
+
+
+
+
+
+
+
+load data /sr213/application_1733153225851_0048/app.log ++
+
+
+
+[Stage 0:> (0 + 1) / 1]+
+
+
+
+
+
+
+
+
+[Stage 1:> (0 + 1) / 1]+
+
+
+
+
+
+
+
+
+[Stage 4:> (0 + 1) / 1]+
+
+
+
+
+
+
+
+
+[Stage 5:> (0 + 1) / 1]+
+
+
+
+
+
+
+
+
+[Stage 17:> (0 + 1) / 1]+
+
+
+
+
+
+
+
+
+[Stage 39:> (0 + 16) / 200] [Stage 39:=> (6 + 16) / 200]+
+
+
+
+[Stage 39:===> (14 + 16) / 200]+
+
+
+
+[Stage 39:========> (33 + 16) / 200] [Stage 39:==================> (67 + 16) / 200]+
+
+
+
+[Stage 39:==========================> (97 + 16) / 200] [Stage 39:==================================> (131 + 16) / 200]+
+
+
+
+
+
[Stage 39:=================================================> (185 + 15) / 200]
+
+
+
+
+
[Stage 42:(177 + 5) / 200][Stage 43:> (0 + 1) / 1][Stage 44:>(0 + 11) / 200]
+
+
+
+
+[Stage 44:(113 + 12) / 200][Stage 45:> (0 + 1) / 1][Stage 46:> (0 + 3) / 200] [Stage 44:(182 + 5) / 200][Stage 46:>(4 + 11) / 200][Stage 47:> (0 + 0) / 200]+
+
+
+
+[Stage 46:(43 + 16) / 200][Stage 47:> (0 + 0) / 200][Stage 48:> (0 + 0) / 200] [Stage 46:(110 + 8) / 200][Stage 47:> (0 + 8) / 200][Stage 48:> (0 + 0) / 200]+
+
+
+
+[Stage 46:(155 + 8) / 200][Stage 47:>(47 + 8) / 200][Stage 48:> (0 + 0) / 200] [Stage 46:(194 + 4) / 200][Stage 47:>(73 + 4) / 200][Stage 48:> (8 + 8) / 200]+
+
+
+
+[Stage 47:(114 + 8) / 200][Stage 48:>(57 + 4) / 200][Stage 49:> (0 + 4) / 16] [Stage 47:(185 + 4) / 200][Stage 48:>(73 + 4) / 200][Stage 49:> (0 + 8) / 16]+
+
+
+
+[Stage 48:(126 + 8) / 200][Stage 49:> (0 + 8) / 16][Stage 51:> (0 + 0) / 1] [Stage 48:(184 + 4) / 200][Stage 49:> (4 + 12) / 16][Stage 51:> (0 + 0) / 1]+
+
+
+
+
+
+
+
+
+
+
+
+| appid | +application_1733153225851_0048 | +
| executor.instances | +4 | +
| executor.cores | +4 | +
| shuffle.partitions | +32 | +
| batch size | +4,096 | +
| real executors | +4 | +
| Failed Tasks | ++ |
| Speculative Tasks | +0 | +
| Speculative Killed Tasks | +0 | +
| Speculative Stage | +0 | +
| runtime | +17.65 | +
| disk spilled | +0.0 | +
| memspilled | +0.0 | +
| local_read | +0.0 | +
| remote_read | +0.0 | +
| shuffle_write | +0.0 | +
| task run time | +6.79 | +
| ser_time | +0.0 | +
| f_wait_time | +0.0 | +
| gc_time | +0.03 | +
| input read | +22.54 | +
| acc_task_time | +13.99 | +
| file read size | +5,951.35 | +
| file write size | +24.52 | +
| disk read size | +5.05 | +
| disk write size | +15.31 | +
| disk cancel size | +0.0 | +
+
+
+
+{'appid': 'application_1733153225851_0048',
+ 'executor.instances': 4,
+ 'executor.cores': 4,
+ 'shuffle.partitions': 32,
+ 'batch size': 4096,
+ 'real executors': 4,
+ 'Failed Tasks': '',
+ 'Speculative Tasks': 0,
+ 'Speculative Killed Tasks': 0,
+ 'Speculative Stage': 0,
+ 'runtime': 17.65,
+ 'disk spilled': 0.0,
+ 'memspilled': 0.0,
+ 'local_read': 0.0,
+ 'remote_read': 0.0,
+ 'shuffle_write': 0.0,
+ 'task run time': 6.79,
+ 'ser_time': 0.0,
+ 'f_wait_time': 0.0,
+ 'gc_time': 0.03,
+ 'input read': 22.54,
+ 'acc_task_time': 13.99,
+ 'file read size': 5951.35,
+ 'file write size': 24.52,
+ 'disk read size': 5.05,
+ 'disk write size': 15.31,
+ 'disk cancel size': 0.0}
+
+
+
+
+
+
+
+
+
+
+[Stage 92:(161 + 4) / 200][Stage 93:>(68 + 9) / 200][Stage 94:> (8 + 4) / 200] [Stage 93:(151 + 4) / 200][Stage 94:>(66 + 8) / 200][Stage 95:> (1 + 4) / 16]+
+
+
+
+
+
+
+
+
+/sr213/application_1733153225851_0048/sr217/emon.parquet is not found, trying to load data ... ++
+
+
+
+[Stage 129:> (0 + 2) / 2][Stage 130:> (0 + 2) / 2] [Stage 129:> (0 + 2) / 2][Stage 130:> (0 + 2) / 2][Stage 131:> (0 + 4) / 4]+
+
+
+
+[Stage 129:=> (1 + 1) / 2][Stage 130:=> (1 + 1) / 2][Stage 131:> (1 + 3) / 4] [Stage 129:========> (1 + 1) / 2][Stage 131:====> (1 + 3) / 4]+
+
+
+
+
+
+
+
+
+[Stage 143:==> (16 + 16) / 400] [Stage 143:==> (17 + 16) / 400]+
+
+
+
+[Stage 143:===> (24 + 16) / 400] [Stage 143:====> (34 + 16) / 400]+
+
+
+
+[Stage 143:=====> (43 + 16) / 400] [Stage 143:=======> (53 + 16) / 400]+
+
+
+
+[Stage 143:========> (65 + 16) / 400] [Stage 143:=========> (72 + 16) / 400]+
+
+
+
+[Stage 143:==========> (83 + 16) / 400] [Stage 143:===========> (90 + 16) / 400]+
+
+
+
+[Stage 143:=============> (99 + 16) / 400] [Stage 143:=============> (106 + 16) / 400]+
+
+
+
+[Stage 143:==============> (113 + 16) / 400] [Stage 143:===============> (118 + 16) / 400]+
+
+
+
+[Stage 143:================> (126 + 16) / 400] [Stage 143:=================> (132 + 16) / 400]+
+
+
+
+[Stage 143:==================> (140 + 16) / 400] [Stage 143:==================> (146 + 16) / 400]+
+
+
+
+[Stage 143:===================> (153 + 16) / 400] [Stage 143:====================> (160 + 16) / 400]+
+
+
+
+[Stage 143:======================> (170 + 16) / 400] [Stage 143:======================> (173 + 16) / 400]+
+
+
+
+[Stage 143:=======================> (182 + 16) / 400] [Stage 143:========================> (186 + 16) / 400]+
+
+
+
+[Stage 143:=========================> (194 + 16) / 400] [Stage 143:=========================> (197 + 16) / 400]+
+
+
+
+[Stage 143:==========================> (204 + 16) / 400] [Stage 143:===========================> (211 + 16) / 400]+
+
+
+
+[Stage 143:===========================> (214 + 16) / 400] [Stage 143:============================> (222 + 16) / 400]+
+
+
+
+[Stage 143:=============================> (230 + 16) / 400] [Stage 143:==============================> (236 + 16) / 400]+
+
+
+
+[Stage 143:===============================> (243 + 16) / 400] [Stage 143:================================> (249 + 16) / 400]+
+
+
+
+[Stage 143:=================================> (256 + 16) / 400] [Stage 143:==================================> (263 + 16) / 400]+
+
+
+
+[Stage 143:===================================> (272 + 16) / 400] [Stage 143:====================================> (279 + 16) / 400]+
+
+
+
+[Stage 143:======================================> (294 + 16) / 400] [Stage 143:======================================> (299 + 16) / 400]+
+
+
+
+[Stage 143:========================================> (311 + 16) / 400] [Stage 143:=========================================> (322 + 16) / 400]+
+
+
+
+[Stage 143:===========================================> (333 + 17) / 400] [Stage 143:=============================================> (348 + 16) / 400]+
+
+
+
+[Stage 143:==============================================> (360 + 16) / 400] [Stage 143:================================================> (372 + 16) / 400]+
+
+
+
+[Stage 143:==================================================> (386 + 14) / 400]+
+
+
+
+[Stage 148:> (0 + 1) / 1]+
+
+
+
+[Stage 154:> (0 + 3) / 3]+
+
+
+
+[Stage 154:===================> (1 + 2) / 3] [Stage 154:======================================> (2 + 1) / 3]+
+
+
+
+
+
+
+
+
+[Stage 157:> (0 + 3) / 3] [Stage 157:===================> (1 + 2) / 3]+
+
+
+
+
[Stage 157:======================================> (2 + 1) / 3]
+
+
+
+
+sar metric ++
+
+
+
+
+
+
+| + | application_1733153225851_0048 | +
|---|---|
| runtime | +17.650000 | +
| disk spilled | +0.000000 | +
| shuffle_write | +0.000000 | +
| f_wait_time | +0.000000 | +
| input read | +22.540000 | +
| acc_task_time | +13.990000 | +
| output rows | +1.180000 | +
| %user>90% | +0.931034 | +
| %kernel>10% | +0.965517 | +
| %iowait>10% | +0.620690 | +
| avg %user | +41.216207 | +
| avg %system | +4.514138 | +
| avg %iowait | +0.743793 | +
| avg disk util | +32.206897 | +
| time more than 90% | +0.000000 | +
| total read (G) | +5.388613 | +
| total write (G) | +1.121773 | +
| avg read bw (MB/s) | +190.273771 | +
| avg write bw (MB/s) | +39.610183 | +
| read bw %75 | +411.578125 | +
| read bw %95 | +484.542969 | +
| read bw max | +510.351562 | +
| time_rd_morethan_95 | +0.034483 | +
| write bw %75 | +1.074219 | +
| write bw %95 | +165.687500 | +
| write bw max | +812.511719 | +
| time_wr_morethan_95 | +0.034483 | +
| cached mean | +93.896552 | +
| cached 75% | +145.000000 | +
| cached max | +188.000000 | +
| used mean | +834.000000 | +
| used 75% | +852.000000 | +
| used max | +859.000000 | +
| rx MB/s 75% | +0.000000 | +
| rx MB/s 95% | +0.000000 | +
| rx MB/s 99% | +0.000000 | +
| pgin mean | +190.206897 | +
| pgin 75% | +412.000000 | +
| pgin max | +509.000000 | +
| pgout mean | +40.965517 | +
| pgout 75% | +1.000000 | +
| pgout max | +840.000000 | +
| fault mean | +117653.310345 | +
| fault 75% | +205151.000000 | +
| fault max | +256538.000000 | +
| cpu%_avg | +0.448817 | +
| cpu freq_avg | +3241.915617 | +
| pathlength_sum | +1933.000000 | +
| ipc_avg | +1.137983 | +
+
+
+
+
+
+
+
+
+
+[Stage 330:> (0 + 1) / 1]+
+
+
+
+
+
+
+
+
+[Stage 341:> (0 + 1) / 1]+
+
+
+
+
+
+
+
+
+DEV in ('nvme0n1')
+
+
+
+
+
+[Stage 388:> (0 + 1) / 1]+
+
+
+
+
+
+
+
+
+[Stage 396:> (0 + 1) / 1]+
+
+
+
+
+
+
+
+
+{'sr217': 200}
+
+
+
+
+
+
[Stage 490:===================> (1 + 2) / 3]
+
+
+
+
+
+
+
+
+gluten tpch_power 6600a1
+
+
+
+
+
+
+
+
+
+
+[Stage 605:==> (11 + 0) / 200] [Stage 531:(174 + 16) / 200][Stage 532:>(0 + 0) / 200][Stage 533:>(0 + 0) / 200]+
+
+
+
+[Stage 532:(102 + 16) / 200][Stage 533:>(0 + 0) / 200][Stage 534:>(0 + 0) / 200] [Stage 533:(72 + 16) / 200][Stage 534:>(0 + 0) / 200][Stage 535:>(0 + 0) / 200]+
+
+
+
+[Stage 534:(63 + 16) / 200][Stage 535:>(0 + 0) / 200][Stage 536:>(0 + 0) / 200] [Stage 535:(75 + 16) / 200][Stage 536:>(0 + 0) / 200][Stage 537:>(0 + 0) / 200]+
+
+
+
+[Stage 536:(102 + 17) / 200][Stage 537:>(0 + 0) / 200][Stage 538:>(3 + 0) / 200] [Stage 537:(114 + 16) / 200][Stage 538:>(3 + 0) / 200][Stage 539:>(0 + 0) / 200]+
+
+
+
+[Stage 538:(105 + 16) / 200][Stage 539:>(0 + 0) / 200][Stage 540:>(0 + 0) / 200] [Stage 539:(67 + 16) / 200][Stage 540:>(0 + 0) / 200][Stage 541:>(0 + 0) / 200]+
+
+
+
+[Stage 540:(59 + 16) / 200][Stage 541:>(0 + 0) / 200][Stage 542:>(0 + 0) / 200] [Stage 541:(104 + 16) / 200][Stage 542:>(0 + 0) / 200][Stage 543:>(0 + 0) / 200]+
+
+
+
+[Stage 542:(115 + 16) / 200][Stage 543:>(0 + 0) / 200][Stage 544:>(0 + 0) / 200] [Stage 543:(148 + 16) / 200][Stage 544:>(0 + 0) / 200][Stage 545:>(0 + 0) / 200]+
+
+
+
+[Stage 545:(0 + 16) / 200][Stage 546:>(0 + 0) / 200][Stage 547:>(0 + 0) / 200] [Stage 546:(38 + 16) / 200][Stage 547:>(0 + 0) / 200][Stage 548:>(0 + 0) / 200]+
+
+
+
+[Stage 547:(58 + 17) / 200][Stage 548:>(0 + 0) / 200][Stage 549:>(0 + 0) / 200] [Stage 548:(94 + 16) / 200][Stage 549:>(0 + 0) / 200][Stage 559:>(0 + 0) / 200]+
+
+
+
+[Stage 549:(113 + 17) / 200][Stage 557:>(0 + 0) / 200][Stage 559:>(0 + 0) / 200] [Stage 555:>(0 + 0) / 200][Stage 557:>(0 + 0) / 200][Stage 559:>(0 + 0) / 200]+
+
+
+
+[Stage 551:(16 + 1) / 200][Stage 553:>(0 + 0) / 200][Stage 555:>(0 + 0) / 200] [Stage 551:(179 + 16) / 200][Stage 553:>(0 + 0) / 200][Stage 555:>(0 + 0) / 200]+
+
+
+
+[Stage 553:(16 + 0) / 200][Stage 555:>(0 + 0) / 200][Stage 557:>(0 + 0) / 200] [Stage 553:(54 + 17) / 200][Stage 555:>(0 + 0) / 200][Stage 557:>(0 + 0) / 200]+
+
+
+
+[Stage 555:>(8 + 8) / 200][Stage 557:>(0 + 0) / 200][Stage 559:>(0 + 0) / 200] [Stage 555:(16 + 0) / 200][Stage 557:>(0 + 0) / 200][Stage 559:>(0 + 0) / 200]+
+
+
+
+[Stage 555:(199 + 1) / 200][Stage 557:(0 + 15) / 200][Stage 559:>(0 + 0) / 200] [Stage 557:(16 + 0) / 200][Stage 559:>(0 + 0) / 200][Stage 560:>(0 + 0) / 200]+
+
+
+
+[Stage 557:(84 + 16) / 200][Stage 559:>(0 + 0) / 200][Stage 560:>(0 + 0) / 200] [Stage 559:(16 + 0) / 200][Stage 560:>(0 + 0) / 200][Stage 564:>(0 + 0) / 200]+
+
+
+
+[Stage 559:(29 + 16) / 200][Stage 560:>(0 + 0) / 200][Stage 564:>(0 + 0) / 200] [Stage 560:(85 + 16) / 200][Stage 564:>(0 + 0) / 200][Stage 566:>(0 + 0) / 200]+
+
+
+
+[Stage 564:>(0 + 0) / 200][Stage 566:>(0 + 0) / 200][Stage 568:>(0 + 0) / 200]+
+
+
+
+[Stage 564:(16 + 0) / 200][Stage 566:>(0 + 0) / 200][Stage 568:>(0 + 0) / 200]+
+
+
+
+[Stage 564:(149 + 16) / 200][Stage 566:>(0 + 0) / 200][Stage 568:>(0 + 0) / 200] [Stage 566:(16 + 0) / 200][Stage 568:>(0 + 0) / 200][Stage 569:>(0 + 0) / 200]+
+
+
+
+[Stage 566:(71 + 16) / 200][Stage 568:>(0 + 0) / 200][Stage 569:>(0 + 0) / 200] [Stage 568:(16 + 0) / 200][Stage 569:>(0 + 0) / 200][Stage 573:>(0 + 0) / 200]+
+
+
+
+[Stage 568:(16 + 2) / 200][Stage 569:>(0 + 0) / 200][Stage 573:>(0 + 0) / 200] [Stage 569:(42 + 18) / 200][Stage 573:>(0 + 0) / 200][Stage 575:>(0 + 0) / 200]+
+
+
+
+[Stage 573:>(0 + 0) / 200][Stage 575:>(0 + 0) / 200][Stage 577:>(0 + 0) / 200]+
+
+
+
+[Stage 573:(0 + 16) / 200][Stage 575:>(0 + 0) / 200][Stage 577:>(0 + 0) / 200] [Stage 573:(16 + 0) / 200][Stage 575:>(0 + 0) / 200][Stage 577:>(0 + 0) / 200]+
+
+
+
+[Stage 573:(67 + 16) / 200][Stage 575:>(0 + 0) / 200][Stage 577:>(0 + 0) / 200] [Stage 575:(16 + 0) / 200][Stage 577:>(0 + 0) / 200][Stage 578:>(0 + 0) / 200]+
+
+
+
+[Stage 575:(144 + 16) / 200][Stage 577:>(0 + 0) / 200][Stage 578:>(0 + 0) / 200] [Stage 577:(16 + 0) / 200][Stage 578:>(0 + 0) / 200][Stage 581:>(0 + 0) / 200]+
+
+
+
+[Stage 577:(184 + 16) / 200][Stage 578:>(0 + 0) / 200][Stage 581:>(0 + 0) / 200] [Stage 578:(197 + 3) / 200][Stage 581:>(0 + 0) / 200][Stage 585:>(0 + 0) / 200]+
+
+
+
+[Stage 581:>(0 + 0) / 200][Stage 585:>(0 + 0) / 200][Stage 587:>(0 + 0) / 200] [Stage 580:(58 + 16) / 200][Stage 581:>(0 + 0) / 200][Stage 585:>(0 + 0) / 200]+
+
+
+
+[Stage 580:(176 + 17) / 200][Stage 581:>(0 + 0) / 200][Stage 585:>(0 + 0) / 200] [Stage 581:(195 + 5) / 200][Stage 585:>(0 + 0) / 200][Stage 587:>(0 + 0) / 200]+
+
+
+
+[Stage 585:>(0 + 0) / 200][Stage 587:>(0 + 0) / 200][Stage 589:>(0 + 0) / 200] [Stage 583:(69 + 17) / 200][Stage 585:>(0 + 0) / 200][Stage 587:>(0 + 0) / 200]+
+
+
+
+[Stage 583:(170 + 16) / 200][Stage 585:>(0 + 0) / 200][Stage 587:>(0 + 0) / 200] [Stage 585:(16 + 0) / 200][Stage 587:>(0 + 0) / 200][Stage 589:>(0 + 0) / 200]+
+
+
+
+[Stage 585:(75 + 16) / 200][Stage 587:>(0 + 0) / 200][Stage 589:>(0 + 0) / 200]+
+
+
+
+[Stage 587:(16 + 0) / 200][Stage 589:>(0 + 0) / 200][Stage 591:>(0 + 0) / 200]+
+
+
+
+[Stage 587:(16 + 6) / 200][Stage 589:>(0 + 0) / 200][Stage 591:>(0 + 0) / 200] [Stage 589:>(8 + 8) / 200][Stage 591:>(0 + 0) / 200][Stage 593:>(0 + 0) / 200]+
+
+
+
+[Stage 589:(16 + 0) / 200][Stage 591:>(0 + 0) / 200][Stage 593:>(0 + 0) / 200]+
+
+
+
+[Stage 589:(182 + 17) / 200][Stage 591:>(0 + 0) / 200][Stage 593:>(0 + 0) / 200] [Stage 591:(16 + 0) / 200][Stage 593:>(0 + 0) / 200][Stage 595:>(0 + 0) / 200]+
+
+
+
+[Stage 591:(133 + 17) / 200][Stage 593:>(0 + 0) / 200][Stage 595:>(0 + 0) / 200] [Stage 593:(16 + 0) / 200][Stage 595:>(0 + 0) / 200][Stage 597:>(0 + 0) / 200]+
+
+
+
+[Stage 593:(39 + 18) / 200][Stage 595:>(0 + 0) / 200][Stage 597:>(0 + 0) / 200] [Stage 595:(16 + 0) / 200][Stage 597:>(0 + 0) / 200][Stage 599:>(0 + 0) / 200]+
+
+
+
+[Stage 597:(16 + 0) / 200][Stage 599:>(0 + 0) / 200][Stage 600:>(0 + 0) / 200]+
+
+
+
+[Stage 599:(0 + 16) / 200][Stage 600:>(0 + 0) / 200][Stage 605:(11 + 0) / 200] [Stage 599:(16 + 0) / 200][Stage 600:>(0 + 0) / 200][Stage 605:(11 + 0) / 200]+
+
+
+
+[Stage 599:(169 + 16) / 200][Stage 600:>(0 + 0) / 200][Stage 605:(11 + 0) / 200] [Stage 605:==> (11 + 0) / 200]+
+
+
+
+[Stage 602:======> (102 + 18) / 200][Stage 605:> (11 + 0) / 200] [Stage 605:==> (11 + 16) / 200]+
+
+
+
+[Stage 606:============================================> (170 + 16) / 200]+
+
+
+
+[Stage 607:===================================> (135 + 16) / 200]+
+
+
+
+
+
+
+
+
+
[Stage 932:====================================================>(197 + 3) / 200]
+
+
+
+
+
+
+
+
+
+ application_1733153225851_0048
+
+
+ query time
+
+
+
+
+
+
+
+| + | runtime | +disk spilled | +memspilled | +local_read | +remote_read | +shuffle_write | +deser_time | +run_time | +ser_time | +f_wait_time | +gc_time | +peak_mem | +queryid | +input read | +acc_task_time | +stages | +output rows | +executors | +core/exec | +task.cpus | +parallelism | +
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| real_queryid | ++ | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + |
| 1 | +17.650000 | +0.000000 | +0.000000 | +0.000000 | +0.000000 | +0.000000 | +0.200000 | +6.790000 | +0.000000 | +0.000000 | +0.030000 | +1.340000 | +8 | +22.540000 | +13.990000 | +[ 8 10 12 15] | +1.180000 | +4 | +4 | +1 | +32 | +
+
+ operator count
+
+
+
+
+
+
+
+| + | 0 | +1 | +
|---|---|---|
| AQEShuffleRead | +0 | +2 | +
| AdaptiveSparkPlan | +0 | +1 | +
| ColumnarExchange | +0 | +2 | +
| FilterExecTransformer | +0 | +1 | +
| FlushableHashAggregateExecTransformer | +0 | +1 | +
| InputAdapter | +0 | +2 | +
| InputIteratorTransformer | +0 | +2 | +
| ProjectExecTransformer | +0 | +2 | +
| RegularHashAggregateExecTransformer | +0 | +1 | +
| Scan parquet | +0 | +1 | +
| ShuffleQueryStage | +0 | +2 | +
| SortExecTransformer | +0 | +1 | +
| VeloxColumnarToRow | +0 | +1 | +
| VeloxResizeBatches | +0 | +2 | +
+
+ operator input row count
+
+
+
+
+
+
+
+| + | 1 | +
|---|---|
| ColumnarExchange | +0.000000 | +
| VeloxResizeBatches | +0.000000 | +
+
+ operator output row count
+
+
+
+
+
+
+
+| + | 1 | +
|---|---|
| ColumnarExchange | +0.000000 | +
| FlushableHashAggregateExecTransformer | +0.000000 | +
| InputIteratorTransformer | +0.000000 | +
| ProjectExecTransformer | +591.600000 | +
| RegularHashAggregateExecTransformer | +0.000000 | +
| Scan parquet | +591.600000 | +
| SortExecTransformer | +0.000000 | +
| VeloxColumnarToRow | +0.000000 | +
| VeloxResizeBatches | +0.000000 | +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
| + | 0 | +
|---|---|
| 44%_time of scan and filter | +7.53 | +
| 36%_time of project | +6.13 | +
| 16%_not_counted | +2.69 | +
| 3%_idle | +0.55 | +
| 0%_time of input iterator | +0.06 | +
| 0%_time of aggregation | +0.03 | +
| 0%_time to append / split batches | +0.00 | +
| 0%_time of rowConstruction | +0.00 | +
| 0%_time to split | +0.00 | +
| 0%_time to deserialize | +0.00 | +
| 0%_time of sort | +0.00 | +
| 0%_time of extraction | +0.00 | +
| 0%_shuffle write time | +0.00 | +
| 0%_time to convert | +0.00 | +
| 0%_time to compress | +0.00 | +
| 0%_time to spill | +0.00 | +
| 0%_time to decompress | +0.00 | +
+
+
+
+
+
+
+
+
+
+Compare to vanilla¶
+
+
+
+
+
+
+
+
+
+
+load data /sr213/application_1733153225851_0029/app.log ++
+
+
+
+emon metric ++
+
+
+
+[Stage 1319:> (0 + 3) / 3]+
+
+
+
+
[Stage 1319:==================> (1 + 2) / 3]
+
+
+
+
+sar metric ++
+
+
+
+time breakdown ++
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+| + | application_1733153225851_0029 | +application_1733153225851_0048 | +diff | +
|---|---|---|---|
| runtime | +132.14 | +17.65 | +648.67% | +
| shuffle_write | +0.00 | +0.00 | +0.00% | +
| f_wait_time | +0.00 | +0.00 | +0.00% | +
| input read | +22.54 | +22.54 | +0.00% | +
| acc_task_time | +128.01 | +13.99 | +815.01% | +
| output rows | +1.79 | +1.18 | +51.69% | +
| %user>90% | +0.99 | +0.93 | +5.91% | +
| %kernel>10% | +0.99 | +0.97 | +2.85% | +
| %iowait>10% | +0.31 | +0.62 | +-49.30% | +
| avg %user | +82.11 | +41.22 | +99.21% | +
| avg %system | +6.10 | +4.51 | +35.11% | +
| avg %iowait | +0.17 | +0.74 | +-76.60% | +
| avg disk util | +7.13 | +32.21 | +-77.85% | +
| time more than 90% | +0.00 | +0.00 | +0.00% | +
| total read (G) | +5.24 | +5.39 | +-2.75% | +
| total write (G) | +0.02 | +1.12 | +-97.81% | +
| avg read bw (MB/s) | +37.52 | +190.27 | +-80.28% | +
| avg write bw (MB/s) | +0.18 | +39.61 | +-99.55% | +
| read bw %75 | +59.27 | +411.58 | +-85.60% | +
| read bw %95 | +173.05 | +484.54 | +-64.29% | +
| read bw max | +236.70 | +510.35 | +-53.62% | +
| time_rd_morethan_95 | +0.05 | +0.03 | +41.96% | +
| write bw %75 | +0.07 | +1.07 | +-93.45% | +
| write bw %95 | +1.23 | +165.69 | +-99.25% | +
| write bw max | +1.70 | +812.51 | +-99.79% | +
| time_wr_morethan_95 | +0.00 | +0.03 | +-100.00% | +
| cached mean | +88.33 | +93.90 | +-5.93% | +
| cached 75% | +132.00 | +145.00 | +-8.97% | +
| cached max | +160.00 | +188.00 | +-14.89% | +
| used mean | +2,060.73 | +834.00 | +147.09% | +
| used 75% | +2,343.00 | +852.00 | +175.00% | +
| used max | +2,346.00 | +859.00 | +173.11% | +
| rx MB/s 75% | +0.00 | +0.00 | +0.00% | +
| rx MB/s 95% | +0.00 | +0.00 | +0.00% | +
| rx MB/s 99% | +0.00 | +0.00 | +0.00% | +
| pgin mean | +37.37 | +190.21 | +-80.35% | +
| pgin 75% | +59.00 | +412.00 | +-85.68% | +
| pgin max | +352.00 | +509.00 | +-30.84% | +
| pgout mean | +0.13 | +40.97 | +-99.68% | +
| pgout 75% | +0.00 | +1.00 | +-100.00% | +
| pgout max | +2.00 | +840.00 | +-99.76% | +
| fault mean | +952,586.87 | +117,653.31 | +709.66% | +
| fault 75% | +1,426,717.00 | +205,151.00 | +595.45% | +
| fault max | +2,628,392.00 | +256,538.00 | +924.56% | +
| cpu%_avg | +0.88 | +0.45 | +96.45% | +
| cpu freq_avg | +3,460.22 | +3,241.92 | +6.73% | +
| pathlength_sum | +17,960.00 | +1,933.00 | +829.13% | +
| ipc_avg | +1.27 | +1.14 | +11.30% | +
| + | runtime | +shuffle_write | +f_wait_time | +input read | +acc_task_time | +output rows | +
|---|---|---|---|---|---|---|
| real_queryid | ++ | + | + | + | + | + |
| 1 | +
+ 132.14
+17.65
+648.67%
+ |
+
+ 0.00
+0.00
+nan%
+ |
+
+ 0.00
+0.00
+nan%
+ |
+
+ 22.54
+22.54
+0.00%
+ |
+
+ 128.01
+13.99
+815.01%
+ |
+
+ 1.79
+1.18
+51.69%
+ |
+
| + | sr217 | +agg | +
|---|---|---|
| 0 | ++ | + |
| %user>90% | +
+ 0.99
+0.93
+5.91%
+ |
+
+ 0.99
+0.93
+5.91%
+ |
+
| %kernel>10% | +
+ 0.99
+0.97
+2.85%
+ |
+
+ 0.99
+0.97
+2.85%
+ |
+
| %iowait>10% | +
+ 0.31
+0.62
+-49.30%
+ |
+
+ 0.31
+0.62
+-49.30%
+ |
+
| avg %user | +
+ 82.11
+41.22
+99.21%
+ |
+
+ 82.11
+41.22
+99.21%
+ |
+
| avg %system | +
+ 6.10
+4.51
+35.11%
+ |
+
+ 6.10
+4.51
+35.11%
+ |
+
| avg %iowait | +
+ 0.17
+0.74
+-76.60%
+ |
+
+ 0.17
+0.74
+-76.60%
+ |
+
| avg disk util | +
+ 7.13
+32.21
+-77.85%
+ |
+
+ 7.13
+32.21
+-77.85%
+ |
+
| time more than 90% | +
+ 0.00
+0.00
+nan%
+ |
+
+ 0.00
+0.00
+nan%
+ |
+
| total read (G) | +
+ 5.24
+5.39
+-2.75%
+ |
+
+ 5.24
+5.39
+-2.75%
+ |
+
| total write (G) | +
+ 0.02
+1.12
+-97.81%
+ |
+
+ 0.02
+1.12
+-97.81%
+ |
+
| avg read bw (MB/s) | +
+ 37.52
+190.27
+-80.28%
+ |
+
+ 37.52
+190.27
+-80.28%
+ |
+
| avg write bw (MB/s) | +
+ 0.18
+39.61
+-99.55%
+ |
+
+ 0.18
+39.61
+-99.55%
+ |
+
| read bw %75 | +
+ 59.27
+411.58
+-85.60%
+ |
+
+ 59.27
+411.58
+-85.60%
+ |
+
| read bw %95 | +
+ 173.05
+484.54
+-64.29%
+ |
+
+ 173.05
+484.54
+-64.29%
+ |
+
| read bw max | +
+ 236.70
+510.35
+-53.62%
+ |
+
+ 236.70
+510.35
+-53.62%
+ |
+
| time_rd_morethan_95 | +
+ 0.05
+0.03
+41.96%
+ |
+
+ 0.05
+0.03
+41.96%
+ |
+
| write bw %75 | +
+ 0.07
+1.07
+-93.45%
+ |
+
+ 0.07
+1.07
+-93.45%
+ |
+
| write bw %95 | +
+ 1.23
+165.69
+-99.25%
+ |
+
+ 1.23
+165.69
+-99.25%
+ |
+
| write bw max | +
+ 1.70
+812.51
+-99.79%
+ |
+
+ 1.70
+812.51
+-99.79%
+ |
+
| time_wr_morethan_95 | +
+ 0.00
+0.03
+-100.00%
+ |
+
+ 0.00
+0.03
+-100.00%
+ |
+
| cached mean | +
+ 88.33
+93.90
+-5.93%
+ |
+
+ 88.33
+93.90
+-5.93%
+ |
+
| cached 75% | +
+ 132.00
+145.00
+-8.97%
+ |
+
+ 132.00
+145.00
+-8.97%
+ |
+
| cached max | +
+ 160.00
+188.00
+-14.89%
+ |
+
+ 160.00
+188.00
+-14.89%
+ |
+
| used mean | +
+ 2,060.73
+834.00
+147.09%
+ |
+
+ 2,060.73
+834.00
+147.09%
+ |
+
| used 75% | +
+ 2,343.00
+852.00
+175.00%
+ |
+
+ 2,343.00
+852.00
+175.00%
+ |
+
| used max | +
+ 2,346.00
+859.00
+173.11%
+ |
+
+ 2,346.00
+859.00
+173.11%
+ |
+
| rx MB/s 75% | +
+ 0.00
+0.00
+nan%
+ |
+
+ 0.00
+0.00
+nan%
+ |
+
| rx MB/s 95% | +
+ 0.00
+0.00
+nan%
+ |
+
+ 0.00
+0.00
+nan%
+ |
+
| rx MB/s 99% | +
+ 0.00
+0.00
+nan%
+ |
+
+ 0.00
+0.00
+nan%
+ |
+
| pgin mean | +
+ 37.37
+190.21
+-80.35%
+ |
+
+ 37.37
+190.21
+-80.35%
+ |
+
| pgin 75% | +
+ 59.00
+412.00
+-85.68%
+ |
+
+ 59.00
+412.00
+-85.68%
+ |
+
| pgin max | +
+ 352.00
+509.00
+-30.84%
+ |
+
+ 352.00
+509.00
+-30.84%
+ |
+
| pgout mean | +
+ 0.13
+40.97
+-99.68%
+ |
+
+ 0.13
+40.97
+-99.68%
+ |
+
| pgout 75% | +
+ 0.00
+1.00
+-100.00%
+ |
+
+ 0.00
+1.00
+-100.00%
+ |
+
| pgout max | +
+ 2.00
+840.00
+-99.76%
+ |
+
+ 2.00
+840.00
+-99.76%
+ |
+
| fault mean | +
+ 952,586.87
+117,653.31
+709.66%
+ |
+
+ 952,586.87
+117,653.31
+709.66%
+ |
+
| fault 75% | +
+ 1,426,717.00
+205,151.00
+595.45%
+ |
+
+ 1,426,717.00
+205,151.00
+595.45%
+ |
+
| fault max | +
+ 2,628,392.00
+256,538.00
+924.56%
+ |
+
+ 2,628,392.00
+256,538.00
+924.56%
+ |
+
| client | +sr217 | +agg | +
|---|---|---|
| cpu%_avg | +
+ 0.88
+0.45
+96.45%
+ |
+
+ 0.88
+0.45
+96.45%
+ |
+
| cpu freq_avg | +
+ 3,460.22
+3,241.92
+6.73%
+ |
+
+ 3,460.22
+3,241.92
+6.73%
+ |
+
| pathlength_sum | +
+ 17,960.00
+1,933.00
+829.13%
+ |
+
+ 17,960.00
+1,933.00
+829.13%
+ |
+
| ipc_avg | +
+ 1.27
+1.14
+11.30%
+ |
+
+ 1.27
+1.14
+11.30%
+ |
+
+
|
+
|
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Config compare¶
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
| + | 0851_0048 | +0851_0029 | +comp | +
|---|---|---|---|
| callSite.short | +collect at /tmp/ipykernel_265482/1936321720.py:117 | +collect at /tmp/ipykernel_234307/1936321720.py:117 | +False | +
| spark.app.submitTime | +1733464301669 | +1733457038427 | +False | +
| spark.executor.extraClassPath | +file:///data0/home/sparkuser/jars/6600a164407ae0e4f5ea5b33dc4b902f23a27730/gluten-velox-bundle-spark3.3_2.12-centos_7_x86_64-1.3.0-snapshot.jar | ++ | False | +
| spark.executor.extraJavaOptions | +-xx:+ignoreunrecognizedvmoptions --add-opens=java.base/java.lang=all-unnamed --add-opens=java.base/java.lang.invoke=all-unnamed --add-opens=java.base/java.lang.reflect=all-unnamed --add-opens=java.base/java.io=all-unnamed --add-opens=java.base/java.net=all-unnamed --add-opens=java.base/java.nio=all-unnamed --add-opens=java.base/java.util=all-unnamed --add-opens=java.base/java.util.concurrent=all-unnamed --add-opens=java.base/java.util.concurrent.atomic=all-unnamed --add-opens=java.base/sun.nio.ch=all-unnamed --add-opens=java.base/sun.nio.cs=all-unnamed --add-opens=java.base/sun.security.action=all-unnamed --add-opens=java.base/sun.util.calendar=all-unnamed --add-opens=java.security.jgss/sun.security.krb5=all-unnamed -xx:+useparalleloldgc -xx:parallelgcthreads=2 -xx:newratio=1 -xx:survivorratio=1 -xx:+usecompressedoops -verbose:gc -xx:+printgcdetails -xx:+printgctimestamps -xx:errorfile=/home/sparkuser/logs/java/hs_err_pid%p.log | +-xx:+ignoreunrecognizedvmoptions --add-opens=java.base/java.lang=all-unnamed --add-opens=java.base/java.lang.invoke=all-unnamed --add-opens=java.base/java.lang.reflect=all-unnamed --add-opens=java.base/java.io=all-unnamed --add-opens=java.base/java.net=all-unnamed --add-opens=java.base/java.nio=all-unnamed --add-opens=java.base/java.util=all-unnamed --add-opens=java.base/java.util.concurrent=all-unnamed --add-opens=java.base/java.util.concurrent.atomic=all-unnamed --add-opens=java.base/sun.nio.ch=all-unnamed --add-opens=java.base/sun.nio.cs=all-unnamed --add-opens=java.base/sun.security.action=all-unnamed --add-opens=java.base/sun.util.calendar=all-unnamed --add-opens=java.security.jgss/sun.security.krb5=all-unnamed -xx:+useparalleloldgc -xx:parallelgcthreads=2 -xx:newratio=1 -xx:survivorratio=1 -xx:+usecompressedoops -verbose:gc -xx:+printgcdetails -xx:+printgctimestamps -xx:errorfile=/data0/home/sparkuser/logs/java/hs_err_pid%p.log | +False | +
| spark.executor.memory | +10944m | +29184m | +False | +
| spark.gluten.memory.conservative.task.offHeap.size.in.bytes | +10041163776 | +NaN | +False | +
| spark.gluten.memory.dynamic.offHeap.sizing.enabled | +false | +NaN | +False | +
| spark.gluten.memory.offHeap.size.in.bytes | +80329310208 | +NaN | +False | +
| spark.gluten.memory.overAcquiredMemoryRatio | +0 | +NaN | +False | +
| spark.gluten.memory.task.offHeap.size.in.bytes | +20082327552 | +NaN | +False | +
| spark.gluten.memoryOverhead.size.in.bytes | +1073741824 | +NaN | +False | +
| spark.gluten.numTaskSlotsPerExecutor | +4 | +NaN | +False | +
| spark.gluten.sql.columnar.backend.lib | +velox | +NaN | +False | +
| spark.gluten.sql.columnar.coalesce.batches | +true | +NaN | +False | +
| spark.gluten.sql.columnar.forceshuffledhashjoin | +true | +NaN | +False | +
| spark.gluten.sql.columnar.maxBatchSize | +4096 | +NaN | +False | +
| spark.gluten.sql.columnar.shuffle.codec | +lz4 | +NaN | +False | +
| spark.gluten.sql.columnar.shuffle.codecBackend | ++ | NaN | +False | +
| spark.gluten.sql.session.timeZone.default | +etc/utc | +NaN | +False | +
| spark.memory.offHeap.size | +80329310208 | +58368m | +False | +
| spark.plugins | +org.apache.gluten.glutenplugin | +NaN | +False | +
| spark.repl.class.outputDir | +/tmp/tmpypqh85b0 | +/tmp/tmpynceqaxd | +False | +
| spark.repl.class.uri | +spark://sr213:40521/classes | +spark://sr213:34951/classes | +False | +
| spark.shuffle.manager | +org.apache.spark.shuffle.sort.columnarshufflemanager | +NaN | +False | +
| spark.sql.adaptive.customCostEvaluatorClass | +org.apache.spark.sql.execution.adaptive.glutencostevaluator | +NaN | +False | +
| spark.sql.extensions | +org.apache.gluten.extension.glutensessionextensions | +NaN | +False | +
| spark.sql.files.maxPartitionBytes | +4g | +NaN | +False | +
| spark.sql.shuffle.partitions | +32 | +64 | +False | +
+
+
+
+Convert to HTML¶
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+