SP1 - Details for Job 3398

Status: SUCCEEDED
Submitted: 2026/10/09 14:04:32
Duration: 11 ms
Associated SQL Query: 2230
Job Group: d770ad83-9d50-4674-8531-813dffacf3f4
Completed Stages: 1
Skipped Stages: 5

Enable zooming

digraph G { subgraph clusterstage_6396 { label="Stage 6396 (skipped)"; subgraph cluster42741 { label="Scan parquet "; 18447 [labelType="html" label="FileScanRDD [18447] count at ColumnStatJob.java:47"]; 18448 [labelType="html" label="MapPartitionsRDD [18448] count at ColumnStatJob.java:47"]; } subgraph cluster42736 { label="WholeStageCodegen (13)"; 18449 [labelType="html" label="MapPartitionsRDD [18449] count at ColumnStatJob.java:47"]; } subgraph cluster42747 { label="Scan parquet "; 18450 [labelType="html" label="FileScanRDD [18450] count at ColumnStatJob.java:47"]; 18451 [labelType="html" label="MapPartitionsRDD [18451] count at ColumnStatJob.java:47"]; } subgraph cluster42742 { label="WholeStageCodegen (14)"; 18452 [labelType="html" label="MapPartitionsRDD [18452] count at ColumnStatJob.java:47"]; } subgraph cluster42753 { label="Scan parquet "; 18453 [labelType="html" label="FileScanRDD [18453] count at ColumnStatJob.java:47"]; 18454 [labelType="html" label="MapPartitionsRDD [18454] count at ColumnStatJob.java:47"]; } subgraph cluster42748 { label="WholeStageCodegen (15)"; 18455 [labelType="html" label="MapPartitionsRDD [18455] count at ColumnStatJob.java:47"]; } subgraph cluster42735 { label="Union"; 18456 [labelType="html" label="UnionRDD [18456] count at ColumnStatJob.java:47"]; } subgraph cluster42734 { label="Exchange"; 18457 [labelType="html" label="MapPartitionsRDD [18457] count at ColumnStatJob.java:47"]; } } 18447->18448; 18448->18449; 18450->18451; 18451->18452; 18453->18454; 18454->18455; 18449->18456; 18452->18456; 18455->18456; 18456->18457; }

digraph G { subgraph clusterstage_6397 { label="Stage 6397 (skipped)"; subgraph cluster42758 { label="Scan csv "; 18458 [labelType="html" label="FileScanRDD [18458] count at ColumnStatJob.java:47"]; 18459 [labelType="html" label="MapPartitionsRDD [18459] count at ColumnStatJob.java:47"]; } subgraph cluster42755 { label="WholeStageCodegen (16)"; 18460 [labelType="html" label="MapPartitionsRDD [18460] count at ColumnStatJob.java:47"]; } subgraph cluster42754 { label="Exchange"; 18461 [labelType="html" label="MapPartitionsRDD [18461] count at ColumnStatJob.java:47"]; } } 18458->18459; 18459->18460; 18460->18461; }

digraph G { subgraph clusterstage_6398 { label="Stage 6398 (skipped)"; subgraph cluster42656 { label="Scan csv "; 18409 [labelType="html" label="FileScanRDD [18409] count at ColumnStatJob.java:47"]; 18410 [labelType="html" label="MapPartitionsRDD [18410] count at ColumnStatJob.java:47"]; } subgraph cluster42651 { label="WholeStageCodegen (1)"; 18411 [labelType="html" label="MapPartitionsRDD [18411] count at ColumnStatJob.java:47"]; } subgraph cluster42665 { label="Scan parquet "; 18412 [labelType="html" label="FileScanRDD [18412] count at ColumnStatJob.java:47"]; 18413 [labelType="html" label="MapPartitionsRDD [18413] count at ColumnStatJob.java:47"]; } subgraph cluster42657 { label="WholeStageCodegen (2)"; 18414 [labelType="html" label="MapPartitionsRDD [18414] count at ColumnStatJob.java:47"]; } subgraph cluster42674 { label="Scan parquet "; 18417 [labelType="html" label="FileScanRDD [18417] count at ColumnStatJob.java:47"]; 18418 [labelType="html" label="MapPartitionsRDD [18418] count at ColumnStatJob.java:47"]; } subgraph cluster42666 { label="WholeStageCodegen (3)"; 18419 [labelType="html" label="MapPartitionsRDD [18419] count at ColumnStatJob.java:47"]; } subgraph cluster42682 { label="Scan parquet "; 18420 [labelType="html" label="FileScanRDD [18420] count at ColumnStatJob.java:47"]; 18421 [labelType="html" label="MapPartitionsRDD [18421] count at ColumnStatJob.java:47"]; } subgraph cluster42676 { label="WholeStageCodegen (4)"; 18422 [labelType="html" label="MapPartitionsRDD [18422] count at ColumnStatJob.java:47"]; } subgraph cluster42689 { label="Scan parquet "; 18423 [labelType="html" label="FileScanRDD [18423] count at ColumnStatJob.java:47"]; 18424 [labelType="html" label="MapPartitionsRDD [18424] count at ColumnStatJob.java:47"]; } subgraph cluster42683 { label="WholeStageCodegen (5)"; 18425 [labelType="html" label="MapPartitionsRDD [18425] count at ColumnStatJob.java:47"]; } subgraph cluster42698 { label="Scan parquet "; 18426 [labelType="html" label="FileScanRDD [18426] count at ColumnStatJob.java:47"]; 18427 [labelType="html" label="MapPartitionsRDD [18427] count at ColumnStatJob.java:47"]; } subgraph cluster42690 { label="WholeStageCodegen (6)"; 18428 [labelType="html" label="MapPartitionsRDD [18428] count at ColumnStatJob.java:47"]; } subgraph cluster42705 { label="Scan parquet "; 18429 [labelType="html" label="FileScanRDD [18429] count at ColumnStatJob.java:47"]; 18430 [labelType="html" label="MapPartitionsRDD [18430] count at ColumnStatJob.java:47"]; } subgraph cluster42699 { label="WholeStageCodegen (7)"; 18431 [labelType="html" label="MapPartitionsRDD [18431] count at ColumnStatJob.java:47"]; } subgraph cluster42712 { label="Scan parquet "; 18432 [labelType="html" label="FileScanRDD [18432] count at ColumnStatJob.java:47"]; 18433 [labelType="html" label="MapPartitionsRDD [18433] count at ColumnStatJob.java:47"]; } subgraph cluster42706 { label="WholeStageCodegen (8)"; 18434 [labelType="html" label="MapPartitionsRDD [18434] count at ColumnStatJob.java:47"]; } subgraph cluster42719 { label="Scan parquet "; 18435 [labelType="html" label="FileScanRDD [18435] count at ColumnStatJob.java:47"]; 18436 [labelType="html" label="MapPartitionsRDD [18436] count at ColumnStatJob.java:47"]; } subgraph cluster42713 { label="WholeStageCodegen (9)"; 18437 [labelType="html" label="MapPartitionsRDD [18437] count at ColumnStatJob.java:47"]; } subgraph cluster42726 { label="Scan parquet "; 18438 [labelType="html" label="FileScanRDD [18438] count at ColumnStatJob.java:47"]; 18439 [labelType="html" label="MapPartitionsRDD [18439] count at ColumnStatJob.java:47"]; } subgraph cluster42720 { label="WholeStageCodegen (10)"; 18440 [labelType="html" label="MapPartitionsRDD [18440] count at ColumnStatJob.java:47"]; } subgraph cluster42733 { label="Scan parquet "; 18441 [labelType="html" label="FileScanRDD [18441] count at ColumnStatJob.java:47"]; 18442 [labelType="html" label="MapPartitionsRDD [18442] count at ColumnStatJob.java:47"]; } subgraph cluster42727 { label="WholeStageCodegen (11)"; 18443 [labelType="html" label="MapPartitionsRDD [18443] count at ColumnStatJob.java:47"]; } subgraph cluster42650 { label="Union"; 18444 [labelType="html" label="UnionRDD [18444] count at ColumnStatJob.java:47"]; } subgraph cluster42640 { label="WholeStageCodegen (12)"; 18445 [labelType="html" label="MapPartitionsRDD [18445] count at ColumnStatJob.java:47"]; } subgraph cluster42639 { label="Exchange"; 18446 [labelType="html" label="MapPartitionsRDD [18446] count at ColumnStatJob.java:47"]; } } 18409->18410; 18410->18411; 18412->18413; 18413->18414; 18417->18418; 18418->18419; 18420->18421; 18421->18422; 18423->18424; 18424->18425; 18426->18427; 18427->18428; 18429->18430; 18430->18431; 18432->18433; 18433->18434; 18435->18436; 18436->18437; 18438->18439; 18439->18440; 18441->18442; 18442->18443; 18411->18444; 18414->18444; 18419->18444; 18422->18444; 18425->18444; 18428->18444; 18431->18444; 18434->18444; 18437->18444; 18440->18444; 18443->18444; 18444->18445; 18445->18446; }

digraph G { subgraph clusterstage_6399 { label="Stage 6399 (skipped)"; subgraph cluster42771 { label="AQEShuffleRead"; 18462 [labelType="html" label="ShuffledRowRDD [18462] [Unordered] count at ColumnStatJob.java:47"]; } subgraph cluster42768 { label="WholeStageCodegen (17)"; 18463 [labelType="html" label="MapPartitionsRDD [18463] [Unordered] count at ColumnStatJob.java:47"]; } subgraph cluster42776 { label="AQEShuffleRead"; 18464 [labelType="html" label="ShuffledRowRDD [18464] [Unordered] count at ColumnStatJob.java:47"]; } subgraph cluster42773 { label="WholeStageCodegen (18)"; 18465 [labelType="html" label="MapPartitionsRDD [18465] [Unordered] count at ColumnStatJob.java:47"]; } subgraph cluster42764 { label="WholeStageCodegen (19)"; 18466 [labelType="html" label="ZippedPartitionsRDD2 [18466] [Unordered] count at ColumnStatJob.java:47"]; 18467 [labelType="html" label="MapPartitionsRDD [18467] [Unordered] count at ColumnStatJob.java:47"]; } subgraph cluster42781 { label="Scan csv "; 18468 [labelType="html" label="FileScanRDD [18468] count at ColumnStatJob.java:47"]; 18469 [labelType="html" label="MapPartitionsRDD [18469] count at ColumnStatJob.java:47"]; } subgraph cluster42777 { label="WholeStageCodegen (20)"; 18470 [labelType="html" label="MapPartitionsRDD [18470] count at ColumnStatJob.java:47"]; } subgraph cluster42763 { label="CartesianProduct"; 18471 [labelType="html" label="UnsafeCartesianRDD [18471] [Unordered] count at ColumnStatJob.java:47"]; 18472 [labelType="html" label="MapPartitionsRDD [18472] [Unordered] count at ColumnStatJob.java:47"]; } subgraph cluster42760 { label="WholeStageCodegen (21)"; 18473 [labelType="html" label="MapPartitionsRDD [18473] [Unordered] count at ColumnStatJob.java:47"]; } subgraph cluster42759 { label="Exchange"; 18474 [labelType="html" label="MapPartitionsRDD [18474] [Unordered] count at ColumnStatJob.java:47"]; } } 18462->18463; 18464->18465; 18463->18466; 18465->18466; 18466->18467; 18468->18469; 18469->18470; 18467->18471; 18470->18471; 18471->18472; 18472->18473; 18473->18474; }

18457,18462

18461,18464

digraph G { subgraph clusterstage_6400 { label="Stage 6400 (skipped)"; subgraph cluster43080 { label="AQEShuffleRead"; 18542 [labelType="html" label="ShuffledRowRDD [18542] [Unordered] count at ColumnStatJob.java:47"]; } subgraph cluster43077 { label="WholeStageCodegen (22)"; 18543 [labelType="html" label="MapPartitionsRDD [18543] [Unordered] count at ColumnStatJob.java:47"]; } subgraph cluster43085 { label="AQEShuffleRead"; 18544 [labelType="html" label="ShuffledRowRDD [18544] [Unordered] count at ColumnStatJob.java:47"]; } subgraph cluster43082 { label="WholeStageCodegen (23)"; 18545 [labelType="html" label="MapPartitionsRDD [18545] [Unordered] count at ColumnStatJob.java:47"]; } subgraph cluster43071 { label="WholeStageCodegen (24)"; 18546 [labelType="html" label="ZippedPartitionsRDD2 [18546] [Unordered] count at ColumnStatJob.java:47"]; 18547 [labelType="html" label="MapPartitionsRDD [18547] [Unordered] count at ColumnStatJob.java:47"]; } subgraph cluster43070 { label="Exchange"; 18548 [labelType="html" label="MapPartitionsRDD [18548] [Unordered] count at ColumnStatJob.java:47"]; } } 18542->18543; 18544->18545; 18543->18546; 18545->18546; 18546->18547; 18547->18548; }

18446,18542

18474,18544

digraph G { subgraph clusterstage_6401 { label="Stage 6401"; subgraph cluster43107 { label="Exchange"; 18556 [labelType="html" label="ShuffledRowRDD [18556] [Unordered] count at ColumnStatJob.java:47"]; } subgraph cluster43103 { label="WholeStageCodegen (25)"; 18557 [labelType="html" label="MapPartitionsRDD [18557] [Unordered] count at ColumnStatJob.java:47"]; } subgraph cluster43108 { label="mapPartitionsInternal"; 18558 [labelType="html" label="MapPartitionsRDD [18558] [Unordered] count at ColumnStatJob.java:47"]; } } 18556->18557; 18557->18558; }

18548,18556

Completed Stages (1)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
6401	default	DataFrameCommonJob_dataExecSummary count at ColumnStatJob.java:47 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3615) com.adb.hdh.spark.job.impl.ColumnStatJob.stats(ColumnStatJob.java:47) com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	2026/10/09 14:04:32	8 ms	1/1			115.0 B

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

6401

default

DataFrameCommonJob_dataExecSummary count at ColumnStatJob.java:47 +details

org.apache.spark.sql.Dataset.count(Dataset.scala:3615)
com.adb.hdh.spark.job.impl.ColumnStatJob.stats(ColumnStatJob.java:47)
com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:35)
com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142)
com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47)
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source)
java.base/java.util.concurrent.FutureTask.run(Unknown Source)
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.base/java.lang.Thread.run(Unknown Source)

2026/10/09 14:04:32

8 ms

1/1

115.0 B

Page:

Skipped Stages (5)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total
6400	default	count at ColumnStatJob.java:47 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3615) com.adb.hdh.spark.job.impl.ColumnStatJob.stats(ColumnStatJob.java:47) com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/2
6399	default	count at ColumnStatJob.java:47 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3615) com.adb.hdh.spark.job.impl.ColumnStatJob.stats(ColumnStatJob.java:47) com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/1
6398	default	count at ColumnStatJob.java:47 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3615) com.adb.hdh.spark.job.impl.ColumnStatJob.stats(ColumnStatJob.java:47) com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/43
6397	default	count at ColumnStatJob.java:47 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3615) com.adb.hdh.spark.job.impl.ColumnStatJob.stats(ColumnStatJob.java:47) com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/1
6396	default	count at ColumnStatJob.java:47 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3615) com.adb.hdh.spark.job.impl.ColumnStatJob.stats(ColumnStatJob.java:47) com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/3

Page: