SP1 - Details for Job 23917

Status: SUCCEEDED
Submitted: 2026/06/10 15:49:45
Duration: 16 s
Associated SQL Query: 17544
Job Group: 1bdb1c40-18dc-468b-b9e8-a12560b3affd
Completed Stages: 1
Skipped Stages: 8

Enable zooming

digraph G { subgraph clusterstage_45066 { label="Stage 45066 (skipped)"; subgraph cluster310334 { label="Scan parquet "; 135872 [labelType="html" label="FileScanRDD [135872] parquet at PersistJob.java:35"]; 135873 [labelType="html" label="MapPartitionsRDD [135873] parquet at PersistJob.java:35"]; } subgraph cluster310329 { label="WholeStageCodegen (4)"; 135874 [labelType="html" label="MapPartitionsRDD [135874] parquet at PersistJob.java:35"]; } subgraph cluster310328 { label="Exchange"; 135875 [labelType="html" label="MapPartitionsRDD [135875] parquet at PersistJob.java:35"]; } } 135872->135873; 135873->135874; 135874->135875; }

digraph G { subgraph clusterstage_45067 { label="Stage 45067 (skipped)"; subgraph cluster310322 { label="Scan parquet "; 135862 [labelType="html" label="FileScanRDD [135862] parquet at PersistJob.java:35"]; 135863 [labelType="html" label="MapPartitionsRDD [135863] parquet at PersistJob.java:35"]; } subgraph cluster310318 { label="WholeStageCodegen (1)"; 135864 [labelType="html" label="MapPartitionsRDD [135864] parquet at PersistJob.java:35"]; } subgraph cluster310327 { label="Scan csv "; 135865 [labelType="html" label="FileScanRDD [135865] parquet at PersistJob.java:35"]; 135866 [labelType="html" label="MapPartitionsRDD [135866] parquet at PersistJob.java:35"]; } subgraph cluster310323 { label="WholeStageCodegen (2)"; 135867 [labelType="html" label="MapPartitionsRDD [135867] parquet at PersistJob.java:35"]; } subgraph cluster310317 { label="CartesianProduct"; 135868 [labelType="html" label="UnsafeCartesianRDD [135868] parquet at PersistJob.java:35"]; 135869 [labelType="html" label="MapPartitionsRDD [135869] parquet at PersistJob.java:35"]; } subgraph cluster310313 { label="WholeStageCodegen (3)"; 135870 [labelType="html" label="MapPartitionsRDD [135870] parquet at PersistJob.java:35"]; } subgraph cluster310312 { label="Exchange"; 135871 [labelType="html" label="MapPartitionsRDD [135871] parquet at PersistJob.java:35"]; } } 135862->135863; 135863->135864; 135865->135866; 135866->135867; 135864->135868; 135867->135868; 135868->135869; 135869->135870; 135870->135871; }

digraph G { subgraph clusterstage_45068 { label="Stage 45068 (skipped)"; subgraph cluster310348 { label="Scan parquet "; 135880 [labelType="html" label="FileScanRDD [135880] parquet at PersistJob.java:35"]; 135881 [labelType="html" label="MapPartitionsRDD [135881] parquet at PersistJob.java:35"]; } subgraph cluster310343 { label="WholeStageCodegen (6)"; 135882 [labelType="html" label="MapPartitionsRDD [135882] parquet at PersistJob.java:35"]; } subgraph cluster310342 { label="Exchange"; 135883 [labelType="html" label="MapPartitionsRDD [135883] parquet at PersistJob.java:35"]; } } 135880->135881; 135881->135882; 135882->135883; }

digraph G { subgraph clusterstage_45069 { label="Stage 45069 (skipped)"; subgraph cluster310377 { label="Exchange"; 135891 [labelType="html" label="ShuffledRowRDD [135891] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310373 { label="WholeStageCodegen (10)"; 135892 [labelType="html" label="MapPartitionsRDD [135892] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310372 { label="Window"; 135893 [labelType="html" label="MapPartitionsRDD [135893] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310368 { label="WholeStageCodegen (11)"; 135894 [labelType="html" label="MapPartitionsRDD [135894] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310367 { label="ObjectHashAggregate"; 135895 [labelType="html" label="MapPartitionsRDD [135895] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310366 { label="Exchange"; 135896 [labelType="html" label="MapPartitionsRDD [135896] [Unordered] parquet at PersistJob.java:35"]; } } 135891->135892; 135892->135893; 135893->135894; 135894->135895; 135895->135896; }

135883,135891

digraph G { subgraph clusterstage_45070 { label="Stage 45070 (skipped)"; subgraph cluster310341 { label="Scan parquet "; 135876 [labelType="html" label="FileScanRDD [135876] parquet at PersistJob.java:35"]; 135877 [labelType="html" label="MapPartitionsRDD [135877] parquet at PersistJob.java:35"]; } subgraph cluster310336 { label="WholeStageCodegen (5)"; 135878 [labelType="html" label="MapPartitionsRDD [135878] parquet at PersistJob.java:35"]; } subgraph cluster310335 { label="Exchange"; 135879 [labelType="html" label="MapPartitionsRDD [135879] parquet at PersistJob.java:35"]; } } 135876->135877; 135877->135878; 135878->135879; }

digraph G { subgraph clusterstage_45071 { label="Stage 45071 (skipped)"; subgraph cluster310359 { label="Exchange"; 135884 [labelType="html" label="ShuffledRowRDD [135884] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310355 { label="WholeStageCodegen (7)"; 135885 [labelType="html" label="MapPartitionsRDD [135885] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310365 { label="Exchange"; 135886 [labelType="html" label="ShuffledRowRDD [135886] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310361 { label="WholeStageCodegen (8)"; 135887 [labelType="html" label="MapPartitionsRDD [135887] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310350 { label="WholeStageCodegen (9)"; 135888 [labelType="html" label="ZippedPartitionsRDD2 [135888] [Unordered] parquet at PersistJob.java:35"]; 135889 [labelType="html" label="MapPartitionsRDD [135889] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310349 { label="Exchange"; 135890 [labelType="html" label="MapPartitionsRDD [135890] [Unordered] parquet at PersistJob.java:35"]; } } 135884->135885; 135886->135887; 135885->135888; 135887->135888; 135888->135889; 135889->135890; }

135871,135884

135875,135886

digraph G { subgraph clusterstage_45072 { label="Stage 45072 (skipped)"; subgraph cluster310390 { label="Exchange"; 135897 [labelType="html" label="ShuffledRowRDD [135897] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310386 { label="WholeStageCodegen (12)"; 135898 [labelType="html" label="MapPartitionsRDD [135898] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310385 { label="Window"; 135899 [labelType="html" label="MapPartitionsRDD [135899] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310381 { label="WholeStageCodegen (13)"; 135900 [labelType="html" label="MapPartitionsRDD [135900] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310402 { label="AQEShuffleRead"; 135901 [labelType="html" label="ShuffledRowRDD [135901] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310399 { label="WholeStageCodegen (14)"; 135902 [labelType="html" label="MapPartitionsRDD [135902] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310409 { label="AQEShuffleRead"; 135903 [labelType="html" label="ShuffledRowRDD [135903] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310408 { label="ObjectHashAggregate"; 135904 [labelType="html" label="MapPartitionsRDD [135904] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310404 { label="WholeStageCodegen (15)"; 135905 [labelType="html" label="MapPartitionsRDD [135905] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310391 { label="WholeStageCodegen (16)"; 135906 [labelType="html" label="ZippedPartitionsRDD2 [135906] [Unordered] parquet at PersistJob.java:35"]; 135907 [labelType="html" label="MapPartitionsRDD [135907] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310380 { label="Union"; 135908 [labelType="html" label="UnionRDD [135908] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310414 { label="Scan csv "; 135909 [labelType="html" label="FileScanRDD [135909] parquet at PersistJob.java:35"]; 135910 [labelType="html" label="MapPartitionsRDD [135910] parquet at PersistJob.java:35"]; } subgraph cluster310410 { label="WholeStageCodegen (17)"; 135911 [labelType="html" label="MapPartitionsRDD [135911] parquet at PersistJob.java:35"]; } subgraph cluster310379 { label="CartesianProduct"; 135912 [labelType="html" label="UnsafeCartesianRDD [135912] [Unordered] parquet at PersistJob.java:35"]; 135913 [labelType="html" label="MapPartitionsRDD [135913] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310378 { label="Exchange"; 135914 [labelType="html" label="MapPartitionsRDD [135914] [Unordered] parquet at PersistJob.java:35"]; } } 135897->135898; 135898->135899; 135899->135900; 135901->135902; 135903->135904; 135904->135905; 135902->135906; 135905->135906; 135906->135907; 135900->135908; 135907->135908; 135909->135910; 135910->135911; 135908->135912; 135911->135912; 135912->135913; 135913->135914; }

135890,135897

135879,135901

135896,135903

digraph G { subgraph clusterstage_45073 { label="Stage 45073 (skipped)"; subgraph cluster310437 { label="Exchange"; 135915 [labelType="html" label="ShuffledRowRDD [135915] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310433 { label="WholeStageCodegen (18)"; 135916 [labelType="html" label="MapPartitionsRDD [135916] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310432 { label="Window"; 135917 [labelType="html" label="MapPartitionsRDD [135917] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310430 { label="Filter"; 135918 [labelType="html" label="MapPartitionsRDD [135918] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310429 { label="Project"; 135919 [labelType="html" label="MapPartitionsRDD [135919] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310428 { label="Project"; 135920 [labelType="html" label="MapPartitionsRDD [135920] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310427 { label="Project"; 135921 [labelType="html" label="MapPartitionsRDD [135921] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310426 { label="Project"; 135922 [labelType="html" label="MapPartitionsRDD [135922] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310425 { label="Project"; 135923 [labelType="html" label="MapPartitionsRDD [135923] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310443 { label="Scan parquet "; 135924 [labelType="html" label="FileScanRDD [135924] parquet at PersistJob.java:35"]; 135925 [labelType="html" label="MapPartitionsRDD [135925] parquet at PersistJob.java:35"]; } subgraph cluster310438 { label="WholeStageCodegen (20)"; 135926 [labelType="html" label="MapPartitionsRDD [135926] parquet at PersistJob.java:35"]; } subgraph cluster310416 { label="Union"; 135927 [labelType="html" label="UnionRDD [135927] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310415 { label="Exchange"; 136014 [labelType="html" label="MapPartitionsRDD [136014] [Unordered] parquet at PersistJob.java:35"]; } } 135915->135916; 135916->135917; 135917->135918; 135918->135919; 135919->135920; 135920->135921; 135921->135922; 135922->135923; 135924->135925; 135925->135926; 135923->135927; 135926->135927; 135927->136014; }

135914,135915

digraph G { subgraph clusterstage_45074 { label="Stage 45074"; subgraph cluster310942 { label="Exchange"; 136098 [labelType="html" label="ShuffledRowRDD [136098] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310937 { label="WholeStageCodegen (21)"; 136099 [labelType="html" label="MapPartitionsRDD [136099] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310936 { label="Coalesce"; 136100 [labelType="html" label="CoalescedRDD [136100] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310935 { label="WriteFiles"; 136101 [labelType="html" label="MapPartitionsRDD [136101] [Unordered] parquet at PersistJob.java:35"]; } } 136098->136099; 136099->136100; 136100->136101; }

136014,136098

Completed Stages (1)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
45074	default	PersistFrame parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	2026/06/10 15:49:45	16 s	1/1		67.2 MiB	106.2 MiB

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

45074

default

PersistFrame parquet at PersistJob.java:35 +details

org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)
com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35)
com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77)
com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47)
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source)
java.base/java.util.concurrent.FutureTask.run(Unknown Source)
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.base/java.lang.Thread.run(Unknown Source)

2026/06/10 15:49:45

16 s

1/1

67.2 MiB

106.2 MiB

Page:

Skipped Stages (8)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total
45073	default	parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/20
45072	default	parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/15
45071	default	parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/10
45070	default	parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/1
45069	default	parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/10
45068	default	parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/10
45067	default	parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/10
45066	default	parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/1

Page: