SP1 - Details for Job 23891

Status: SUCCEEDED
Submitted: 2026/06/10 15:49:39
Duration: 6 s
Associated SQL Query: 17544
Job Group: 1bdb1c40-18dc-468b-b9e8-a12560b3affd
Completed Stages: 1
Skipped Stages: 7

Enable zooming

digraph G { subgraph clusterstage_45025 { label="Stage 45025 (skipped)"; subgraph cluster310334 { label="Scan parquet "; 135872 [labelType="html" label="FileScanRDD [135872] parquet at PersistJob.java:35"]; 135873 [labelType="html" label="MapPartitionsRDD [135873] parquet at PersistJob.java:35"]; } subgraph cluster310329 { label="WholeStageCodegen (4)"; 135874 [labelType="html" label="MapPartitionsRDD [135874] parquet at PersistJob.java:35"]; } subgraph cluster310328 { label="Exchange"; 135875 [labelType="html" label="MapPartitionsRDD [135875] parquet at PersistJob.java:35"]; } } 135872->135873; 135873->135874; 135874->135875; }

digraph G { subgraph clusterstage_45026 { label="Stage 45026 (skipped)"; subgraph cluster310322 { label="Scan parquet "; 135862 [labelType="html" label="FileScanRDD [135862] parquet at PersistJob.java:35"]; 135863 [labelType="html" label="MapPartitionsRDD [135863] parquet at PersistJob.java:35"]; } subgraph cluster310318 { label="WholeStageCodegen (1)"; 135864 [labelType="html" label="MapPartitionsRDD [135864] parquet at PersistJob.java:35"]; } subgraph cluster310327 { label="Scan csv "; 135865 [labelType="html" label="FileScanRDD [135865] parquet at PersistJob.java:35"]; 135866 [labelType="html" label="MapPartitionsRDD [135866] parquet at PersistJob.java:35"]; } subgraph cluster310323 { label="WholeStageCodegen (2)"; 135867 [labelType="html" label="MapPartitionsRDD [135867] parquet at PersistJob.java:35"]; } subgraph cluster310317 { label="CartesianProduct"; 135868 [labelType="html" label="UnsafeCartesianRDD [135868] parquet at PersistJob.java:35"]; 135869 [labelType="html" label="MapPartitionsRDD [135869] parquet at PersistJob.java:35"]; } subgraph cluster310313 { label="WholeStageCodegen (3)"; 135870 [labelType="html" label="MapPartitionsRDD [135870] parquet at PersistJob.java:35"]; } subgraph cluster310312 { label="Exchange"; 135871 [labelType="html" label="MapPartitionsRDD [135871] parquet at PersistJob.java:35"]; } } 135862->135863; 135863->135864; 135865->135866; 135866->135867; 135864->135868; 135867->135868; 135868->135869; 135869->135870; 135870->135871; }

digraph G { subgraph clusterstage_45027 { label="Stage 45027 (skipped)"; subgraph cluster310348 { label="Scan parquet "; 135880 [labelType="html" label="FileScanRDD [135880] parquet at PersistJob.java:35"]; 135881 [labelType="html" label="MapPartitionsRDD [135881] parquet at PersistJob.java:35"]; } subgraph cluster310343 { label="WholeStageCodegen (6)"; 135882 [labelType="html" label="MapPartitionsRDD [135882] parquet at PersistJob.java:35"]; } subgraph cluster310342 { label="Exchange"; 135883 [labelType="html" label="MapPartitionsRDD [135883] parquet at PersistJob.java:35"]; } } 135880->135881; 135881->135882; 135882->135883; }

digraph G { subgraph clusterstage_45028 { label="Stage 45028 (skipped)"; subgraph cluster310377 { label="Exchange"; 135891 [labelType="html" label="ShuffledRowRDD [135891] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310373 { label="WholeStageCodegen (10)"; 135892 [labelType="html" label="MapPartitionsRDD [135892] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310372 { label="Window"; 135893 [labelType="html" label="MapPartitionsRDD [135893] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310368 { label="WholeStageCodegen (11)"; 135894 [labelType="html" label="MapPartitionsRDD [135894] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310367 { label="ObjectHashAggregate"; 135895 [labelType="html" label="MapPartitionsRDD [135895] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310366 { label="Exchange"; 135896 [labelType="html" label="MapPartitionsRDD [135896] [Unordered] parquet at PersistJob.java:35"]; } } 135891->135892; 135892->135893; 135893->135894; 135894->135895; 135895->135896; }

135883,135891

digraph G { subgraph clusterstage_45029 { label="Stage 45029 (skipped)"; subgraph cluster310341 { label="Scan parquet "; 135876 [labelType="html" label="FileScanRDD [135876] parquet at PersistJob.java:35"]; 135877 [labelType="html" label="MapPartitionsRDD [135877] parquet at PersistJob.java:35"]; } subgraph cluster310336 { label="WholeStageCodegen (5)"; 135878 [labelType="html" label="MapPartitionsRDD [135878] parquet at PersistJob.java:35"]; } subgraph cluster310335 { label="Exchange"; 135879 [labelType="html" label="MapPartitionsRDD [135879] parquet at PersistJob.java:35"]; } } 135876->135877; 135877->135878; 135878->135879; }

digraph G { subgraph clusterstage_45030 { label="Stage 45030 (skipped)"; subgraph cluster310359 { label="Exchange"; 135884 [labelType="html" label="ShuffledRowRDD [135884] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310355 { label="WholeStageCodegen (7)"; 135885 [labelType="html" label="MapPartitionsRDD [135885] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310365 { label="Exchange"; 135886 [labelType="html" label="ShuffledRowRDD [135886] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310361 { label="WholeStageCodegen (8)"; 135887 [labelType="html" label="MapPartitionsRDD [135887] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310350 { label="WholeStageCodegen (9)"; 135888 [labelType="html" label="ZippedPartitionsRDD2 [135888] [Unordered] parquet at PersistJob.java:35"]; 135889 [labelType="html" label="MapPartitionsRDD [135889] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310349 { label="Exchange"; 135890 [labelType="html" label="MapPartitionsRDD [135890] [Unordered] parquet at PersistJob.java:35"]; } } 135884->135885; 135886->135887; 135885->135888; 135887->135888; 135888->135889; 135889->135890; }

135871,135884

135875,135886

digraph G { subgraph clusterstage_45031 { label="Stage 45031 (skipped)"; subgraph cluster310390 { label="Exchange"; 135897 [labelType="html" label="ShuffledRowRDD [135897] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310386 { label="WholeStageCodegen (12)"; 135898 [labelType="html" label="MapPartitionsRDD [135898] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310385 { label="Window"; 135899 [labelType="html" label="MapPartitionsRDD [135899] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310381 { label="WholeStageCodegen (13)"; 135900 [labelType="html" label="MapPartitionsRDD [135900] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310402 { label="AQEShuffleRead"; 135901 [labelType="html" label="ShuffledRowRDD [135901] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310399 { label="WholeStageCodegen (14)"; 135902 [labelType="html" label="MapPartitionsRDD [135902] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310409 { label="AQEShuffleRead"; 135903 [labelType="html" label="ShuffledRowRDD [135903] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310408 { label="ObjectHashAggregate"; 135904 [labelType="html" label="MapPartitionsRDD [135904] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310404 { label="WholeStageCodegen (15)"; 135905 [labelType="html" label="MapPartitionsRDD [135905] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310391 { label="WholeStageCodegen (16)"; 135906 [labelType="html" label="ZippedPartitionsRDD2 [135906] [Unordered] parquet at PersistJob.java:35"]; 135907 [labelType="html" label="MapPartitionsRDD [135907] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310380 { label="Union"; 135908 [labelType="html" label="UnionRDD [135908] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310414 { label="Scan csv "; 135909 [labelType="html" label="FileScanRDD [135909] parquet at PersistJob.java:35"]; 135910 [labelType="html" label="MapPartitionsRDD [135910] parquet at PersistJob.java:35"]; } subgraph cluster310410 { label="WholeStageCodegen (17)"; 135911 [labelType="html" label="MapPartitionsRDD [135911] parquet at PersistJob.java:35"]; } subgraph cluster310379 { label="CartesianProduct"; 135912 [labelType="html" label="UnsafeCartesianRDD [135912] [Unordered] parquet at PersistJob.java:35"]; 135913 [labelType="html" label="MapPartitionsRDD [135913] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310378 { label="Exchange"; 135914 [labelType="html" label="MapPartitionsRDD [135914] [Unordered] parquet at PersistJob.java:35"]; } } 135897->135898; 135898->135899; 135899->135900; 135901->135902; 135903->135904; 135904->135905; 135902->135906; 135905->135906; 135906->135907; 135900->135908; 135907->135908; 135909->135910; 135910->135911; 135908->135912; 135911->135912; 135912->135913; 135913->135914; }

135890,135897

135879,135901

135896,135903

digraph G { subgraph clusterstage_45032 { label="Stage 45032"; subgraph cluster310437 { label="Exchange"; 135915 [labelType="html" label="ShuffledRowRDD [135915] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310433 { label="WholeStageCodegen (18)"; 135916 [labelType="html" label="MapPartitionsRDD [135916] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310432 { label="Window"; 135917 [labelType="html" label="MapPartitionsRDD [135917] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310430 { label="Filter"; 135918 [labelType="html" label="MapPartitionsRDD [135918] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310429 { label="Project"; 135919 [labelType="html" label="MapPartitionsRDD [135919] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310428 { label="Project"; 135920 [labelType="html" label="MapPartitionsRDD [135920] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310427 { label="Project"; 135921 [labelType="html" label="MapPartitionsRDD [135921] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310426 { label="Project"; 135922 [labelType="html" label="MapPartitionsRDD [135922] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310425 { label="Project"; 135923 [labelType="html" label="MapPartitionsRDD [135923] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310443 { label="Scan parquet "; 135924 [labelType="html" label="FileScanRDD [135924] parquet at PersistJob.java:35"]; 135925 [labelType="html" label="MapPartitionsRDD [135925] parquet at PersistJob.java:35"]; } subgraph cluster310438 { label="WholeStageCodegen (20)"; 135926 [labelType="html" label="MapPartitionsRDD [135926] parquet at PersistJob.java:35"]; } subgraph cluster310416 { label="Union"; 135927 [labelType="html" label="UnionRDD [135927] [Unordered] parquet at PersistJob.java:35"]; } subgraph cluster310415 { label="Exchange"; 136014 [labelType="html" label="MapPartitionsRDD [136014] [Unordered] parquet at PersistJob.java:35"]; } } 135915->135916; 135916->135917; 135917->135918; 135918->135919; 135919->135920; 135920->135921; 135921->135922; 135922->135923; 135924->135925; 135925->135926; 135923->135927; 135926->135927; 135927->136014; }

135914,135915

Completed Stages (1)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
45032	default	PersistFrame parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	2026/06/10 15:49:39	6 s	20/20	86.7 MiB		16.9 MiB	106.2 MiB

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

45032

default

PersistFrame parquet at PersistJob.java:35 +details

org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)
com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35)
com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77)
com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47)
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source)
java.base/java.util.concurrent.FutureTask.run(Unknown Source)
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.base/java.lang.Thread.run(Unknown Source)

2026/06/10 15:49:39

6 s

20/20

86.7 MiB

16.9 MiB

106.2 MiB

Page:

Skipped Stages (7)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total
45031	default	parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/15
45030	default	parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/10
45029	default	parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/1
45028	default	parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/10
45027	default	parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/10
45026	default	parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/10
45025	default	parquet at PersistJob.java:35 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802) com.adb.hdh.spark.job.impl.PersistJob.exec(PersistJob.java:35) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:77) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/1

Page: