SP1 - Details for Query 6080

Submitted Time: 2026/03/27 14:33:24
Duration: 3 s
Succeeded Jobs: 9245 9246 9247 9248

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; 1 [labelType="html" label="SortAggregate number of output rows: 1"]; 2 [labelType="html" label="Exchange shuffle records written: 1 local merged chunks fetched: 0 shuffle write time: 0 ms remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 1 local bytes read: 96.0 B fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 1 remote merged chunks fetched: 0 remote blocks read: 0 data size: 72.0 B local merged bytes read: 0.0 B number of partitions: 1 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written: 96.0 B"]; 3 [labelType="html" label="SortAggregate number of output rows: 1"]; subgraph cluster4 { isCluster="true"; label="WholeStageCodegen (5)\n \nduration: 2.4 s"; 5 [labelType="html" label=" Project "]; 6 [labelType="html" label="SortMergeJoin number of output rows: 2,231,123 spill size: 0.0 B"]; } subgraph cluster7 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: 2.4 s"; 8 [labelType="html" label="Sort sort time: 100 ms peak memory: 192.0 MiB spill size: 0.0 B"]; } 9 [labelType="html" label="AQEShuffleRead number of partitions: 1 partition data size: 905.4 KiB number of coalesced partitions: 1"]; 10 [labelType="html" label="Exchange shuffle records written: 2,231,123 local merged chunks fetched: 0 shuffle write time total (min, med, max (stageId: taskId)) 11 ms (0 ms, 0 ms, 2 ms (stage 17416.0: task 24231)) remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 2,231,123 local bytes read: 847.7 KiB fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 10 remote merged chunks fetched: 0 remote blocks read: 0 data size total (min, med, max (stageId: taskId)) 119.2 MiB (11.8 MiB, 11.9 MiB, 12.1 MiB (stage 17416.0: task 24230)) local merged bytes read: 0.0 B number of partitions: 10 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written total (min, med, max (stageId: taskId)) 847.7 KiB (83.7 KiB, 84.6 KiB, 86.0 KiB (stage 17416.0: task 24230))"]; subgraph cluster11 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n2.2 s (214 ms, 219 ms, 231 ms (stage 17416.0: task 24237))"; 12 [labelType="html" label=" Project "]; 13 [labelType="html" label="Filter number of output rows: 2,231,123"]; 14 [labelType="html" label="ColumnarToRow number of output rows: 2,231,123 number of input batches: 551"]; } 15 [labelType="html" label="Scan parquet number of files read: 10 scan time total (min, med, max (stageId: taskId)) 57 ms (3 ms, 5 ms, 11 ms (stage 17416.0: task 24232)) metadata time: 0 ms size of files read: 105.5 MiB number of output rows: 2,231,123"]; subgraph cluster16 { isCluster="true"; label="WholeStageCodegen (4)\n \nduration: 985 ms"; 17 [labelType="html" label="Sort sort time: 0 ms peak memory: 16.1 MiB spill size: 0.0 B"]; } 18 [labelType="html" label="AQEShuffleRead number of partitions: 1 partition data size: 80.0 B number of coalesced partitions: 1"]; 19 [labelType="html" label="Exchange shuffle records written: 1 local merged chunks fetched: 0 shuffle write time: 0 ms remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 1 local bytes read: 76.0 B fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 1 remote merged chunks fetched: 0 remote blocks read: 0 data size: 32.0 B local merged bytes read: 0.0 B number of partitions: 10 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written: 76.0 B"]; subgraph cluster20 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: 4 ms"; 21 [labelType="html" label=" Project "]; 22 [labelType="html" label="Filter number of output rows: 1"]; 23 [labelType="html" label="ColumnarToRow number of output rows: 1 number of input batches: 1"]; } 24 [labelType="html" label="Scan parquet number of files read: 1 scan time: 2 ms metadata time: 0 ms size of files read: 2044.0 B number of output rows: 1"]; 1->0; 2->1; 3->2; 5->3; 6->5; 8->6; 9->8; 10->9; 12->10; 13->12; 14->13; 15->14; 17->6; 18->17; 19->18; 21->19; 22->21; 23->22; 24->23; }

AdaptiveSparkPlan isFinalPlan=true

SortAggregate(key=[], functions=[count(1), count(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018), min(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018), max(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)])

Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=719090]

SortAggregate(key=[], functions=[partial_count(1), partial_count(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018), partial_min(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018), partial_max(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)])

Project [COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018]

SortMergeJoin [COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008], [09_INFORMATION__204665_FINESS_204667#1391042], Inner

WholeStageCodegen (5)

Sort [COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008 ASC NULLS FIRST], false, 0

WholeStageCodegen (3)

AQEShuffleRead coalesced

Exchange hashpartitioning(COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008, 10), ENSURE_REQUIREMENTS, [plan_id=718975]

Project [COL_788D4A66_31DF_4E0F_911A_8BD9F9464719#1390865 AS COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008, COL_FC10C112_88B1_45B9_AA4E_B279540F2D09#1390873 AS COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018]

Filter isnotnull(COL_788D4A66_31DF_4E0F_911A_8BD9F9464719#1390865)

ColumnarToRow

WholeStageCodegen (1)

FileScan parquet [COL_788D4A66_31DF_4E0F_911A_8BD9F9464719#1390865,COL_FC10C112_88B1_45B9_AA4E_B279540F2D09#1390873] Batched: true, DataFilters: [isnotnull(COL_788D4A66_31DF_4E0F_911A_8BD9F9464719#1390865)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/00_D_DETERMINAT_204986], PartitionFilters: [], PushedFilters: [IsNotNull(COL_788D4A66_31DF_4E0F_911A_8BD9F9464719)], ReadSchema: struct<COL_788D4A66_31DF_4E0F_911A_8BD9F9464719:string,COL_FC10C112_88B1_45B9_AA4E_B279540F2D09:s...

Sort [09_INFORMATION__204665_FINESS_204667#1391042 ASC NULLS FIRST], false, 0

WholeStageCodegen (4)

AQEShuffleRead coalesced

Exchange hashpartitioning(09_INFORMATION__204665_FINESS_204667#1391042, 10), ENSURE_REQUIREMENTS, [plan_id=718996]

Project [Finess#1390812 AS 09_INFORMATION__204665_FINESS_204667#1391042]

Filter isnotnull(Finess#1390812)

ColumnarToRow

WholeStageCodegen (2)

FileScan parquet [Finess#1390812] Batched: true, DataFilters: [isnotnull(Finess#1390812)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/excel/cache/09_INFORMATION__204529_26897fa91350..., PartitionFilters: [], PushedFilters: [IsNotNull(Finess)], ReadSchema: struct<Finess:string>

Details

== Physical Plan ==
AdaptiveSparkPlan (36)
+- == Final Plan ==
   SortAggregate (22)
   +- ShuffleQueryStage (21), Statistics(sizeInBytes=72.0 B, rowCount=1)
      +- Exchange (20)
         +- SortAggregate (19)
            +- * Project (18)
               +- * SortMergeJoin Inner (17)
                  :- * Sort (8)
                  :  +- AQEShuffleRead (7)
                  :     +- ShuffleQueryStage (6), Statistics(sizeInBytes=119.2 MiB, rowCount=2.23E+6)
                  :        +- Exchange (5)
                  :           +- * Project (4)
                  :              +- * Filter (3)
                  :                 +- * ColumnarToRow (2)
                  :                    +- Scan parquet  (1)
                  +- * Sort (16)
                     +- AQEShuffleRead (15)
                        +- ShuffleQueryStage (14), Statistics(sizeInBytes=32.0 B, rowCount=1)
                           +- Exchange (13)
                              +- * Project (12)
                                 +- * Filter (11)
                                    +- * ColumnarToRow (10)
                                       +- Scan parquet  (9)
+- == Initial Plan ==
   SortAggregate (35)
   +- Exchange (34)
      +- SortAggregate (33)
         +- Project (32)
            +- SortMergeJoin Inner (31)
               :- Sort (26)
               :  +- Exchange (25)
               :     +- Project (24)
               :        +- Filter (23)
               :           +- Scan parquet  (1)
               +- Sort (30)
                  +- Exchange (29)
                     +- Project (28)
                        +- Filter (27)
                           +- Scan parquet  (9)


(1) Scan parquet 
Output [2]: [COL_788D4A66_31DF_4E0F_911A_8BD9F9464719#1390865, COL_FC10C112_88B1_45B9_AA4E_B279540F2D09#1390873]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/00_D_DETERMINAT_204986]
PushedFilters: [IsNotNull(COL_788D4A66_31DF_4E0F_911A_8BD9F9464719)]
ReadSchema: struct<COL_788D4A66_31DF_4E0F_911A_8BD9F9464719:string,COL_FC10C112_88B1_45B9_AA4E_B279540F2D09:string>

(2) ColumnarToRow [codegen id : 1]
Input [2]: [COL_788D4A66_31DF_4E0F_911A_8BD9F9464719#1390865, COL_FC10C112_88B1_45B9_AA4E_B279540F2D09#1390873]

(3) Filter [codegen id : 1]
Input [2]: [COL_788D4A66_31DF_4E0F_911A_8BD9F9464719#1390865, COL_FC10C112_88B1_45B9_AA4E_B279540F2D09#1390873]
Condition : isnotnull(COL_788D4A66_31DF_4E0F_911A_8BD9F9464719#1390865)

(4) Project [codegen id : 1]
Output [2]: [COL_788D4A66_31DF_4E0F_911A_8BD9F9464719#1390865 AS COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008, COL_FC10C112_88B1_45B9_AA4E_B279540F2D09#1390873 AS COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018]
Input [2]: [COL_788D4A66_31DF_4E0F_911A_8BD9F9464719#1390865, COL_FC10C112_88B1_45B9_AA4E_B279540F2D09#1390873]

(5) Exchange
Input [2]: [COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008, COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018]
Arguments: hashpartitioning(COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008, 10), ENSURE_REQUIREMENTS, [plan_id=718975]

(6) ShuffleQueryStage
Output [2]: [COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008, COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018]
Arguments: 0

(7) AQEShuffleRead
Input [2]: [COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008, COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018]
Arguments: coalesced

(8) Sort [codegen id : 3]
Input [2]: [COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008, COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018]
Arguments: [COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008 ASC NULLS FIRST], false, 0

(9) Scan parquet 
Output [1]: [Finess#1390812]
Batched: true
Location: InMemoryFileIndex [file:/data/input/depot/excel/cache/09_INFORMATION__204529_26897fa913503b7c2ff6c2e0bd71df5c/Info_Etab_init_v1.1 CHSF.xlsx_41dc788345dd715eba9a9d7feff1aa68]
PushedFilters: [IsNotNull(Finess)]
ReadSchema: struct<Finess:string>

(10) ColumnarToRow [codegen id : 2]
Input [1]: [Finess#1390812]

(11) Filter [codegen id : 2]
Input [1]: [Finess#1390812]
Condition : isnotnull(Finess#1390812)

(12) Project [codegen id : 2]
Output [1]: [Finess#1390812 AS 09_INFORMATION__204665_FINESS_204667#1391042]
Input [1]: [Finess#1390812]

(13) Exchange
Input [1]: [09_INFORMATION__204665_FINESS_204667#1391042]
Arguments: hashpartitioning(09_INFORMATION__204665_FINESS_204667#1391042, 10), ENSURE_REQUIREMENTS, [plan_id=718996]

(14) ShuffleQueryStage
Output [1]: [09_INFORMATION__204665_FINESS_204667#1391042]
Arguments: 1

(15) AQEShuffleRead
Input [1]: [09_INFORMATION__204665_FINESS_204667#1391042]
Arguments: coalesced

(16) Sort [codegen id : 4]
Input [1]: [09_INFORMATION__204665_FINESS_204667#1391042]
Arguments: [09_INFORMATION__204665_FINESS_204667#1391042 ASC NULLS FIRST], false, 0

(17) SortMergeJoin [codegen id : 5]
Left keys [1]: [COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008]
Right keys [1]: [09_INFORMATION__204665_FINESS_204667#1391042]
Join type: Inner
Join condition: None

(18) Project [codegen id : 5]
Output [1]: [COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018]
Input [3]: [COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008, COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018, 09_INFORMATION__204665_FINESS_204667#1391042]

(19) SortAggregate
Input [1]: [COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018]
Keys: []
Functions [4]: [partial_count(1), partial_count(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018), partial_min(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018), partial_max(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)]
Aggregate Attributes [4]: [count#1391176L, count#1391177L, min#1391178, max#1391179]
Results [4]: [count#1391180L, count#1391181L, min#1391182, max#1391183]

(20) Exchange
Input [4]: [count#1391180L, count#1391181L, min#1391182, max#1391183]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=719090]

(21) ShuffleQueryStage
Output [4]: [count#1391180L, count#1391181L, min#1391182, max#1391183]
Arguments: 2

(22) SortAggregate
Input [4]: [count#1391180L, count#1391181L, min#1391182, max#1391183]
Keys: []
Functions [4]: [count(1), count(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018), min(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018), max(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)]
Aggregate Attributes [4]: [count(1)#1391140L, count(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)#1391143L, min(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)#1391146, max(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)#1391148]
Results [4]: [count(1)#1391140L AS COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD1#1391142L, count(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)#1391143L AS COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD2#1391145L, min(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)#1391146 AS COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD4#1391147, max(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)#1391148 AS COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD5#1391149]

(23) Filter
Input [2]: [COL_788D4A66_31DF_4E0F_911A_8BD9F9464719#1390865, COL_FC10C112_88B1_45B9_AA4E_B279540F2D09#1390873]
Condition : isnotnull(COL_788D4A66_31DF_4E0F_911A_8BD9F9464719#1390865)

(24) Project
Output [2]: [COL_788D4A66_31DF_4E0F_911A_8BD9F9464719#1390865 AS COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008, COL_FC10C112_88B1_45B9_AA4E_B279540F2D09#1390873 AS COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018]
Input [2]: [COL_788D4A66_31DF_4E0F_911A_8BD9F9464719#1390865, COL_FC10C112_88B1_45B9_AA4E_B279540F2D09#1390873]

(25) Exchange
Input [2]: [COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008, COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018]
Arguments: hashpartitioning(COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008, 10), ENSURE_REQUIREMENTS, [plan_id=718931]

(26) Sort
Input [2]: [COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008, COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018]
Arguments: [COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008 ASC NULLS FIRST], false, 0

(27) Filter
Input [1]: [Finess#1390812]
Condition : isnotnull(Finess#1390812)

(28) Project
Output [1]: [Finess#1390812 AS 09_INFORMATION__204665_FINESS_204667#1391042]
Input [1]: [Finess#1390812]

(29) Exchange
Input [1]: [09_INFORMATION__204665_FINESS_204667#1391042]
Arguments: hashpartitioning(09_INFORMATION__204665_FINESS_204667#1391042, 10), ENSURE_REQUIREMENTS, [plan_id=718932]

(30) Sort
Input [1]: [09_INFORMATION__204665_FINESS_204667#1391042]
Arguments: [09_INFORMATION__204665_FINESS_204667#1391042 ASC NULLS FIRST], false, 0

(31) SortMergeJoin
Left keys [1]: [COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008]
Right keys [1]: [09_INFORMATION__204665_FINESS_204667#1391042]
Join type: Inner
Join condition: None

(32) Project
Output [1]: [COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018]
Input [3]: [COL_F522EF71_EF5D_46C0_B0EC_B3EB638B97F7#1391008, COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018, 09_INFORMATION__204665_FINESS_204667#1391042]

(33) SortAggregate
Input [1]: [COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018]
Keys: []
Functions [4]: [partial_count(1), partial_count(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018), partial_min(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018), partial_max(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)]
Aggregate Attributes [4]: [count#1391176L, count#1391177L, min#1391178, max#1391179]
Results [4]: [count#1391180L, count#1391181L, min#1391182, max#1391183]

(34) Exchange
Input [4]: [count#1391180L, count#1391181L, min#1391182, max#1391183]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=718939]

(35) SortAggregate
Input [4]: [count#1391180L, count#1391181L, min#1391182, max#1391183]
Keys: []
Functions [4]: [count(1), count(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018), min(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018), max(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)]
Aggregate Attributes [4]: [count(1)#1391140L, count(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)#1391143L, min(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)#1391146, max(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)#1391148]
Results [4]: [count(1)#1391140L AS COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD1#1391142L, count(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)#1391143L AS COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD2#1391145L, min(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)#1391146 AS COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD4#1391147, max(COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD#1391018)#1391148 AS COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD5#1391149]

(36) AdaptiveSparkPlan
Output [4]: [COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD1#1391142L, COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD2#1391145L, COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD4#1391147, COL_2C09E3B7_4086_45D3_8676_F1D58494C2DD5#1391149]
Arguments: isFinalPlan=true

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet