SP1 - Details for Query 2512

Submitted Time: 2026/06/11 09:55:35
Duration: 0.3 s
Succeeded Jobs: 3814 3815 3816

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (6)\n \nduration: 0 ms"; 2 [labelType="html" label="HashAggregate time in aggregation build: 0 ms number of output rows: 1"]; } 3 [labelType="html" label="Exchange shuffle records written: 1 local merged chunks fetched: 0 shuffle write time: 0 ms remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 1 local bytes read: 59.0 B fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 1 remote merged chunks fetched: 0 remote blocks read: 0 data size: 16.0 B local merged bytes read: 0.0 B number of partitions: 1 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written: 59.0 B"]; subgraph cluster4 { isCluster="true"; label="WholeStageCodegen (5)\n \nduration: 0 ms"; 5 [labelType="html" label="HashAggregate time in aggregation build: 0 ms number of output rows: 1"]; 6 [labelType="html" label=" Project "]; 7 [labelType="html" label="Filter number of output rows: 2"]; } 8 [labelType="html" label="Window spill size: 0.0 B"]; subgraph cluster9 { isCluster="true"; label="WholeStageCodegen (4)\n \nduration: 22 ms"; 10 [labelType="html" label="Sort sort time: 0 ms peak memory: 16.1 MiB spill size: 0.0 B"]; } 11 [labelType="html" label="AQEShuffleRead number of partitions: 1 partition data size: 271.0 B number of coalesced partitions: 1"]; 12 [labelType="html" label="Exchange shuffle records written: 7 local merged chunks fetched: 0 shuffle write time: 0 ms remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 7 local bytes read: 252.0 B fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 1 remote merged chunks fetched: 0 remote blocks read: 0 data size: 512.0 B local merged bytes read: 0.0 B number of partitions: 10 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written: 252.0 B"]; subgraph cluster13 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: 19 ms"; 14 [labelType="html" label=" Project "]; } 15 [labelType="html" label="CartesianProduct number of output rows: 7"]; subgraph cluster16 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: 34 ms"; 17 [labelType="html" label=" Project "]; 18 [labelType="html" label="Filter number of output rows: 7"]; 19 [labelType="html" label="ColumnarToRow number of output rows: 7 number of input batches: 1"]; } 20 [labelType="html" label="Scan parquet number of files read: 1 scan time: 3 ms metadata time: 0 ms size of files read: 20.3 KiB number of output rows: 7"]; subgraph cluster21 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: 6 ms"; 22 [labelType="html" label=" Project "]; 23 [labelType="html" label="Filter number of output rows: 1"]; } 24 [labelType="html" label="Scan csv number of output rows: 1 number of files read: 1 metadata time: 0 ms size of files read: 174.0 B"]; 2->0; 3->2; 5->3; 6->5; 7->6; 8->7; 10->8; 11->10; 12->11; 14->12; 15->14; 17->15; 18->17; 19->18; 20->19; 22->15; 23->22; 24->23; }

AdaptiveSparkPlan isFinalPlan=true

HashAggregate(keys=[], functions=[count(1)])

WholeStageCodegen (6)

Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=488535]

HashAggregate(keys=[], functions=[partial_count(1)])

Project

Filter (COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864 <=> 1)

WholeStageCodegen (5)

Window [row_number() windowspecdefinition(COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864], [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792], [COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST]

Sort [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618 ASC NULLS FIRST, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792 ASC NULLS FIRST, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST], false, 0

WholeStageCodegen (4)

AQEShuffleRead coalesced

Exchange hashpartitioning(COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, 10), ENSURE_REQUIREMENTS, [plan_id=488458]

Project [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, str_extract_by_str(COL_1BD1CF4D_D1E1_4C54_94F8_0754D4126CAC#791620, *, RIGHT) AS COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]

WholeStageCodegen (3)

CartesianProduct

Project [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291 AS COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333 AS COL_1BD1CF4D_D1E1_4C54_94F8_0754D4126CAC#791620, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313 AS COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]

Filter (COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291 <=> 011040518)

ColumnarToRow

WholeStageCodegen (1)

FileScan parquet [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291,COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313,COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333] Batched: true, DataFilters: [(COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291 <=> 011040518)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/00_B_ACTES_PATI_210234], PartitionFilters: [], PushedFilters: [EqualNullSafe(COL_64ED17E5_A991_405B_AEED_8A2977283F76,011040518)], ReadSchema: struct<COL_64ED17E5_A991_405B_AEED_8A2977283F76:string,COL_7D270645_B926_42AF_A110_EB3A1A53031B:s...

Project

Filter ((isnotnull(REGLES#791416) AND (RC-SSN-SSF = REGLES#791416)) AND (APPLICABLE#791417 <=> O))

WholeStageCodegen (2)

FileScan csv [REGLES#791416,APPLICABLE#791417] Batched: false, DataFilters: [isnotnull(REGLES#791416), (RC-SSN-SSF = REGLES#791416), (APPLICABLE#791417 <=> O)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/tmp/REGLES_APPLICAB_209983/REGLES-A-TTT new..., PartitionFilters: [], PushedFilters: [IsNotNull(REGLES), EqualTo(REGLES,RC-SSN-SSF), EqualNullSafe(APPLICABLE,O)], ReadSchema: struct<REGLES:string,APPLICABLE:string>

Details

== Physical Plan ==
AdaptiveSparkPlan (35)
+- == Final Plan ==
   * HashAggregate (20)
   +- ShuffleQueryStage (19), Statistics(sizeInBytes=16.0 B, rowCount=1)
      +- Exchange (18)
         +- * HashAggregate (17)
            +- * Project (16)
               +- * Filter (15)
                  +- Window (14)
                     +- * Sort (13)
                        +- AQEShuffleRead (12)
                           +- ShuffleQueryStage (11), Statistics(sizeInBytes=512.0 B, rowCount=7)
                              +- Exchange (10)
                                 +- * Project (9)
                                    +- CartesianProduct Inner (8)
                                       :- * Project (4)
                                       :  +- * Filter (3)
                                       :     +- * ColumnarToRow (2)
                                       :        +- Scan parquet  (1)
                                       +- * Project (7)
                                          +- * Filter (6)
                                             +- Scan csv  (5)
+- == Initial Plan ==
   HashAggregate (34)
   +- Exchange (33)
      +- HashAggregate (32)
         +- Project (31)
            +- Filter (30)
               +- Window (29)
                  +- Sort (28)
                     +- Exchange (27)
                        +- Project (26)
                           +- CartesianProduct Inner (25)
                              :- Project (22)
                              :  +- Filter (21)
                              :     +- Scan parquet  (1)
                              +- Project (24)
                                 +- Filter (23)
                                    +- Scan csv  (5)


(1) Scan parquet 
Output [3]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/00_B_ACTES_PATI_210234]
PushedFilters: [EqualNullSafe(COL_64ED17E5_A991_405B_AEED_8A2977283F76,011040518)]
ReadSchema: struct<COL_64ED17E5_A991_405B_AEED_8A2977283F76:string,COL_7D270645_B926_42AF_A110_EB3A1A53031B:string,COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28:string>

(2) ColumnarToRow [codegen id : 1]
Input [3]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333]

(3) Filter [codegen id : 1]
Input [3]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333]
Condition : (COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291 <=> 011040518)

(4) Project [codegen id : 1]
Output [3]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291 AS COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333 AS COL_1BD1CF4D_D1E1_4C54_94F8_0754D4126CAC#791620, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313 AS COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Input [3]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333]

(5) Scan csv 
Output [2]: [REGLES#791416, APPLICABLE#791417]
Batched: false
Location: InMemoryFileIndex [file:/data/input/depot/csv/tmp/REGLES_APPLICAB_209983/REGLES-A-TTT new.csv]
PushedFilters: [IsNotNull(REGLES), EqualTo(REGLES,RC-SSN-SSF), EqualNullSafe(APPLICABLE,O)]
ReadSchema: struct<REGLES:string,APPLICABLE:string>

(6) Filter [codegen id : 2]
Input [2]: [REGLES#791416, APPLICABLE#791417]
Condition : ((isnotnull(REGLES#791416) AND (RC-SSN-SSF = REGLES#791416)) AND (APPLICABLE#791417 <=> O))

(7) Project [codegen id : 2]
Output: []
Input [2]: [REGLES#791416, APPLICABLE#791417]

(8) CartesianProduct
Join type: Inner
Join condition: None

(9) Project [codegen id : 3]
Output [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, str_extract_by_str(COL_1BD1CF4D_D1E1_4C54_94F8_0754D4126CAC#791620, *, RIGHT) AS COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_1BD1CF4D_D1E1_4C54_94F8_0754D4126CAC#791620, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]

(10) Exchange
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Arguments: hashpartitioning(COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, 10), ENSURE_REQUIREMENTS, [plan_id=488458]

(11) ShuffleQueryStage
Output [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Arguments: 0

(12) AQEShuffleRead
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Arguments: coalesced

(13) Sort [codegen id : 4]
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Arguments: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618 ASC NULLS FIRST, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792 ASC NULLS FIRST, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST], false, 0

(14) Window
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Arguments: [row_number() windowspecdefinition(COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864], [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792], [COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST]

(15) Filter [codegen id : 5]
Input [4]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623, COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864]
Condition : (COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864 <=> 1)

(16) Project [codegen id : 5]
Output: []
Input [4]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623, COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864]

(17) HashAggregate [codegen id : 5]
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#791995L]
Results [1]: [count#791996L]

(18) Exchange
Input [1]: [count#791996L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=488535]

(19) ShuffleQueryStage
Output [1]: [count#791996L]
Arguments: 1

(20) HashAggregate [codegen id : 6]
Input [1]: [count#791996L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#791992L]
Results [1]: [count(1)#791992L AS count#791993L]

(21) Filter
Input [3]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333]
Condition : (COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291 <=> 011040518)

(22) Project
Output [3]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291 AS COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333 AS COL_1BD1CF4D_D1E1_4C54_94F8_0754D4126CAC#791620, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313 AS COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Input [3]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333]

(23) Filter
Input [2]: [REGLES#791416, APPLICABLE#791417]
Condition : ((isnotnull(REGLES#791416) AND (RC-SSN-SSF = REGLES#791416)) AND (APPLICABLE#791417 <=> O))

(24) Project
Output: []
Input [2]: [REGLES#791416, APPLICABLE#791417]

(25) CartesianProduct
Join type: Inner
Join condition: None

(26) Project
Output [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, str_extract_by_str(COL_1BD1CF4D_D1E1_4C54_94F8_0754D4126CAC#791620, *, RIGHT) AS COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_1BD1CF4D_D1E1_4C54_94F8_0754D4126CAC#791620, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]

(27) Exchange
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Arguments: hashpartitioning(COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, 10), ENSURE_REQUIREMENTS, [plan_id=488406]

(28) Sort
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Arguments: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618 ASC NULLS FIRST, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792 ASC NULLS FIRST, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST], false, 0

(29) Window
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Arguments: [row_number() windowspecdefinition(COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864], [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792], [COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST]

(30) Filter
Input [4]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623, COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864]
Condition : (COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864 <=> 1)

(31) Project
Output: []
Input [4]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623, COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864]

(32) HashAggregate
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#791995L]
Results [1]: [count#791996L]

(33) Exchange
Input [1]: [count#791996L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=488413]

(34) HashAggregate
Input [1]: [count#791996L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#791992L]
Results [1]: [count(1)#791992L AS count#791993L]

(35) AdaptiveSparkPlan
Output [1]: [count#791993L]
Arguments: isFinalPlan=true

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet