SP1 - Details for Query 4438

Submitted Time: 2026/03/25 06:38:16
Duration: 0.1 s
Succeeded Jobs: 6836 6838 6839

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" CollectLimit "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n70 ms (5 ms, 7 ms, 9 ms (stage 13300.0: task 18853))"; 2 [labelType="html" label=" Project "]; 3 [labelType="html" label="Filter number of output rows: 0"]; 4 [labelType="html" label="ColumnarToRow number of output rows: 0 number of input batches: 0"]; } 5 [labelType="html" label="Scan parquet number of files read: 10 scan time total (min, med, max (stageId: taskId)) 68 ms (5 ms, 7 ms, 9 ms (stage 13300.0: task 18853)) metadata time: 0 ms size of files read: 9.6 MiB number of output rows: 0"]; 2->0; 3->2; 4->3; 5->4; }

CollectLimit 500000

Project [COL_4ECA949F_2DA4_41C5_80E1_520DEFC8D1EA#1056654 AS COL_C8642779_1624_4BD4_B482_48F42FD0AA1A#1057298, COL_F76265AB_8748_4114_92B4_00E37DE25D34#1056663 AS COL_410A59DB_8913_453E_A7DE_094E71CF93CF#1057299, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#1056676 AS COL_FD29262D_E254_4F93_9270_D614DBF333CF#1057300, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#1056686 AS COL_B8C4BF90_6934_4D20_A082_4DE58551A79B#1057301, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#1056685 AS COL_BD662B0A_72EE_4962_930C_1D34EABDD684#1057302, COL_580D5890_2839_4108_A636_8E25F97E50E0#1056666 AS COL_D102E1C0_6469_45E9_B151_12452931BD68#1057303, COL_97360DFB_D20B_4DFF_A68A_57C41867D04E#1056675 AS COL_30016DE3_D7A1_48B1_ABDF_1A3F0862D29B#1057304, COL_1DDA21A4_0127_416B_A5DB_7EDD9CC61B56#1056658 AS COL_E3C81250_89D0_4F3A_8B7B_D6EFAE425D6D#1057305]

Filter ((COL_F76265AB_8748_4114_92B4_00E37DE25D34#1056663 <=> 250000001) AND ((COL_1DDA21A4_0127_416B_A5DB_7EDD9CC61B56#1056658 <=> C) OR (COL_1DDA21A4_0127_416B_A5DB_7EDD9CC61B56#1056658 <=> B)))

ColumnarToRow

WholeStageCodegen (1)

FileScan parquet [COL_4ECA949F_2DA4_41C5_80E1_520DEFC8D1EA#1056654,COL_1DDA21A4_0127_416B_A5DB_7EDD9CC61B56#1056658,COL_F76265AB_8748_4114_92B4_00E37DE25D34#1056663,COL_580D5890_2839_4108_A636_8E25F97E50E0#1056666,COL_97360DFB_D20B_4DFF_A68A_57C41867D04E#1056675,COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#1056676,COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#1056685,COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#1056686] Batched: true, DataFilters: [(COL_F76265AB_8748_4114_92B4_00E37DE25D34#1056663 <=> 250000001), ((COL_1DDA21A4_0127_416B_A5DB_..., Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/02_A_ELIGIBLE_203080], PartitionFilters: [], PushedFilters: [EqualNullSafe(COL_F76265AB_8748_4114_92B4_00E37DE25D34,250000001), Or(EqualNullSafe(COL_1DDA21A4..., ReadSchema: struct<COL_4ECA949F_2DA4_41C5_80E1_520DEFC8D1EA:string,COL_1DDA21A4_0127_416B_A5DB_7EDD9CC61B56:s...

Details

== Physical Plan ==
CollectLimit (5)
+- * Project (4)
   +- * Filter (3)
      +- * ColumnarToRow (2)
         +- Scan parquet  (1)


(1) Scan parquet 
Output [8]: [COL_4ECA949F_2DA4_41C5_80E1_520DEFC8D1EA#1056654, COL_1DDA21A4_0127_416B_A5DB_7EDD9CC61B56#1056658, COL_F76265AB_8748_4114_92B4_00E37DE25D34#1056663, COL_580D5890_2839_4108_A636_8E25F97E50E0#1056666, COL_97360DFB_D20B_4DFF_A68A_57C41867D04E#1056675, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#1056676, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#1056685, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#1056686]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/02_A_ELIGIBLE_203080]
PushedFilters: [EqualNullSafe(COL_F76265AB_8748_4114_92B4_00E37DE25D34,250000001), Or(EqualNullSafe(COL_1DDA21A4_0127_416B_A5DB_7EDD9CC61B56,C),EqualNullSafe(COL_1DDA21A4_0127_416B_A5DB_7EDD9CC61B56,B))]
ReadSchema: struct<COL_4ECA949F_2DA4_41C5_80E1_520DEFC8D1EA:string,COL_1DDA21A4_0127_416B_A5DB_7EDD9CC61B56:string,COL_F76265AB_8748_4114_92B4_00E37DE25D34:string,COL_580D5890_2839_4108_A636_8E25F97E50E0:double,COL_97360DFB_D20B_4DFF_A68A_57C41867D04E:double,COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC:string,COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4:string,COL_B2D69078_23A7_4B36_BBF2_3F9C29840464:string>

(2) ColumnarToRow [codegen id : 1]
Input [8]: [COL_4ECA949F_2DA4_41C5_80E1_520DEFC8D1EA#1056654, COL_1DDA21A4_0127_416B_A5DB_7EDD9CC61B56#1056658, COL_F76265AB_8748_4114_92B4_00E37DE25D34#1056663, COL_580D5890_2839_4108_A636_8E25F97E50E0#1056666, COL_97360DFB_D20B_4DFF_A68A_57C41867D04E#1056675, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#1056676, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#1056685, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#1056686]

(3) Filter [codegen id : 1]
Input [8]: [COL_4ECA949F_2DA4_41C5_80E1_520DEFC8D1EA#1056654, COL_1DDA21A4_0127_416B_A5DB_7EDD9CC61B56#1056658, COL_F76265AB_8748_4114_92B4_00E37DE25D34#1056663, COL_580D5890_2839_4108_A636_8E25F97E50E0#1056666, COL_97360DFB_D20B_4DFF_A68A_57C41867D04E#1056675, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#1056676, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#1056685, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#1056686]
Condition : ((COL_F76265AB_8748_4114_92B4_00E37DE25D34#1056663 <=> 250000001) AND ((COL_1DDA21A4_0127_416B_A5DB_7EDD9CC61B56#1056658 <=> C) OR (COL_1DDA21A4_0127_416B_A5DB_7EDD9CC61B56#1056658 <=> B)))

(4) Project [codegen id : 1]
Output [8]: [COL_4ECA949F_2DA4_41C5_80E1_520DEFC8D1EA#1056654 AS COL_C8642779_1624_4BD4_B482_48F42FD0AA1A#1057298, COL_F76265AB_8748_4114_92B4_00E37DE25D34#1056663 AS COL_410A59DB_8913_453E_A7DE_094E71CF93CF#1057299, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#1056676 AS COL_FD29262D_E254_4F93_9270_D614DBF333CF#1057300, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#1056686 AS COL_B8C4BF90_6934_4D20_A082_4DE58551A79B#1057301, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#1056685 AS COL_BD662B0A_72EE_4962_930C_1D34EABDD684#1057302, COL_580D5890_2839_4108_A636_8E25F97E50E0#1056666 AS COL_D102E1C0_6469_45E9_B151_12452931BD68#1057303, COL_97360DFB_D20B_4DFF_A68A_57C41867D04E#1056675 AS COL_30016DE3_D7A1_48B1_ABDF_1A3F0862D29B#1057304, COL_1DDA21A4_0127_416B_A5DB_7EDD9CC61B56#1056658 AS COL_E3C81250_89D0_4F3A_8B7B_D6EFAE425D6D#1057305]
Input [8]: [COL_4ECA949F_2DA4_41C5_80E1_520DEFC8D1EA#1056654, COL_1DDA21A4_0127_416B_A5DB_7EDD9CC61B56#1056658, COL_F76265AB_8748_4114_92B4_00E37DE25D34#1056663, COL_580D5890_2839_4108_A636_8E25F97E50E0#1056666, COL_97360DFB_D20B_4DFF_A68A_57C41867D04E#1056675, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#1056676, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#1056685, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#1056686]

(5) CollectLimit
Input [8]: [COL_C8642779_1624_4BD4_B482_48F42FD0AA1A#1057298, COL_410A59DB_8913_453E_A7DE_094E71CF93CF#1057299, COL_FD29262D_E254_4F93_9270_D614DBF333CF#1057300, COL_B8C4BF90_6934_4D20_A082_4DE58551A79B#1057301, COL_BD662B0A_72EE_4962_930C_1D34EABDD684#1057302, COL_D102E1C0_6469_45E9_B151_12452931BD68#1057303, COL_30016DE3_D7A1_48B1_ABDF_1A3F0862D29B#1057304, COL_E3C81250_89D0_4F3A_8B7B_D6EFAE425D6D#1057305]
Arguments: 500000

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet