SP1 - Details for Query 12038

Submitted Time: 2026/01/24 21:07:45
Duration: 0.4 s
Succeeded Jobs: 17910

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label="Execute InsertIntoHadoopFsRelationCommand task commit time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 32516.0: task 39480)) number of written files: 1 job commit time: 4 ms number of output rows: 0 number of dynamic part: 0 written output: 1305.0 B"]; 1 [labelType="html" label=" WriteFiles "]; subgraph cluster2 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n578 ms (18 ms, 48 ms, 138 ms (stage 32516.0: task 39481))"; 3 [labelType="html" label=" Project "]; 4 [labelType="html" label=" Project "]; } 5 [labelType="html" label="CartesianProduct number of output rows: 0"]; subgraph cluster6 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n343 ms (10 ms, 31 ms, 72 ms (stage 32516.0: task 39477))"; 7 [labelType="html" label=" Project "]; 8 [labelType="html" label="Filter number of output rows: 0"]; 9 [labelType="html" label="ColumnarToRow number of output rows: 474,437 number of input batches: 120"]; } 10 [labelType="html" label="Scan parquet number of files read: 10 scan time total (min, med, max (stageId: taskId)) 39 ms (3 ms, 4 ms, 8 ms (stage 32516.0: task 39481)) metadata time: 0 ms size of files read: 21.5 MiB number of output rows: 474,437"]; subgraph cluster11 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n24 ms (2 ms, 2 ms, 5 ms (stage 32516.0: task 39481))"; 12 [labelType="html" label=" Project "]; 13 [labelType="html" label="Filter number of output rows: 10"]; } 14 [labelType="html" label="Scan csv number of output rows: 10 number of files read: 1 metadata time: 0 ms size of files read: 163.0 B"]; 1->0; 3->1; 4->3; 5->4; 7->5; 8->7; 9->8; 10->9; 12->5; 13->12; 14->13; }

Execute InsertIntoHadoopFsRelationCommand file:/data/output/cache/parquet/datablock/05_GR8_DOUBLON__161603, false, Parquet, [path=file:///data/output/cache/parquet/datablock/05_GR8_DOUBLON__161603], Overwrite, [COL_6A75989B_D287_4B6C_893B_7D78A3023D30, COL_129360FD_BB12_46AD_A00B_165330B80A60, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C, COL_910DC838_241D_4365_A8A3_070714C6238B, COL_481586E2_E506_44E5_9DC4_79601A1736D6, COL_48A39647_D8F8_4138_9D64_310E44B62D64, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015]

WriteFiles

Project [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2463532, str_if_with_rule(((COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048 <=> SUN) OR (COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048 <=> SUF)), str_concat(array(COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, -!, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, , )), str_concat(array(COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, -!, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, _, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058))) AS COL_910DC838_241D_4365_A8A3_070714C6238B#2463783, str_concat(array(Suppression de Forfait@, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, @, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058)) AS COL_481586E2_E506_44E5_9DC4_79601A1736D6#2463784, str_concat([Suppression Doublon du RSF]) AS COL_48A39647_D8F8_4138_9D64_310E44B62D64#2463785, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475]

Project [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058, str_join(array(str_extract(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, RIGHT, 2), str_extract_middle(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, 5, 2), str_extract(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, LEFT, 4)), ) AS COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, str_create(RC DBL-RSF) AS COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2463532]

WholeStageCodegen (3)

CartesianProduct

Project [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2454796 AS COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2454800 AS COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2454813 AS COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2454822 AS COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2454823 AS COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058]

Filter ((((CASE WHEN isnull(COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840) THEN false ELSE (COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840 > 1) END OR CASE WHEN isnull(COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841) THEN false ELSE (COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841 > 1) END) OR CASE WHEN isnull(COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842) THEN false ELSE (COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842 > 1) END) OR CASE WHEN isnull(COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843) THEN false ELSE (COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843 > 1) END) OR CASE WHEN isnull(COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844) THEN false ELSE (COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844 > 1) END)

ColumnarToRow

WholeStageCodegen (1)

FileScan parquet [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2454796,COL_F76265AB_8748_4114_92B4_00E37DE25D34#2454800,COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2454813,COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2454822,COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2454823,COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840,COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841,COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842,COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843,COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844] Batched: true, DataFilters: [((((CASE WHEN isnull(COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840) THEN false ELSE (COL_E7D4..., Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/02_A_ELIGIBLE_161654], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB:string,COL_F76265AB_8748_4114_92B4_00E37DE25D34:s...

Project

Filter ((isnotnull(REGLES#2462609) AND (RC-DBL-RSF = REGLES#2462609)) AND (APPLICABLE#2462610 <=> O))

WholeStageCodegen (2)

FileScan csv [REGLES#2462609,APPLICABLE#2462610] Batched: false, DataFilters: [isnotnull(REGLES#2462609), (RC-DBL-RSF = REGLES#2462609), (APPLICABLE#2462610 <=> O)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/tmp/REGLES_APPLICAB_161411/REGLES-A-TTT.csv], PartitionFilters: [], PushedFilters: [IsNotNull(REGLES), EqualTo(REGLES,RC-DBL-RSF), EqualNullSafe(APPLICABLE,O)], ReadSchema: struct<REGLES:string,APPLICABLE:string>

Details

== Physical Plan ==
Execute InsertIntoHadoopFsRelationCommand (12)
+- WriteFiles (11)
   +- * Project (10)
      +- * Project (9)
         +- CartesianProduct Inner (8)
            :- * Project (4)
            :  +- * Filter (3)
            :     +- * ColumnarToRow (2)
            :        +- Scan parquet  (1)
            +- * Project (7)
               +- * Filter (6)
                  +- Scan csv  (5)


(1) Scan parquet 
Output [10]: [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2454796, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2454800, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2454813, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2454822, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2454823, COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840, COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841, COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842, COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843, COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/02_A_ELIGIBLE_161654]
ReadSchema: struct<COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB:string,COL_F76265AB_8748_4114_92B4_00E37DE25D34:string,COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC:string,COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4:string,COL_B2D69078_23A7_4B36_BBF2_3F9C29840464:string,COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846:int,COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B:int,COL_B8D74F46_C755_4417_9E18_FAA760CD8457:int,COL_51335C75_C8A9_4932_8985_BA9E68B80F3A:int,COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C:int>

(2) ColumnarToRow [codegen id : 1]
Input [10]: [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2454796, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2454800, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2454813, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2454822, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2454823, COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840, COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841, COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842, COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843, COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844]

(3) Filter [codegen id : 1]
Input [10]: [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2454796, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2454800, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2454813, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2454822, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2454823, COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840, COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841, COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842, COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843, COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844]
Condition : ((((CASE WHEN isnull(COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840) THEN false ELSE (COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840 > 1) END OR CASE WHEN isnull(COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841) THEN false ELSE (COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841 > 1) END) OR CASE WHEN isnull(COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842) THEN false ELSE (COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842 > 1) END) OR CASE WHEN isnull(COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843) THEN false ELSE (COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843 > 1) END) OR CASE WHEN isnull(COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844) THEN false ELSE (COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844 > 1) END)

(4) Project [codegen id : 1]
Output [5]: [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2454796 AS COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2454800 AS COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2454813 AS COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2454822 AS COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2454823 AS COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058]
Input [10]: [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2454796, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2454800, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2454813, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2454822, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2454823, COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840, COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841, COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842, COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843, COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844]

(5) Scan csv 
Output [2]: [REGLES#2462609, APPLICABLE#2462610]
Batched: false
Location: InMemoryFileIndex [file:/data/input/depot/csv/tmp/REGLES_APPLICAB_161411/REGLES-A-TTT.csv]
PushedFilters: [IsNotNull(REGLES), EqualTo(REGLES,RC-DBL-RSF), EqualNullSafe(APPLICABLE,O)]
ReadSchema: struct<REGLES:string,APPLICABLE:string>

(6) Filter [codegen id : 2]
Input [2]: [REGLES#2462609, APPLICABLE#2462610]
Condition : ((isnotnull(REGLES#2462609) AND (RC-DBL-RSF = REGLES#2462609)) AND (APPLICABLE#2462610 <=> O))

(7) Project [codegen id : 2]
Output: []
Input [2]: [REGLES#2462609, APPLICABLE#2462610]

(8) CartesianProduct
Join type: Inner
Join condition: None

(9) Project [codegen id : 3]
Output [6]: [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058, str_join(array(str_extract(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, RIGHT, 2), str_extract_middle(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, 5, 2), str_extract(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, LEFT, 4)), ) AS COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, str_create(RC DBL-RSF) AS COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2463532]
Input [5]: [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058]

(10) Project [codegen id : 3]
Output [7]: [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2463532, str_if_with_rule(((COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048 <=> SUN) OR (COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048 <=> SUF)), str_concat(array(COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, -!, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, , )), str_concat(array(COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, -!, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, _, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058))) AS COL_910DC838_241D_4365_A8A3_070714C6238B#2463783, str_concat(array(Suppression de Forfait@, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, @, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058)) AS COL_481586E2_E506_44E5_9DC4_79601A1736D6#2463784, str_concat([Suppression Doublon du RSF]) AS COL_48A39647_D8F8_4138_9D64_310E44B62D64#2463785, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475]
Input [6]: [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2463532]

(11) WriteFiles
Input [7]: [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2463532, COL_910DC838_241D_4365_A8A3_070714C6238B#2463783, COL_481586E2_E506_44E5_9DC4_79601A1736D6#2463784, COL_48A39647_D8F8_4138_9D64_310E44B62D64#2463785, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475]

(12) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: file:/data/output/cache/parquet/datablock/05_GR8_DOUBLON__161603, false, Parquet, [path=file:///data/output/cache/parquet/datablock/05_GR8_DOUBLON__161603], Overwrite, [COL_6A75989B_D287_4B6C_893B_7D78A3023D30, COL_129360FD_BB12_46AD_A00B_165330B80A60, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C, COL_910DC838_241D_4365_A8A3_070714C6238B, COL_481586E2_E506_44E5_9DC4_79601A1736D6, COL_48A39647_D8F8_4138_9D64_310E44B62D64, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015]

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet