SP1 - Details for Query 11343

Submitted Time: 2026/01/24 15:06:57
Duration: 0.5 s
Succeeded Jobs: 16883

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label="Execute InsertIntoHadoopFsRelationCommand task commit time total (min, med, max (stageId: taskId)) 10 ms (0 ms, 0 ms, 7 ms (stage 30825.0: task 36691)) number of written files: 1 job commit time: 5 ms number of output rows: 0 number of dynamic part: 0 written output: 1305.0 B"]; 1 [labelType="html" label=" WriteFiles "]; subgraph cluster2 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n988 ms (30 ms, 94 ms, 198 ms (stage 30825.0: task 36683))"; 3 [labelType="html" label=" Project "]; 4 [labelType="html" label=" Project "]; } 5 [labelType="html" label="CartesianProduct number of output rows: 0"]; subgraph cluster6 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n699 ms (16 ms, 88 ms, 131 ms (stage 30825.0: task 36682))"; 7 [labelType="html" label=" Project "]; 8 [labelType="html" label="Filter number of output rows: 0"]; 9 [labelType="html" label="ColumnarToRow number of output rows: 474,437 number of input batches: 120"]; } 10 [labelType="html" label="Scan parquet number of files read: 10 scan time total (min, med, max (stageId: taskId)) 120 ms (5 ms, 12 ms, 25 ms (stage 30825.0: task 36692)) metadata time: 0 ms size of files read: 21.5 MiB number of output rows: 474,437"]; subgraph cluster11 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n62 ms (3 ms, 7 ms, 13 ms (stage 30825.0: task 36694))"; 12 [labelType="html" label=" Project "]; 13 [labelType="html" label="Filter number of output rows: 10"]; } 14 [labelType="html" label="Scan csv number of output rows: 10 number of files read: 1 metadata time: 0 ms size of files read: 163.0 B"]; 1->0; 3->1; 4->3; 5->4; 7->5; 8->7; 9->8; 10->9; 12->5; 13->12; 14->13; }

Execute InsertIntoHadoopFsRelationCommand file:/data/output/cache/parquet/datablock/05_GR8_DOUBLON__161603, false, Parquet, [path=file:///data/output/cache/parquet/datablock/05_GR8_DOUBLON__161603], Overwrite, [COL_6A75989B_D287_4B6C_893B_7D78A3023D30, COL_129360FD_BB12_46AD_A00B_165330B80A60, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C, COL_910DC838_241D_4365_A8A3_070714C6238B, COL_481586E2_E506_44E5_9DC4_79601A1736D6, COL_48A39647_D8F8_4138_9D64_310E44B62D64, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015]

WriteFiles

Project [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2346802, COL_129360FD_BB12_46AD_A00B_165330B80A60#2346806, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2347317, str_if_with_rule(((COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2346819 <=> SUN) OR (COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2346819 <=> SUF)), str_concat(array(COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2346819, -!, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2347258, , )), str_concat(array(COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2346819, -!, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2347258, _, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2346829))) AS COL_910DC838_241D_4365_A8A3_070714C6238B#2347593, str_concat(array(Suppression de Forfait@, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2347258, @, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2346829)) AS COL_481586E2_E506_44E5_9DC4_79601A1736D6#2347594, str_concat([Suppression Doublon du RSF]) AS COL_48A39647_D8F8_4138_9D64_310E44B62D64#2347595, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2347258]

Project [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2346802, COL_129360FD_BB12_46AD_A00B_165330B80A60#2346806, COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2346819, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2346829, str_join(array(str_extract(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2346828, RIGHT, 2), str_extract_middle(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2346828, 5, 2), str_extract(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2346828, LEFT, 4)), ) AS COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2347258, str_create(RC DBL-RSF) AS COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2347317]

WholeStageCodegen (3)

CartesianProduct

Project [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2338433 AS COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2346802, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2338437 AS COL_129360FD_BB12_46AD_A00B_165330B80A60#2346806, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2338450 AS COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2346819, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2338459 AS COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2346828, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2338460 AS COL_D15FD825_13D6_4F72_901C_E623B5A02916#2346829]

Filter ((((CASE WHEN isnull(COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2338477) THEN false ELSE (COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2338477 > 1) END OR CASE WHEN isnull(COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2338478) THEN false ELSE (COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2338478 > 1) END) OR CASE WHEN isnull(COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2338479) THEN false ELSE (COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2338479 > 1) END) OR CASE WHEN isnull(COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2338480) THEN false ELSE (COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2338480 > 1) END) OR CASE WHEN isnull(COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2338481) THEN false ELSE (COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2338481 > 1) END)

ColumnarToRow

WholeStageCodegen (1)

FileScan parquet [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2338433,COL_F76265AB_8748_4114_92B4_00E37DE25D34#2338437,COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2338450,COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2338459,COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2338460,COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2338477,COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2338478,COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2338479,COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2338480,COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2338481] Batched: true, DataFilters: [((((CASE WHEN isnull(COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2338477) THEN false ELSE (COL_E7D4..., Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/02_A_ELIGIBLE_161654], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB:string,COL_F76265AB_8748_4114_92B4_00E37DE25D34:s...

Project

Filter ((isnotnull(REGLES#2346446) AND (RC-DBL-RSF = REGLES#2346446)) AND (APPLICABLE#2346447 <=> O))

WholeStageCodegen (2)

FileScan csv [REGLES#2346446,APPLICABLE#2346447] Batched: false, DataFilters: [isnotnull(REGLES#2346446), (RC-DBL-RSF = REGLES#2346446), (APPLICABLE#2346447 <=> O)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/tmp/REGLES_APPLICAB_161411/REGLES-A-TTT.csv], PartitionFilters: [], PushedFilters: [IsNotNull(REGLES), EqualTo(REGLES,RC-DBL-RSF), EqualNullSafe(APPLICABLE,O)], ReadSchema: struct<REGLES:string,APPLICABLE:string>

Details

== Physical Plan ==
Execute InsertIntoHadoopFsRelationCommand (12)
+- WriteFiles (11)
   +- * Project (10)
      +- * Project (9)
         +- CartesianProduct Inner (8)
            :- * Project (4)
            :  +- * Filter (3)
            :     +- * ColumnarToRow (2)
            :        +- Scan parquet  (1)
            +- * Project (7)
               +- * Filter (6)
                  +- Scan csv  (5)


(1) Scan parquet 
Output [10]: [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2338433, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2338437, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2338450, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2338459, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2338460, COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2338477, COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2338478, COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2338479, COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2338480, COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2338481]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/02_A_ELIGIBLE_161654]
ReadSchema: struct<COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB:string,COL_F76265AB_8748_4114_92B4_00E37DE25D34:string,COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC:string,COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4:string,COL_B2D69078_23A7_4B36_BBF2_3F9C29840464:string,COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846:int,COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B:int,COL_B8D74F46_C755_4417_9E18_FAA760CD8457:int,COL_51335C75_C8A9_4932_8985_BA9E68B80F3A:int,COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C:int>

(2) ColumnarToRow [codegen id : 1]
Input [10]: [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2338433, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2338437, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2338450, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2338459, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2338460, COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2338477, COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2338478, COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2338479, COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2338480, COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2338481]

(3) Filter [codegen id : 1]
Input [10]: [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2338433, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2338437, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2338450, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2338459, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2338460, COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2338477, COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2338478, COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2338479, COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2338480, COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2338481]
Condition : ((((CASE WHEN isnull(COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2338477) THEN false ELSE (COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2338477 > 1) END OR CASE WHEN isnull(COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2338478) THEN false ELSE (COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2338478 > 1) END) OR CASE WHEN isnull(COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2338479) THEN false ELSE (COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2338479 > 1) END) OR CASE WHEN isnull(COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2338480) THEN false ELSE (COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2338480 > 1) END) OR CASE WHEN isnull(COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2338481) THEN false ELSE (COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2338481 > 1) END)

(4) Project [codegen id : 1]
Output [5]: [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2338433 AS COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2346802, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2338437 AS COL_129360FD_BB12_46AD_A00B_165330B80A60#2346806, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2338450 AS COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2346819, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2338459 AS COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2346828, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2338460 AS COL_D15FD825_13D6_4F72_901C_E623B5A02916#2346829]
Input [10]: [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2338433, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2338437, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2338450, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2338459, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2338460, COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2338477, COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2338478, COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2338479, COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2338480, COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2338481]

(5) Scan csv 
Output [2]: [REGLES#2346446, APPLICABLE#2346447]
Batched: false
Location: InMemoryFileIndex [file:/data/input/depot/csv/tmp/REGLES_APPLICAB_161411/REGLES-A-TTT.csv]
PushedFilters: [IsNotNull(REGLES), EqualTo(REGLES,RC-DBL-RSF), EqualNullSafe(APPLICABLE,O)]
ReadSchema: struct<REGLES:string,APPLICABLE:string>

(6) Filter [codegen id : 2]
Input [2]: [REGLES#2346446, APPLICABLE#2346447]
Condition : ((isnotnull(REGLES#2346446) AND (RC-DBL-RSF = REGLES#2346446)) AND (APPLICABLE#2346447 <=> O))

(7) Project [codegen id : 2]
Output: []
Input [2]: [REGLES#2346446, APPLICABLE#2346447]

(8) CartesianProduct
Join type: Inner
Join condition: None

(9) Project [codegen id : 3]
Output [6]: [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2346802, COL_129360FD_BB12_46AD_A00B_165330B80A60#2346806, COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2346819, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2346829, str_join(array(str_extract(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2346828, RIGHT, 2), str_extract_middle(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2346828, 5, 2), str_extract(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2346828, LEFT, 4)), ) AS COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2347258, str_create(RC DBL-RSF) AS COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2347317]
Input [5]: [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2346802, COL_129360FD_BB12_46AD_A00B_165330B80A60#2346806, COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2346819, COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2346828, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2346829]

(10) Project [codegen id : 3]
Output [7]: [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2346802, COL_129360FD_BB12_46AD_A00B_165330B80A60#2346806, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2347317, str_if_with_rule(((COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2346819 <=> SUN) OR (COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2346819 <=> SUF)), str_concat(array(COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2346819, -!, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2347258, , )), str_concat(array(COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2346819, -!, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2347258, _, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2346829))) AS COL_910DC838_241D_4365_A8A3_070714C6238B#2347593, str_concat(array(Suppression de Forfait@, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2347258, @, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2346829)) AS COL_481586E2_E506_44E5_9DC4_79601A1736D6#2347594, str_concat([Suppression Doublon du RSF]) AS COL_48A39647_D8F8_4138_9D64_310E44B62D64#2347595, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2347258]
Input [6]: [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2346802, COL_129360FD_BB12_46AD_A00B_165330B80A60#2346806, COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2346819, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2346829, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2347258, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2347317]

(11) WriteFiles
Input [7]: [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2346802, COL_129360FD_BB12_46AD_A00B_165330B80A60#2346806, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2347317, COL_910DC838_241D_4365_A8A3_070714C6238B#2347593, COL_481586E2_E506_44E5_9DC4_79601A1736D6#2347594, COL_48A39647_D8F8_4138_9D64_310E44B62D64#2347595, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2347258]

(12) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: file:/data/output/cache/parquet/datablock/05_GR8_DOUBLON__161603, false, Parquet, [path=file:///data/output/cache/parquet/datablock/05_GR8_DOUBLON__161603], Overwrite, [COL_6A75989B_D287_4B6C_893B_7D78A3023D30, COL_129360FD_BB12_46AD_A00B_165330B80A60, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C, COL_910DC838_241D_4365_A8A3_070714C6238B, COL_481586E2_E506_44E5_9DC4_79601A1736D6, COL_48A39647_D8F8_4138_9D64_310E44B62D64, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015]

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet