digraph G {
0 [labelType="html" label="<b>Execute InsertIntoHadoopFsRelationCommand</b><br><br>task commit time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 32516.0: task 39480))<br>number of written files: 1<br>job commit time: 4 ms<br>number of output rows: 0<br>number of dynamic part: 0<br>written output: 1305.0 B"];
1 [labelType="html" label="<br><b>WriteFiles</b><br><br>"];
subgraph cluster2 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n578 ms (18 ms, 48 ms, 138 ms (stage 32516.0: task 39481))";
3 [labelType="html" label="<br><b>Project</b><br><br>"];
4 [labelType="html" label="<br><b>Project</b><br><br>"];
}
5 [labelType="html" label="<b>CartesianProduct</b><br><br>number of output rows: 0"];
subgraph cluster6 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n343 ms (10 ms, 31 ms, 72 ms (stage 32516.0: task 39477))";
7 [labelType="html" label="<br><b>Project</b><br><br>"];
8 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 0"];
9 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 474,437<br>number of input batches: 120"];
}
10 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 10<br>scan time total (min, med, max (stageId: taskId))<br>39 ms (3 ms, 4 ms, 8 ms (stage 32516.0: task 39481))<br>metadata time: 0 ms<br>size of files read: 21.5 MiB<br>number of output rows: 474,437"];
subgraph cluster11 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n24 ms (2 ms, 2 ms, 5 ms (stage 32516.0: task 39481))";
12 [labelType="html" label="<br><b>Project</b><br><br>"];
13 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 10"];
}
14 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 10<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 163.0 B"];
1->0;
3->1;
4->3;
5->4;
7->5;
8->7;
9->8;
10->9;
12->5;
13->12;
14->13;
}
15
Execute InsertIntoHadoopFsRelationCommand file:/data/output/cache/parquet/datablock/05_GR8_DOUBLON__161603, false, Parquet, [path=file:///data/output/cache/parquet/datablock/05_GR8_DOUBLON__161603], Overwrite, [COL_6A75989B_D287_4B6C_893B_7D78A3023D30, COL_129360FD_BB12_46AD_A00B_165330B80A60, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C, COL_910DC838_241D_4365_A8A3_070714C6238B, COL_481586E2_E506_44E5_9DC4_79601A1736D6, COL_48A39647_D8F8_4138_9D64_310E44B62D64, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015]
WriteFiles
Project [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2463532, str_if_with_rule(((COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048 <=> SUN) OR (COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048 <=> SUF)), str_concat(array(COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, -!, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, , )), str_concat(array(COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, -!, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, _, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058))) AS COL_910DC838_241D_4365_A8A3_070714C6238B#2463783, str_concat(array(Suppression de Forfait@, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, @, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058)) AS COL_481586E2_E506_44E5_9DC4_79601A1736D6#2463784, str_concat([Suppression Doublon du RSF]) AS COL_48A39647_D8F8_4138_9D64_310E44B62D64#2463785, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475]
Project [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058, str_join(array(str_extract(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, RIGHT, 2), str_extract_middle(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, 5, 2), str_extract(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, LEFT, 4)), ) AS COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, str_create(RC DBL-RSF) AS COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2463532]
WholeStageCodegen (3)
CartesianProduct
Project [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2454796 AS COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2454800 AS COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2454813 AS COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2454822 AS COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2454823 AS COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058]
Filter ((((CASE WHEN isnull(COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840) THEN false ELSE (COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840 > 1) END OR CASE WHEN isnull(COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841) THEN false ELSE (COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841 > 1) END) OR CASE WHEN isnull(COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842) THEN false ELSE (COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842 > 1) END) OR CASE WHEN isnull(COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843) THEN false ELSE (COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843 > 1) END) OR CASE WHEN isnull(COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844) THEN false ELSE (COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844 > 1) END)
ColumnarToRow
WholeStageCodegen (1)
FileScan parquet [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2454796,COL_F76265AB_8748_4114_92B4_00E37DE25D34#2454800,COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2454813,COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2454822,COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2454823,COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840,COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841,COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842,COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843,COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844] Batched: true, DataFilters: [((((CASE WHEN isnull(COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840) THEN false ELSE (COL_E7D4..., Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/02_A_ELIGIBLE_161654], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB:string,COL_F76265AB_8748_4114_92B4_00E37DE25D34:s...
Project
Filter ((isnotnull(REGLES#2462609) AND (RC-DBL-RSF = REGLES#2462609)) AND (APPLICABLE#2462610 <=> O))
WholeStageCodegen (2)
FileScan csv [REGLES#2462609,APPLICABLE#2462610] Batched: false, DataFilters: [isnotnull(REGLES#2462609), (RC-DBL-RSF = REGLES#2462609), (APPLICABLE#2462610 <=> O)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/tmp/REGLES_APPLICAB_161411/REGLES-A-TTT.csv], PartitionFilters: [], PushedFilters: [IsNotNull(REGLES), EqualTo(REGLES,RC-DBL-RSF), EqualNullSafe(APPLICABLE,O)], ReadSchema: struct<REGLES:string,APPLICABLE:string>
== Physical Plan ==
Execute InsertIntoHadoopFsRelationCommand (12)
+- WriteFiles (11)
+- * Project (10)
+- * Project (9)
+- CartesianProduct Inner (8)
:- * Project (4)
: +- * Filter (3)
: +- * ColumnarToRow (2)
: +- Scan parquet (1)
+- * Project (7)
+- * Filter (6)
+- Scan csv (5)
(1) Scan parquet
Output [10]: [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2454796, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2454800, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2454813, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2454822, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2454823, COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840, COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841, COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842, COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843, COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/02_A_ELIGIBLE_161654]
ReadSchema: struct<COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB:string,COL_F76265AB_8748_4114_92B4_00E37DE25D34:string,COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC:string,COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4:string,COL_B2D69078_23A7_4B36_BBF2_3F9C29840464:string,COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846:int,COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B:int,COL_B8D74F46_C755_4417_9E18_FAA760CD8457:int,COL_51335C75_C8A9_4932_8985_BA9E68B80F3A:int,COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C:int>
(2) ColumnarToRow [codegen id : 1]
Input [10]: [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2454796, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2454800, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2454813, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2454822, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2454823, COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840, COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841, COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842, COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843, COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844]
(3) Filter [codegen id : 1]
Input [10]: [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2454796, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2454800, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2454813, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2454822, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2454823, COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840, COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841, COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842, COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843, COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844]
Condition : ((((CASE WHEN isnull(COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840) THEN false ELSE (COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840 > 1) END OR CASE WHEN isnull(COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841) THEN false ELSE (COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841 > 1) END) OR CASE WHEN isnull(COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842) THEN false ELSE (COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842 > 1) END) OR CASE WHEN isnull(COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843) THEN false ELSE (COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843 > 1) END) OR CASE WHEN isnull(COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844) THEN false ELSE (COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844 > 1) END)
(4) Project [codegen id : 1]
Output [5]: [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2454796 AS COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2454800 AS COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2454813 AS COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2454822 AS COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2454823 AS COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058]
Input [10]: [COL_F44B0D19_6A77_4497_88B3_B9698DF17BCB#2454796, COL_F76265AB_8748_4114_92B4_00E37DE25D34#2454800, COL_8F13405A_6271_4B4F_BE3D_AFBE0E624AFC#2454813, COL_50D051CB_B5A3_4A22_99CF_B636DE0B08A4#2454822, COL_B2D69078_23A7_4B36_BBF2_3F9C29840464#2454823, COL_E7D4B3BC_F96E_4431_BC95_1D1CCE75D846#2454840, COL_B057EBBF_57E7_449C_9DE3_CD56F041B86B#2454841, COL_B8D74F46_C755_4417_9E18_FAA760CD8457#2454842, COL_51335C75_C8A9_4932_8985_BA9E68B80F3A#2454843, COL_03AE34A5_99F1_4525_9BD2_BF7BAFE47F5C#2454844]
(5) Scan csv
Output [2]: [REGLES#2462609, APPLICABLE#2462610]
Batched: false
Location: InMemoryFileIndex [file:/data/input/depot/csv/tmp/REGLES_APPLICAB_161411/REGLES-A-TTT.csv]
PushedFilters: [IsNotNull(REGLES), EqualTo(REGLES,RC-DBL-RSF), EqualNullSafe(APPLICABLE,O)]
ReadSchema: struct<REGLES:string,APPLICABLE:string>
(6) Filter [codegen id : 2]
Input [2]: [REGLES#2462609, APPLICABLE#2462610]
Condition : ((isnotnull(REGLES#2462609) AND (RC-DBL-RSF = REGLES#2462609)) AND (APPLICABLE#2462610 <=> O))
(7) Project [codegen id : 2]
Output: []
Input [2]: [REGLES#2462609, APPLICABLE#2462610]
(8) CartesianProduct
Join type: Inner
Join condition: None
(9) Project [codegen id : 3]
Output [6]: [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058, str_join(array(str_extract(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, RIGHT, 2), str_extract_middle(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, 5, 2), str_extract(COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, LEFT, 4)), ) AS COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, str_create(RC DBL-RSF) AS COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2463532]
Input [5]: [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, COL_AB04B4E3_0BF7_4EFA_A42C_8B76A316E9CA#2463057, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058]
(10) Project [codegen id : 3]
Output [7]: [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2463532, str_if_with_rule(((COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048 <=> SUN) OR (COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048 <=> SUF)), str_concat(array(COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, -!, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, , )), str_concat(array(COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, -!, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, _, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058))) AS COL_910DC838_241D_4365_A8A3_070714C6238B#2463783, str_concat(array(Suppression de Forfait@, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, @, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058)) AS COL_481586E2_E506_44E5_9DC4_79601A1736D6#2463784, str_concat([Suppression Doublon du RSF]) AS COL_48A39647_D8F8_4138_9D64_310E44B62D64#2463785, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475]
Input [6]: [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_46C6EB6C_534E_4982_ABBA_A69BDE364588#2463048, COL_D15FD825_13D6_4F72_901C_E623B5A02916#2463058, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2463532]
(11) WriteFiles
Input [7]: [COL_6A75989B_D287_4B6C_893B_7D78A3023D30#2463031, COL_129360FD_BB12_46AD_A00B_165330B80A60#2463035, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C#2463532, COL_910DC838_241D_4365_A8A3_070714C6238B#2463783, COL_481586E2_E506_44E5_9DC4_79601A1736D6#2463784, COL_48A39647_D8F8_4138_9D64_310E44B62D64#2463785, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015#2463475]
(12) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: file:/data/output/cache/parquet/datablock/05_GR8_DOUBLON__161603, false, Parquet, [path=file:///data/output/cache/parquet/datablock/05_GR8_DOUBLON__161603], Overwrite, [COL_6A75989B_D287_4B6C_893B_7D78A3023D30, COL_129360FD_BB12_46AD_A00B_165330B80A60, COL_1DB4A2D9_3B6B_4DB7_ACEB_6FF7B071AD7C, COL_910DC838_241D_4365_A8A3_070714C6238B, COL_481586E2_E506_44E5_9DC4_79601A1736D6, COL_48A39647_D8F8_4138_9D64_310E44B62D64, COL_B1229C2A_C44E_41C8_BC2A_E56A7EE27015]