digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: 0 ms";
2 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 0 ms<br>number of output rows: 1"];
}
3 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 1<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 1<br>local bytes read: 59.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 16.0 B<br>local merged bytes read: 0.0 B<br>number of partitions: 1<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 59.0 B"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: 455 ms";
5 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 455 ms<br>peak memory: 0.0 B<br>number of output rows: 1<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
6 [labelType="html" label="<br><b>Project</b><br><br>"];
7 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 323,604<br>spill size: 0.0 B"];
}
subgraph cluster8 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: 470 ms";
9 [labelType="html" label="<b>Sort</b><br><br>sort time: 196 ms<br>peak memory: 24.0 MiB<br>spill size: 0.0 B"];
}
10 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 1599.7 KiB<br>number of coalesced partitions: 1"];
11 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 323,604<br>local merged chunks fetched: 0<br>shuffle write time: 6 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 323,604<br>local bytes read: 1513.8 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 6.4 MiB<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 1513.8 KiB"];
subgraph cluster12 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n626 ms (2 ms, 9 ms, 594 ms (stage 15335.0: task 21243))";
13 [labelType="html" label="<br><b>Project</b><br><br>"];
14 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 323,604<br>number of input batches: 80"];
}
15 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 1<br>scan time total (min, med, max (stageId: taskId))<br>134 ms (2 ms, 7 ms, 109 ms (stage 15335.0: task 21243))<br>metadata time: 0 ms<br>size of files read: 20.4 MiB<br>number of output rows: 323,604"];
subgraph cluster16 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: 2.39 h";
17 [labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 16.1 MiB<br>spill size: 0.0 B"];
}
18 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 840.0 B<br>number of coalesced partitions: 1"];
19 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 26<br>local merged chunks fetched: 0<br>shuffle write time: 1 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 26<br>local bytes read: 795.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 624.0 B<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 795.0 B"];
subgraph cluster20 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: 10 ms";
21 [labelType="html" label="<br><b>Project</b><br><br>"];
22 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 26"];
}
23 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 26<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 119.0 B"];
2->0;
3->2;
5->3;
6->5;
7->6;
9->7;
10->9;
11->10;
13->11;
14->13;
15->14;
17->7;
18->17;
19->18;
21->19;
22->21;
23->22;
}
24
AdaptiveSparkPlan isFinalPlan=true
HashAggregate(keys=[], functions=[count(1)])
WholeStageCodegen (6)
Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=673720]
HashAggregate(keys=[], functions=[partial_count(1)])
Project
SortMergeJoin [COL_A4AF3049_C967_4B58_9D36_40EE5D79F6AC#1196184], [FORFAITURGENCE_208492_FORFAIT_URGENCE_208493#1196686], LeftOuter
WholeStageCodegen (5)
Sort [COL_A4AF3049_C967_4B58_9D36_40EE5D79F6AC#1196184 ASC NULLS FIRST], false, 0
WholeStageCodegen (3)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_A4AF3049_C967_4B58_9D36_40EE5D79F6AC#1196184, 10), ENSURE_REQUIREMENTS, [plan_id=673589]
Project [str_remove_space(str_if_with_rule((COL_2EE853AF_4512_4E8D_ABAE_19143D81E357#1195897 <=> M), str_extract_middle(COL_E76BCE3C_BF78_45B5_AC59_423A3A530FDF#1195894, 75, 13), str_if_with_rule(((COL_2EE853AF_4512_4E8D_ABAE_19143D81E357#1195897 <=> B) OR (COL_2EE853AF_4512_4E8D_ABAE_19143D81E357#1195897 <=> C)), str_extract_middle(COL_E76BCE3C_BF78_45B5_AC59_423A3A530FDF#1195894, 78, 5), )), ONLY_ONE) AS COL_A4AF3049_C967_4B58_9D36_40EE5D79F6AC#1196184]
ColumnarToRow
WholeStageCodegen (1)
FileScan parquet [COL_E76BCE3C_BF78_45B5_AC59_423A3A530FDF#1195894,COL_2EE853AF_4512_4E8D_ABAE_19143D81E357#1195897] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/00_A_RSF_SOURCE_208498], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<COL_E76BCE3C_BF78_45B5_AC59_423A3A530FDF:string,COL_2EE853AF_4512_4E8D_ABAE_19143D81E357:s...
Sort [FORFAITURGENCE_208492_FORFAIT_URGENCE_208493#1196686 ASC NULLS FIRST], false, 0
WholeStageCodegen (4)
AQEShuffleRead coalesced
Exchange hashpartitioning(FORFAITURGENCE_208492_FORFAIT_URGENCE_208493#1196686, 10), ENSURE_REQUIREMENTS, [plan_id=673603]
Project [Forfait Urgence#1195884 AS FORFAITURGENCE_208492_FORFAIT_URGENCE_208493#1196686]
Filter isnotnull(Forfait Urgence#1195884)
WholeStageCodegen (2)
FileScan csv [Forfait Urgence#1195884] Batched: false, DataFilters: [isnotnull(Forfait Urgence#1195884)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/tmp/FORFAITS_URGENC_208433/Forfait_Urgence...., PartitionFilters: [], PushedFilters: [IsNotNull(Forfait Urgence)], ReadSchema: struct<Forfait Urgence:string>