digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
1 [labelType="html" label="<b>SortAggregate</b><br><br>number of output rows: 1"];
2 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 1<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 1<br>local bytes read: 102.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 72.0 B<br>local merged bytes read: 0.0 B<br>number of partitions: 1<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 102.0 B"];
3 [labelType="html" label="<b>SortAggregate</b><br><br>number of output rows: 1"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (13)\n \nduration: 166 ms";
5 [labelType="html" label="<br><b>Project</b><br><br>"];
6 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 21,131<br>spill size: 0.0 B"];
}
subgraph cluster7 {
isCluster="true";
label="WholeStageCodegen (11)\n \nduration: 206 ms";
8 [labelType="html" label="<b>Sort</b><br><br>sort time: 2 ms<br>peak memory: 16.5 MiB<br>spill size: 0.0 B"];
}
9 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 148.0 KiB<br>number of coalesced partitions: 1"];
10 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 21,131<br>local merged chunks fetched: 0<br>shuffle write time: 1 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 21,131<br>local bytes read: 141.2 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 660.2 KiB<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 141.2 KiB"];
subgraph cluster11 {
isCluster="true";
label="WholeStageCodegen (10)\n \nduration: 39 ms";
12 [labelType="html" label="<br><b>Project</b><br><br>"];
13 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 21,131<br>spill size: 0.0 B"];
}
subgraph cluster14 {
isCluster="true";
label="WholeStageCodegen (8)\n \nduration: 170 ms";
15 [labelType="html" label="<b>Sort</b><br><br>sort time: 2 ms<br>peak memory: 16.5 MiB<br>spill size: 0.0 B"];
}
16 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 148.0 KiB<br>number of coalesced partitions: 1"];
17 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 21,127<br>local merged chunks fetched: 0<br>shuffle write time: 1 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 21,127<br>local bytes read: 141.1 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 660.1 KiB<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 141.1 KiB"];
subgraph cluster18 {
isCluster="true";
label="WholeStageCodegen (7)\n \nduration: 108 ms";
19 [labelType="html" label="<br><b>Project</b><br><br>"];
20 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 21,127<br>spill size: 0.0 B"];
}
subgraph cluster21 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: 306 ms";
22 [labelType="html" label="<b>Sort</b><br><br>sort time: 7 ms<br>peak memory: 16.5 MiB<br>spill size: 0.0 B"];
}
23 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 175.9 KiB<br>number of coalesced partitions: 1"];
24 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 21,121<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>6 ms (1 ms, 1 ms, 3 ms (stage 20821.0: task 27126))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 21,121<br>local bytes read: 166.3 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 3<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>660.0 KiB (20.7 KiB, 319.7 KiB, 319.7 KiB (stage 20821.0: task 27126))<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>166.3 KiB (6.7 KiB, 79.7 KiB, 79.9 KiB (stage 20821.0: task 27125))"];
subgraph cluster25 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n221 ms (17 ms, 77 ms, 127 ms (stage 20821.0: task 27125))";
26 [labelType="html" label="<br><b>Project</b><br><br>"];
}
27 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 21,121<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 8.3 MiB"];
subgraph cluster28 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: 2.7 m";
29 [labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 16.1 MiB<br>spill size: 0.0 B"];
30 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 0 ms<br>peak memory: 256.0 KiB<br>number of output rows: 445<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
}
31 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 5.0 KiB<br>number of coalesced partitions: 1"];
32 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 445<br>local merged chunks fetched: 0<br>shuffle write time: 1 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 445<br>local bytes read: 4.8 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 13.9 KiB<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 4.8 KiB"];
subgraph cluster33 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: 9 ms";
34 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 6 ms<br>peak memory: 256.0 KiB<br>number of output rows: 445<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
35 [labelType="html" label="<br><b>Project</b><br><br>"];
}
36 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 1,216<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 111.6 KiB"];
subgraph cluster37 {
isCluster="true";
label="WholeStageCodegen (9)\n \nduration: 98 ms";
38 [labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 16.1 MiB<br>spill size: 0.0 B"];
39 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 0 ms<br>peak memory: 256.0 KiB<br>number of output rows: 464<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
}
40 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 5.2 KiB<br>number of coalesced partitions: 1"];
41 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 464<br>local merged chunks fetched: 0<br>shuffle write time: 1 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 464<br>local bytes read: 5.0 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 14.5 KiB<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 5.0 KiB"];
subgraph cluster42 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: 9 ms";
43 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 6 ms<br>peak memory: 256.0 KiB<br>number of output rows: 464<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
44 [labelType="html" label="<br><b>Project</b><br><br>"];
}
45 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 486<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 50.8 KiB"];
subgraph cluster46 {
isCluster="true";
label="WholeStageCodegen (12)\n \nduration: 177 ms";
47 [labelType="html" label="<b>Sort</b><br><br>sort time: 5 ms<br>peak memory: 16.5 MiB<br>spill size: 0.0 B"];
48 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 15 ms<br>peak memory: 16.3 MiB<br>number of output rows: 21,121<br>number of sort fallback tasks: 0<br>avg hash probes per key: 1"];
}
49 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 159.0 KiB<br>number of coalesced partitions: 1"];
50 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 21,122<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>5 ms (1 ms, 1 ms, 2 ms (stage 20824.0: task 27132))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 21,122<br>local bytes read: 150.8 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 3<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>660.1 KiB (5.3 KiB, 316.2 KiB, 338.6 KiB (stage 20824.0: task 27131))<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>150.8 KiB (2.3 KiB, 71.5 KiB, 77.0 KiB (stage 20824.0: task 27131))"];
subgraph cluster51 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n417 ms (15 ms, 165 ms, 237 ms (stage 20824.0: task 27131))";
52 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build total (min, med, max (stageId: taskId))<br>388 ms (8 ms, 152 ms, 228 ms (stage 20824.0: task 27131))<br>peak memory total (min, med, max (stageId: taskId))<br>32.8 MiB (256.0 KiB, 16.3 MiB, 16.3 MiB (stage 20824.0: task 27130))<br>number of output rows: 21,122<br>number of sort fallback tasks: 0<br>avg hash probes per key (min, med, max (stageId: taskId)):<br>(1, 1, 1 (stage 20824.0: task 27130))"];
53 [labelType="html" label="<br><b>Project</b><br><br>"];
54 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 26,044"];
}
55 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 26,044<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 8.0 MiB"];
1->0;
2->1;
3->2;
5->3;
6->5;
8->6;
9->8;
10->9;
12->10;
13->12;
15->13;
16->15;
17->16;
19->17;
20->19;
22->20;
23->22;
24->23;
26->24;
27->26;
29->20;
30->29;
31->30;
32->31;
34->32;
35->34;
36->35;
38->13;
39->38;
40->39;
41->40;
43->41;
44->43;
45->44;
47->6;
48->47;
49->48;
50->49;
52->50;
53->52;
54->53;
55->54;
}
56
AdaptiveSparkPlan isFinalPlan=true
SortAggregate(key=[], functions=[count(1), count(COL_1D43C4C0_AE5C_43DC_ABA0_79E72CBFE6D4#1647816), min(COL_1D43C4C0_AE5C_43DC_ABA0_79E72CBFE6D4#1647816), max(COL_1D43C4C0_AE5C_43DC_ABA0_79E72CBFE6D4#1647816)])
Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1025413]
SortAggregate(key=[], functions=[partial_count(1), partial_count(COL_1D43C4C0_AE5C_43DC_ABA0_79E72CBFE6D4#1647816), partial_min(COL_1D43C4C0_AE5C_43DC_ABA0_79E72CBFE6D4#1647816), partial_max(COL_1D43C4C0_AE5C_43DC_ABA0_79E72CBFE6D4#1647816)])
Project [RSS_DECOUPAGE_237747_COL_5588E97C_F5A9_4AFC_AD58_2B0A6381409A#1647255 AS COL_1D43C4C0_AE5C_43DC_ABA0_79E72CBFE6D4#1647816]
SortMergeJoin [COL_7948E047_29EE_472A_BFFC_723098878DAD#1646993], [RSS_DECOUPAGE_237747_COL_5588E97C_F5A9_4AFC_AD58_2B0A6381409A#1647255], LeftOuter
WholeStageCodegen (13)
Sort [COL_7948E047_29EE_472A_BFFC_723098878DAD#1646993 ASC NULLS FIRST], false, 0
WholeStageCodegen (11)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_7948E047_29EE_472A_BFFC_723098878DAD#1646993, 10), ENSURE_REQUIREMENTS, [plan_id=1025301]
Project [COL_7948E047_29EE_472A_BFFC_723098878DAD#1646993]
SortMergeJoin [COL_7948E047_29EE_472A_BFFC_723098878DAD#1646993], [01_MOLECULES_ON_237708_COL_EE4AB0B4_F37E_40C0_9292_9E68BF166E36#1647118], FullOuter
WholeStageCodegen (10)
Sort [COL_7948E047_29EE_472A_BFFC_723098878DAD#1646993 ASC NULLS FIRST], false, 0
WholeStageCodegen (8)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_7948E047_29EE_472A_BFFC_723098878DAD#1646993, 10), ENSURE_REQUIREMENTS, [plan_id=1025174]
Project [COL_7948E047_29EE_472A_BFFC_723098878DAD#1646993]
SortMergeJoin [COL_7948E047_29EE_472A_BFFC_723098878DAD#1646993], [01_DMI_DECOUPAG_237714_COL_1D1ABD09_1E00_4FA7_81FE_A558E22F6A6D#1647007], FullOuter
WholeStageCodegen (7)
Sort [COL_7948E047_29EE_472A_BFFC_723098878DAD#1646993 ASC NULLS FIRST], false, 0
WholeStageCodegen (5)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_7948E047_29EE_472A_BFFC_723098878DAD#1646993, 10), ENSURE_REQUIREMENTS, [plan_id=1024938]
Project [str_remove_space(NO_ADMIN#1641333, ALL) AS COL_7948E047_29EE_472A_BFFC_723098878DAD#1646993]
WholeStageCodegen (1)
FileScan csv [NO_ADMIN#1641333] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/tmp/VISUALVALOSEJOU_237544/420780652.2025.1..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<NO_ADMIN:string>
Sort [01_DMI_DECOUPAG_237714_COL_1D1ABD09_1E00_4FA7_81FE_A558E22F6A6D#1647007 ASC NULLS FIRST], false, 0
HashAggregate(keys=[COL_9D7AAEDE_94EE_40AF_8530_BC9D1F27742A#1645469], functions=[])
WholeStageCodegen (6)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_9D7AAEDE_94EE_40AF_8530_BC9D1F27742A#1645469, 10), ENSURE_REQUIREMENTS, [plan_id=1024951]
HashAggregate(keys=[COL_9D7AAEDE_94EE_40AF_8530_BC9D1F27742A#1645469], functions=[])
Project [str_remove_space(str_extract_middle(_c0#1645364, 12, 20), ALL) AS COL_9D7AAEDE_94EE_40AF_8530_BC9D1F27742A#1645469]
WholeStageCodegen (2)
FileScan csv [_c0#1645364] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/tmp/DMI_M12_2025_237543/DMI-M12-2025 - BIS...., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<_c0:string>
Sort [01_MOLECULES_ON_237708_COL_EE4AB0B4_F37E_40C0_9292_9E68BF166E36#1647118 ASC NULLS FIRST], false, 0
HashAggregate(keys=[COL_45066C9F_40FD_4BE1_ACAF_6139C274CECB#1643711], functions=[])
WholeStageCodegen (9)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_45066C9F_40FD_4BE1_ACAF_6139C274CECB#1643711, 10), ENSURE_REQUIREMENTS, [plan_id=1024971]
HashAggregate(keys=[COL_45066C9F_40FD_4BE1_ACAF_6139C274CECB#1643711], functions=[])
Project [str_remove_space(str_extract_middle(_c0#1643592, 12, 20), ALL) AS COL_45066C9F_40FD_4BE1_ACAF_6139C274CECB#1643711]
WholeStageCodegen (3)
FileScan csv [_c0#1643592] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/tmp/MOLECULES_ONERE_237545/MO-M12-2025.txt], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<_c0:string>
Sort [RSS_DECOUPAGE_237747_COL_5588E97C_F5A9_4AFC_AD58_2B0A6381409A#1647255 ASC NULLS FIRST], false, 0
HashAggregate(keys=[COL_0FDBEAA7_683C_4647_A06F_34AC7E6418C2#1641217], functions=[])
WholeStageCodegen (12)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_0FDBEAA7_683C_4647_A06F_34AC7E6418C2#1641217, 10), ENSURE_REQUIREMENTS, [plan_id=1024992]
HashAggregate(keys=[COL_0FDBEAA7_683C_4647_A06F_34AC7E6418C2#1641217], functions=[])
Project [str_remove_space(str_extract_middle(_c0#1640638, 48, 20), ALL) AS COL_0FDBEAA7_683C_4647_A06F_34AC7E6418C2#1641217]
Filter isnotnull(str_remove_space(str_extract_middle(_c0#1640638, 48, 20), ALL))
WholeStageCodegen (4)
FileScan csv [_c0#1640638] Batched: false, DataFilters: [isnotnull(str_remove_space(str_extract_middle(_c0#1640638, 48, 20), ALL))], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/tmp/RSS_M12_2025_237542/RSS- 420780652.txt], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<_c0:string>