digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (7)\n \nduration: 0 ms";
2 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 0 ms<br>number of output rows: 1"];
}
3 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 3<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>1 ms (0 ms, 0 ms, 0 ms (stage 17440.0: task 24277))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 3<br>local bytes read: 177.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 3<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>48.0 B (16.0 B, 16.0 B, 16.0 B (stage 17440.0: task 24277))<br>local merged bytes read: 0.0 B<br>number of partitions: 1<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>177.0 B (59.0 B, 59.0 B, 59.0 B (stage 17440.0: task 24277))"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: total (min, med, max (stageId: taskId))\n7.1 s (2.0 s, 2.1 s, 3.0 s (stage 17440.0: task 24279))";
5 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build total (min, med, max (stageId: taskId))<br>7.1 s (2.0 s, 2.1 s, 3.0 s (stage 17440.0: task 24279))<br>peak memory: 0.0 B<br>number of output rows: 3<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
6 [labelType="html" label="<br><b>Project</b><br><br>"];
7 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 2,231,123<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 17440.0: task 24277))"];
}
subgraph cluster8 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n7.3 s (2.1 s, 2.1 s, 3.1 s (stage 17440.0: task 24279))";
9 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>6.0 s (1.7 s, 1.7 s, 2.6 s (stage 17440.0: task 24279))<br>peak memory total (min, med, max (stageId: taskId))<br>160.0 MiB (48.0 MiB, 48.0 MiB, 64.0 MiB (stage 17440.0: task 24279))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 17440.0: task 24277))"];
}
10 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 3<br>partition data size total (min, med, max (stageId: taskId))<br>3.7 MiB (1134.6 KiB, 1134.6 KiB, 1512.8 KiB (driver))<br>number of coalesced partitions: 3"];
11 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 2,231,123<br>local merged chunks fetched: 0<br>shuffle write time: 12 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 2,231,123<br>local bytes read total (min, med, max (stageId: taskId))<br>3.6 MiB (1112.1 KiB, 1113.3 KiB, 1483.6 KiB (stage 17440.0: task 24279))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 17440.0: task 24277))<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 3<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 68.1 MiB<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 3.6 MiB"];
subgraph cluster12 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n929 ms (2 ms, 7 ms, 872 ms (stage 17434.0: task 24267))";
13 [labelType="html" label="<br><b>Project</b><br><br>"];
14 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 2,231,123<br>number of input batches: 545"];
}
15 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 1<br>scan time total (min, med, max (stageId: taskId))<br>67 ms (1 ms, 6 ms, 22 ms (stage 17434.0: task 24267))<br>metadata time: 0 ms<br>size of files read: 98.8 MiB<br>number of output rows: 2,231,123"];
subgraph cluster16 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: 0 ms";
17 [labelType="html" label="<br><b>Project</b><br><br>"];
18 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 3"];
}
19 [labelType="html" label="<b>Window</b><br><br>spill size: 0.0 B"];
subgraph cluster20 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: 0 ms";
21 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>34 ms (7 ms, 9 ms, 18 ms (stage 17440.0: task 24279))<br>peak memory total (min, med, max (stageId: taskId))<br>52.0 MiB (17.0 MiB, 17.0 MiB, 18.0 MiB (stage 17440.0: task 24279))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 17440.0: task 24277))"];
}
22 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 3<br>partition data size total (min, med, max (stageId: taskId))<br>903.7 KiB (270.4 KiB, 273.3 KiB, 360.0 KiB (driver))<br>number of coalesced partitions: 3"];
23 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 137,144<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>6 ms (2 ms, 3 ms, 3 ms (stage 17435.0: task 24273))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 137,144<br>local bytes read total (min, med, max (stageId: taskId))<br>849.5 KiB (253.6 KiB, 255.6 KiB, 340.3 KiB (stage 17440.0: task 24279))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 17440.0: task 24277))<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 6<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>3.1 MiB (551.1 KiB, 2.6 MiB, 2.6 MiB (stage 17435.0: task 24273))<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>849.5 KiB (147.2 KiB, 702.3 KiB, 702.3 KiB (stage 17435.0: task 24273))"];
subgraph cluster24 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n346 ms (94 ms, 252 ms, 252 ms (stage 17435.0: task 24273))";
25 [labelType="html" label="<br><b>Project</b><br><br>"];
26 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 137,144"];
}
27 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 137,144<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 4.8 MiB"];
2->0;
3->2;
5->3;
6->5;
7->6;
9->7;
10->9;
11->10;
13->11;
14->13;
15->14;
17->7;
18->17;
19->18;
21->19;
22->21;
23->22;
25->23;
26->25;
27->26;
}
28
AdaptiveSparkPlan isFinalPlan=true
HashAggregate(keys=[], functions=[count(1)])
WholeStageCodegen (7)
Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=719464]
HashAggregate(keys=[], functions=[partial_count(1)])
Project
SortMergeJoin [COL_AF155DAB_420D_45B6_A9A9_56AA8D2E3D5C#1391817], [00_C_PASSAGE_PA_205245_COL_2AF4FE05_74B1_4A24_9BC1_7612D6ED731D#1391937], LeftOuter
WholeStageCodegen (6)
Sort [COL_AF155DAB_420D_45B6_A9A9_56AA8D2E3D5C#1391817 ASC NULLS FIRST], false, 0
WholeStageCodegen (3)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_AF155DAB_420D_45B6_A9A9_56AA8D2E3D5C#1391817, 10), ENSURE_REQUIREMENTS, [plan_id=719290]
Project [COL_1CCAED33_3E69_435E_B522_DAE3B9E63F9D#1391703 AS COL_AF155DAB_420D_45B6_A9A9_56AA8D2E3D5C#1391817]
ColumnarToRow
WholeStageCodegen (1)
FileScan parquet [COL_1CCAED33_3E69_435E_B522_DAE3B9E63F9D#1391703] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/00_A_LIGNES_RSF_205155], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<COL_1CCAED33_3E69_435E_B522_DAE3B9E63F9D:string>
Project [COL_2AF4FE05_74B1_4A24_9BC1_7612D6ED731D#1391640 AS 00_C_PASSAGE_PA_205245_COL_2AF4FE05_74B1_4A24_9BC1_7612D6ED731D#1391937]
Filter (COL_6B340AEB_6908_4A50_8AD6_FE561D33FBE0#1391656 <=> 1)
WholeStageCodegen (5)
Window [row_number() windowspecdefinition(COL_2AF4FE05_74B1_4A24_9BC1_7612D6ED731D#1391640, 1 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_6B340AEB_6908_4A50_8AD6_FE561D33FBE0#1391656], [COL_2AF4FE05_74B1_4A24_9BC1_7612D6ED731D#1391640], [1 DESC NULLS LAST]
Sort [COL_2AF4FE05_74B1_4A24_9BC1_7612D6ED731D#1391640 ASC NULLS FIRST, 1 DESC NULLS LAST], false, 0
WholeStageCodegen (4)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_2AF4FE05_74B1_4A24_9BC1_7612D6ED731D#1391640, 10), ENSURE_REQUIREMENTS, [plan_id=719304]
Project [Numéro Séjour#1391524 AS COL_2AF4FE05_74B1_4A24_9BC1_7612D6ED731D#1391640]
Filter (NOT CASE WHEN isnull(StartsWith(Numéro Séjour#1391524, #)) THEN false ELSE StartsWith(Numéro Séjour#1391524, #) END AND isnotnull(Numéro Séjour#1391524))
WholeStageCodegen (2)
FileScan csv [Numéro Séjour#1391524] Batched: false, DataFilters: [NOT CASE WHEN isnull(StartsWith(Numéro Séjour#1391524, #)) THEN false ELSE StartsWith(Numéro Séj..., Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/tmp/02_PASSAGE_PATI_204519/passage_Patient_..., PartitionFilters: [], PushedFilters: [IsNotNull(Numéro Séjour)], ReadSchema: struct<Numéro Séjour:string>