digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: 0 ms";
2 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 0 ms<br>number of output rows: 1"];
}
3 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 1<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 1<br>local bytes read: 59.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 16.0 B<br>local merged bytes read: 0.0 B<br>number of partitions: 1<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 59.0 B"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: 0 ms";
5 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 0 ms<br>number of output rows: 1"];
6 [labelType="html" label="<br><b>Project</b><br><br>"];
7 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 2"];
}
8 [labelType="html" label="<b>Window</b><br><br>spill size: 0.0 B"];
subgraph cluster9 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: 22 ms";
10 [labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 16.1 MiB<br>spill size: 0.0 B"];
}
11 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 271.0 B<br>number of coalesced partitions: 1"];
12 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 7<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 7<br>local bytes read: 252.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 512.0 B<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 252.0 B"];
subgraph cluster13 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: 19 ms";
14 [labelType="html" label="<br><b>Project</b><br><br>"];
}
15 [labelType="html" label="<b>CartesianProduct</b><br><br>number of output rows: 7"];
subgraph cluster16 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: 34 ms";
17 [labelType="html" label="<br><b>Project</b><br><br>"];
18 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 7"];
19 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 7<br>number of input batches: 1"];
}
20 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 1<br>scan time: 3 ms<br>metadata time: 0 ms<br>size of files read: 20.3 KiB<br>number of output rows: 7"];
subgraph cluster21 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: 6 ms";
22 [labelType="html" label="<br><b>Project</b><br><br>"];
23 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 1"];
}
24 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 1<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 174.0 B"];
2->0;
3->2;
5->3;
6->5;
7->6;
8->7;
10->8;
11->10;
12->11;
14->12;
15->14;
17->15;
18->17;
19->18;
20->19;
22->15;
23->22;
24->23;
}
25
AdaptiveSparkPlan isFinalPlan=true
HashAggregate(keys=[], functions=[count(1)])
WholeStageCodegen (6)
Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=488535]
HashAggregate(keys=[], functions=[partial_count(1)])
Project
Filter (COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864 <=> 1)
WholeStageCodegen (5)
Window [row_number() windowspecdefinition(COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864], [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792], [COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST]
Sort [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618 ASC NULLS FIRST, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792 ASC NULLS FIRST, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST], false, 0
WholeStageCodegen (4)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, 10), ENSURE_REQUIREMENTS, [plan_id=488458]
Project [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, str_extract_by_str(COL_1BD1CF4D_D1E1_4C54_94F8_0754D4126CAC#791620, *, RIGHT) AS COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
WholeStageCodegen (3)
CartesianProduct
Project [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291 AS COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333 AS COL_1BD1CF4D_D1E1_4C54_94F8_0754D4126CAC#791620, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313 AS COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Filter (COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291 <=> 011040518)
ColumnarToRow
WholeStageCodegen (1)
FileScan parquet [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291,COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313,COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333] Batched: true, DataFilters: [(COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291 <=> 011040518)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/00_B_ACTES_PATI_210234], PartitionFilters: [], PushedFilters: [EqualNullSafe(COL_64ED17E5_A991_405B_AEED_8A2977283F76,011040518)], ReadSchema: struct<COL_64ED17E5_A991_405B_AEED_8A2977283F76:string,COL_7D270645_B926_42AF_A110_EB3A1A53031B:s...
Project
Filter ((isnotnull(REGLES#791416) AND (RC-SSN-SSF = REGLES#791416)) AND (APPLICABLE#791417 <=> O))
WholeStageCodegen (2)
FileScan csv [REGLES#791416,APPLICABLE#791417] Batched: false, DataFilters: [isnotnull(REGLES#791416), (RC-SSN-SSF = REGLES#791416), (APPLICABLE#791417 <=> O)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/tmp/REGLES_APPLICAB_209983/REGLES-A-TTT new..., PartitionFilters: [], PushedFilters: [IsNotNull(REGLES), EqualTo(REGLES,RC-SSN-SSF), EqualNullSafe(APPLICABLE,O)], ReadSchema: struct<REGLES:string,APPLICABLE:string>
== Physical Plan ==
AdaptiveSparkPlan (35)
+- == Final Plan ==
* HashAggregate (20)
+- ShuffleQueryStage (19), Statistics(sizeInBytes=16.0 B, rowCount=1)
+- Exchange (18)
+- * HashAggregate (17)
+- * Project (16)
+- * Filter (15)
+- Window (14)
+- * Sort (13)
+- AQEShuffleRead (12)
+- ShuffleQueryStage (11), Statistics(sizeInBytes=512.0 B, rowCount=7)
+- Exchange (10)
+- * Project (9)
+- CartesianProduct Inner (8)
:- * Project (4)
: +- * Filter (3)
: +- * ColumnarToRow (2)
: +- Scan parquet (1)
+- * Project (7)
+- * Filter (6)
+- Scan csv (5)
+- == Initial Plan ==
HashAggregate (34)
+- Exchange (33)
+- HashAggregate (32)
+- Project (31)
+- Filter (30)
+- Window (29)
+- Sort (28)
+- Exchange (27)
+- Project (26)
+- CartesianProduct Inner (25)
:- Project (22)
: +- Filter (21)
: +- Scan parquet (1)
+- Project (24)
+- Filter (23)
+- Scan csv (5)
(1) Scan parquet
Output [3]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/00_B_ACTES_PATI_210234]
PushedFilters: [EqualNullSafe(COL_64ED17E5_A991_405B_AEED_8A2977283F76,011040518)]
ReadSchema: struct<COL_64ED17E5_A991_405B_AEED_8A2977283F76:string,COL_7D270645_B926_42AF_A110_EB3A1A53031B:string,COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28:string>
(2) ColumnarToRow [codegen id : 1]
Input [3]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333]
(3) Filter [codegen id : 1]
Input [3]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333]
Condition : (COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291 <=> 011040518)
(4) Project [codegen id : 1]
Output [3]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291 AS COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333 AS COL_1BD1CF4D_D1E1_4C54_94F8_0754D4126CAC#791620, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313 AS COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Input [3]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333]
(5) Scan csv
Output [2]: [REGLES#791416, APPLICABLE#791417]
Batched: false
Location: InMemoryFileIndex [file:/data/input/depot/csv/tmp/REGLES_APPLICAB_209983/REGLES-A-TTT new.csv]
PushedFilters: [IsNotNull(REGLES), EqualTo(REGLES,RC-SSN-SSF), EqualNullSafe(APPLICABLE,O)]
ReadSchema: struct<REGLES:string,APPLICABLE:string>
(6) Filter [codegen id : 2]
Input [2]: [REGLES#791416, APPLICABLE#791417]
Condition : ((isnotnull(REGLES#791416) AND (RC-SSN-SSF = REGLES#791416)) AND (APPLICABLE#791417 <=> O))
(7) Project [codegen id : 2]
Output: []
Input [2]: [REGLES#791416, APPLICABLE#791417]
(8) CartesianProduct
Join type: Inner
Join condition: None
(9) Project [codegen id : 3]
Output [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, str_extract_by_str(COL_1BD1CF4D_D1E1_4C54_94F8_0754D4126CAC#791620, *, RIGHT) AS COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_1BD1CF4D_D1E1_4C54_94F8_0754D4126CAC#791620, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
(10) Exchange
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Arguments: hashpartitioning(COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, 10), ENSURE_REQUIREMENTS, [plan_id=488458]
(11) ShuffleQueryStage
Output [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Arguments: 0
(12) AQEShuffleRead
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Arguments: coalesced
(13) Sort [codegen id : 4]
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Arguments: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618 ASC NULLS FIRST, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792 ASC NULLS FIRST, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST], false, 0
(14) Window
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Arguments: [row_number() windowspecdefinition(COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864], [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792], [COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST]
(15) Filter [codegen id : 5]
Input [4]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623, COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864]
Condition : (COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864 <=> 1)
(16) Project [codegen id : 5]
Output: []
Input [4]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623, COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864]
(17) HashAggregate [codegen id : 5]
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#791995L]
Results [1]: [count#791996L]
(18) Exchange
Input [1]: [count#791996L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=488535]
(19) ShuffleQueryStage
Output [1]: [count#791996L]
Arguments: 1
(20) HashAggregate [codegen id : 6]
Input [1]: [count#791996L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#791992L]
Results [1]: [count(1)#791992L AS count#791993L]
(21) Filter
Input [3]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333]
Condition : (COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291 <=> 011040518)
(22) Project
Output [3]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291 AS COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333 AS COL_1BD1CF4D_D1E1_4C54_94F8_0754D4126CAC#791620, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313 AS COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Input [3]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#791291, COL_7D270645_B926_42AF_A110_EB3A1A53031B#791313, COL_DC641C8D_1EAF_49BD_A02D_E74816E44C28#791333]
(23) Filter
Input [2]: [REGLES#791416, APPLICABLE#791417]
Condition : ((isnotnull(REGLES#791416) AND (RC-SSN-SSF = REGLES#791416)) AND (APPLICABLE#791417 <=> O))
(24) Project
Output: []
Input [2]: [REGLES#791416, APPLICABLE#791417]
(25) CartesianProduct
Join type: Inner
Join condition: None
(26) Project
Output [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, str_extract_by_str(COL_1BD1CF4D_D1E1_4C54_94F8_0754D4126CAC#791620, *, RIGHT) AS COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_1BD1CF4D_D1E1_4C54_94F8_0754D4126CAC#791620, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
(27) Exchange
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Arguments: hashpartitioning(COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, 10), ENSURE_REQUIREMENTS, [plan_id=488406]
(28) Sort
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Arguments: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618 ASC NULLS FIRST, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792 ASC NULLS FIRST, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST], false, 0
(29) Window
Input [3]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623]
Arguments: [row_number() windowspecdefinition(COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864], [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792], [COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623 ASC NULLS LAST]
(30) Filter
Input [4]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623, COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864]
Condition : (COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864 <=> 1)
(31) Project
Output: []
Input [4]: [COL_568DEDCB_F936_41B1_A0EC_986E8205D003#791618, COL_C97CE078_7AD5_4D54_A5FF_290B4E6B18FF#791792, COL_7453C4CB_1AAF_4D1F_A079_968F5F790238#791623, COL_B2D8469D_E32C_4A51_BA8A_9172B340952C#791864]
(32) HashAggregate
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#791995L]
Results [1]: [count#791996L]
(33) Exchange
Input [1]: [count#791996L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=488413]
(34) HashAggregate
Input [1]: [count#791996L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#791992L]
Results [1]: [count(1)#791992L AS count#791993L]
(35) AdaptiveSparkPlan
Output [1]: [count#791993L]
Arguments: isFinalPlan=true