Show the Stage ID and Task ID that corresponds to the max metric
digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
1 [labelType="html" label="<br><b>CollectLimit</b><br><br>"];
subgraph cluster2 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: 49 ms";
3 [labelType="html" label="<br><b>Project</b><br><br>"];
4 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 1"];
}
5 [labelType="html" label="<b>ObjectHashAggregate</b><br><br>number of output rows: 1,733<br>time in aggregation build: 4 ms<br>spill size: 0.0 B<br>number of sort fallback tasks: 1"];
6 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 57.3 KiB<br>number of coalesced partitions: 1"];
7 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 1,733<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>4 ms (0 ms, 0 ms, 0 ms (stage 5402.0: task 5970))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 1,733<br>local bytes read: 54.8 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 10<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>347.0 KiB (30.5 KiB, 35.0 KiB, 38.3 KiB (stage 5402.0: task 5974))<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>54.8 KiB (4.7 KiB, 5.6 KiB, 6.0 KiB (stage 5402.0: task 5974))"];
8 [labelType="html" label="<b>ObjectHashAggregate</b><br><br>number of output rows: 1,733<br>time in aggregation build total (min, med, max (stageId: taskId))<br>5.9 s (550 ms, 561 ms, 649 ms (stage 5402.0: task 5970))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 5402.0: task 5972))<br>number of sort fallback tasks: 10"];
subgraph cluster9 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n5.9 s (550 ms, 561 ms, 649 ms (stage 5402.0: task 5970))";
10 [labelType="html" label="<br><b>Project</b><br><br>"];
11 [labelType="html" label="<br><b>Project</b><br><br>"];
}
12 [labelType="html" label="<b>CartesianProduct</b><br><br>number of output rows: 2,139"];
13 [labelType="html" label="<b>CartesianProduct</b><br><br>number of output rows: 2,139"];
subgraph cluster14 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n5.9 s (554 ms, 565 ms, 654 ms (stage 5402.0: task 5976))";
15 [labelType="html" label="<br><b>Project</b><br><br>"];
16 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 1,457,791"];
17 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 2,199,023<br>number of input batches: 541"];
}
18 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 10<br>scan time total (min, med, max (stageId: taskId))<br>206 ms (16 ms, 20 ms, 33 ms (stage 5402.0: task 5972))<br>metadata time: 0 ms<br>size of files read: 146.8 MiB<br>number of output rows: 2,199,023"];
19 [labelType="html" label="<b>ObjectHashAggregate</b><br><br>number of output rows: 10<br>time in aggregation build total (min, med, max (stageId: taskId))<br>11 ms (1 ms, 1 ms, 2 ms (stage 5402.0: task 5978))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 5402.0: task 5972))<br>number of sort fallback tasks: 0"];
20 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 1<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 10<br>local bytes read total (min, med, max (stageId: taskId))<br>1840.0 B (184.0 B, 184.0 B, 184.0 B (stage 5402.0: task 5972))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 5402.0: task 5972))<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 10<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 192.0 B<br>local merged bytes read: 0.0 B<br>number of partitions: 1<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 184.0 B"];
21 [labelType="html" label="<b>ObjectHashAggregate</b><br><br>number of output rows: 1<br>time in aggregation build: 6 ms<br>spill size: 0.0 B<br>number of sort fallback tasks: 0"];
subgraph cluster22 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: 6 ms";
23 [labelType="html" label="<br><b>Project</b><br><br>"];
}
24 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 9<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 50.0 B"];
subgraph cluster25 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n43 ms (3 ms, 4 ms, 6 ms (stage 5402.0: task 5976))";
26 [labelType="html" label="<br><b>Project</b><br><br>"];
27 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 10"];
}
28 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 10<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 110.0 B"];
1->0;
3->1;
4->3;
5->4;
6->5;
7->6;
8->7;
10->8;
11->10;
12->11;
13->12;
15->13;
16->15;
17->16;
18->17;
19->13;
20->19;
21->20;
23->21;
24->23;
26->12;
27->26;
28->27;
}
29
AdaptiveSparkPlan isFinalPlan=true
CollectLimit 1500
Project [COL_DE03DDD5_DFD7_4921_8E39_F0B46AC88716#474925, str_list_get(COL_D333C639_088D_43C6_B7CE_3AFF1AB0255F#474955, 0) AS COL_6841DD78_881F_4FF5_8139_B45CB0535307#474990, str_remove_extend(str_list_as_str(COL_3A253AA5_CE2C_4C9B_9EE6_E66674D9388A#474957), [[,],"]) AS COL_B71AED96_D44A_4496_B1D2_F66096545E56#474991]
Filter ((CASE WHEN isnull(Contains(str_remove_accent(upper(COL_DE03DDD5_DFD7_4921_8E39_F0B46AC88716#474925)), 030027795)) THEN false ELSE Contains(str_remove_accent(upper(COL_DE03DDD5_DFD7_4921_8E39_F0B46AC88716#474925)), 030027795) END OR CASE WHEN isnull(Contains(str_remove_accent(upper(str_list_get(COL_D333C639_088D_43C6_B7CE_3AFF1AB0255F#474955, 0))), 030027795)) THEN false ELSE Contains(str_remove_accent(upper(str_list_get(COL_D333C639_088D_43C6_B7CE_3AFF1AB0255F#474955, 0))), 030027795) END) OR CASE WHEN isnull(Contains(str_remove_accent(upper(str_remove_extend(str_list_as_str(COL_3A253AA5_CE2C_4C9B_9EE6_E66674D9388A#474957), [[,],"]))), 030027795)) THEN false ELSE Contains(str_remove_accent(upper(str_remove_extend(str_list_as_str(COL_3A253AA5_CE2C_4C9B_9EE6_E66674D9388A#474957), [[,],"]))), 030027795) END)
Project [COL_131C1042_88F0_40B5_A05D_6AC9BF440E17#474286, COL_3931E036_315E_4AC8_A56D_37AE79FDF6B7#474287, COL_F65F0EE2_FCC8_4E9F_B23F_0D97C40F54FD#474297, COL_B9A76B6D_E3EA_49F5_A0E9_DE474A8CE2B4#474300, str_if_with_rule(((((((COL_E43C6574_A3E3_4056_BB55_EA935E04502C#474293 <=> 1) AND (COL_10A975F7_B67F_438C_B8DE_9E0F00F273DC#474294 <=> 1)) AND (COL_318510CA_23E7_4678_9FCA_D34FF51ADC39#474302 <=> 0)) AND (size(str_list_intersect_fixed(COL_3377534C_2BCD_43B6_853A_869D44C151B3#474301, [MCS]), true) <=> 0)) AND (size(str_list_intersect_fixed(99_ACTES_ELIGIB_231708_COL_61C476F6_706A_4DB7_97C1_3E8CEB931839#474656, array(COL_B9A76B6D_E3EA_49F5_A0E9_DE474A8CE2B4#474300)), true) > 0)) AND (((CASE WHEN isnull(COL_414BA164_8DC3_4834_BDD7_434B2278858B#474298L) THEN false ELSE (COL_414BA164_8DC3_4834_BDD7_434B2278858B#474298L < 16) END AND NOT (COL_3931E036_315E_4AC8_A56D_37AE79FDF6B7#474287 <=> 01)) OR (CASE WHEN isnull(COL_414BA164_8DC3_4834_BDD7_434B2278858B#474298L) THEN false ELSE (COL_414BA164_8DC3_4834_BDD7_434B2278858B#474298L < 26) END AND (((COL_3931E036_315E_4AC8_A56D_37AE79FDF6B7#474287 <=> 17) OR (COL_3931E036_315E_4AC8_A56D_37AE79FDF6B7#474287 <=> 33)) OR (COL_3931E036_315E_4AC8_A56D_37AE79FDF6B7#474287 <=> 75)))) OR (COL_085E5798_40D8_4726_B01C_7D83841C9C25#474299 <=> O))), MCS, ) AS COL_5BD3501A_EDB6_4CA3_97B2_111DB176847B#474865]
WholeStageCodegen (4)
CartesianProduct
CartesianProduct (str_if_with_rule(((((((COL_E43C6574_A3E3_4056_BB55_EA935E04502C#474293 <=> 1) AND (COL_10A975F7_B67F_438C_B8DE_9E0F00F273DC#474294 <=> 1)) AND (COL_318510CA_23E7_4678_9FCA_D34FF51ADC39#474302 <=> 0)) AND (size(str_list_intersect_fixed(COL_3377534C_2BCD_43B6_853A_869D44C151B3#474301, [MCS]), true) <=> 0)) AND (size(str_list_intersect_fixed(99_ACTES_ELIGIB_231708_COL_61C476F6_706A_4DB7_97C1_3E8CEB931839#474656, array(COL_B9A76B6D_E3EA_49F5_A0E9_DE474A8CE2B4#474300)), true) > 0)) AND (((CASE WHEN isnull(COL_414BA164_8DC3_4834_BDD7_434B2278858B#474298L) THEN false ELSE (COL_414BA164_8DC3_4834_BDD7_434B2278858B#474298L < 16) END AND NOT (COL_3931E036_315E_4AC8_A56D_37AE79FDF6B7#474287 <=> 01)) OR (CASE WHEN isnull(COL_414BA164_8DC3_4834_BDD7_434B2278858B#474298L) THEN false ELSE (COL_414BA164_8DC3_4834_BDD7_434B2278858B#474298L < 26) END AND (((COL_3931E036_315E_4AC8_A56D_37AE79FDF6B7#474287 <=> 17) OR (COL_3931E036_315E_4AC8_A56D_37AE79FDF6B7#474287 <=> 33)) OR (COL_3931E036_315E_4AC8_A56D_37AE79FDF6B7#474287 <=> 75)))) OR (COL_085E5798_40D8_4726_B01C_7D83841C9C25#474299 <=> O))), MCS, ) <=> MCS)
Project [COL_7153B3F6_7E41_45CB_84CC_CF466AE1BE86#474042 AS COL_131C1042_88F0_40B5_A05D_6AC9BF440E17#474286, COL_4587DB83_72BD_4F2D_BA18_AAD14178DC7F#474043 AS COL_3931E036_315E_4AC8_A56D_37AE79FDF6B7#474287, COL_346C27DC_6D06_4FCF_A022_7FF177053D81#474049 AS COL_E43C6574_A3E3_4056_BB55_EA935E04502C#474293, COL_75DE8325_2CC3_4E7A_A718_80AA4F2A9D62#474050 AS COL_10A975F7_B67F_438C_B8DE_9E0F00F273DC#474294, COL_A5765544_357B_4395_B6BC_21D7D5C2D0E0#474053 AS COL_F65F0EE2_FCC8_4E9F_B23F_0D97C40F54FD#474297, COL_969EAFA1_AD39_4285_AD05_6146CD6F9342#474054L AS COL_414BA164_8DC3_4834_BDD7_434B2278858B#474298L, COL_19CE074B_1F54_459C_ABA2_D9FCF9DC7487#474055 AS COL_085E5798_40D8_4726_B01C_7D83841C9C25#474299, COL_A4AF3049_C967_4B58_9D36_40EE5D79F6AC#474056 AS COL_B9A76B6D_E3EA_49F5_A0E9_DE474A8CE2B4#474300, COL_714AB906_7002_4ADC_AA1A_38467A3EDE10#474057 AS COL_3377534C_2BCD_43B6_853A_869D44C151B3#474301, COL_D986422B_41E3_4D4B_9F25_9D68ED522BE6#474059 AS COL_318510CA_23E7_4678_9FCA_D34FF51ADC39#474302]
Filter (NOT (COL_A4AF3049_C967_4B58_9D36_40EE5D79F6AC#474056 <=> ) AND isnotnull(COL_A4AF3049_C967_4B58_9D36_40EE5D79F6AC#474056))