SP1 - Details for Query 2362

Submitted Time: 2026/09/17 16:20:36
Duration: 0.1 s
Succeeded Jobs: 3435

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" CollectLimit "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: 0 ms"; 2 [labelType="html" label=" Project "]; } 3 [labelType="html" label="CartesianProduct number of output rows: 1,500"]; subgraph cluster4 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: 0 ms"; 5 [labelType="html" label=" LocalLimit "]; 6 [labelType="html" label=" Project "]; 7 [labelType="html" label="ColumnarToRow number of output rows: 4,096 number of input batches: 1"]; } 8 [labelType="html" label="Scan parquet number of files read: 2 scan time: 4 ms metadata time: 0 ms size of files read: 2.3 MiB number of output rows: 4,096"]; subgraph cluster9 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: 4 ms"; 10 [labelType="html" label=" LocalLimit "]; 11 [labelType="html" label=" Project "]; 12 [labelType="html" label="Filter number of output rows: 1"]; } 13 [labelType="html" label="Scan csv number of output rows: 1 number of files read: 1 metadata time: 0 ms size of files read: 175.0 B"]; 2->0; 3->2; 5->3; 6->5; 7->6; 8->7; 10->3; 11->10; 12->11; 13->12; }

CollectLimit 1500

Project [COL_32F739B2_1E09_4FE7_8AEE_4EAB7D20E586#589915, COL_FDC28A6E_072C_4A38_860E_0858AFE166FD#589916, COL_1C67BACF_1B65_4A59_B0AD_E458D86429D5#589917, COL_658DDA2B_EB4A_460F_BDE0_BEB40D7FFD26#589918, COL_032696D7_7058_44F0_ABC3_DB207100F75A#589919, REGLES_A_APPLIQ_225642_APPLICABLE_225644#589927 AS COL_E6D2D4F2_F182_4E39_AE7F_5F932052634B#589950]

WholeStageCodegen (3)

CartesianProduct

LocalLimit 1500

Project [COL_64ED17E5_A991_405B_AEED_8A2977283F76#589635 AS COL_32F739B2_1E09_4FE7_8AEE_4EAB7D20E586#589915, COL_46E1DB98_C579_4CA8_91D3_EC36371C7D62#589636 AS COL_FDC28A6E_072C_4A38_860E_0858AFE166FD#589916, COL_81B86243_0BBB_463E_BE7E_73AEE16D2DD0#589676 AS COL_1C67BACF_1B65_4A59_B0AD_E458D86429D5#589917, COL_F45E0732_AB66_4936_AECE_1F9475288E1E#589668 AS COL_658DDA2B_EB4A_460F_BDE0_BEB40D7FFD26#589918, COL_F23CCCC8_5C4A_4FB8_A318_67291E42693A#589683 AS COL_032696D7_7058_44F0_ABC3_DB207100F75A#589919]

ColumnarToRow

WholeStageCodegen (1)

FileScan parquet [COL_64ED17E5_A991_405B_AEED_8A2977283F76#589635,COL_46E1DB98_C579_4CA8_91D3_EC36371C7D62#589636,COL_F45E0732_AB66_4936_AECE_1F9475288E1E#589668,COL_81B86243_0BBB_463E_BE7E_73AEE16D2DD0#589676,COL_F23CCCC8_5C4A_4FB8_A318_67291E42693A#589683] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/datablock/00_B_ACTES_PATI_225863], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<COL_64ED17E5_A991_405B_AEED_8A2977283F76:string,COL_46E1DB98_C579_4CA8_91D3_EC36371C7D62:s...

LocalLimit 1500

Project [APPLICABLE#589617 AS REGLES_A_APPLIQ_225642_APPLICABLE_225644#589927]

Filter ((isnotnull(REGLES#589616) AND (RC-SIC/SIM = REGLES#589616)) AND (APPLICABLE#589617 <=> O))

WholeStageCodegen (2)

FileScan csv [REGLES#589616,APPLICABLE#589617] Batched: false, DataFilters: [isnotnull(REGLES#589616), (RC-SIC/SIM = REGLES#589616), (APPLICABLE#589617 <=> O)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/tmp/REGLES_APPLICAB_225515/REGLES-A-TTT new..., PartitionFilters: [], PushedFilters: [IsNotNull(REGLES), EqualTo(REGLES,RC-SIC/SIM), EqualNullSafe(APPLICABLE,O)], ReadSchema: struct<REGLES:string,APPLICABLE:string>

Details

== Physical Plan ==
CollectLimit (11)
+- * Project (10)
   +- CartesianProduct Inner (9)
      :- * LocalLimit (4)
      :  +- * Project (3)
      :     +- * ColumnarToRow (2)
      :        +- Scan parquet  (1)
      +- * LocalLimit (8)
         +- * Project (7)
            +- * Filter (6)
               +- Scan csv  (5)


(1) Scan parquet 
Output [5]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#589635, COL_46E1DB98_C579_4CA8_91D3_EC36371C7D62#589636, COL_F45E0732_AB66_4936_AECE_1F9475288E1E#589668, COL_81B86243_0BBB_463E_BE7E_73AEE16D2DD0#589676, COL_F23CCCC8_5C4A_4FB8_A318_67291E42693A#589683]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/datablock/00_B_ACTES_PATI_225863]
ReadSchema: struct<COL_64ED17E5_A991_405B_AEED_8A2977283F76:string,COL_46E1DB98_C579_4CA8_91D3_EC36371C7D62:string,COL_F45E0732_AB66_4936_AECE_1F9475288E1E:string,COL_81B86243_0BBB_463E_BE7E_73AEE16D2DD0:string,COL_F23CCCC8_5C4A_4FB8_A318_67291E42693A:string>

(2) ColumnarToRow [codegen id : 1]
Input [5]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#589635, COL_46E1DB98_C579_4CA8_91D3_EC36371C7D62#589636, COL_F45E0732_AB66_4936_AECE_1F9475288E1E#589668, COL_81B86243_0BBB_463E_BE7E_73AEE16D2DD0#589676, COL_F23CCCC8_5C4A_4FB8_A318_67291E42693A#589683]

(3) Project [codegen id : 1]
Output [5]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#589635 AS COL_32F739B2_1E09_4FE7_8AEE_4EAB7D20E586#589915, COL_46E1DB98_C579_4CA8_91D3_EC36371C7D62#589636 AS COL_FDC28A6E_072C_4A38_860E_0858AFE166FD#589916, COL_81B86243_0BBB_463E_BE7E_73AEE16D2DD0#589676 AS COL_1C67BACF_1B65_4A59_B0AD_E458D86429D5#589917, COL_F45E0732_AB66_4936_AECE_1F9475288E1E#589668 AS COL_658DDA2B_EB4A_460F_BDE0_BEB40D7FFD26#589918, COL_F23CCCC8_5C4A_4FB8_A318_67291E42693A#589683 AS COL_032696D7_7058_44F0_ABC3_DB207100F75A#589919]
Input [5]: [COL_64ED17E5_A991_405B_AEED_8A2977283F76#589635, COL_46E1DB98_C579_4CA8_91D3_EC36371C7D62#589636, COL_F45E0732_AB66_4936_AECE_1F9475288E1E#589668, COL_81B86243_0BBB_463E_BE7E_73AEE16D2DD0#589676, COL_F23CCCC8_5C4A_4FB8_A318_67291E42693A#589683]

(4) LocalLimit [codegen id : 1]
Input [5]: [COL_32F739B2_1E09_4FE7_8AEE_4EAB7D20E586#589915, COL_FDC28A6E_072C_4A38_860E_0858AFE166FD#589916, COL_1C67BACF_1B65_4A59_B0AD_E458D86429D5#589917, COL_658DDA2B_EB4A_460F_BDE0_BEB40D7FFD26#589918, COL_032696D7_7058_44F0_ABC3_DB207100F75A#589919]
Arguments: 1500

(5) Scan csv 
Output [2]: [REGLES#589616, APPLICABLE#589617]
Batched: false
Location: InMemoryFileIndex [file:/data/input/depot/csv/tmp/REGLES_APPLICAB_225515/REGLES-A-TTT new.csv]
PushedFilters: [IsNotNull(REGLES), EqualTo(REGLES,RC-SIC/SIM), EqualNullSafe(APPLICABLE,O)]
ReadSchema: struct<REGLES:string,APPLICABLE:string>

(6) Filter [codegen id : 2]
Input [2]: [REGLES#589616, APPLICABLE#589617]
Condition : ((isnotnull(REGLES#589616) AND (RC-SIC/SIM = REGLES#589616)) AND (APPLICABLE#589617 <=> O))

(7) Project [codegen id : 2]
Output [1]: [APPLICABLE#589617 AS REGLES_A_APPLIQ_225642_APPLICABLE_225644#589927]
Input [2]: [REGLES#589616, APPLICABLE#589617]

(8) LocalLimit [codegen id : 2]
Input [1]: [REGLES_A_APPLIQ_225642_APPLICABLE_225644#589927]
Arguments: 1500

(9) CartesianProduct
Join type: Inner
Join condition: None

(10) Project [codegen id : 3]
Output [6]: [COL_32F739B2_1E09_4FE7_8AEE_4EAB7D20E586#589915, COL_FDC28A6E_072C_4A38_860E_0858AFE166FD#589916, COL_1C67BACF_1B65_4A59_B0AD_E458D86429D5#589917, COL_658DDA2B_EB4A_460F_BDE0_BEB40D7FFD26#589918, COL_032696D7_7058_44F0_ABC3_DB207100F75A#589919, REGLES_A_APPLIQ_225642_APPLICABLE_225644#589927 AS COL_E6D2D4F2_F182_4E39_AE7F_5F932052634B#589950]
Input [6]: [COL_32F739B2_1E09_4FE7_8AEE_4EAB7D20E586#589915, COL_FDC28A6E_072C_4A38_860E_0858AFE166FD#589916, COL_1C67BACF_1B65_4A59_B0AD_E458D86429D5#589917, COL_658DDA2B_EB4A_460F_BDE0_BEB40D7FFD26#589918, COL_032696D7_7058_44F0_ABC3_DB207100F75A#589919, REGLES_A_APPLIQ_225642_APPLICABLE_225644#589927]

(11) CollectLimit
Input [6]: [COL_32F739B2_1E09_4FE7_8AEE_4EAB7D20E586#589915, COL_FDC28A6E_072C_4A38_860E_0858AFE166FD#589916, COL_1C67BACF_1B65_4A59_B0AD_E458D86429D5#589917, COL_658DDA2B_EB4A_460F_BDE0_BEB40D7FFD26#589918, COL_032696D7_7058_44F0_ABC3_DB207100F75A#589919, COL_E6D2D4F2_F182_4E39_AE7F_5F932052634B#589950]
Arguments: 1500

SQL / DataFrame Properties

Name	Value
spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources	parquet,orc,geoparquet