From 6ca10b905544360c770645acb7a0e7f02f3542a8 Mon Sep 17 00:00:00 2001 From: David Kimura Date: Mon, 3 Oct 2022 16:21:52 -0700 Subject: [PATCH] [ORCA] Avoid pushdown of predicate with set-returning function (#14201) Issue is that if a predicate with a set-returning function is pushed down then it can lead to bad execution because there exist cases where the below context cannot accept a set. Logic already existed to prevent such a case, but it wasn't sufficient. Following example demonstrates such a case: ```sql CREATE TABLE t(a int[]) ; INSERT INTO t VALUES (ARRAY[1, 2]); SELECT * FROM (SELECT unnest(t1.a) a_unnest FROM t t1, t t2) s WHERE a_unnest IS NOT NULL; ``` Also similarly, do not push down a filter through a projected column containing a subquery. --- .../JoinOnViewWithSetReturningColumn.mdp | 148 ++-- .../JoinWithSubqueryProjectColumn.mdp | 678 ++++++++++++++++++ .../src/operators/CExpressionPreprocessor.cpp | 12 +- src/backend/gporca/server/CMakeLists.txt | 2 +- src/test/regress/expected/qp_subquery.out | 76 ++ .../expected/qp_subquery_optimizer.out | 79 ++ .../expected/qp_with_clause_optimizer.out | 18 - src/test/regress/sql/qp_subquery.sql | 34 + 8 files changed, 958 insertions(+), 89 deletions(-) create mode 100644 src/backend/gporca/data/dxl/minidump/JoinWithSubqueryProjectColumn.mdp diff --git a/src/backend/gporca/data/dxl/minidump/JoinOnViewWithSetReturningColumn.mdp b/src/backend/gporca/data/dxl/minidump/JoinOnViewWithSetReturningColumn.mdp index 6499730ff18..4ffbdba0540 100644 --- a/src/backend/gporca/data/dxl/minidump/JoinOnViewWithSetReturningColumn.mdp +++ b/src/backend/gporca/data/dxl/minidump/JoinOnViewWithSetReturningColumn.mdp @@ -318,7 +318,7 @@ - + @@ -338,14 +338,11 @@ - + - - - - + @@ -357,19 +354,27 @@ - + + + + + + - + - + + + + + + + - - - @@ -378,16 +383,10 @@ - - - - - - - - + + - + @@ -396,35 +395,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -433,21 +403,69 @@ - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/backend/gporca/data/dxl/minidump/JoinWithSubqueryProjectColumn.mdp b/src/backend/gporca/data/dxl/minidump/JoinWithSubqueryProjectColumn.mdp new file mode 100644 index 00000000000..b850b47cbdc --- /dev/null +++ b/src/backend/gporca/data/dxl/minidump/JoinWithSubqueryProjectColumn.mdp @@ -0,0 +1,678 @@ + + + Result + Filter: ((text((SubPlan 1))) = 'dd'::text) + -> Nested Loop + Join Filter: true + -> Seq Scan on foo + -> Materialize + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on bar + SubPlan 1 + -> Result + -> Result + -> Materialize + -> Broadcast Motion 3:3 (slice3; segments: 3) + -> Seq Scan on bar bar_1 + Optimizer: Pivotal Optimizer (GPORCA) + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/backend/gporca/libgpopt/src/operators/CExpressionPreprocessor.cpp b/src/backend/gporca/libgpopt/src/operators/CExpressionPreprocessor.cpp index d570a5197f7..f591450b391 100644 --- a/src/backend/gporca/libgpopt/src/operators/CExpressionPreprocessor.cpp +++ b/src/backend/gporca/libgpopt/src/operators/CExpressionPreprocessor.cpp @@ -2840,18 +2840,20 @@ CExpressionPreprocessor::PexprTransposeSelectAndProject(CMemoryPool *mp, CExpression *pprojexpr = CUtils::PNthProjectElementExpr(pproject, ul); - CExpressionHandle exprhdl(mp); - exprhdl.Attach(pprojexpr); - exprhdl.DeriveProps(nullptr /*pdpctxt*/); - - if (exprhdl.Arity() > 1 && exprhdl.DeriveHasNonScalarFunction(1)) + if (pprojexpr->DeriveHasNonScalarFunction() || + pprojexpr->DeriveHasSubquery()) { // Bail if project expression contains a set-returning function + // or subquery pdrgpexpr->Release(); pexpr->AddRef(); return pexpr; } + CExpressionHandle exprhdl(mp); + exprhdl.Attach(pprojexpr); + exprhdl.DeriveProps(nullptr /*pdpctxt*/); + if (exprhdl.FChildrenHaveVolatileFunc()) { // Bail if project expression contains a volatile function diff --git a/src/backend/gporca/server/CMakeLists.txt b/src/backend/gporca/server/CMakeLists.txt index 924219e0c5e..016893fd329 100644 --- a/src/backend/gporca/server/CMakeLists.txt +++ b/src/backend/gporca/server/CMakeLists.txt @@ -169,7 +169,7 @@ CJoinPredTest: MultipleDampedPredJoinCardinality MultipleIndependentPredJoinCardinality MultiDistKeyJoinCardinality MultiDistKeyWithOtherPredsJoinCardinality NoDistKeyMultiPredJoinCardinality OneDistKeyMultiPredJoinCardinality JoinOnViewWithCastedColumn JoinOnViewWithCastedColumnAndSubqueryInPredicate JoinOnViewWithVolatileColumn -JoinOnViewWithMixOfPushableAndNonpushablePredicates JoinOnViewWithSetReturningColumn OuterJoinOnViewWithCastedColumn; +JoinOnViewWithMixOfPushableAndNonpushablePredicates JoinOnViewWithSetReturningColumn OuterJoinOnViewWithCastedColumn JoinWithSubqueryProjectColumn; CArrayCmpTest: ArrayConcat ArrayRef FoldedArrayCmp ArrayCmpAll diff --git a/src/test/regress/expected/qp_subquery.out b/src/test/regress/expected/qp_subquery.out index 5e3031aea94..f1d8fd389d5 100644 --- a/src/test/regress/expected/qp_subquery.out +++ b/src/test/regress/expected/qp_subquery.out @@ -1677,5 +1677,81 @@ select sum(case when b in (select b from temp_b where EXISTS (select sum(d) from 4 | 6 (1 row) +-- Check that predicate with set-returning function is not pushed down +create table table_with_array_column (an_array_column double precision[]); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'an_array_column' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +insert into table_with_array_column values (array[1.1, 2.2]); +explain (costs off) +select * +from ( + select unnest(t1.an_array_column) unnested_array_column + from table_with_array_column t1, table_with_array_column t2) zz +where unnested_array_column is not null; + QUERY PLAN +---------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Subquery Scan on zz + Filter: (zz.unnested_array_column IS NOT NULL) + -> ProjectSet + -> Nested Loop + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on table_with_array_column t2 + -> Materialize + -> Seq Scan on table_with_array_column t1 + Optimizer: Postgres query optimizer +(10 rows) + +select * +from ( + select unnest(t1.an_array_column) unnested_array_column + from table_with_array_column t1, table_with_array_column t2) zz +where unnested_array_column is not null; + unnested_array_column +----------------------- + 1.1 + 2.2 +(2 rows) + +-- check that predicate is not pushed through a projected non-correlated subquery +create table subquery_nonpush_through_1(a int, b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +create table subquery_nonpush_through_2(a int, b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +explain (costs off) +select * +from( + select (subquery_nonpush_through_1.a in (select a from subquery_nonpush_through_2))::text as xx, subquery_nonpush_through_1.b + from subquery_nonpush_through_1,subquery_nonpush_through_2) t +where xx='dd'; + QUERY PLAN +----------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Nested Loop + -> Broadcast Motion 3:3 (slice3; segments: 3) + -> Seq Scan on subquery_nonpush_through_2 + -> Materialize + -> Seq Scan on subquery_nonpush_through_1 + Filter: (((hashed SubPlan 2))::text = 'dd'::text) + SubPlan 2 + -> Broadcast Motion 3:3 (slice4; segments: 3) + -> Seq Scan on subquery_nonpush_through_2 subquery_nonpush_through_2_2 + SubPlan 1 + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on subquery_nonpush_through_2 subquery_nonpush_through_2_1 + Optimizer: Postgres query optimizer +(14 rows) + +select * +from( + select (subquery_nonpush_through_1.a in (select a from subquery_nonpush_through_2))::text as xx, subquery_nonpush_through_1.b + from subquery_nonpush_through_1,subquery_nonpush_through_2) t +where xx='dd'; + xx | b +----+--- +(0 rows) + set client_min_messages='warning'; drop schema qp_subquery cascade; diff --git a/src/test/regress/expected/qp_subquery_optimizer.out b/src/test/regress/expected/qp_subquery_optimizer.out index 35ff4d71d7a..5414069ba69 100644 --- a/src/test/regress/expected/qp_subquery_optimizer.out +++ b/src/test/regress/expected/qp_subquery_optimizer.out @@ -1675,5 +1675,84 @@ select sum(case when b in (select b from temp_b where EXISTS (select sum(d) from 4 | 6 (1 row) +-- Check that predicate with set-returning function is not pushed down +create table table_with_array_column (an_array_column double precision[]); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'an_array_column' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +insert into table_with_array_column values (array[1.1, 2.2]); +explain (costs off) +select * +from ( + select unnest(t1.an_array_column) unnested_array_column + from table_with_array_column t1, table_with_array_column t2) zz +where unnested_array_column is not null; + QUERY PLAN +--------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Result + Filter: (NOT ((unnest(table_with_array_column_1.an_array_column)) IS NULL)) + -> ProjectSet + -> Nested Loop + Join Filter: true + -> Seq Scan on table_with_array_column table_with_array_column_1 + -> Materialize + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on table_with_array_column + Optimizer: Pivotal Optimizer (GPORCA) +(11 rows) + +select * +from ( + select unnest(t1.an_array_column) unnested_array_column + from table_with_array_column t1, table_with_array_column t2) zz +where unnested_array_column is not null; + unnested_array_column +----------------------- + 1.1 + 2.2 +(2 rows) + +-- check that predicate is not pushed through a projected non-correlated subquery +create table subquery_nonpush_through_1(a int, b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +create table subquery_nonpush_through_2(a int, b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +explain (costs off) +select * +from( + select (subquery_nonpush_through_1.a in (select a from subquery_nonpush_through_2))::text as xx, subquery_nonpush_through_1.b + from subquery_nonpush_through_1,subquery_nonpush_through_2) t +where xx='dd'; + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Result + Filter: ((((SubPlan 1))::text) = 'dd'::text) + -> Nested Loop + Join Filter: true + -> Seq Scan on subquery_nonpush_through_1 + -> Materialize + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on subquery_nonpush_through_2 + SubPlan 1 + -> Result + -> Result + -> Materialize + -> Broadcast Motion 3:3 (slice3; segments: 3) + -> Seq Scan on subquery_nonpush_through_2 subquery_nonpush_through_2_1 + Optimizer: Pivotal Optimizer (GPORCA) +(19 rows) + +select * +from( + select (subquery_nonpush_through_1.a in (select a from subquery_nonpush_through_2))::text as xx, subquery_nonpush_through_1.b + from subquery_nonpush_through_1,subquery_nonpush_through_2) t +where xx='dd'; + xx | b +----+--- +(0 rows) + set client_min_messages='warning'; drop schema qp_subquery cascade; diff --git a/src/test/regress/expected/qp_with_clause_optimizer.out b/src/test/regress/expected/qp_with_clause_optimizer.out index 12b1ef76cde..0815400f4cc 100644 --- a/src/test/regress/expected/qp_with_clause_optimizer.out +++ b/src/test/regress/expected/qp_with_clause_optimizer.out @@ -6417,15 +6417,6 @@ CREATE TABLE countrylanguage_ao ( isofficial boolean NOT NULL, percentage real NOT NULL ) with (appendonly=true) distributed by (countrycode,language); -ALTER TABLE ONLY city_ao - ADD CONSTRAINT city_ao_pkey PRIMARY KEY (id); -ERROR: append-only tables do not support unique indexes -ALTER TABLE ONLY country_ao - ADD CONSTRAINT country_ao_pkey PRIMARY KEY (code); -ERROR: append-only tables do not support unique indexes -ALTER TABLE ONLY countrylanguage_ao - ADD CONSTRAINT countrylanguage_ao_pkey PRIMARY KEY (countrycode, "language"); -ERROR: append-only tables do not support unique indexes create index bitmap_city_ao_countrycode on city_ao using bitmap(countrycode); create index bitmap_country_ao_gf on country_ao using bitmap(governmentform); create index bitmap_country_ao_region on country_ao using bitmap(region); @@ -7323,15 +7314,6 @@ CREATE TABLE countrylanguage_co ( isofficial boolean NOT NULL, percentage real NOT NULL ) with (appendonly=true,orientation=column) distributed by (countrycode,language); -ALTER TABLE ONLY city_co - ADD CONSTRAINT city_co_pkey PRIMARY KEY (id); -ERROR: append-only tables do not support unique indexes -ALTER TABLE ONLY country_co - ADD CONSTRAINT country_co_pkey PRIMARY KEY (code); -ERROR: append-only tables do not support unique indexes -ALTER TABLE ONLY countrylanguage_co - ADD CONSTRAINT countrylanguage_co_pkey PRIMARY KEY (countrycode, "language"); -ERROR: append-only tables do not support unique indexes create index bitmap_city_co_countrycode on city_co using bitmap(countrycode); create index bitmap_country_co_gf on country_co using bitmap(governmentform); create index bitmap_country_co_region on country_co using bitmap(region); diff --git a/src/test/regress/sql/qp_subquery.sql b/src/test/regress/sql/qp_subquery.sql index 5f8bc099c1b..e73998150b1 100644 --- a/src/test/regress/sql/qp_subquery.sql +++ b/src/test/regress/sql/qp_subquery.sql @@ -670,5 +670,39 @@ select sum(case when b in (select b from temp_b where t.a>c) then 1 else 0 end), select sum(case when b in (select b from temp_b where EXISTS (select sum(d) from temp_c where t.a > d)) then 1 else 0 end),sum(case when not( b in (select b from temp_b where t.a>c)) then 1 else 0 end) from temp_a t; select sum(case when b in (select b from temp_b where EXISTS (select sum(d) from temp_c where t.a > d or t.a > temp_b.c)) then 1 else 0 end),sum(case when not( b in (select b from temp_b, temp_c where t.a>temp_b.c or t.a > temp_c.d)) then 1 else 0 end) from temp_a t; +-- Check that predicate with set-returning function is not pushed down +create table table_with_array_column (an_array_column double precision[]); +insert into table_with_array_column values (array[1.1, 2.2]); + +explain (costs off) +select * +from ( + select unnest(t1.an_array_column) unnested_array_column + from table_with_array_column t1, table_with_array_column t2) zz +where unnested_array_column is not null; + +select * +from ( + select unnest(t1.an_array_column) unnested_array_column + from table_with_array_column t1, table_with_array_column t2) zz +where unnested_array_column is not null; + +-- check that predicate is not pushed through a projected non-correlated subquery +create table subquery_nonpush_through_1(a int, b int); +create table subquery_nonpush_through_2(a int, b int); + +explain (costs off) +select * +from( + select (subquery_nonpush_through_1.a in (select a from subquery_nonpush_through_2))::text as xx, subquery_nonpush_through_1.b + from subquery_nonpush_through_1,subquery_nonpush_through_2) t +where xx='dd'; + +select * +from( + select (subquery_nonpush_through_1.a in (select a from subquery_nonpush_through_2))::text as xx, subquery_nonpush_through_1.b + from subquery_nonpush_through_1,subquery_nonpush_through_2) t +where xx='dd'; + set client_min_messages='warning'; drop schema qp_subquery cascade;