From 60c5e148e2ee1c30a1a28eb8fcc1fc42af6bf8d2 Mon Sep 17 00:00:00 2001 From: Drew Kimball Date: Mon, 8 Jan 2024 03:38:42 -0700 Subject: [PATCH 1/4] opt: add method to add strict dependency to FuncDepSet This commit adds a new method, `AddStrictDependency`, to `FuncDepSet`. This will be used in the following commit to add a dependency between a Window operator's partition columns and its functions. Existing methods don't work for this because the dependency is not a key. Epic: None Release note: None --- pkg/sql/opt/props/func_dep.go | 6 +++ pkg/sql/opt/props/func_dep_rand_test.go | 63 +++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/pkg/sql/opt/props/func_dep.go b/pkg/sql/opt/props/func_dep.go index 0b8ac2aa3e28..136084c3a977 100644 --- a/pkg/sql/opt/props/func_dep.go +++ b/pkg/sql/opt/props/func_dep.go @@ -929,6 +929,12 @@ func (f *FuncDepSet) AddEquivalency(a, b opt.ColumnID) { f.tryToReduceKey(opt.ColSet{} /* notNullCols */) } +// AddStrictDependency adds a new strict dependency to the set. +func (f *FuncDepSet) AddStrictDependency(from, to opt.ColSet) { + f.addDependency(from, to, true /* strict */, false /* equiv */) + f.tryToReduceKey(opt.ColSet{} /* notNullCols */) +} + // AddConstants adds a strict FD to the set that declares each given column as // having the same constant value for all rows. If a column is nullable, then // its value may be NULL, but then the column must be NULL for all rows. For diff --git a/pkg/sql/opt/props/func_dep_rand_test.go b/pkg/sql/opt/props/func_dep_rand_test.go index 36af140e00de..729c36d5da25 100644 --- a/pkg/sql/opt/props/func_dep_rand_test.go +++ b/pkg/sql/opt/props/func_dep_rand_test.go @@ -690,6 +690,67 @@ func (o *addSynthOp) ApplyToFDs(fd FuncDepSet) FuncDepSet { return out } +// addStrictDepOp is a test operation corresponding to AddStrictDependency. +type addStrictDepOp struct { + from, to opt.ColSet +} + +func genAddStrictDep(minCols, maxCols int) testOpGenerator { + return func(tc *testConfig) testOp { + from := tc.randColSet(minCols, maxCols) + to := tc.randColSet(minCols, maxCols) + from.DifferenceWith(to) + return &addStrictDepOp{ + from: from, + to: to, + } + } +} + +func (o *addStrictDepOp) String() string { + return fmt.Sprintf("AddStrictDependency(%s, %s)", o.from, o.to) +} + +func (o *addStrictDepOp) FilterRelation(tr testRelation) testRelation { + // Filter out rows where the from->to FD doesn't hold. The code here parallels + // that in testRelation.checkKey. + // + // We split the rows into groups (keyed on the `from` columns), picking the + // first row in each group as the "representative" of that group. All other + // rows in the group are checked against the representative row. + var out testRelation + m := make(map[rowKey]testRow) + perm := rand.Perm(len(tr)) + for _, rowIdx := range perm { + r := tr[rowIdx] + k, _ := r.key(o.from) + if first, ok := m[k]; ok { + shouldFilter := false + for col, ok := o.to.Next(0); ok; col, ok = o.to.Next(col + 1) { + if first.value(col) != r.value(col) { + // Filter out row. + shouldFilter = true + break + } + } + if shouldFilter { + continue + } + } else { + m[k] = r + } + out = append(out, r) + } + return out +} + +func (o *addStrictDepOp) ApplyToFDs(fd FuncDepSet) FuncDepSet { + var out FuncDepSet + out.CopyFrom(&fd) + out.AddStrictDependency(o.from, o.to) + return out +} + // testState corresponds to a chain of applied test operations. The head of a // testStates chain has no parent and no op and just corresponds to the initial // (empty) FDs and test relation. @@ -803,6 +864,7 @@ func TestFuncDepOpsRandom(t *testing.T) { genAddConst(1 /* minCols */, 3 /* maxCols */), genAddEquiv(), genAddSynth(0 /* minCols */, 3 /* maxCols */), + genAddStrictDep(0 /* minCols */, 3 /* maxCols */), }, }, @@ -819,6 +881,7 @@ func TestFuncDepOpsRandom(t *testing.T) { genAddConst(1 /* minCols */, 4 /* maxCols */), genAddEquiv(), genAddSynth(0 /* minCols */, 3 /* maxCols */), + genAddStrictDep(0 /* minCols */, 3 /* maxCols */), }, }, } From b3aba93858f8cf1c8b98b2ac5f1b765299422864 Mon Sep 17 00:00:00 2001 From: Yahor Yuzefovich Date: Mon, 9 Dec 2024 17:13:46 -0800 Subject: [PATCH 2/4] opt: add implicit SELECT FOR UPDATE to initial scan of DELETE This commit makes it so that we apply implicit SELECT FOR UPDATE locking behavior to the initial scan of the DELETE operation in some cases. Namely, we do so when the input to the DELETE is either a scan or an index join on top of a scan (with any number of renders on top) - these conditions mean that all filters were pushed down into the scan, so we won't lock any unnecessary rows. I think it's only possible to have at most one render expression on top of the scan, but I chose to be defensive and allowed nested renders too. In such form the conditions are exactly the same as we use for adding SFU to UPDATEs, so the same function is reused. Existing `enable_implicit_select_for_update` session variable is consulted. Release note (sql change): DELETE statements now acquire locks using the FOR UPDATE locking mode during their initial row scan in some comes, which improves performance for contended workloads. This behavior is configurable using the `enable_implicit_select_for_update` session variable. --- .../logic_test/regional_by_row_query_behavior | 1 + pkg/sql/opt/exec/execbuilder/mutation.go | 48 +++++++------------ pkg/sql/opt/exec/execbuilder/testdata/cascade | 4 ++ pkg/sql/opt/exec/execbuilder/testdata/delete | 7 +++ .../exec/execbuilder/testdata/explain_redact | 2 + pkg/sql/opt/exec/execbuilder/testdata/fk | 7 +++ .../execbuilder/testdata/fk_read_committed | 1 + .../exec/execbuilder/testdata/inverted_index | 4 +- .../testdata/inverted_index_multi_column | 8 ++-- .../execbuilder/testdata/not_visible_index | 3 ++ pkg/sql/opt/exec/execbuilder/testdata/orderby | 1 + .../exec/execbuilder/testdata/partial_index | 14 +++--- .../opt/exec/execbuilder/testdata/show_trace | 2 +- pkg/sql/opt/exec/execbuilder/testdata/spool | 2 + .../testdata/sql_activity_stats_compaction | 4 ++ .../exec/execbuilder/testdata/virtual_columns | 7 +++ pkg/sql/opt/exec/explain/testdata/gists | 2 + .../index_mutations/delete_preserving_delete | 10 ++-- .../delete_preserving_unique_index | 2 +- 19 files changed, 79 insertions(+), 50 deletions(-) diff --git a/pkg/ccl/logictestccl/testdata/logic_test/regional_by_row_query_behavior b/pkg/ccl/logictestccl/testdata/logic_test/regional_by_row_query_behavior index 951264a09eb1..0e5b5bbff503 100644 --- a/pkg/ccl/logictestccl/testdata/logic_test/regional_by_row_query_behavior +++ b/pkg/ccl/logictestccl/testdata/logic_test/regional_by_row_query_behavior @@ -3171,6 +3171,7 @@ vectorized: true │ missing stats │ table: user_settings_cascades@user_settings_cascades_user_id_idx │ spans: [/'ap-southeast-2'/'5ebfedee-0dcf-41e6-a315-5fa0b51b9882' - /'ap-southeast-2'/'5ebfedee-0dcf-41e6-a315-5fa0b51b9882'] [/'ca-central-1'/'5ebfedee-0dcf-41e6-a315-5fa0b51b9882' - /'ca-central-1'/'5ebfedee-0dcf-41e6-a315-5fa0b51b9882'] [/'us-east-1'/'5ebfedee-0dcf-41e6-a315-5fa0b51b9882' - /'us-east-1'/'5ebfedee-0dcf-41e6-a315-5fa0b51b9882'] +│ locking strength: for update │ └── • constraint-check │ diff --git a/pkg/sql/opt/exec/execbuilder/mutation.go b/pkg/sql/opt/exec/execbuilder/mutation.go index cab0fcd2afe9..f1d8df8d973a 100644 --- a/pkg/sql/opt/exec/execbuilder/mutation.go +++ b/pkg/sql/opt/exec/execbuilder/mutation.go @@ -1131,6 +1131,10 @@ var forUpdateLocking = opt.Locking{ func (b *Builder) shouldApplyImplicitLockingToMutationInput( mutExpr memo.RelExpr, ) (opt.TableID, error) { + if !b.evalCtx.SessionData().ImplicitSelectForUpdate { + return 0, nil + } + switch t := mutExpr.(type) { case *memo.InsertExpr: // Unlike with the other three mutation expressions, it never makes @@ -1140,23 +1144,23 @@ func (b *Builder) shouldApplyImplicitLockingToMutationInput( return 0, nil case *memo.UpdateExpr: - return b.shouldApplyImplicitLockingToUpdateInput(t), nil + return shouldApplyImplicitLockingToUpdateOrDeleteInput(t), nil case *memo.UpsertExpr: - return b.shouldApplyImplicitLockingToUpsertInput(t), nil + return shouldApplyImplicitLockingToUpsertInput(t), nil case *memo.DeleteExpr: - return b.shouldApplyImplicitLockingToDeleteInput(t), nil + return shouldApplyImplicitLockingToUpdateOrDeleteInput(t), nil default: return 0, errors.AssertionFailedf("unexpected mutation expression %T", t) } } -// shouldApplyImplicitLockingToUpdateInput determines whether or not the builder -// should apply a FOR UPDATE row-level locking mode to the initial row scan of -// an UPDATE statement. If the builder should lock the initial row scan, it -// returns the TableID of the scan, otherwise it returns 0. +// shouldApplyImplicitLockingToUpdateOrDeleteInput determines whether the +// builder should apply a FOR UPDATE row-level locking mode to the initial row +// scan of an UPDATE statement or a DELETE. If the builder should lock the +// initial row scan, it returns the TableID of the scan, otherwise it returns 0. // // Conceptually, if we picture an UPDATE statement as the composition of a // SELECT statement and an INSERT statement (with loosened semantics around @@ -1176,16 +1180,15 @@ func (b *Builder) shouldApplyImplicitLockingToMutationInput( // is strictly a performance optimization for contended writes. Therefore, it is // not worth risking the transformation being a pessimization, so it is only // applied when doing so does not risk creating artificial contention. -func (b *Builder) shouldApplyImplicitLockingToUpdateInput(upd *memo.UpdateExpr) opt.TableID { - if !b.evalCtx.SessionData().ImplicitSelectForUpdate { - return 0 - } - - // Try to match the Update's input expression against the pattern: +// +// UPDATEs and DELETEs happen to have exactly the same matching pattern, so we +// reuse this function for both. +func shouldApplyImplicitLockingToUpdateOrDeleteInput(mutExpr memo.RelExpr) opt.TableID { + // Try to match the mutation's input expression against the pattern: // // [Project]* [IndexJoin] Scan // - input := upd.Input + input := mutExpr.Child(0).(memo.RelExpr) input = unwrapProjectExprs(input) if idxJoin, ok := input.(*memo.IndexJoinExpr); ok { input = idxJoin.Input @@ -1200,11 +1203,7 @@ func (b *Builder) shouldApplyImplicitLockingToUpdateInput(upd *memo.UpdateExpr) // should apply a FOR UPDATE row-level locking mode to the initial row scan of // an UPSERT statement. If the builder should lock the initial row scan, it // returns the TableID of the scan, otherwise it returns 0. -func (b *Builder) shouldApplyImplicitLockingToUpsertInput(ups *memo.UpsertExpr) opt.TableID { - if !b.evalCtx.SessionData().ImplicitSelectForUpdate { - return 0 - } - +func shouldApplyImplicitLockingToUpsertInput(ups *memo.UpsertExpr) opt.TableID { // Try to match the Upsert's input expression against the pattern: // // [Project]* (LeftJoin Scan | LookupJoin) [Project]* Values @@ -1235,17 +1234,6 @@ func (b *Builder) shouldApplyImplicitLockingToUpsertInput(ups *memo.UpsertExpr) return 0 } -// tryApplyImplicitLockingToDeleteInput determines whether or not the builder -// should apply a FOR UPDATE row-level locking mode to the initial row scan of a -// DELETE statement. If the builder should lock the initial row scan, it returns -// the TableID of the scan, otherwise it returns 0. -// -// TODO(nvanbenschoten): implement this method to match on appropriate Delete -// expression trees and apply a row-level locking mode. -func (b *Builder) shouldApplyImplicitLockingToDeleteInput(del *memo.DeleteExpr) opt.TableID { - return 0 -} - // unwrapProjectExprs unwraps zero or more nested ProjectExprs. It returns the // first non-ProjectExpr in the chain, or the input if it is not a ProjectExpr. func unwrapProjectExprs(input memo.RelExpr) memo.RelExpr { diff --git a/pkg/sql/opt/exec/execbuilder/testdata/cascade b/pkg/sql/opt/exec/execbuilder/testdata/cascade index 9d275e9e42f4..23ca5d3178d8 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/cascade +++ b/pkg/sql/opt/exec/execbuilder/testdata/cascade @@ -900,6 +900,7 @@ vectorized: true │ missing stats │ table: self@self_pkey │ spans: [/4 - /4] +│ locking strength: for update │ └── • fk-cascade │ fk: self_b_fkey @@ -984,6 +985,7 @@ vectorized: true │ missing stats │ table: loop_a@loop_a_pkey │ spans: [/'loop_a-pk1' - /'loop_a-pk1'] +│ locking strength: for update │ └── • fk-cascade │ fk: loop_b_cascade_delete_fkey @@ -1085,6 +1087,7 @@ quality of service: regular │ missing stats │ table: loop_a@loop_a_pkey │ spans: [/'loop_a-pk1' - /'loop_a-pk1'] +│ locking strength: for update │ └── • fk-cascade │ fk: loop_b_cascade_delete_fkey @@ -1235,6 +1238,7 @@ vectorized: true │ missing stats │ table: t3@t3_pkey │ spans: [/1 - /1] +│ locking strength: for update │ ├── • fk-cascade │ │ fk: fk1 diff --git a/pkg/sql/opt/exec/execbuilder/testdata/delete b/pkg/sql/opt/exec/execbuilder/testdata/delete index 845438ed6996..a6724af86c45 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/delete +++ b/pkg/sql/opt/exec/execbuilder/testdata/delete @@ -141,6 +141,7 @@ vectorized: true table: unindexed@unindexed_pkey spans: [/1 - ] limit: 1 + locking strength: for update query T EXPLAIN DELETE FROM indexed WHERE value = 5 LIMIT 10 @@ -157,6 +158,7 @@ vectorized: true table: indexed@indexed_value_idx spans: [/5 - /5] limit: 10 + locking strength: for update query T EXPLAIN DELETE FROM indexed LIMIT 10 @@ -173,6 +175,7 @@ vectorized: true table: indexed@indexed_value_idx spans: LIMITED SCAN limit: 10 + locking strength: for update # TODO(andyk): Prune columns so that index-join is not necessary. query T @@ -190,6 +193,7 @@ vectorized: true table: indexed@indexed_value_idx spans: [/5 - /5] limit: 10 + locking strength: for update # Ensure that index hints in DELETE statements force the choice of a specific index # as described in #38799. @@ -213,6 +217,7 @@ vectorized: true estimated row count: 1,000 (missing stats) table: t38799@foo spans: FULL SCAN + locking strength: for update # Tracing tests for fast delete. statement ok @@ -331,12 +336,14 @@ vectorized: true │ estimated row count: 990 (missing stats) │ table: xyz@xyz_pkey │ key columns: x + │ locking strength: for update │ └── • scan columns: (x, y) estimated row count: 990 (missing stats) table: xyz@xyz_y_idx spans: /1-/1000 /2001-/3000 + locking strength: for update # Testcase for issue 105803. diff --git a/pkg/sql/opt/exec/execbuilder/testdata/explain_redact b/pkg/sql/opt/exec/execbuilder/testdata/explain_redact index d9bdf3b233c3..8291ed8aa126 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/explain_redact +++ b/pkg/sql/opt/exec/execbuilder/testdata/explain_redact @@ -1458,6 +1458,7 @@ vectorized: true missing stats table: f@f_f_idx (partial index) spans: 1 span + locking strength: for update query T EXPLAIN (VERBOSE, REDACT) DELETE FROM f WHERE f = 8.5 @@ -1482,6 +1483,7 @@ vectorized: true estimated row count: 10 (missing stats) table: f@f_f_idx (partial index) spans: 1 span + locking strength: for update query T EXPLAIN (OPT, REDACT) DELETE FROM f WHERE f = 8.5 diff --git a/pkg/sql/opt/exec/execbuilder/testdata/fk b/pkg/sql/opt/exec/execbuilder/testdata/fk index ca3a775c95ee..06af7e7c434c 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/fk +++ b/pkg/sql/opt/exec/execbuilder/testdata/fk @@ -418,6 +418,7 @@ vectorized: true │ missing stats │ table: parent@parent_pkey │ spans: [/3 - /3] +│ locking strength: for update │ ├── • constraint-check │ │ @@ -464,6 +465,7 @@ vectorized: true │ missing stats │ table: parent@parent_pkey │ spans: [/3 - /3] +│ locking strength: for update │ ├── • constraint-check │ │ @@ -511,6 +513,7 @@ vectorized: true │ missing stats │ table: parent@parent_pkey │ spans: [/3 - /3] +│ locking strength: for update │ ├── • constraint-check │ │ @@ -575,6 +578,7 @@ vectorized: true │ missing stats │ table: doubleparent@doubleparent_pkey │ spans: [/10 - /10] +│ locking strength: for update │ └── • constraint-check │ @@ -1982,6 +1986,7 @@ vectorized: true │ estimated row count: 1 (100% of the table; stats collected ago) │ table: p@p_pkey │ spans: [/1 - ] +│ locking strength: for update │ └── • constraint-check │ @@ -2071,6 +2076,7 @@ vectorized: true │ estimated row count: 1 (100% of the table; stats collected ago) │ table: p@p_pkey │ spans: [/1 - ] +│ locking strength: for update │ └── • constraint-check │ @@ -2181,6 +2187,7 @@ vectorized: true │ missing stats │ table: partial_parent@partial_parent_pkey │ spans: [/1 - /1] +│ locking strength: for update │ └── • constraint-check │ diff --git a/pkg/sql/opt/exec/execbuilder/testdata/fk_read_committed b/pkg/sql/opt/exec/execbuilder/testdata/fk_read_committed index 031de7ec33d1..b1e0d9acf3e3 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/fk_read_committed +++ b/pkg/sql/opt/exec/execbuilder/testdata/fk_read_committed @@ -322,6 +322,7 @@ vectorized: true │ estimated row count: 1 (missing stats) │ table: jars@jars_pkey │ spans: /1/0 +│ locking strength: for update │ └── • constraint-check │ diff --git a/pkg/sql/opt/exec/execbuilder/testdata/inverted_index b/pkg/sql/opt/exec/execbuilder/testdata/inverted_index index 755a2b10e0bd..2acb5310c1a2 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/inverted_index +++ b/pkg/sql/opt/exec/execbuilder/testdata/inverted_index @@ -49,7 +49,7 @@ InitPut /Table/106/2/Arr/7/1/0 -> /BYTES/ query T kvtrace DELETE FROM d WHERE a=1 ---- -Scan /Table/106/1/1/0 +Scan /Table/106/1/1/0 lock Exclusive (Block, Unreplicated) Del /Table/106/2/Arr/0/1/0 Del /Table/106/2/Arr/7/1/0 Del /Table/106/1/1/0 @@ -96,7 +96,7 @@ Del /Table/106/2/Arr/1/4/0 query T kvtrace DELETE FROM d WHERE a=4 ---- -Scan /Table/106/1/4/0 +Scan /Table/106/1/4/0 lock Exclusive (Block, Unreplicated) Del /Table/106/1/4/0 # Tests for array inverted indexes. diff --git a/pkg/sql/opt/exec/execbuilder/testdata/inverted_index_multi_column b/pkg/sql/opt/exec/execbuilder/testdata/inverted_index_multi_column index 78be7fade70c..36e2db63f394 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/inverted_index_multi_column +++ b/pkg/sql/opt/exec/execbuilder/testdata/inverted_index_multi_column @@ -65,7 +65,7 @@ InitPut /Table/106/3/333/"foo"/Arr/"a"/"b"/3/0 -> /BYTES/ query T kvtrace DELETE FROM t WHERE k = 2 ---- -Scan /Table/106/1/2/0 +Scan /Table/106/1/2/0 lock Exclusive (Block, Unreplicated) Del /Table/106/2/333/Arr/0/2/0 Del /Table/106/2/333/Arr/7/2/0 Del /Table/106/3/333/"foo"/Arr/0/2/0 @@ -75,7 +75,7 @@ Del /Table/106/1/2/0 query T kvtrace DELETE FROM t WHERE k = 3 ---- -Scan /Table/106/1/3/0 +Scan /Table/106/1/3/0 lock Exclusive (Block, Unreplicated) Del /Table/106/2/333/Arr/3/3/0 Del /Table/106/2/333/Arr/"a"/"b"/3/0 Del /Table/106/3/333/"foo"/Arr/3/3/0 @@ -110,7 +110,7 @@ Del /Table/106/3/333/"foo"/Arr/1/4/0 query T kvtrace DELETE FROM t WHERE k = 4 ---- -Scan /Table/106/1/4/0 +Scan /Table/106/1/4/0 lock Exclusive (Block, Unreplicated) Del /Table/106/1/4/0 # Insert NULL non-inverted value. @@ -147,7 +147,7 @@ InitPut /Table/106/3/NULL/"foo"/"a"/"b"/5/0 -> /BYTES/ query T kvtrace DELETE FROM t WHERE k = 5 ---- -Scan /Table/106/1/5/0 +Scan /Table/106/1/5/0 lock Exclusive (Block, Unreplicated) Del /Table/106/2/NULL/"a"/"b"/5/0 Del /Table/106/3/NULL/"foo"/"a"/"b"/5/0 Del /Table/106/1/5/0 diff --git a/pkg/sql/opt/exec/execbuilder/testdata/not_visible_index b/pkg/sql/opt/exec/execbuilder/testdata/not_visible_index index 041359667359..429918cd8698 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/not_visible_index +++ b/pkg/sql/opt/exec/execbuilder/testdata/not_visible_index @@ -853,6 +853,7 @@ vectorized: true │ missing stats │ table: parent@parent_pkey │ spans: [/2 - /2] +│ locking strength: for update │ └── • constraint-check │ @@ -1043,6 +1044,7 @@ vectorized: true │ missing stats │ table: parent@parent_pkey │ spans: [/2 - /2] +│ locking strength: for update │ ├── • fk-cascade │ │ fk: child_delete_p_fkey @@ -1054,6 +1056,7 @@ vectorized: true │ missing stats │ table: child_delete@c_delete_idx_invisible │ spans: [/2 - /2] +│ locking strength: for update │ └── • constraint-check │ diff --git a/pkg/sql/opt/exec/execbuilder/testdata/orderby b/pkg/sql/opt/exec/execbuilder/testdata/orderby index 254189ed9f4c..91e554740116 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/orderby +++ b/pkg/sql/opt/exec/execbuilder/testdata/orderby @@ -736,6 +736,7 @@ vectorized: true estimated row count: 1 (missing stats) table: t@t_pkey spans: /3/0 + locking strength: for update query T EXPLAIN (VERBOSE) UPDATE t SET c = TRUE RETURNING b diff --git a/pkg/sql/opt/exec/execbuilder/testdata/partial_index b/pkg/sql/opt/exec/execbuilder/testdata/partial_index index eb0c74182812..eaa0a1673c5b 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/partial_index +++ b/pkg/sql/opt/exec/execbuilder/testdata/partial_index @@ -265,7 +265,7 @@ InitPut /Table/113/2/3/2/0 -> /BYTES/ query T kvtrace DELETE FROM t WHERE a = 5 ---- -Scan /Table/112/1/5/0 +Scan /Table/112/1/5/0 lock Exclusive (Block, Unreplicated) Del /Table/112/2/4/5/0 Del /Table/112/1/5/0 @@ -273,7 +273,7 @@ Del /Table/112/1/5/0 query T kvtrace DELETE FROM t WHERE a = 6 ---- -Scan /Table/112/1/6/0 +Scan /Table/112/1/6/0 lock Exclusive (Block, Unreplicated) Del /Table/112/2/11/6/0 Del /Table/112/3/11/6/0 Del /Table/112/1/6/0 @@ -282,7 +282,7 @@ Del /Table/112/1/6/0 query T kvtrace DELETE FROM t WHERE a = 12 ---- -Scan /Table/112/1/12/0 +Scan /Table/112/1/12/0 lock Exclusive (Block, Unreplicated) Del /Table/112/2/11/12/0 Del /Table/112/3/11/12/0 Del /Table/112/4/"foo"/12/0 @@ -293,7 +293,7 @@ Del /Table/112/1/12/0 query T kvtrace DELETE FROM u WHERE k = 1 ---- -Scan /Table/113/1/1/0 +Scan /Table/113/1/1/0 lock Exclusive (Block, Unreplicated) Del /Table/113/1/1/0 # Deleted row matches partial index with predicate column that is not @@ -301,7 +301,7 @@ Del /Table/113/1/1/0 query T kvtrace DELETE FROM u WHERE k = 2 ---- -Scan /Table/113/1/2/0 +Scan /Table/113/1/2/0 lock Exclusive (Block, Unreplicated) Del /Table/113/2/3/2/0 Del /Table/113/1/2/0 @@ -890,7 +890,7 @@ CPut /Table/107/1/2/0 -> /TUPLE/ query T kvtrace DELETE FROM inv WHERE a = 1 ---- -Scan /Table/107/1/1/0 +Scan /Table/107/1/1/0 lock Exclusive (Block, Unreplicated) Del /Table/107/2/"num"/1/1/0 Del /Table/107/2/"x"/"y"/1/0 Del /Table/107/1/1/0 @@ -898,7 +898,7 @@ Del /Table/107/1/1/0 query T kvtrace DELETE FROM inv WHERE a = 2 ---- -Scan /Table/107/1/2/0 +Scan /Table/107/1/2/0 lock Exclusive (Block, Unreplicated) Del /Table/107/1/2/0 # --------------------------------------------------------- diff --git a/pkg/sql/opt/exec/execbuilder/testdata/show_trace b/pkg/sql/opt/exec/execbuilder/testdata/show_trace index 16e8d7582a3c..552d2e4f429f 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/show_trace +++ b/pkg/sql/opt/exec/execbuilder/testdata/show_trace @@ -177,7 +177,7 @@ SET tracing = off query TT $trace_query ---- -colbatchscan Scan /Table/108/{1-2} +colbatchscan Scan /Table/108/{1-2} lock Exclusive (Block, Unreplicated) colbatchscan fetched: /kv/kv_pkey/1/v -> /2 count Del /Table/108/2/2/0 count Del /Table/108/1/1/0 diff --git a/pkg/sql/opt/exec/execbuilder/testdata/spool b/pkg/sql/opt/exec/execbuilder/testdata/spool index 2b77d65a56e5..56b58cc8b437 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/spool +++ b/pkg/sql/opt/exec/execbuilder/testdata/spool @@ -68,6 +68,7 @@ vectorized: true missing stats table: t@t_pkey spans: FULL SCAN + locking strength: for update query T @@ -196,6 +197,7 @@ vectorized: true missing stats table: t@t_pkey spans: FULL SCAN + locking strength: for update query T EXPLAIN SELECT * FROM [UPDATE t SET x = x + 1 RETURNING x] LIMIT 1 diff --git a/pkg/sql/opt/exec/execbuilder/testdata/sql_activity_stats_compaction b/pkg/sql/opt/exec/execbuilder/testdata/sql_activity_stats_compaction index ec1a1118fcd0..86dfa8f016ca 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/sql_activity_stats_compaction +++ b/pkg/sql/opt/exec/execbuilder/testdata/sql_activity_stats_compaction @@ -264,6 +264,7 @@ vectorized: true table: statement_statistics@primary spans: /0-/0/2022-05-04T15:59:59.999999001Z limit: 1024 + locking strength: for update statement ok ALTER TABLE system.transaction_statistics INJECT STATISTICS '[ @@ -448,6 +449,7 @@ vectorized: true table: transaction_statistics@primary spans: /0-/0/2022-05-04T15:59:59.999999001Z limit: 1024 + locking strength: for update query T EXPLAIN (VERBOSE) @@ -507,6 +509,7 @@ vectorized: true table: statement_statistics@primary spans: /0/2022-05-04T14:00:00Z/"123"/"234"/"345"/"test"/1-/0/2022-05-04T15:59:59.999999001Z limit: 1024 + locking strength: for update query T EXPLAIN (VERBOSE) @@ -561,6 +564,7 @@ vectorized: true table: transaction_statistics@primary spans: /0/2022-05-04T14:00:00Z/"123"/"test"/2-/0/2022-05-04T15:59:59.999999001Z limit: 1024 + locking strength: for update statement ok RESET CLUSTER SETTING sql.stats.flush.interval diff --git a/pkg/sql/opt/exec/execbuilder/testdata/virtual_columns b/pkg/sql/opt/exec/execbuilder/testdata/virtual_columns index ff32313e4987..daf585017889 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/virtual_columns +++ b/pkg/sql/opt/exec/execbuilder/testdata/virtual_columns @@ -286,6 +286,7 @@ vectorized: true estimated row count: 333 (missing stats) table: t@t_pkey spans: /2- + locking strength: for update query T EXPLAIN (VERBOSE) DELETE FROM t WHERE v = 1 @@ -336,6 +337,7 @@ vectorized: true estimated row count: 333 (missing stats) table: t_idx@t_idx_pkey spans: /2- + locking strength: for update query T EXPLAIN (VERBOSE) DELETE FROM t_idx WHERE a > 1 RETURNING v @@ -363,6 +365,7 @@ vectorized: true estimated row count: 333 (missing stats) table: t_idx@t_idx_pkey spans: /2- + locking strength: for update query T EXPLAIN (VERBOSE) DELETE FROM t_idx WHERE v = 1 @@ -387,12 +390,14 @@ vectorized: true │ estimated row count: 333 (missing stats) │ table: t_idx@t_idx_pkey │ key columns: a + │ locking strength: for update │ └── • scan columns: (a) estimated row count: 10 (missing stats) table: t_idx@t_idx_v_idx spans: /1-/2 + locking strength: for update query T EXPLAIN (VERBOSE) DELETE FROM t_idx WHERE a + b = 1 @@ -417,12 +422,14 @@ vectorized: true │ estimated row count: 333 (missing stats) │ table: t_idx@t_idx_pkey │ key columns: a + │ locking strength: for update │ └── • scan columns: (a) estimated row count: 10 (missing stats) table: t_idx@t_idx_v_idx spans: /1-/2 + locking strength: for update subtest Update diff --git a/pkg/sql/opt/exec/explain/testdata/gists b/pkg/sql/opt/exec/explain/testdata/gists index 6058020f760f..29f16b3838d5 100644 --- a/pkg/sql/opt/exec/explain/testdata/gists +++ b/pkg/sql/opt/exec/explain/testdata/gists @@ -710,6 +710,7 @@ explain(shape): └── • scan table: foo@foo_pkey spans: FULL SCAN + locking strength: for update explain(gist): • delete │ from: foo @@ -733,6 +734,7 @@ explain(shape): └── • scan table: foo@foo_pkey spans: 1+ spans + locking strength: for update explain(gist): • delete │ from: foo diff --git a/pkg/sql/testdata/index_mutations/delete_preserving_delete b/pkg/sql/testdata/index_mutations/delete_preserving_delete index 45ba98072737..11b2b1d281bd 100644 --- a/pkg/sql/testdata/index_mutations/delete_preserving_delete +++ b/pkg/sql/testdata/index_mutations/delete_preserving_delete @@ -21,7 +21,7 @@ INSERT INTO ti VALUES (1, 2, 100) kvtrace DELETE FROM ti WHERE a = 1 ---- -Scan /Table/106/1/1/0 +Scan /Table/106/1/1/0 lock Exclusive (Block, Unreplicated) Put (delete) /Table/106/2/2/100/1/0 Del /Table/106/1/1/0 @@ -49,14 +49,14 @@ INSERT INTO tpi VALUES (1, 2, 'bar'), (2, 3, 'bar'), (3, 2, 'foo') kvtrace DELETE FROM tpi WHERE a = 1 ---- -Scan /Table/107/1/1/0 +Scan /Table/107/1/1/0 lock Exclusive (Block, Unreplicated) Del /Table/107/1/1/0 # Delete a row that matches the partial index. kvtrace DELETE FROM tpi WHERE a = 3 ---- -Scan /Table/107/1/3/0 +Scan /Table/107/1/3/0 lock Exclusive (Block, Unreplicated) Put (delete) /Table/107/2/"foo"/3/0 Del /Table/107/1/3/0 @@ -110,7 +110,7 @@ INSERT INTO tii VALUES (1, ARRAY[1, 2, 3, 2, 2, NULL, 3]) kvtrace DELETE FROM tii WHERE a = 1 ---- -Scan /Table/109/1/1/0 +Scan /Table/109/1/1/0 lock Exclusive (Block, Unreplicated) Put (delete) /Table/109/2/NULL/1/0 Put (delete) /Table/109/2/1/1/0 Put (delete) /Table/109/2/2/1/0 @@ -141,7 +141,7 @@ INSERT INTO tmi VALUES (1, 2, '{"a": "foo", "b": "bar"}'::json) kvtrace DELETE FROM tmi WHERE a = 1 ---- -Scan /Table/110/1/1/0 +Scan /Table/110/1/1/0 lock Exclusive (Block, Unreplicated) Put (delete) /Table/110/2/2/"a"/"foo"/1/0 Put (delete) /Table/110/2/2/"b"/"bar"/1/0 Del /Table/110/1/1/0 diff --git a/pkg/sql/testdata/index_mutations/delete_preserving_unique_index b/pkg/sql/testdata/index_mutations/delete_preserving_unique_index index 968119262ca3..7af06d99e72d 100644 --- a/pkg/sql/testdata/index_mutations/delete_preserving_unique_index +++ b/pkg/sql/testdata/index_mutations/delete_preserving_unique_index @@ -23,7 +23,7 @@ INSERT INTO ti VALUES (1, 1, 100), (2, 2, 1) kvtrace DELETE FROM ti WHERE a = 1 ---- -Scan /Table/106/1/1/0 +Scan /Table/106/1/1/0 lock Exclusive (Block, Unreplicated) Put (delete) /Table/106/2/1/100/0 Del /Table/106/1/1/0 From 6a3221fe5a536b673213f3f79082859e9749470b Mon Sep 17 00:00:00 2001 From: Drew Kimball Date: Mon, 8 Jan 2024 03:41:16 -0700 Subject: [PATCH 3/4] opt: infer functional dependencies for window functions This commit adds logic to infer strict functional dependencies from a Window operator's partition column(s) to some or all of its window functions when the following conditions are satisfied: 1. The window function must be an aggregate, or first_value or last_value. 2. The window frame must be unbounded. The above conditions ensure that the window function always produces the same result given the same window frame, as well as that every row in a partition has the same window frame. This means that the window function produces the same output for every row in the partition, and therefore, the partition columns functionally determine the output of the window function. This patch also fixes a small omission made in the window function FD calculation, which caused the window's FDs to preserve an input key without extending the key to apply to all the window's output cols. Epic: None Release note: None --- .../colexecwindow/window_aggregator.eg.go | 15 + .../colexecwindow/window_aggregator_tmpl.go | 5 + pkg/sql/opt/memo/logical_props_builder.go | 53 ++- pkg/sql/opt/memo/testdata/logprops/window | 311 +++++++++++++++++- pkg/sql/opt/memo/testdata/stats/window | 2 +- pkg/sql/opt/norm/testdata/rules/decorrelate | 36 +- pkg/sql/opt/norm/testdata/rules/prune_cols | 2 +- pkg/sql/opt/norm/testdata/rules/window | 67 ++-- pkg/sql/opt/norm/testdata/rules/with | 2 +- pkg/sql/opt/xform/testdata/external/tpce | 22 +- .../opt/xform/testdata/external/tpce-no-stats | 22 +- pkg/sql/opt/xform/testdata/physprops/ordering | 2 + pkg/sql/sem/builtins/window_builtins.go | 5 + 13 files changed, 465 insertions(+), 79 deletions(-) diff --git a/pkg/sql/colexec/colexecwindow/window_aggregator.eg.go b/pkg/sql/colexec/colexecwindow/window_aggregator.eg.go index 9b6435968e34..31a5ee79e5d6 100644 --- a/pkg/sql/colexec/colexecwindow/window_aggregator.eg.go +++ b/pkg/sql/colexec/colexecwindow/window_aggregator.eg.go @@ -311,11 +311,26 @@ func (a *slidingWindowAggregator) processBatch(batch coldata.Batch, startIdx, en }) } +// INVARIANT: the rows within a window frame are always processed in the same +// order, regardless of whether the user specified an ordering. This means that +// two rows with the exact same frame will produce the same result for a given +// aggregation. +// // execgen:inline const _ = "template_aggregateOverIntervals" +// INVARIANT: the rows within a window frame are always processed in the same +// order, regardless of whether the user specified an ordering. This means that +// two rows with the exact same frame will produce the same result for a given +// aggregation. +// // execgen:inline const _ = "inlined_aggregateOverIntervals_false" +// INVARIANT: the rows within a window frame are always processed in the same +// order, regardless of whether the user specified an ordering. This means that +// two rows with the exact same frame will produce the same result for a given +// aggregation. +// // execgen:inline const _ = "inlined_aggregateOverIntervals_true" diff --git a/pkg/sql/colexec/colexecwindow/window_aggregator_tmpl.go b/pkg/sql/colexec/colexecwindow/window_aggregator_tmpl.go index 35d1c539725e..066416eed24f 100644 --- a/pkg/sql/colexec/colexecwindow/window_aggregator_tmpl.go +++ b/pkg/sql/colexec/colexecwindow/window_aggregator_tmpl.go @@ -247,6 +247,11 @@ func (a *slidingWindowAggregator) processBatch(batch coldata.Batch, startIdx, en }) } +// INVARIANT: the rows within a window frame are always processed in the same +// order, regardless of whether the user specified an ordering. This means that +// two rows with the exact same frame will produce the same result for a given +// aggregation. +// // execgen:inline // execgen:template func aggregateOverIntervals(intervals []windowInterval, removeRows bool) { diff --git a/pkg/sql/opt/memo/logical_props_builder.go b/pkg/sql/opt/memo/logical_props_builder.go index ad6c691c7cf0..f564e8dec264 100644 --- a/pkg/sql/opt/memo/logical_props_builder.go +++ b/pkg/sql/opt/memo/logical_props_builder.go @@ -17,6 +17,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sem/cast" "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" + "github.com/cockroachdb/cockroach/pkg/sql/sem/tree/treewindow" "github.com/cockroachdb/cockroach/pkg/sql/types" "github.com/cockroachdb/cockroach/pkg/util/buildutil" "github.com/cockroachdb/cockroach/pkg/util/intsets" @@ -1395,8 +1396,22 @@ func (b *logicalPropsBuilder) buildWindowProps(window *WindowExpr, rel *props.Re // examples include: // * row_number+the partition is a key. // * rank is determined by the partition and the value being ordered by. - // * aggregations/first_value/last_value are determined by the partition. rel.FuncDeps.CopyFrom(&inputProps.FuncDeps) + if inputProps.FuncDeps.ColsAreStrictKey(window.Partition) { + // Special case: when the partition columns form a strict key over the + // input, each partition will only have a single row. Therefore, the window + // function output columns are trivially determined by the partition cols. + rel.FuncDeps.AddStrictKey(window.Partition, rel.OutputCols) + } else { + // It may still be possible to infer functional dependencies based on the + // window frames and window function types. + determinedCols := getWindowPartitionDeps(window, &inputProps.FuncDeps) + if !determinedCols.Empty() { + // The partition columns determine some of the window function outputs. + rel.FuncDeps.AddStrictDependency(window.Partition, determinedCols) + } + } + rel.FuncDeps.ProjectCols(rel.OutputCols) // Cardinality // ----------- @@ -2971,3 +2986,39 @@ func CanBeCompositeSensitive(e opt.Expr) bool { isCompositeInsensitive, _ := check(e) return !isCompositeInsensitive } + +// getWindowPartitionDeps returns the set of window function output columns that +// are functionally determined by the Window operator's partition columns +// (which may be empty) based on the window frame and function type. +// +// NOTE: getWindowPartitionDeps assumes that execution performs aggregation in +// the same order for every row in the window, even when there is no explicit +// ORDER BY. +func getWindowPartitionDeps(window *WindowExpr, inputFDs *props.FuncDepSet) opt.ColSet { + var determinedCols opt.ColSet + for i := range window.Windows { + // Ensure that the window frame extends to the entire partition. This + // ensures that every row in the partition has the exact same frame. + item := &window.Windows[i] + if item.Frame.FrameExclusion != treewindow.NoExclusion || + item.Frame.StartBoundType != treewindow.UnboundedPreceding || + item.Frame.EndBoundType != treewindow.UnboundedFollowing { + continue + } + // Aggregations, first_value, and last_value functions always produce the + // same result for any row given the same frame. + if !opt.IsAggregateOp(item.Function) { + switch item.Function.Op() { + case opt.FirstValueOp, opt.LastValueOp: + default: + continue + } + } + // Since we determined that this function always produces the same result + // for a given window frame, as well as that the frame is the same for all + // rows in a given partition, there is a dependency from the partition + // columns to the output of this window function. + determinedCols.Add(item.Col) + } + return determinedCols +} diff --git a/pkg/sql/opt/memo/testdata/logprops/window b/pkg/sql/opt/memo/testdata/logprops/window index c1a10381b936..658a49fd0223 100644 --- a/pkg/sql/opt/memo/testdata/logprops/window +++ b/pkg/sql/opt/memo/testdata/logprops/window @@ -30,7 +30,7 @@ project ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) rank:10(int) ├── cardinality: [0 - 10] ├── key: (1) - ├── fd: (1)-->(2-7) + ├── fd: (1)-->(2-7,10) ├── prune: (1-7,10) ├── limit │ ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) @@ -70,7 +70,7 @@ project ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) rank:10(int) ├── cardinality: [0 - 10] ├── key: (1) - ├── fd: (1)-->(2-7) + ├── fd: (1)-->(2-7,10) ├── prune: (1,3,5-7,10) ├── limit │ ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) @@ -138,7 +138,7 @@ project │ ├── outer: (1) │ ├── cardinality: [1 - 1] │ ├── key: () - │ ├── fd: ()-->(10) + │ ├── fd: ()-->(10,11) │ ├── prune: (10,11) │ ├── project │ │ ├── columns: x:10(int) @@ -171,7 +171,7 @@ project ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) lag:10(string) lag:11(int) lag_1_arg1:12(string!null) lag_1_arg2:13(int!null) lag_1_arg3:14(string) lag_2_arg3:15(int) ├── immutable ├── key: (1) - ├── fd: ()-->(12-15), (1)-->(2-9) + ├── fd: ()-->(12-15), (1)-->(2-11) ├── prune: (1-11) ├── project │ ├── columns: lag_1_arg1:12(string!null) lag_1_arg2:13(int!null) lag_1_arg3:14(string) lag_2_arg3:15(int) k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) @@ -202,3 +202,306 @@ project ├── variable: lag_1_arg2:13 [type=int] ├── variable: lag_1_arg2:13 [type=int] └── variable: lag_2_arg3:15 [type=int] + +# Given an unbounded window frame, aggregates, first_value, and last_value +# are functionally determined by the partition column(s). +build +SELECT + sum(v) OVER w, + array_agg(s) OVER w, + first_value(d) OVER w, + last_value(f) OVER w +FROM kv +WINDOW w AS ( + PARTITION BY w ORDER BY v, s, d, f + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING +) +---- +project + ├── columns: sum:10(decimal) array_agg:11(string[]) first_value:12(decimal) last_value:13(float) + ├── prune: (10-13) + └── window partition=(3) ordering=+2,+6,+5,+4 + ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) sum:10(decimal) array_agg:11(string[]) first_value:12(decimal) last_value:13(float) + ├── key: (1) + ├── fd: (1)-->(2-9), (3)-->(10-13) + ├── prune: (1,7-13) + ├── scan kv + │ ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) + │ ├── key: (1) + │ ├── fd: (1)-->(2-9) + │ ├── prune: (1-9) + │ └── interesting orderings: (+1) + └── windows + ├── sum [as=sum:10, frame="rows from unbounded to unbounded", type=decimal, outer=(2)] + │ └── variable: v:2 [type=int] + ├── array-agg [as=array_agg:11, frame="rows from unbounded to unbounded", type=string[], outer=(6)] + │ └── variable: s:6 [type=string] + ├── first-value [as=first_value:12, frame="rows from unbounded to unbounded", type=decimal, outer=(5)] + │ └── variable: d:5 [type=decimal] + └── last-value [as=last_value:13, frame="rows from unbounded to unbounded", type=float, outer=(4)] + └── variable: f:4 [type=float] + +# Case with no partition columns. +build +SELECT + sum(v) OVER w, + array_agg(s) OVER w, + first_value(d) OVER w, + last_value(f) OVER w +FROM kv +WINDOW w AS ( + ORDER BY v, s, d, f + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING +) +---- +project + ├── columns: sum:10(decimal) array_agg:11(string[]) first_value:12(decimal) last_value:13(float) + ├── fd: ()-->(10-13) + ├── prune: (10-13) + └── window partition=() ordering=+2,+6,+5,+4 + ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) sum:10(decimal) array_agg:11(string[]) first_value:12(decimal) last_value:13(float) + ├── key: (1) + ├── fd: ()-->(10-13), (1)-->(2-9) + ├── prune: (1,3,7-13) + ├── scan kv + │ ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) + │ ├── key: (1) + │ ├── fd: (1)-->(2-9) + │ ├── prune: (1-9) + │ └── interesting orderings: (+1) + └── windows + ├── sum [as=sum:10, frame="rows from unbounded to unbounded", type=decimal, outer=(2)] + │ └── variable: v:2 [type=int] + ├── array-agg [as=array_agg:11, frame="rows from unbounded to unbounded", type=string[], outer=(6)] + │ └── variable: s:6 [type=string] + ├── first-value [as=first_value:12, frame="rows from unbounded to unbounded", type=decimal, outer=(5)] + │ └── variable: d:5 [type=decimal] + └── last-value [as=last_value:13, frame="rows from unbounded to unbounded", type=float, outer=(4)] + └── variable: f:4 [type=float] + +# Case with multiple partition columns. +build +SELECT + sum(v) OVER w, + array_agg(s) OVER w, + first_value(d) OVER w, + last_value(f) OVER w +FROM kv +WINDOW w AS ( + PARTITION BY b, w ORDER BY v, s, d, f + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING +) +---- +project + ├── columns: sum:10(decimal) array_agg:11(string[]) first_value:12(decimal) last_value:13(float) + ├── prune: (10-13) + └── window partition=(3,7) ordering=+2,+6,+5,+4 + ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) sum:10(decimal) array_agg:11(string[]) first_value:12(decimal) last_value:13(float) + ├── key: (1) + ├── fd: (1)-->(2-9), (3,7)-->(10-13) + ├── prune: (1,8-13) + ├── scan kv + │ ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) + │ ├── key: (1) + │ ├── fd: (1)-->(2-9) + │ ├── prune: (1-9) + │ └── interesting orderings: (+1) + └── windows + ├── sum [as=sum:10, frame="rows from unbounded to unbounded", type=decimal, outer=(2)] + │ └── variable: v:2 [type=int] + ├── array-agg [as=array_agg:11, frame="rows from unbounded to unbounded", type=string[], outer=(6)] + │ └── variable: s:6 [type=string] + ├── first-value [as=first_value:12, frame="rows from unbounded to unbounded", type=decimal, outer=(5)] + │ └── variable: d:5 [type=decimal] + └── last-value [as=last_value:13, frame="rows from unbounded to unbounded", type=float, outer=(4)] + └── variable: f:4 [type=float] + +# The frame must be unbounded in order to infer an FD from partition columns to +# window function output. +build +SELECT + sum(v) OVER w, + array_agg(s) OVER w, + first_value(d) OVER w, + last_value(f) OVER w +FROM kv +WINDOW w AS ( + PARTITION BY w ORDER BY v, s, d, f +) +---- +project + ├── columns: sum:10(decimal) array_agg:11(string[]) first_value:12(decimal) last_value:13(float) + ├── prune: (10-13) + └── window partition=(3) ordering=+2,+6,+5,+4 + ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) sum:10(decimal) array_agg:11(string[]) first_value:12(decimal) last_value:13(float) + ├── key: (1) + ├── fd: (1)-->(2-13) + ├── prune: (1,7-13) + ├── scan kv + │ ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) + │ ├── key: (1) + │ ├── fd: (1)-->(2-9) + │ ├── prune: (1-9) + │ └── interesting orderings: (+1) + └── windows + ├── sum [as=sum:10, type=decimal, outer=(2)] + │ └── variable: v:2 [type=int] + ├── array-agg [as=array_agg:11, type=string[], outer=(6)] + │ └── variable: s:6 [type=string] + ├── first-value [as=first_value:12, type=decimal, outer=(5)] + │ └── variable: d:5 [type=decimal] + └── last-value [as=last_value:13, type=float, outer=(4)] + └── variable: f:4 [type=float] + +# The default window frame is unbounded in the absence of ORDER BY. +# Currently, we cannot see that this is the case, because the window +# frame is not normalized to unbounded preceding and following. +# This is tracked in #117716. +build +SELECT + sum(v) OVER w, + array_agg(s) OVER w, + first_value(d) OVER w, + last_value(f) OVER w +FROM kv +WINDOW w AS ( + PARTITION BY w +) +---- +project + ├── columns: sum:10(decimal) array_agg:11(string[]) first_value:12(decimal) last_value:13(float) + ├── prune: (10-13) + └── window partition=(3) + ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) sum:10(decimal) array_agg:11(string[]) first_value:12(decimal) last_value:13(float) + ├── key: (1) + ├── fd: (1)-->(2-13) + ├── prune: (1,7-13) + ├── scan kv + │ ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) + │ ├── key: (1) + │ ├── fd: (1)-->(2-9) + │ ├── prune: (1-9) + │ └── interesting orderings: (+1) + └── windows + ├── sum [as=sum:10, type=decimal, outer=(2)] + │ └── variable: v:2 [type=int] + ├── array-agg [as=array_agg:11, type=string[], outer=(6)] + │ └── variable: s:6 [type=string] + ├── first-value [as=first_value:12, type=decimal, outer=(5)] + │ └── variable: d:5 [type=decimal] + └── last-value [as=last_value:13, type=float, outer=(4)] + └── variable: f:4 [type=float] + +# The results of other window functions can depend on things other than the +# window frame (e.g. position for rank, or the value of a column for nth_value). +build +SELECT + rank() OVER w, + row_number() OVER w, + nth_value(v, 2) OVER w, + lead(v, 2) OVER w +FROM kv +WINDOW w AS ( + PARTITION BY w ORDER BY v, s, d, f + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING +) +---- +project + ├── columns: rank:10(int) row_number:11(int) nth_value:12(int) lead:13(int) + ├── immutable + ├── prune: (10-13) + └── window partition=(3) ordering=+2,+6,+5,+4 + ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) rank:10(int) row_number:11(int) nth_value:12(int) lead:13(int) nth_value_3_arg2:14(int!null) lead_4_arg3:15(int) + ├── immutable + ├── key: (1) + ├── fd: ()-->(14,15), (1)-->(2-13) + ├── prune: (1,7-13) + ├── project + │ ├── columns: nth_value_3_arg2:14(int!null) lead_4_arg3:15(int) k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) + │ ├── immutable + │ ├── key: (1) + │ ├── fd: ()-->(14,15), (1)-->(2-9) + │ ├── prune: (1-9,14,15) + │ ├── interesting orderings: (+1 opt(14,15)) + │ ├── scan kv + │ │ ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) + │ │ ├── key: (1) + │ │ ├── fd: (1)-->(2-9) + │ │ ├── prune: (1-9) + │ │ └── interesting orderings: (+1) + │ └── projections + │ ├── const: 2 [as=nth_value_3_arg2:14, type=int] + │ └── cast: INT8 [as=lead_4_arg3:15, type=int, immutable] + │ └── null [type=unknown] + └── windows + ├── rank [as=rank:10, frame="rows from unbounded to unbounded", type=int] + ├── row-number [as=row_number:11, frame="rows from unbounded to unbounded", type=int] + ├── nth-value [as=nth_value:12, frame="rows from unbounded to unbounded", type=int, outer=(2,14)] + │ ├── variable: v:2 [type=int] + │ └── variable: nth_value_3_arg2:14 [type=int] + └── lead [as=lead:13, frame="rows from unbounded to unbounded", type=int, outer=(2,14,15)] + ├── variable: v:2 [type=int] + ├── variable: nth_value_3_arg2:14 [type=int] + └── variable: lead_4_arg3:15 [type=int] + +# If the partition columns are a strict key over the input, then window frames +# and functions don't matter - the partition columns are also a strict key over +# the output. +build +SELECT + sum(v) OVER w, + array_agg(s) OVER w, + first_value(d) OVER w, + last_value(f) OVER w, + rank() OVER w, + row_number() OVER w, + nth_value(v, 2) OVER w, + lead(v, 2) OVER w +FROM kv +WINDOW w AS (PARTITION BY k ORDER BY s, d) +---- +project + ├── columns: sum:10(decimal) array_agg:11(string[]) first_value:12(decimal) last_value:13(float) rank:14(int) row_number:15(int) nth_value:16(int) lead:17(int) + ├── immutable + ├── prune: (10-17) + └── window partition=(1) ordering=+6,+5 + ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) sum:10(decimal) array_agg:11(string[]) first_value:12(decimal) last_value:13(float) rank:14(int) row_number:15(int) nth_value:16(int) lead:17(int) nth_value_7_arg2:18(int!null) lead_8_arg3:19(int) + ├── immutable + ├── key: (1) + ├── fd: ()-->(18,19), (1)-->(2-17) + ├── prune: (3,7-17) + ├── project + │ ├── columns: nth_value_7_arg2:18(int!null) lead_8_arg3:19(int) k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) + │ ├── immutable + │ ├── key: (1) + │ ├── fd: ()-->(18,19), (1)-->(2-9) + │ ├── prune: (1-9,18,19) + │ ├── interesting orderings: (+1 opt(18,19)) + │ ├── scan kv + │ │ ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) crdb_internal_mvcc_timestamp:8(decimal) tableoid:9(oid) + │ │ ├── key: (1) + │ │ ├── fd: (1)-->(2-9) + │ │ ├── prune: (1-9) + │ │ └── interesting orderings: (+1) + │ └── projections + │ ├── const: 2 [as=nth_value_7_arg2:18, type=int] + │ └── cast: INT8 [as=lead_8_arg3:19, type=int, immutable] + │ └── null [type=unknown] + └── windows + ├── sum [as=sum:10, type=decimal, outer=(2)] + │ └── variable: v:2 [type=int] + ├── array-agg [as=array_agg:11, type=string[], outer=(6)] + │ └── variable: s:6 [type=string] + ├── first-value [as=first_value:12, type=decimal, outer=(5)] + │ └── variable: d:5 [type=decimal] + ├── last-value [as=last_value:13, type=float, outer=(4)] + │ └── variable: f:4 [type=float] + ├── rank [as=rank:14, type=int] + ├── row-number [as=row_number:15, type=int] + ├── nth-value [as=nth_value:16, type=int, outer=(2,18)] + │ ├── variable: v:2 [type=int] + │ └── variable: nth_value_7_arg2:18 [type=int] + └── lead [as=lead:17, type=int, outer=(2,18,19)] + ├── variable: v:2 [type=int] + ├── variable: nth_value_7_arg2:18 [type=int] + └── variable: lead_8_arg3:19 [type=int] diff --git a/pkg/sql/opt/memo/testdata/stats/window b/pkg/sql/opt/memo/testdata/stats/window index 571bf2565764..ef7c997d7282 100644 --- a/pkg/sql/opt/memo/testdata/stats/window +++ b/pkg/sql/opt/memo/testdata/stats/window @@ -27,7 +27,7 @@ project ├── cardinality: [0 - 10] ├── stats: [rows=10, distinct(10)=10, null(10)=0] ├── key: (1) - ├── fd: (1)-->(2-7) + ├── fd: (1)-->(2-7,10) ├── limit │ ├── columns: k:1(int!null) v:2(int) w:3(int) f:4(float) d:5(decimal) s:6(string) b:7(bool) │ ├── cardinality: [0 - 10] diff --git a/pkg/sql/opt/norm/testdata/rules/decorrelate b/pkg/sql/opt/norm/testdata/rules/decorrelate index 7df944b0bac1..96616a7f2874 100644 --- a/pkg/sql/opt/norm/testdata/rules/decorrelate +++ b/pkg/sql/opt/norm/testdata/rules/decorrelate @@ -277,12 +277,12 @@ WHERE i = 3 ---- project ├── columns: u:1!null v:2 rank:12 i:6!null - ├── key: (1,12) - ├── fd: ()-->(6), (1)-->(2) + ├── key: (1) + ├── fd: ()-->(6), (1)-->(2,12) └── window partition=(1) ├── columns: u:1!null v:2 k:5!null i:6!null rank:12 ├── key: (5) - ├── fd: ()-->(6), (1)-->(2), (1)==(5), (5)==(1) + ├── fd: ()-->(6), (1)-->(2,5,12), (1)==(5), (5)==(1) ├── inner-join (hash) │ ├── columns: u:1!null v:2 k:5!null i:6!null │ ├── multiplicity: left-rows(zero-or-one), right-rows(zero-or-one) @@ -377,7 +377,7 @@ project ├── columns: column1:1!null k:2!null i:3 row_number:9 rownum:10!null ├── cardinality: [0 - 3] ├── key: (10) - ├── fd: (10)-->(1), (2)-->(3), (1)==(2), (2)==(1) + ├── fd: (10)-->(1-3,9), (2)-->(3), (1)==(2), (2)==(1) ├── inner-join (hash) │ ├── columns: column1:1!null k:2!null i:3 rownum:10!null │ ├── cardinality: [0 - 3] @@ -460,7 +460,7 @@ project ├── columns: column1:1!null i:3 row_number:9 rownum:11!null ├── cardinality: [0 - 3] ├── key: (11) - ├── fd: (11)-->(1), (1)-->(3) + ├── fd: (11)-->(1,3,9), (1)-->(3) ├── project │ ├── columns: column1:1!null i:3 rownum:11!null │ ├── cardinality: [0 - 3] @@ -507,7 +507,7 @@ project ├── columns: column1:1!null column2:2!null i:4 row_number:10 rownum:13!null ├── cardinality: [0 - 3] ├── key: (13) - ├── fd: (13)-->(1,2), (1)-->(4) + ├── fd: (13)-->(1,2,4,10), (1)-->(4) ├── project │ ├── columns: column1:1!null column2:2!null i:4 rownum:13!null │ ├── cardinality: [0 - 3] @@ -549,7 +549,7 @@ FROM window partition=(1) ├── columns: u:1!null v:2 row_number:12 i:6 ├── key: (1) - ├── fd: (1)-->(2,6) + ├── fd: (1)-->(2,6,12) ├── project │ ├── columns: u:1!null v:2 i:6 │ ├── key: (1) @@ -581,12 +581,12 @@ FROM ---- project ├── columns: u:1!null v:2 row_number:12 i:6 - ├── key: (1,12) - ├── fd: (1)-->(2,6) + ├── key: (1) + ├── fd: (1)-->(2,6,12) └── window partition=(1) ├── columns: u:1!null v:2 k:5!null i:6 row_number:12 ├── key: (5) - ├── fd: (1)-->(2), (5)-->(6), (1)==(5), (5)==(1) + ├── fd: (1)-->(2,5,6,12), (5)-->(6), (1)==(5), (5)==(1) ├── inner-join (hash) │ ├── columns: u:1!null v:2 k:5!null i:6 │ ├── multiplicity: left-rows(zero-or-one), right-rows(zero-or-one) @@ -3234,11 +3234,11 @@ group-by (hash) ├── select │ ├── columns: k:1!null i:2!null f:3 s:4 j:5 x:8!null u:12!null v:13!null row_num:16!null │ ├── key: (1,12) - │ ├── fd: (1)-->(2-5), (12)-->(13), (2)==(13), (13)==(2), (8)==(12), (12)==(8), (1,12)-->(16) + │ ├── fd: (1)-->(2-5), (12)-->(13), (2)==(13), (13)==(2), (1,8,12)-->(16), (8)==(12), (12)==(8) │ ├── window partition=(1,8) │ │ ├── columns: k:1!null i:2!null f:3 s:4 j:5 x:8!null u:12!null v:13!null row_num:16 │ │ ├── key: (1,8,12) - │ │ ├── fd: (1)-->(2-5), (12)-->(13), (2)==(13), (13)==(2) + │ │ ├── fd: (1)-->(2-5), (12)-->(13), (2)==(13), (13)==(2), (1,8,12)-->(16) │ │ ├── inner-join (cross) │ │ │ ├── columns: k:1!null i:2!null f:3 s:4 j:5 x:8!null u:12!null v:13!null │ │ │ ├── key: (1,8,12) @@ -3299,7 +3299,7 @@ project │ ├── left-join-apply │ │ ├── columns: x:1!null u:5 uv.v:6 k:9 i:10 f:11 row_num:16 │ │ ├── key: (1,9) - │ │ ├── fd: (1,5)-->(6), (1,9)-->(10,11,16), (5)==(9), (9)==(5) + │ │ ├── fd: (1,5)-->(6), (1,9)-->(10,11), (1,5,9)-->(16), (5)==(9), (9)==(5) │ │ ├── scan xy │ │ │ ├── columns: x:1!null │ │ │ └── key: (1) @@ -3308,18 +3308,18 @@ project │ │ │ ├── outer: (1) │ │ │ ├── cardinality: [0 - 3] │ │ │ ├── key: (9) - │ │ │ ├── fd: ()-->(10), (5)-->(6), (9)-->(11,16), (5)==(9), (9)==(5) + │ │ │ ├── fd: ()-->(10), (5)-->(6), (9)-->(11), (5,9)-->(16), (5)==(9), (9)==(5) │ │ │ ├── select │ │ │ │ ├── columns: u:5!null uv.v:6 k:9!null i:10!null f:11 row_num:16!null │ │ │ │ ├── outer: (1) │ │ │ │ ├── key: (9) - │ │ │ │ ├── fd: ()-->(10), (5)-->(6), (9)-->(11,16), (5)==(9), (9)==(5) + │ │ │ │ ├── fd: ()-->(10), (5)-->(6), (9)-->(11), (5,9)-->(16), (5)==(9), (9)==(5) │ │ │ │ ├── limit hint: 3.00 │ │ │ │ ├── window partition=(5) ordering=+11 opt(5-8,10) │ │ │ │ │ ├── columns: u:5!null uv.v:6 k:9!null i:10!null f:11 row_num:16 │ │ │ │ │ ├── outer: (1) │ │ │ │ │ ├── key: (5,9) - │ │ │ │ │ ├── fd: ()-->(10), (5)-->(6), (9)-->(11) + │ │ │ │ │ ├── fd: ()-->(10), (5)-->(6), (9)-->(11), (5,9)-->(16) │ │ │ │ │ ├── limit hint: 8999.57 │ │ │ │ │ ├── inner-join (cross) │ │ │ │ │ │ ├── columns: u:5!null uv.v:6 k:9!null i:10!null f:11 @@ -3479,12 +3479,12 @@ project ├── columns: x:1!null y:2 xy.crdb_internal_mvcc_timestamp:3 xy.tableoid:4 u:5!null v:6 foo:9 row_num:10!null ├── immutable ├── key: (5) - ├── fd: (1)-->(2-4), (5)-->(6,10), (2,6)-->(9), (1)==(5), (5)==(1) + ├── fd: (1)-->(2-4), (5)-->(6), (2,6)-->(9), (1,5)-->(10), (1)==(5), (5)==(1) ├── window partition=(1) │ ├── columns: x:1!null y:2 xy.crdb_internal_mvcc_timestamp:3 xy.tableoid:4 u:5!null v:6 foo:9 row_num:10 │ ├── immutable │ ├── key: (1,5) - │ ├── fd: (1)-->(2-4), (5)-->(6), (2,6)-->(9) + │ ├── fd: (1)-->(2-4), (5)-->(6), (2,6)-->(9), (1,5)-->(10) │ ├── project │ │ ├── columns: foo:9 x:1!null y:2 xy.crdb_internal_mvcc_timestamp:3 xy.tableoid:4 u:5!null v:6 │ │ ├── immutable diff --git a/pkg/sql/opt/norm/testdata/rules/prune_cols b/pkg/sql/opt/norm/testdata/rules/prune_cols index b77a4acc9f6d..48a8bdddd47b 100644 --- a/pkg/sql/opt/norm/testdata/rules/prune_cols +++ b/pkg/sql/opt/norm/testdata/rules/prune_cols @@ -1938,7 +1938,7 @@ sort ├── window partition=(3) ordering=+4 opt(3) │ ├── columns: k:1!null f:3 s:4 avg:7 │ ├── key: (1) - │ ├── fd: (1)-->(3,4) + │ ├── fd: (1)-->(3,4,7) │ ├── scan a │ │ ├── columns: k:1!null f:3 s:4 │ │ ├── key: (1) diff --git a/pkg/sql/opt/norm/testdata/rules/window b/pkg/sql/opt/norm/testdata/rules/window index 904d67ded3f8..48eae452339f 100644 --- a/pkg/sql/opt/norm/testdata/rules/window +++ b/pkg/sql/opt/norm/testdata/rules/window @@ -14,6 +14,7 @@ project └── window partition=(1) ├── columns: k:1!null rank:8 ├── key: (1) + ├── fd: (1)-->(8) ├── scan a │ ├── columns: k:1!null │ └── key: (1) @@ -44,6 +45,7 @@ project └── window partition=() ordering=+1 ├── columns: k:1!null rank:8 ├── key: (1) + ├── fd: (1)-->(8) ├── scan a │ ├── columns: k:1!null │ └── key: (1) @@ -62,6 +64,7 @@ project └── window partition=(1) ├── columns: k:1!null rank:8 ├── key: (1) + ├── fd: (1)-->(8) ├── scan a │ ├── columns: k:1!null │ └── key: (1) @@ -237,7 +240,7 @@ project └── window partition=(1) ├── columns: k:1!null i:2!null rank:8 ├── key: (1) - ├── fd: ()-->(2) + ├── fd: ()-->(2), (1)-->(8) ├── select │ ├── columns: k:1!null i:2!null │ ├── key: (1) @@ -261,7 +264,7 @@ project ├── columns: k:1!null i:2 f:3 rank:8 ├── immutable ├── key: (1) - ├── fd: (1)-->(2,3) + ├── fd: (1)-->(2,3,8) ├── select │ ├── columns: k:1!null i:2 f:3 │ ├── immutable @@ -289,7 +292,7 @@ project ├── window partition=(1) │ ├── columns: k:1!null i:2 f:3 rank:8 │ ├── key: (1) - │ ├── fd: (1)-->(2,3) + │ ├── fd: (1)-->(2,3,8) │ ├── scan a │ │ ├── columns: k:1!null i:2 f:3 │ │ ├── key: (1) @@ -402,9 +405,11 @@ project ├── columns: k:1!null avg:8 ├── cardinality: [0 - 10] ├── key: (1) + ├── fd: (1)-->(8) ├── window partition=() │ ├── columns: k:1!null avg:8 │ ├── key: (1) + │ ├── fd: (1)-->(8) │ ├── limit hint: 10.00 │ ├── scan a │ │ ├── columns: k:1!null @@ -490,11 +495,11 @@ project ├── columns: k:1!null i:2 f:3 rank:8 avg:9 ├── cardinality: [0 - 10] ├── key: (1) - ├── fd: (1)-->(2,3) + ├── fd: (1)-->(2,3,8,9) ├── window partition=(2) ordering=+3 opt(2) │ ├── columns: k:1!null i:2 f:3 rank:8 avg:9 │ ├── key: (1) - │ ├── fd: (1)-->(2,3) + │ ├── fd: (1)-->(2,3,8,9) │ ├── limit hint: 10.00 │ ├── scan a │ │ ├── columns: k:1!null i:2 f:3 @@ -518,18 +523,18 @@ limit ├── internal-ordering: +3 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── ordering: +3 ├── sort │ ├── columns: w:1!null x:2 y:3 z:4 rank:7 │ ├── key: (1) - │ ├── fd: (1)-->(2-4) + │ ├── fd: (1)-->(2-4,7) │ ├── ordering: +3 │ ├── limit hint: 2.00 │ └── window partition=(4) ordering=+3 opt(4) │ ├── columns: w:1!null x:2 y:3 z:4 rank:7 │ ├── key: (1) - │ ├── fd: (1)-->(2-4) + │ ├── fd: (1)-->(2-4,7) │ ├── scan wxyz │ │ ├── columns: w:1!null x:2 y:3 z:4 │ │ ├── key: (1) @@ -545,13 +550,13 @@ sort ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── ordering: +3 └── window partition=(1) ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── limit │ ├── columns: w:1!null x:2 y:3 z:4 │ ├── internal-ordering: +3,+1 @@ -579,13 +584,13 @@ sort ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── ordering: +1 └── window partition=(1) ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── limit │ ├── columns: w:1!null x:2 y:3 z:4 │ ├── internal-ordering: +1 @@ -609,13 +614,13 @@ sort ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── ordering: +1 └── window partition=(1) ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── limit │ ├── columns: w:1!null x:2 y:3 z:4 │ ├── internal-ordering: +1 @@ -639,13 +644,13 @@ sort ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── ordering: +3,+1 └── window partition=(1) ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── limit │ ├── columns: w:1!null x:2 y:3 z:4 │ ├── internal-ordering: +3,+1 @@ -673,13 +678,13 @@ sort ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── ordering: +1 └── window partition=(1) ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── limit │ ├── columns: w:1!null x:2 y:3 z:4 │ ├── internal-ordering: +1 @@ -703,13 +708,13 @@ sort ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── ordering: +4,+2 └── window partition=(2,4) ordering=+3 opt(2,4) ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── limit │ ├── columns: w:1!null x:2 y:3 z:4 │ ├── internal-ordering: +4,+2,+3 @@ -737,13 +742,13 @@ sort ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── ordering: +4,+3 └── window partition=(4) ordering=+3 opt(4) ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── limit │ ├── columns: w:1!null x:2 y:3 z:4 │ ├── internal-ordering: +4,+3 @@ -772,18 +777,18 @@ limit ├── internal-ordering: +3 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── ordering: +3 ├── sort │ ├── columns: w:1!null x:2 y:3 z:4 rank:7 │ ├── key: (1) - │ ├── fd: (1)-->(2-4) + │ ├── fd: (1)-->(2-4,7) │ ├── ordering: +3 │ ├── limit hint: 2.00 │ └── window partition=(4) ordering=+3 opt(4) │ ├── columns: w:1!null x:2 y:3 z:4 rank:7 │ ├── key: (1) - │ ├── fd: (1)-->(2-4) + │ ├── fd: (1)-->(2-4,7) │ ├── scan wxyz │ │ ├── columns: w:1!null x:2 y:3 z:4 │ │ ├── key: (1) @@ -799,13 +804,13 @@ sort ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── ordering: +1 └── window partition=(1) ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── limit │ ├── columns: w:1!null x:2 y:3 z:4 │ ├── internal-ordering: +1 @@ -829,13 +834,13 @@ sort ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── ordering: +4,+1 └── window partition=(1) ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── limit │ ├── columns: w:1!null x:2 y:3 z:4 │ ├── internal-ordering: +4,+1 @@ -863,13 +868,13 @@ sort ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── ordering: +4 └── window partition=(1) ├── columns: w:1!null x:2 y:3 z:4 rank:7 ├── cardinality: [0 - 2] ├── key: (1) - ├── fd: (1)-->(2-4) + ├── fd: (1)-->(2-4,7) ├── limit │ ├── columns: w:1!null x:2 y:3 z:4 │ ├── internal-ordering: +4,+1 diff --git a/pkg/sql/opt/norm/testdata/rules/with b/pkg/sql/opt/norm/testdata/rules/with index d8964ea559dd..92c336d452bc 100644 --- a/pkg/sql/opt/norm/testdata/rules/with +++ b/pkg/sql/opt/norm/testdata/rules/with @@ -1856,7 +1856,7 @@ project │ │ ├── columns: i:5!null sum:7 │ │ ├── cardinality: [0 - 1] │ │ ├── key: () - │ │ ├── fd: ()-->(5) + │ │ ├── fd: ()-->(5,7) │ │ ├── select │ │ │ ├── columns: i:5!null │ │ │ ├── cardinality: [0 - 1] diff --git a/pkg/sql/opt/xform/testdata/external/tpce b/pkg/sql/opt/xform/testdata/external/tpce index fbfcde4e913b..380e8395fd39 100644 --- a/pkg/sql/opt/xform/testdata/external/tpce +++ b/pkg/sql/opt/xform/testdata/external/tpce @@ -4649,19 +4649,19 @@ left-join (cross) ├── cardinality: [0 - 50] ├── multiplicity: left-rows(exactly-one), right-rows(zero-or-more) ├── immutable - ├── key: (98,99,106) - ├── fd: (99)-->(100-105,107,108), (103)-->(107,108), (98,99,106)-->(109-111) + ├── key: (99,106) + ├── fd: (99)-->(98,100-105,107,108), (103)-->(107,108), (99,106)-->(109-111) ├── project │ ├── columns: row_number:98 t_id:99!null t_dts:100!null st_name:101!null tt_name:102!null t_s_symb:103!null t_qty:104!null t_exec_name:105!null t_chrg:106!null s_name:107!null ex_name:108!null │ ├── cardinality: [0 - 50] │ ├── immutable - │ ├── key: (98,99,106) - │ ├── fd: (99)-->(100-105,107,108), (103)-->(107,108) + │ ├── key: (99,106) + │ ├── fd: (99)-->(98,100-105,107,108), (103)-->(107,108) │ ├── window partition=() │ │ ├── columns: trade.t_id:1!null trade.t_dts:2!null t_st_id:3!null t_tt_id:4!null trade.t_s_symb:6!null trade.t_qty:7!null t_ca_id:9!null trade.t_exec_name:10!null trade.t_chrg:12!null s_symb:18!null security.s_name:21!null s_ex_id:22!null ex_id:36!null exchange.ex_name:37!null st_id:45!null status_type.st_name:46!null tt_id:49!null trade_type.tt_name:50!null row_number:55 │ │ ├── cardinality: [0 - 50] │ │ ├── key: (1) - │ │ ├── fd: ()-->(9), (1)-->(2-4,6,7,10,12), (18)-->(21,22), (6)==(18), (18)==(6), (36)-->(37), (22)==(36), (36)==(22), (45)-->(46), (3)==(45), (45)==(3), (49)-->(50), (4)==(49), (49)==(4) + │ │ ├── fd: ()-->(9), (1)-->(2-4,6,7,10,12,55), (18)-->(21,22), (6)==(18), (18)==(6), (36)-->(37), (22)==(36), (36)==(22), (45)-->(46), (3)==(45), (45)==(3), (49)-->(50), (4)==(49), (49)==(4) │ │ ├── inner-join (hash) │ │ │ ├── columns: trade.t_id:1!null trade.t_dts:2!null t_st_id:3!null t_tt_id:4!null trade.t_s_symb:6!null trade.t_qty:7!null t_ca_id:9!null trade.t_exec_name:10!null trade.t_chrg:12!null s_symb:18!null security.s_name:21!null s_ex_id:22!null ex_id:36!null exchange.ex_name:37!null st_id:45!null status_type.st_name:46!null tt_id:49!null trade_type.tt_name:50!null │ │ │ ├── cardinality: [0 - 50] @@ -4800,19 +4800,19 @@ left-join (cross) ├── cardinality: [0 - 50] ├── multiplicity: left-rows(exactly-one), right-rows(zero-or-more) ├── immutable - ├── key: (98,99,106) - ├── fd: (99)-->(100-105,107,108), (103)-->(107,108), (98,99,106)-->(109-111) + ├── key: (99,106) + ├── fd: (99)-->(98,100-105,107,108), (103)-->(107,108), (99,106)-->(109-111) ├── project │ ├── columns: row_number:98 t_id:99!null t_dts:100!null st_name:101!null tt_name:102!null t_s_symb:103!null t_qty:104!null t_exec_name:105!null t_chrg:106!null s_name:107!null ex_name:108!null │ ├── cardinality: [0 - 50] │ ├── immutable - │ ├── key: (98,99,106) - │ ├── fd: (99)-->(100-105,107,108), (103)-->(107,108) + │ ├── key: (99,106) + │ ├── fd: (99)-->(98,100-105,107,108), (103)-->(107,108) │ ├── window partition=() │ │ ├── columns: trade.t_id:1!null trade.t_dts:2!null t_st_id:3!null t_tt_id:4!null trade.t_s_symb:6!null trade.t_qty:7!null t_ca_id:9!null trade.t_exec_name:10!null trade.t_chrg:12!null s_symb:18!null security.s_name:21!null s_ex_id:22!null ex_id:36!null exchange.ex_name:37!null st_id:45!null status_type.st_name:46!null tt_id:49!null trade_type.tt_name:50!null row_number:55 │ │ ├── cardinality: [0 - 50] │ │ ├── key: (1) - │ │ ├── fd: ()-->(9), (1)-->(2-4,6,7,10,12), (18)-->(21,22), (6)==(18), (18)==(6), (36)-->(37), (22)==(36), (36)==(22), (45)-->(46), (3)==(45), (45)==(3), (49)-->(50), (4)==(49), (49)==(4) + │ │ ├── fd: ()-->(9), (1)-->(2-4,6,7,10,12,55), (18)-->(21,22), (6)==(18), (18)==(6), (36)-->(37), (22)==(36), (36)==(22), (45)-->(46), (3)==(45), (45)==(3), (49)-->(50), (4)==(49), (49)==(4) │ │ ├── inner-join (hash) │ │ │ ├── columns: trade.t_id:1!null trade.t_dts:2!null t_st_id:3!null t_tt_id:4!null trade.t_s_symb:6!null trade.t_qty:7!null t_ca_id:9!null trade.t_exec_name:10!null trade.t_chrg:12!null s_symb:18!null security.s_name:21!null s_ex_id:22!null ex_id:36!null exchange.ex_name:37!null st_id:45!null status_type.st_name:46!null tt_id:49!null trade_type.tt_name:50!null │ │ │ ├── flags: force hash join (store right side) @@ -6063,7 +6063,7 @@ project ├── window partition=() │ ├── columns: wi_wl_id:1!null wi_s_symb:2!null wl_id:5!null wl_c_id:6!null row_number:9 │ ├── key: (2,5) - │ ├── fd: ()-->(6), (1)==(5), (5)==(1) + │ ├── fd: ()-->(6), (1)==(5), (5)==(1), (2,5)-->(9) │ ├── inner-join (lookup watch_item) │ │ ├── columns: wi_wl_id:1!null wi_s_symb:2!null wl_id:5!null wl_c_id:6!null │ │ ├── key columns: [5] = [1] diff --git a/pkg/sql/opt/xform/testdata/external/tpce-no-stats b/pkg/sql/opt/xform/testdata/external/tpce-no-stats index f74dd7fda6d0..4a9eb83c19a6 100644 --- a/pkg/sql/opt/xform/testdata/external/tpce-no-stats +++ b/pkg/sql/opt/xform/testdata/external/tpce-no-stats @@ -4679,19 +4679,19 @@ left-join (cross) ├── cardinality: [0 - 50] ├── multiplicity: left-rows(exactly-one), right-rows(zero-or-more) ├── immutable - ├── key: (98,99,106) - ├── fd: (99)-->(100-105,107,108), (103)-->(107,108), (98,99,106)-->(109-111) + ├── key: (99,106) + ├── fd: (99)-->(98,100-105,107,108), (103)-->(107,108), (99,106)-->(109-111) ├── project │ ├── columns: row_number:98 t_id:99!null t_dts:100!null st_name:101!null tt_name:102!null t_s_symb:103!null t_qty:104!null t_exec_name:105!null t_chrg:106!null s_name:107!null ex_name:108!null │ ├── cardinality: [0 - 50] │ ├── immutable - │ ├── key: (98,99,106) - │ ├── fd: (99)-->(100-105,107,108), (103)-->(107,108) + │ ├── key: (99,106) + │ ├── fd: (99)-->(98,100-105,107,108), (103)-->(107,108) │ ├── window partition=() │ │ ├── columns: trade.t_id:1!null trade.t_dts:2!null t_st_id:3!null t_tt_id:4!null trade.t_s_symb:6!null trade.t_qty:7!null t_ca_id:9!null trade.t_exec_name:10!null trade.t_chrg:12!null s_symb:18!null security.s_name:21!null s_ex_id:22!null ex_id:36!null exchange.ex_name:37!null st_id:45!null status_type.st_name:46!null tt_id:49!null trade_type.tt_name:50!null row_number:55 │ │ ├── cardinality: [0 - 50] │ │ ├── key: (1) - │ │ ├── fd: ()-->(9), (1)-->(2-4,6,7,10,12), (18)-->(21,22), (6)==(18), (18)==(6), (36)-->(37), (22)==(36), (36)==(22), (45)-->(46), (3)==(45), (45)==(3), (49)-->(50), (4)==(49), (49)==(4) + │ │ ├── fd: ()-->(9), (1)-->(2-4,6,7,10,12,55), (18)-->(21,22), (6)==(18), (18)==(6), (36)-->(37), (22)==(36), (36)==(22), (45)-->(46), (3)==(45), (45)==(3), (49)-->(50), (4)==(49), (49)==(4) │ │ ├── inner-join (lookup trade_type) │ │ │ ├── columns: trade.t_id:1!null trade.t_dts:2!null t_st_id:3!null t_tt_id:4!null trade.t_s_symb:6!null trade.t_qty:7!null t_ca_id:9!null trade.t_exec_name:10!null trade.t_chrg:12!null s_symb:18!null security.s_name:21!null s_ex_id:22!null ex_id:36!null exchange.ex_name:37!null st_id:45!null status_type.st_name:46!null tt_id:49!null trade_type.tt_name:50!null │ │ │ ├── key columns: [4] = [49] @@ -4818,19 +4818,19 @@ left-join (cross) ├── cardinality: [0 - 50] ├── multiplicity: left-rows(exactly-one), right-rows(zero-or-more) ├── immutable - ├── key: (98,99,106) - ├── fd: (99)-->(100-105,107,108), (103)-->(107,108), (98,99,106)-->(109-111) + ├── key: (99,106) + ├── fd: (99)-->(98,100-105,107,108), (103)-->(107,108), (99,106)-->(109-111) ├── project │ ├── columns: row_number:98 t_id:99!null t_dts:100!null st_name:101!null tt_name:102!null t_s_symb:103!null t_qty:104!null t_exec_name:105!null t_chrg:106!null s_name:107!null ex_name:108!null │ ├── cardinality: [0 - 50] │ ├── immutable - │ ├── key: (98,99,106) - │ ├── fd: (99)-->(100-105,107,108), (103)-->(107,108) + │ ├── key: (99,106) + │ ├── fd: (99)-->(98,100-105,107,108), (103)-->(107,108) │ ├── window partition=() │ │ ├── columns: trade.t_id:1!null trade.t_dts:2!null t_st_id:3!null t_tt_id:4!null trade.t_s_symb:6!null trade.t_qty:7!null t_ca_id:9!null trade.t_exec_name:10!null trade.t_chrg:12!null s_symb:18!null security.s_name:21!null s_ex_id:22!null ex_id:36!null exchange.ex_name:37!null st_id:45!null status_type.st_name:46!null tt_id:49!null trade_type.tt_name:50!null row_number:55 │ │ ├── cardinality: [0 - 50] │ │ ├── key: (1) - │ │ ├── fd: ()-->(9), (1)-->(2-4,6,7,10,12), (18)-->(21,22), (6)==(18), (18)==(6), (36)-->(37), (22)==(36), (36)==(22), (45)-->(46), (3)==(45), (45)==(3), (49)-->(50), (4)==(49), (49)==(4) + │ │ ├── fd: ()-->(9), (1)-->(2-4,6,7,10,12,55), (18)-->(21,22), (6)==(18), (18)==(6), (36)-->(37), (22)==(36), (36)==(22), (45)-->(46), (3)==(45), (45)==(3), (49)-->(50), (4)==(49), (49)==(4) │ │ ├── inner-join (hash) │ │ │ ├── columns: trade.t_id:1!null trade.t_dts:2!null t_st_id:3!null t_tt_id:4!null trade.t_s_symb:6!null trade.t_qty:7!null t_ca_id:9!null trade.t_exec_name:10!null trade.t_chrg:12!null s_symb:18!null security.s_name:21!null s_ex_id:22!null ex_id:36!null exchange.ex_name:37!null st_id:45!null status_type.st_name:46!null tt_id:49!null trade_type.tt_name:50!null │ │ │ ├── flags: force hash join (store right side) @@ -6080,7 +6080,7 @@ project ├── window partition=() │ ├── columns: wi_wl_id:1!null wi_s_symb:2!null wl_id:5!null wl_c_id:6!null row_number:9 │ ├── key: (2,5) - │ ├── fd: ()-->(6), (1)==(5), (5)==(1) + │ ├── fd: ()-->(6), (1)==(5), (5)==(1), (2,5)-->(9) │ ├── inner-join (lookup watch_item) │ │ ├── columns: wi_wl_id:1!null wi_s_symb:2!null wl_id:5!null wl_c_id:6!null │ │ ├── key columns: [5] = [1] diff --git a/pkg/sql/opt/xform/testdata/physprops/ordering b/pkg/sql/opt/xform/testdata/physprops/ordering index 6bfe3c1a1f4d..b5efa0c94bee 100644 --- a/pkg/sql/opt/xform/testdata/physprops/ordering +++ b/pkg/sql/opt/xform/testdata/physprops/ordering @@ -2493,10 +2493,12 @@ SELECT *, row_number() OVER() FROM abc ORDER BY a sort ├── columns: a:1!null b:2!null c:3!null row_number:6 ├── key: (1-3) + ├── fd: (1-3)-->(6) ├── ordering: +1 └── window partition=() ├── columns: a:1!null b:2!null c:3!null row_number:6 ├── key: (1-3) + ├── fd: (1-3)-->(6) ├── scan abc │ ├── columns: a:1!null b:2!null c:3!null │ └── key: (1-3) diff --git a/pkg/sql/sem/builtins/window_builtins.go b/pkg/sql/sem/builtins/window_builtins.go index 8b375ff47950..e436afdbd1b8 100644 --- a/pkg/sql/sem/builtins/window_builtins.go +++ b/pkg/sql/sem/builtins/window_builtins.go @@ -223,6 +223,11 @@ var _ eval.WindowFunc = &nthValueWindow{} // aggregateWindowFunc aggregates over the current row's window frame, using // the internal eval.AggregateFunc to perform the aggregation. +// +// INVARIANT: the rows within a window frame are always processed in the same +// order, regardless of whether the user specified an ordering. This means that +// two rows with the exact same frame will produce the same result for a given +// aggregation. type aggregateWindowFunc struct { agg eval.AggregateFunc peerRes tree.Datum From 147054d9843c4ed803d36931fa71ef7b8c455d2e Mon Sep 17 00:00:00 2001 From: Drew Kimball Date: Mon, 8 Jan 2024 04:50:17 -0700 Subject: [PATCH 4/4] opt: add rule to merge GroupBy and Window This commit adds a new norm rule, `FoldGroupByAndWindow`, which can merge a Window operator with a parent GroupBy operator when the grouping columns are the same as the partition columns. See the rule comment for the complete list of conditions. In addition to removing a potentially expensive Window operator, this transformation makes way for other rules to match. Fixes #113292 Release note: None --- .../testdata/benchmark_expectations | 2 +- .../logictest/testdata/logic_test/pg_catalog | 1 + pkg/sql/opt/exec/execbuilder/testdata/explain | 65 +-- pkg/sql/opt/norm/groupby_funcs.go | 76 +++ pkg/sql/opt/norm/rules/groupby.opt | 80 +++ pkg/sql/opt/norm/testdata/rules/groupby | 467 ++++++++++++++++++ pkg/sql/opt/norm/testdata/rules/prune_cols | 74 ++- pkg/sql/opt/norm/window_funcs.go | 21 + 8 files changed, 713 insertions(+), 73 deletions(-) diff --git a/pkg/bench/rttanalysis/testdata/benchmark_expectations b/pkg/bench/rttanalysis/testdata/benchmark_expectations index 5cccd470a0ea..d2b5b1b8d619 100644 --- a/pkg/bench/rttanalysis/testdata/benchmark_expectations +++ b/pkg/bench/rttanalysis/testdata/benchmark_expectations @@ -74,7 +74,7 @@ exp,benchmark 3,Jobs/show_job 3-5,Jobs/show_jobs 3,ORMQueries/activerecord_type_introspection_query -0,ORMQueries/asyncpg_types +4,ORMQueries/asyncpg_types 6,ORMQueries/column_descriptions_json_agg 4,ORMQueries/django_column_introspection_1_table 4,ORMQueries/django_column_introspection_4_tables diff --git a/pkg/sql/logictest/testdata/logic_test/pg_catalog b/pkg/sql/logictest/testdata/logic_test/pg_catalog index 208ceee38332..73a846d18e1d 100644 --- a/pkg/sql/logictest/testdata/logic_test/pg_catalog +++ b/pkg/sql/logictest/testdata/logic_test/pg_catalog @@ -4490,6 +4490,7 @@ FROM ( WHERE c.relname = 'indexes_table' ) s2 GROUP BY indexname, indisunique, indisprimary, amname, exprdef, attoptions +ORDER BY indexname ---- indexname array_agg indisunique indisprimary array_agg amname exprdef attoptions indexes_include_idx {a,c,d} false false {ASC,ASC,ASC} prefix NULL NULL diff --git a/pkg/sql/opt/exec/execbuilder/testdata/explain b/pkg/sql/opt/exec/execbuilder/testdata/explain index 3269b51d61b6..fbc30e575fe1 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/explain +++ b/pkg/sql/opt/exec/execbuilder/testdata/explain @@ -560,7 +560,8 @@ vectorized: true └── • group (hash) │ group by: column_name, ordinal_position, column_default, is_nullable, generation_expression, is_hidden, crdb_sql_type │ - └── • window + └── • sort + │ order: +index_name │ └── • hash join (left outer) │ equality: (column_name) = (column_name) @@ -734,39 +735,41 @@ vectorized: true │ estimated row count: 3 │ order: +"role" │ - └── • hash join (left outer) + └── • merge join (right outer) │ estimated row count: 3 - │ equality: (username) = (member) - │ left cols are key + │ equality: (member) = (username) + │ right cols are key │ - ├── • group (hash) - │ │ estimated row count: 3 - │ │ group by: username - │ │ - │ └── • window - │ │ estimated row count: 3 - │ │ - │ └── • render - │ │ - │ └── • hash join (left outer) - │ │ estimated row count: 3 - │ │ equality: (username) = (username) - │ │ left cols are key - │ │ - │ ├── • scan - │ │ estimated row count: 3 (100% of the table; stats collected ago) - │ │ table: users@users_user_id_idx - │ │ spans: FULL SCAN - │ │ - │ └── • scan - │ estimated row count: 1 (100% of the table; stats collected ago) - │ table: role_options@primary - │ spans: FULL SCAN + ├── • scan + │ estimated row count: 1 (100% of the table; stats collected ago) + │ table: role_members@role_members_member_idx + │ spans: FULL SCAN │ - └── • scan - estimated row count: 1 (100% of the table; stats collected ago) - table: role_members@role_members_role_idx - spans: FULL SCAN + └── • group (streaming) + │ estimated row count: 3 + │ group by: username + │ ordered: +username + │ + └── • sort + │ estimated row count: 3 + │ order: +username,+option + │ + └── • render + │ + └── • hash join (left outer) + │ estimated row count: 3 + │ equality: (username) = (username) + │ left cols are key + │ + ├── • scan + │ estimated row count: 3 (100% of the table; stats collected ago) + │ table: users@users_user_id_idx + │ spans: FULL SCAN + │ + └── • scan + estimated row count: 1 (100% of the table; stats collected ago) + table: role_options@primary + spans: FULL SCAN # EXPLAIN selecting from a sequence. statement ok diff --git a/pkg/sql/opt/norm/groupby_funcs.go b/pkg/sql/opt/norm/groupby_funcs.go index 78a601d0d6d9..99e0568c2095 100644 --- a/pkg/sql/opt/norm/groupby_funcs.go +++ b/pkg/sql/opt/norm/groupby_funcs.go @@ -382,6 +382,82 @@ func (c *CustomFuncs) MergeAggs( return newAggs } +// CanMergeAggsAndWindow returns true if all the given aggregations satisfy one +// of the following conditions: +// 1. Reference only columns from the input of the Window operator. +// 2. Is a ConstAgg (or similar) that references an input aggregate window +// function. +// +// CanMergeAggsAndWindow expects that all the window functions have been +// verified to be aggregate functions. +func (c *CustomFuncs) CanMergeAggsAndWindow( + aggs memo.AggregationsExpr, windows memo.WindowsExpr, inputCols opt.ColSet, +) bool { + // Collect the columns produced by the window functions. + var windowCols opt.ColSet + for i := range windows { + windowCols.Add(windows[i].Col) + } + for i := range aggs { + if memo.ExtractAggInputColumns(aggs[i].Agg).SubsetOf(inputCols) { + // Condition 1: the aggregate function only references columns from the + // input of the Window operator. It will not be affected by a merge. + // In this case, it doesn't matter what the aggregate is, since it won't + // be modified in any way. + // + // Note that unlike for CanMergeAggs, it is not necessary to check for + // duplicate sensitivity. This is because window operators do not group or + // duplicate rows. + continue + } + // Condition 2: the aggregate function must be a AnyNotNullAgg, ConstAgg, + // ConstNotNullAgg, or FirstAggOp that references a window function. + switch aggs[i].Agg.Op() { + case opt.AnyNotNullAggOp, opt.ConstAggOp, opt.ConstNotNullAggOp, opt.FirstAggOp: + // Ensure that the input to the aggregation is a direct reference to a + // window function, with no intervening logic. + ref, ok := aggs[i].Agg.Child(0).(*memo.VariableExpr) + if !ok { + return false + } + if !windowCols.Contains(ref.Col) { + return false + } + default: + return false + } + } + return true +} + +// MergeAggsAndWindow returns an AggregationsExpr that is equivalent to the +// combination of the given (outer) aggregations and (inner) window functions. +// ConstAgg-like outer aggregations that reference a window function are +// replaced with that window function. +// +// MergeAggs will panic if CanMergeAggs is false. It also expects the given +// window functions to all be aggregate functions. +func (c *CustomFuncs) MergeAggsAndWindow( + aggs memo.AggregationsExpr, windows memo.WindowsExpr, inputCols opt.ColSet, +) memo.AggregationsExpr { + // Create a mapping from column IDs to the window functions that produce them. + colsToWindowFuncs := map[opt.ColumnID]opt.ScalarExpr{} + for i := range windows { + colsToWindowFuncs[windows[i].Col] = windows[i].Function + } + newAggs := make(memo.AggregationsExpr, len(aggs)) + for i := range aggs { + aggCols := memo.ExtractAggInputColumns(aggs[i].Agg) + if aggCols.SubsetOf(inputCols) { + newAggs[i] = aggs[i] + continue + } + windowFunc := colsToWindowFuncs[aggCols.SingleColumn()] + newAggs[i] = c.f.ConstructAggregationsItem(windowFunc, aggs[i].Col) + } + return newAggs +} + // CanEliminateJoinUnderGroupByLeft returns true if the given join can be // eliminated and replaced by its left input. It should be called only when the // join is under a grouping operator that is only using columns from the join's diff --git a/pkg/sql/opt/norm/rules/groupby.opt b/pkg/sql/opt/norm/rules/groupby.opt index 824510ed567b..d87df2926771 100644 --- a/pkg/sql/opt/norm/rules/groupby.opt +++ b/pkg/sql/opt/norm/rules/groupby.opt @@ -587,3 +587,83 @@ (MergeAggs $innerAggs $outerAggs $innerGroupingCols) (MakeGrouping $outerGroupingCols (EmptyOrdering)) ) + +# FoldGroupByAndWindow merges a GroupBy operator with an input Window operator. +# This is possible when the following conditions are satisfied: +# +# 1. The GroupBy is unordered. This may not technically be necessary, but +# avoids complication in determining the correctness of ordering-sensitive +# aggregations. +# +# 2. The window function output cols are functionally determined by the +# partition-by cols. This means that the window function outputs the +# same value for every row in the partition (group). +# +# 3. The Window operator partition-by cols and grouping cols are the same. +# This ensures that an aggregate operator will act on the same set of rows, +# whether it is part of the Window operator or the GroupBy operator. +# +# 4. The window functions are all aggregate functions. This ensures they are +# compatible with GroupBy operators. +# +# 5. Finally, all of the GroupBy's aggregations must satisfy one of two cases: +# a. The aggregate only references cols from the Window operator's input. +# b. The aggregate is a ConstAgg (or ConstNotNull, AnyNotNull, or FirstAgg) +# that passes through the result of a window function. +# +# Assuming all of the above are satisfied, each GroupBy aggregate that only +# references the Window's input can be left alone (5a). Then, each ConstAgg +# referencing a window function can be replaced by that function (5b). +# +# Here's an example with slightly altered SQL syntax: +# +# SELECT max(b), const_agg(foo), const_agg(bar) +# FROM +# ( +# SELECT *, count(c) OVER w AS foo, array_agg(d) OVER w AS bar +# FROM abcd +# WINDOW w AS ( +# PARTITION BY a ORDER BY d +# RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING +# ) +# ) +# GROUP BY a; +# => +# SELECT max(b), count(c), array_agg(d ORDER BY d) FROM abcd GROUP BY a; +# +# Note also that the Window's ordering should be preserved by the GroupBy to +# ensure that ordering-sensitive aggregates produce correct results. +[FoldGroupByAndWindow, Normalize] +(GroupBy | ScalarGroupBy + $window:(Window + $input:* + $windows:* & (WindowsAreAggregations $windows) + $windowPrivate:* + ) & + (ColsAreDeterminedBy + (WindowFuncOutputCols $windows) + $partitionByCols:(WindowPartition $windowPrivate) + $window + ) + $aggs:* & + (CanMergeAggsAndWindow + $aggs + $windows + $inputCols:(OutputCols $input) + ) + $groupingPrivate:* & + (IsUnorderedGrouping $groupingPrivate) & + (ColsAreEqual + $groupingCols:(GroupingCols $groupingPrivate) + $partitionByCols + ) +) +=> +((OpName) + $input + (MergeAggsAndWindow $aggs $windows $inputCols) + (MakeGrouping + (GroupingCols $groupingPrivate) + (WindowOrdering $windowPrivate) + ) +) diff --git a/pkg/sql/opt/norm/testdata/rules/groupby b/pkg/sql/opt/norm/testdata/rules/groupby index 2b4780392d3a..f8ed3b2e2b8c 100644 --- a/pkg/sql/opt/norm/testdata/rules/groupby +++ b/pkg/sql/opt/norm/testdata/rules/groupby @@ -4408,3 +4408,470 @@ scalar-group-by └── aggregations └── sum [as=sum:6, outer=(1)] └── a:1 + +# -------------------------------------------------- +# FoldGroupByAndWindow +# -------------------------------------------------- + +# Case with one partition column and an aggregate that references an input col. +# NOTE: the "foo" and "bar" grouping columns are simplified to ConstAgg. +norm expect=FoldGroupByAndWindow +SELECT sum(v), foo, bar +FROM ( + SELECT *, count(w) OVER w AS foo, array_agg(z) OVER w AS bar + FROM uvwz + WINDOW w AS ( + PARTITION BY u ORDER BY z + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) +) +GROUP BY u, foo, bar; +---- +project + ├── columns: sum:10!null foo:8!null bar:9!null + └── group-by (hash) + ├── columns: u:1!null count:8!null array_agg:9!null sum:10!null + ├── grouping columns: u:1!null + ├── internal-ordering: +4 opt(1) + ├── key: (1) + ├── fd: (1)-->(8-10) + ├── sort + │ ├── columns: u:1!null v:2!null z:4!null + │ ├── key: (1,2) + │ ├── fd: (1,2)-->(4) + │ ├── ordering: +4 opt(1) [actual: +4] + │ └── scan uvwz + │ ├── columns: u:1!null v:2!null z:4!null + │ ├── key: (1,2) + │ └── fd: (1,2)-->(4) + └── aggregations + ├── sum [as=sum:10, outer=(2)] + │ └── v:2 + ├── count-rows [as=count:8] + └── array-agg [as=array_agg:9, outer=(4)] + └── z:4 + +# Case with a ConstAgg referencing an input column. +norm expect=FoldGroupByAndWindow +SELECT sum(v), foo, bar +FROM ( + SELECT *, count(w) OVER w AS foo + FROM (SELECT *, 100 AS bar FROM uvwz) + WINDOW w AS ( + PARTITION BY u ORDER BY z + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) +) +GROUP BY u, foo, bar; +---- +project + ├── columns: sum:10!null foo:9!null bar:8!null + ├── fd: ()-->(8) + └── group-by (hash) + ├── columns: u:1!null bar:8!null count:9!null sum:10!null + ├── grouping columns: u:1!null + ├── internal-ordering: +4 opt(1,8) + ├── key: (1) + ├── fd: ()-->(8), (1)-->(8-10) + ├── project + │ ├── columns: bar:8!null u:1!null v:2!null z:4!null + │ ├── key: (1,2) + │ ├── fd: ()-->(8), (1,2)-->(4) + │ ├── ordering: +4 opt(1,8) [actual: +4] + │ ├── sort + │ │ ├── columns: u:1!null v:2!null z:4!null + │ │ ├── key: (1,2) + │ │ ├── fd: (1,2)-->(4) + │ │ ├── ordering: +4 opt(1) [actual: +4] + │ │ └── scan uvwz + │ │ ├── columns: u:1!null v:2!null z:4!null + │ │ ├── key: (1,2) + │ │ └── fd: (1,2)-->(4) + │ └── projections + │ └── 100 [as=bar:8] + └── aggregations + ├── sum [as=sum:10, outer=(2)] + │ └── v:2 + ├── const-agg [as=bar:8, outer=(8)] + │ └── bar:8 + └── count-rows [as=count:9] + +# Case with no partition columns. +norm expect=FoldGroupByAndWindow +SELECT sum(v), foo, bar +FROM ( + SELECT *, count(w) OVER w AS foo, array_agg(z) OVER w AS bar + FROM uvwz + WINDOW w AS ( + ORDER BY z + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) +) +GROUP BY foo, bar; +---- +group-by (streaming) + ├── columns: sum:10!null foo:8!null bar:9!null + ├── internal-ordering: +4 + ├── cardinality: [0 - 1] + ├── key: () + ├── fd: ()-->(8-10) + ├── sort + │ ├── columns: v:2!null z:4!null + │ ├── ordering: +4 + │ └── scan uvwz + │ └── columns: v:2!null z:4!null + └── aggregations + ├── sum [as=sum:10, outer=(2)] + │ └── v:2 + ├── count-rows [as=count:8] + └── array-agg [as=array_agg:9, outer=(4)] + └── z:4 + +# Case with multiple partition columns. +norm expect=FoldGroupByAndWindow +SELECT sum(v), foo, bar +FROM ( + SELECT *, count(v) OVER w AS foo, array_agg(z) OVER w AS bar + FROM uvwz + WINDOW w AS ( + PARTITION BY u, w ORDER BY z + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) +) +GROUP BY u, w, foo, bar; +---- +project + ├── columns: sum:10!null foo:8!null bar:9!null + └── group-by (hash) + ├── columns: u:1!null w:3!null count:8!null array_agg:9!null sum:10!null + ├── grouping columns: u:1!null w:3!null + ├── internal-ordering: +4 opt(1,3) + ├── key: (1,3) + ├── fd: (1,3)-->(8-10) + ├── sort + │ ├── columns: u:1!null v:2!null w:3!null z:4!null + │ ├── key: (2,3) + │ ├── fd: (1,2)-->(3,4), (2,3)-->(1,4) + │ ├── ordering: +4 opt(1,3) [actual: +4] + │ └── scan uvwz + │ ├── columns: u:1!null v:2!null w:3!null z:4!null + │ ├── key: (2,3) + │ └── fd: (1,2)-->(3,4), (2,3)-->(1,4) + └── aggregations + ├── sum [as=sum:10, outer=(2)] + │ └── v:2 + ├── count-rows [as=count:8] + └── array-agg [as=array_agg:9, outer=(4)] + └── z:4 + +# Case with grouping/partitioning on key columns. +norm expect=FoldGroupByAndWindow +SELECT sum(w), foo, bar +FROM ( + SELECT *, count(w) OVER w AS foo, array_agg(z) OVER w AS bar + FROM uvwz + WINDOW w AS ( + PARTITION BY u, v ORDER BY z + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) +) +GROUP BY u, v, foo, bar; +---- +project + ├── columns: sum:10!null foo:8!null bar:9!null + └── group-by (hash) + ├── columns: u:1!null v:2!null count:8!null array_agg:9!null sum:10!null + ├── grouping columns: u:1!null v:2!null + ├── key: (1,2) + ├── fd: (1,2)-->(8-10) + ├── scan uvwz + │ ├── columns: u:1!null v:2!null w:3!null z:4!null + │ ├── key: (2,3) + │ └── fd: (1,2)-->(3,4), (2,3)-->(1,4) + └── aggregations + ├── sum [as=sum:10, outer=(3)] + │ └── w:3 + ├── count-rows [as=count:8] + └── array-agg [as=array_agg:9, outer=(4)] + └── z:4 + +# No-op because of an ordered GroupBy. +norm expect-not=FoldGroupByAndWindow +SELECT array_agg(v), foo, bar +FROM ( + SELECT * FROM ( + SELECT *, count(w) OVER w AS foo, array_agg(z) OVER w AS bar + FROM uvwz + WINDOW w AS ( + PARTITION BY u ORDER BY z + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) + ) + ORDER BY v DESC +) +GROUP BY u, foo, bar; +---- +project + ├── columns: array_agg:10!null foo:8 bar:9 + └── group-by (hash) + ├── columns: u:1!null count:8 array_agg:9 array_agg:10!null + ├── grouping columns: u:1!null + ├── internal-ordering: -2 opt(1,8,9) + ├── key: (1) + ├── fd: (1)-->(8-10) + ├── sort + │ ├── columns: u:1!null v:2!null w:3!null z:4!null count:8 array_agg:9 + │ ├── key: (2,3) + │ ├── fd: (1,2)-->(3,4), (2,3)-->(1,4), (1)-->(8,9) + │ ├── ordering: -2 opt(1,8,9) [actual: -2] + │ └── window partition=(1) ordering=+4 opt(1) + │ ├── columns: u:1!null v:2!null w:3!null z:4!null count:8 array_agg:9 + │ ├── key: (2,3) + │ ├── fd: (1,2)-->(3,4), (2,3)-->(1,4), (1)-->(8,9) + │ ├── scan uvwz + │ │ ├── columns: u:1!null v:2!null w:3!null z:4!null + │ │ ├── key: (2,3) + │ │ └── fd: (1,2)-->(3,4), (2,3)-->(1,4) + │ └── windows + │ ├── count [as=count:8, frame="rows from unbounded to unbounded", outer=(3)] + │ │ └── w:3 + │ └── array-agg [as=array_agg:9, frame="rows from unbounded to unbounded", outer=(4)] + │ └── z:4 + └── aggregations + ├── array-agg [as=array_agg:10, outer=(2)] + │ └── v:2 + ├── const-agg [as=count:8, outer=(8)] + │ └── count:8 + └── const-agg [as=array_agg:9, outer=(9)] + └── array_agg:9 + +# No-op because the grouping columns reference a window function. +norm expect-not=FoldGroupByAndWindow +SELECT sum(v), foo, bar +FROM ( + SELECT *, count(w) OVER w AS foo, array_agg(z) OVER w AS bar, row_number() OVER w AS baz + FROM uvwz + WINDOW w AS ( + PARTITION BY u ORDER BY z + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) +) +GROUP BY u, foo, bar, baz; +---- +project + ├── columns: sum:11!null foo:8 bar:9 + └── group-by (hash) + ├── columns: u:1!null count:8 array_agg:9 row_number:10 sum:11!null + ├── grouping columns: u:1!null row_number:10 + ├── key: (1,10) + ├── fd: (1)-->(8,9), (1,10)-->(8,9,11) + ├── window partition=(1) ordering=+4 opt(1) + │ ├── columns: u:1!null v:2!null w:3!null z:4!null count:8 array_agg:9 row_number:10 + │ ├── key: (2,3) + │ ├── fd: (1,2)-->(3,4), (2,3)-->(1,4,10), (1)-->(8,9) + │ ├── scan uvwz + │ │ ├── columns: u:1!null v:2!null w:3!null z:4!null + │ │ ├── key: (2,3) + │ │ └── fd: (1,2)-->(3,4), (2,3)-->(1,4) + │ └── windows + │ ├── count [as=count:8, frame="rows from unbounded to unbounded", outer=(3)] + │ │ └── w:3 + │ ├── array-agg [as=array_agg:9, frame="rows from unbounded to unbounded", outer=(4)] + │ │ └── z:4 + │ └── row-number [as=row_number:10, frame="rows from unbounded to unbounded"] + └── aggregations + ├── sum [as=sum:11, outer=(2)] + │ └── v:2 + ├── const-agg [as=count:8, outer=(8)] + │ └── count:8 + └── const-agg [as=array_agg:9, outer=(9)] + └── array_agg:9 + +# No-op because the grouping columns are not the same as the partition columns. +norm expect-not=FoldGroupByAndWindow +SELECT sum(u), foo, bar +FROM ( + SELECT *, count(w) OVER w AS foo, array_agg(z) OVER w AS bar + FROM uvwz + WINDOW w AS ( + PARTITION BY u ORDER BY z + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) +) +GROUP BY v, foo, bar; +---- +project + ├── columns: sum:10!null foo:8 bar:9 + └── group-by (hash) + ├── columns: v:2!null count:8 array_agg:9 sum:10!null + ├── grouping columns: v:2!null count:8 array_agg:9 + ├── key: (2,8,9) + ├── fd: (2,8,9)-->(10) + ├── window partition=(1) ordering=+4 opt(1) + │ ├── columns: u:1!null v:2!null w:3!null z:4!null count:8 array_agg:9 + │ ├── key: (2,3) + │ ├── fd: (1,2)-->(3,4), (2,3)-->(1,4), (1)-->(8,9) + │ ├── scan uvwz + │ │ ├── columns: u:1!null v:2!null w:3!null z:4!null + │ │ ├── key: (2,3) + │ │ └── fd: (1,2)-->(3,4), (2,3)-->(1,4) + │ └── windows + │ ├── count [as=count:8, frame="rows from unbounded to unbounded", outer=(3)] + │ │ └── w:3 + │ └── array-agg [as=array_agg:9, frame="rows from unbounded to unbounded", outer=(4)] + │ └── z:4 + └── aggregations + └── sum [as=sum:10, outer=(1)] + └── u:1 + +# No-op because the grouping columns are not the same as the partition columns. +norm expect-not=FoldGroupByAndWindow +SELECT sum(v), foo, bar +FROM ( + SELECT *, count(w) OVER w AS foo, array_agg(z) OVER w AS bar + FROM uvwz + WINDOW w AS ( + ORDER BY z + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) +) +GROUP BY u, foo, bar; +---- +project + ├── columns: sum:10!null foo:8 bar:9 + ├── fd: ()-->(8,9) + └── group-by (hash) + ├── columns: u:1!null count:8 array_agg:9 sum:10!null + ├── grouping columns: u:1!null + ├── key: (1) + ├── fd: ()-->(8,9), (1)-->(8-10) + ├── window partition=() ordering=+4 + │ ├── columns: u:1!null v:2!null w:3!null z:4!null count:8 array_agg:9 + │ ├── key: (2,3) + │ ├── fd: ()-->(8,9), (1,2)-->(3,4), (2,3)-->(1,4) + │ ├── scan uvwz + │ │ ├── columns: u:1!null v:2!null w:3!null z:4!null + │ │ ├── key: (2,3) + │ │ └── fd: (1,2)-->(3,4), (2,3)-->(1,4) + │ └── windows + │ ├── count [as=count:8, frame="rows from unbounded to unbounded", outer=(3)] + │ │ └── w:3 + │ └── array-agg [as=array_agg:9, frame="rows from unbounded to unbounded", outer=(4)] + │ └── z:4 + └── aggregations + ├── sum [as=sum:10, outer=(2)] + │ └── v:2 + ├── const-agg [as=count:8, outer=(8)] + │ └── count:8 + └── const-agg [as=array_agg:9, outer=(9)] + └── array_agg:9 + +# No-op because the window frame is not unbounded. +norm expect-not=FoldGroupByAndWindow +SELECT sum(v), foo, bar +FROM ( + SELECT *, count(w) OVER w AS foo, array_agg(z) OVER w AS bar + FROM uvwz + WINDOW w AS ( + PARTITION BY u ORDER BY z + ) +) +GROUP BY u, foo, bar; +---- +project + ├── columns: sum:10!null foo:8 bar:9 + └── group-by (hash) + ├── columns: u:1!null count:8 array_agg:9 sum:10!null + ├── grouping columns: u:1!null count:8 array_agg:9 + ├── key: (1,8,9) + ├── fd: (1,8,9)-->(10) + ├── window partition=(1) ordering=+4 opt(1) + │ ├── columns: u:1!null v:2!null w:3!null z:4!null count:8 array_agg:9 + │ ├── key: (2,3) + │ ├── fd: (1,2)-->(3,4), (2,3)-->(1,4,8,9) + │ ├── scan uvwz + │ │ ├── columns: u:1!null v:2!null w:3!null z:4!null + │ │ ├── key: (2,3) + │ │ └── fd: (1,2)-->(3,4), (2,3)-->(1,4) + │ └── windows + │ ├── count [as=count:8, outer=(3)] + │ │ └── w:3 + │ └── array-agg [as=array_agg:9, outer=(4)] + │ └── z:4 + └── aggregations + └── sum [as=sum:10, outer=(2)] + └── v:2 + +# No-op because the (non ConstAgg-like) Sum GroupBy aggregate references a +# Window aggregate. +norm expect-not=FoldGroupByAndWindow +SELECT sum(foo), foo, bar +FROM ( + SELECT *, count(w) OVER w AS foo, array_agg(z) OVER w AS bar + FROM uvwz + WINDOW w AS ( + PARTITION BY u ORDER BY z + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) +) +GROUP BY u, foo, bar; +---- +project + ├── columns: sum:10 foo:8 bar:9 + └── group-by (hash) + ├── columns: u:1!null count:8 array_agg:9 sum:10 + ├── grouping columns: u:1!null + ├── key: (1) + ├── fd: (1)-->(8-10) + ├── window partition=(1) ordering=+4 opt(1) + │ ├── columns: u:1!null w:3!null z:4!null count:8 array_agg:9 + │ ├── fd: (1)-->(8,9) + │ ├── scan uvwz + │ │ └── columns: u:1!null w:3!null z:4!null + │ └── windows + │ ├── count [as=count:8, frame="rows from unbounded to unbounded", outer=(3)] + │ │ └── w:3 + │ └── array-agg [as=array_agg:9, frame="rows from unbounded to unbounded", outer=(4)] + │ └── z:4 + └── aggregations + ├── sum [as=sum:10, outer=(8)] + │ └── count:8 + ├── const-agg [as=count:8, outer=(8)] + │ └── count:8 + └── const-agg [as=array_agg:9, outer=(9)] + └── array_agg:9 + +# No-op case with row_number, which can produce a different result for each row +# in a window frame. +norm expect-not=FoldGroupByAndWindow +SELECT sum(v), foo +FROM ( + SELECT *, row_number() OVER w AS foo + FROM uvwz + WINDOW w AS ( + PARTITION BY u ORDER BY z + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) +) +GROUP BY u, foo; +---- +project + ├── columns: sum:9!null foo:8 + └── group-by (hash) + ├── columns: u:1!null row_number:8 sum:9!null + ├── grouping columns: u:1!null row_number:8 + ├── key: (1,8) + ├── fd: (1,8)-->(9) + ├── window partition=(1) ordering=+4 opt(1) + │ ├── columns: u:1!null v:2!null z:4!null row_number:8 + │ ├── key: (1,2) + │ ├── fd: (1,2)-->(4,8) + │ ├── scan uvwz + │ │ ├── columns: u:1!null v:2!null z:4!null + │ │ ├── key: (1,2) + │ │ └── fd: (1,2)-->(4) + │ └── windows + │ └── row-number [as=row_number:8, frame="rows from unbounded to unbounded"] + └── aggregations + └── sum [as=sum:9, outer=(2)] + └── v:2 diff --git a/pkg/sql/opt/norm/testdata/rules/prune_cols b/pkg/sql/opt/norm/testdata/rules/prune_cols index 48a8bdddd47b..d0fdeab60191 100644 --- a/pkg/sql/opt/norm/testdata/rules/prune_cols +++ b/pkg/sql/opt/norm/testdata/rules/prune_cols @@ -5382,47 +5382,39 @@ SELECT concat_agg(tab_1739.col3_7::STRING ORDER BY tab_1739.col3_7 DESC)::STRING FROM table86308@[0] AS tab_1739 GROUP BY tab_1739.col3_7 HAVING bool_and(false::BOOL)::BOOL ---- project - └── group-by (hash) - ├── select - │ ├── window partition=(8) - │ │ ├── window partition=(8) - │ │ │ ├── project - │ │ │ │ ├── scan table86308 - │ │ │ │ │ ├── computed column expressions - │ │ │ │ │ │ ├── col3_9 - │ │ │ │ │ │ │ └── col3_3 + 0.7530620098114014 - │ │ │ │ │ │ ├── col3_10 - │ │ │ │ │ │ │ └── lower(col3_0::STRING) - │ │ │ │ │ │ ├── col3_11 - │ │ │ │ │ │ │ └── col3_3 + 0.8790965676307678 - │ │ │ │ │ │ ├── col3_12 - │ │ │ │ │ │ │ └── lower(col3_0::STRING) - │ │ │ │ │ │ ├── col3_13 - │ │ │ │ │ │ │ └── lower(col3_6::STRING) - │ │ │ │ │ │ ├── col3_14 - │ │ │ │ │ │ │ └── col3_3 + -0.27059364318847656 - │ │ │ │ │ │ └── col3_15 - │ │ │ │ │ │ └── lower(col3_5::STRING) - │ │ │ │ │ └── partial index predicates - │ │ │ │ │ └── table3_col3_4_col3_11_col3_1_col3_0_col3_3_key: filters - │ │ │ │ │ ├── ((col3_9 = 5e-324) OR (col3_1 = -32768)) OR (col3_15 = '') - │ │ │ │ │ ├── col3_3 > 3.4028234663852886e+38 - │ │ │ │ │ └── lower(col3_0::STRING) != '""' - │ │ │ │ └── projections - │ │ │ │ ├── false - │ │ │ │ ├── col3_3 + 0.8790965676307678 - │ │ │ │ └── lower(col3_0::STRING) - │ │ │ └── windows - │ │ │ └── concat-agg [frame="range from unbounded to unbounded"] - │ │ │ └── col3_7 - │ │ └── windows - │ │ └── bool-and [frame="range from unbounded to unbounded"] - │ │ └── column21 - │ └── filters - │ └── bool_and - └── aggregations - └── const-agg - └── concat_agg + └── select + ├── group-by (hash) + │ ├── project + │ │ ├── scan table86308 + │ │ │ ├── computed column expressions + │ │ │ │ ├── col3_9 + │ │ │ │ │ └── col3_3 + 0.7530620098114014 + │ │ │ │ ├── col3_10 + │ │ │ │ │ └── lower(col3_0::STRING) + │ │ │ │ ├── col3_11 + │ │ │ │ │ └── col3_3 + 0.8790965676307678 + │ │ │ │ ├── col3_12 + │ │ │ │ │ └── lower(col3_0::STRING) + │ │ │ │ ├── col3_13 + │ │ │ │ │ └── lower(col3_6::STRING) + │ │ │ │ ├── col3_14 + │ │ │ │ │ └── col3_3 + -0.27059364318847656 + │ │ │ │ └── col3_15 + │ │ │ │ └── lower(col3_5::STRING) + │ │ │ └── partial index predicates + │ │ │ └── table3_col3_4_col3_11_col3_1_col3_0_col3_3_key: filters + │ │ │ ├── ((col3_9 = 5e-324) OR (col3_1 = -32768)) OR (col3_15 = '') + │ │ │ ├── col3_3 > 3.4028234663852886e+38 + │ │ │ └── lower(col3_0::STRING) != '""' + │ │ └── projections + │ │ └── false + │ └── aggregations + │ ├── concat-agg + │ │ └── col3_7 + │ └── bool-and + │ └── column21 + └── filters + └── bool_and exec-ddl CREATE TABLE p100478 ( diff --git a/pkg/sql/opt/norm/window_funcs.go b/pkg/sql/opt/norm/window_funcs.go index 783a7e316e6c..61a2050c10db 100644 --- a/pkg/sql/opt/norm/window_funcs.go +++ b/pkg/sql/opt/norm/window_funcs.go @@ -197,3 +197,24 @@ func (c *CustomFuncs) LimitToRowNumberFilter( ), } } + +// WindowsAreAggregations returns true if all the window functions are aggregate +// functions. +func (c *CustomFuncs) WindowsAreAggregations(windows memo.WindowsExpr) bool { + for i := range windows { + if !opt.IsAggregateOp(windows[i].Function) { + return false + } + } + return true +} + +// WindowFuncOutputCols collects all columns projected by the given set of +// window functions. +func (c *CustomFuncs) WindowFuncOutputCols(windows memo.WindowsExpr) opt.ColSet { + var cols opt.ColSet + for i := range windows { + cols.Add(windows[i].Col) + } + return cols +}