diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/mv/MVTimelinessListPartitionArbiter.java b/fe/fe-core/src/main/java/com/starrocks/catalog/mv/MVTimelinessListPartitionArbiter.java index 2125c43e36a0be..0cfd298ae1a209 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/mv/MVTimelinessListPartitionArbiter.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/mv/MVTimelinessListPartitionArbiter.java @@ -91,7 +91,7 @@ public MvUpdateInfo getMVTimelinessUpdateInfoInChecked() throws AnalysisExceptio }); ListPartitionDiffResult result = ListPartitionDiffer.computeListPartitionDiff(mv, refBaseTablePartitionMap, - allBasePartitionItems, tableRefIdxes); + allBasePartitionItems, tableRefIdxes, isQueryRewrite); if (result == null) { logMVPrepare(mv, "Partitioned mv compute list diff failed"); return new MvUpdateInfo(MvUpdateInfo.MvToRefreshType.FULL); @@ -131,7 +131,7 @@ public MvUpdateInfo getMVTimelinessUpdateInfoInLoose() { TableProperty.QueryRewriteConsistencyMode.LOOSE); ListPartitionDiff listPartitionDiff = null; try { - ListPartitionDiffResult result = ListPartitionDiffer.computeListPartitionDiff(mv); + ListPartitionDiffResult result = ListPartitionDiffer.computeListPartitionDiff(mv, isQueryRewrite); if (result == null) { logMVPrepare(mv, "Partitioned mv compute list diff failed"); return new MvUpdateInfo(MvUpdateInfo.MvToRefreshType.FULL); diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/mv/MVTimelinessRangePartitionArbiter.java b/fe/fe-core/src/main/java/com/starrocks/catalog/mv/MVTimelinessRangePartitionArbiter.java index 6c2c5c0bd4ced9..ea45779c00d2bb 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/mv/MVTimelinessRangePartitionArbiter.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/mv/MVTimelinessRangePartitionArbiter.java @@ -15,7 +15,6 @@ package com.starrocks.catalog.mv; import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; import com.google.common.collect.Range; import com.google.common.collect.Sets; import com.starrocks.analysis.Expr; @@ -83,9 +82,8 @@ protected MvUpdateInfo getMVTimelinessUpdateInfoInChecked() throws AnalysisExcep mvTimelinessInfo); // collect all ref base table's partition range map - Map>> extBaseToMVPartitionNameMap = Maps.newHashMap(); Map>> basePartitionNameToRangeMap = - RangePartitionDiffer.syncBaseTablePartitionInfos(mv, partitionExpr, extBaseToMVPartitionNameMap); + RangePartitionDiffer.syncBaseTablePartitionInfos(mv, partitionExpr); // If base table is materialized view, add partition name to cell mapping into base table partition mapping, // otherwise base table(mv) may lose partition names of the real base table changed partitions. @@ -100,7 +98,7 @@ protected MvUpdateInfo getMVTimelinessUpdateInfoInChecked() throws AnalysisExcep // There may be a performance issue here, because it will fetch all partitions of base tables and mv partitions. RangePartitionDiffResult differ = RangePartitionDiffer.computeRangePartitionDiff(mv, null, - extBaseToMVPartitionNameMap, basePartitionNameToRangeMap, isQueryRewrite); + basePartitionNameToRangeMap, isQueryRewrite); if (differ == null) { throw new AnalysisException(String.format("Compute partition difference of mv %s with base table failed.", mv.getName())); diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVPCTRefreshListPartitioner.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVPCTRefreshListPartitioner.java index fdd5de0b303d1a..8ea8314897bb0c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVPCTRefreshListPartitioner.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVPCTRefreshListPartitioner.java @@ -80,7 +80,7 @@ public boolean syncAddOrDropPartitions() throws LockTimeoutException { ListPartitionDiffResult result; try { - result = ListPartitionDiffer.computeListPartitionDiff(mv); + result = ListPartitionDiffer.computeListPartitionDiff(mv, false); if (result == null) { LOG.warn("compute list partition diff failed: mv: {}", mv.getName()); return false; @@ -128,6 +128,7 @@ public boolean syncAddOrDropPartitions() throws LockTimeoutException { mvContext.setRefBaseTableMVIntersectedPartitions(baseToMvNameRef); mvContext.setMvRefBaseTableIntersectedPartitions(mvToBaseNameRef); mvContext.setRefBaseTableListPartitionMap(refBaseTablePartitionMap); + mvContext.setExternalRefBaseTableMVPartitionMap(result.getRefBaseTableMVPartitionMap()); } return true; } diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVPCTRefreshRangePartitioner.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVPCTRefreshRangePartitioner.java index e7e788fd2c0c53..b3a6c6012fcf3d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVPCTRefreshRangePartitioner.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/MVPCTRefreshRangePartitioner.java @@ -130,7 +130,7 @@ public boolean syncAddOrDropPartitions() throws AnalysisException { mvContext.setRefBaseTableMVIntersectedPartitions(baseToMvNameRef); mvContext.setMvRefBaseTableIntersectedPartitions(mvToBaseNameRef); mvContext.setRefBaseTableRangePartitionMap(result.refBaseTablePartitionMap); - mvContext.setExternalRefBaseTableMVPartitionMap(result.refBaseTableMVPartitionMap); + mvContext.setExternalRefBaseTableMVPartitionMap(result.getRefBaseTableMVPartitionMap()); return true; } diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index e16c3488f982ad..ffa0169283b0a9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -86,7 +86,6 @@ import com.starrocks.catalog.PartitionType; import com.starrocks.catalog.PhysicalPartition; import com.starrocks.catalog.PhysicalPartitionImpl; -import com.starrocks.catalog.PrimitiveType; import com.starrocks.catalog.RandomDistributionInfo; import com.starrocks.catalog.RangePartitionInfo; import com.starrocks.catalog.Replica; @@ -3007,7 +3006,7 @@ public void createMaterializedView(CreateMaterializedViewStatement stmt) materializedView.getWarehouseId()); materializedView.addPartition(partition); } else { - Expr partitionExpr = stmt.getPartitionExpDesc().getExpr(); + Expr partitionExpr = stmt.getPartitionByExpr(); Map partitionExprMaps = MVPartitionExprResolver.getMVPartitionExprsChecked(partitionExpr, stmt.getQueryStatement(), stmt.getBaseTableInfos()); LOG.info("Generate mv {} partition exprs: {}", mvName, partitionExprMaps); @@ -3051,30 +3050,35 @@ private long getRandomStart(IntervalLiteral interval, long randomizeStart) throw } public static PartitionInfo buildPartitionInfo(CreateMaterializedViewStatement stmt) throws DdlException { - ExpressionPartitionDesc expressionPartitionDesc = stmt.getPartitionExpDesc(); - if (expressionPartitionDesc != null) { - Expr expr = expressionPartitionDesc.getExpr(); - if (expr instanceof SlotRef) { - SlotRef slotRef = (SlotRef) expr; - if (slotRef.getType().getPrimitiveType() == PrimitiveType.VARCHAR) { - return new ListPartitionInfo(PartitionType.LIST, - Collections.singletonList(stmt.getPartitionColumn())); + Expr partitionByExpr = stmt.getPartitionByExpr(); + PartitionType partitionType = stmt.getPartitionType(); + if (partitionByExpr != null) { + if (partitionType == PartitionType.LIST) { + if (!(partitionByExpr instanceof SlotRef)) { + throw new DdlException("List partition only support partition by slot ref column:" + + partitionByExpr.toSql()); } - } - if ((expr instanceof FunctionCallExpr)) { - FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr; - if (functionCallExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.STR2DATE)) { - Column partitionColumn = new Column(stmt.getPartitionColumn()); - partitionColumn.setType(com.starrocks.catalog.Type.DATE); - return expressionPartitionDesc.toPartitionInfo( - Collections.singletonList(partitionColumn), - Maps.newHashMap(), false); + return new ListPartitionInfo(PartitionType.LIST, Collections.singletonList(stmt.getPartitionColumn())); + } else { + ExpressionPartitionDesc expressionPartitionDesc = new ExpressionPartitionDesc(partitionByExpr); + if ((partitionByExpr instanceof FunctionCallExpr)) { + FunctionCallExpr functionCallExpr = (FunctionCallExpr) partitionByExpr; + if (functionCallExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.STR2DATE)) { + Column partitionColumn = new Column(stmt.getPartitionColumn()); + partitionColumn.setType(com.starrocks.catalog.Type.DATE); + return expressionPartitionDesc.toPartitionInfo( + Collections.singletonList(partitionColumn), + Maps.newHashMap(), false); + } } + return expressionPartitionDesc.toPartitionInfo( + Collections.singletonList(stmt.getPartitionColumn()), + Maps.newHashMap(), false); } - return expressionPartitionDesc.toPartitionInfo( - Collections.singletonList(stmt.getPartitionColumn()), - Maps.newHashMap(), false); } else { + if (partitionType != PartitionType.UNPARTITIONED && partitionType != null) { + throw new DdlException("Partition type is " + stmt.getPartitionType() + ", but partition by expr is null"); + } return new SinglePartitionInfo(); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java index e3288f563c800f..59a9611b44101a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java @@ -47,6 +47,7 @@ import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.PaimonTable; import com.starrocks.catalog.PartitionInfo; +import com.starrocks.catalog.PartitionType; import com.starrocks.catalog.PrimitiveType; import com.starrocks.catalog.RangePartitionInfo; import com.starrocks.catalog.Table; @@ -68,7 +69,6 @@ import com.starrocks.sql.ast.CreateMaterializedViewStatement; import com.starrocks.sql.ast.DistributionDesc; import com.starrocks.sql.ast.DropMaterializedViewStmt; -import com.starrocks.sql.ast.ExpressionPartitionDesc; import com.starrocks.sql.ast.HashDistributionDesc; import com.starrocks.sql.ast.IndexDef; import com.starrocks.sql.ast.PartitionRangeDesc; @@ -93,6 +93,7 @@ import com.starrocks.sql.optimizer.transformer.RelationTransformer; import com.starrocks.sql.plan.ExecPlan; import com.starrocks.sql.plan.PlanFragmentBuilder; +import org.apache.commons.collections.map.CaseInsensitiveMap; import org.apache.commons.collections4.CollectionUtils; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; @@ -336,7 +337,7 @@ public Void visitCreateMaterializedViewStatement(CreateMaterializedViewStatement columnExprMap.put(pair.first, outputExpressions.get(pair.second)); } // some check if partition exp exists - if (statement.getPartitionExpDesc() != null) { + if (statement.getPartitionByExpr() != null) { // check partition expression all in column list and // write the expr into partitionExpDesc if partition expression exists checkExpInColumn(statement); @@ -346,7 +347,10 @@ public Void visitCreateMaterializedViewStatement(CreateMaterializedViewStatement checkPartitionExpPatterns(statement); // check partition column must be base table's partition column checkPartitionColumnWithBaseTable(statement, aliasTableMap); + // check window function can be used in partitioned mv checkWindowFunctions(statement, columnExprMap); + // determine mv partition 's type: list or range + determinePartitionType(statement, aliasTableMap); } // check and analyze distribution checkDistribution(statement, aliasTableMap); @@ -365,9 +369,8 @@ public Void visitCreateMaterializedViewStatement(CreateMaterializedViewStatement */ private Map getNormalizedBaseTables(QueryStatement queryStatement, ConnectContext context) { Map aliasTableMap = getAllBaseTables(queryStatement, context); - // do normalization if catalog is null - Map result = Maps.newHashMap(); + Map result = new CaseInsensitiveMap(); for (Map.Entry entry : aliasTableMap.entrySet()) { entry.getKey().normalization(context); result.put(entry.getKey(), entry.getValue()); @@ -634,12 +637,12 @@ private List genMaterializedViewIndexes(CreateMaterializedViewStatement s } private void checkExpInColumn(CreateMaterializedViewStatement statement) { - ExpressionPartitionDesc expressionPartitionDesc = statement.getPartitionExpDesc(); + Expr partitionByExpr = statement.getPartitionByExpr(); List columns = statement.getMvColumnItems(); - SlotRef slotRef = getSlotRef(expressionPartitionDesc.getExpr()); + SlotRef slotRef = getSlotRef(partitionByExpr); if (slotRef.getTblNameWithoutAnalyzed() != null) { throw new SemanticException("Materialized view partition exp: " - + slotRef.toSql() + " must related to column", expressionPartitionDesc.getExpr().getPos()); + + slotRef.toSql() + " must related to column", partitionByExpr.getPos()); } int columnId = 0; for (Column column : columns) { @@ -674,8 +677,11 @@ private boolean isValidPartitionExpr(Expr partitionExpr) { private void checkPartitionColumnExprs(CreateMaterializedViewStatement statement, Map columnExprMap, ConnectContext connectContext) throws SemanticException { - ExpressionPartitionDesc expressionPartitionDesc = statement.getPartitionExpDesc(); + Expr partitionByExpr = statement.getPartitionByExpr(); Column partitionColumn = statement.getPartitionColumn(); + if (partitionByExpr == null) { + return; + } // partition column expr from input query Expr partitionColumnExpr = columnExprMap.get(partitionColumn); @@ -684,13 +690,13 @@ private void checkPartitionColumnExprs(CreateMaterializedViewStatement statement } catch (Exception e) { LOG.warn("resolve partition column failed", e); throw new SemanticException("Resolve partition column failed:" + e.getMessage(), - statement.getPartitionExpDesc().getPos()); + statement.getPartitionByExpr().getPos()); } // set partition-ref into statement - if (expressionPartitionDesc.isFunction()) { + if (partitionByExpr instanceof FunctionCallExpr) { // e.g. partition by date_trunc('month', dt) - FunctionCallExpr functionCallExpr = (FunctionCallExpr) expressionPartitionDesc.getExpr(); + FunctionCallExpr functionCallExpr = (FunctionCallExpr) partitionByExpr; String functionName = functionCallExpr.getFnName().getFunction(); if (!PartitionFunctionChecker.FN_NAME_TO_PATTERN.containsKey(functionName)) { throw new SemanticException("Materialized view partition function " + @@ -718,7 +724,7 @@ private void checkPartitionColumnExprs(CreateMaterializedViewStatement statement statement.setPartitionRefTableExpr(partitionColumnExpr); } else { throw new SemanticException("Materialized view partition function must related with column", - expressionPartitionDesc.getPos()); + partitionByExpr.getPos()); } } } @@ -809,6 +815,61 @@ private void checkPartitionColumnWithBaseTable(CreateMaterializedViewStatement s replaceTableAlias(slotRef, statement, tableNameTableMap); } + private void determinePartitionType(CreateMaterializedViewStatement statement, + Map tableNameTableMap) { + Expr partitionRefTableExpr = statement.getPartitionRefTableExpr(); + if (partitionRefTableExpr == null) { + statement.setPartitionType(PartitionType.UNPARTITIONED); + return; + } + + SlotRef slotRef = getSlotRef(partitionRefTableExpr); + Table refBaseTable = tableNameTableMap.get(slotRef.getTblNameWithoutAnalyzed()); + if (refBaseTable == null) { + LOG.warn("Materialized view partition expression %s could only ref to base table", + slotRef.toSql()); + statement.setPartitionType(PartitionType.UNPARTITIONED); + return; + } + + if (refBaseTable.isNativeTableOrMaterializedView()) { + // To olap table, determine mv's partition by its ref base table's partition type. + OlapTable olapTable = (OlapTable) refBaseTable; + PartitionInfo partitionInfo = olapTable.getPartitionInfo(); + if (partitionInfo.isRangePartition()) { + // set the partition type + statement.setPartitionType(PartitionType.RANGE); + } else if (partitionInfo.isListPartition()) { + // set the partition type + statement.setPartitionType(PartitionType.LIST); + } + } else { + List refPartitionCols = refBaseTable.getPartitionColumns(); + Optional refPartitionColOpt = refPartitionCols.stream() + .filter(col -> col.getName().equals(slotRef.getColumnName())) + .findFirst(); + if (refPartitionColOpt.isEmpty()) { + throw new SemanticException("Materialized view partition column in partition exp " + + "must be base table partition column", partitionRefTableExpr.getPos()); + } + Column refPartitionCol = refPartitionColOpt.get(); + Type partitionExprType = refPartitionCol.getType(); + Expr partitionByExpr = statement.getPartitionByExpr(); + // To olap table, determine mv's partition by its ref base table's partition column type: + // - if the partition column is string type && no use `str2date`, use list partition. + // - otherwise use range partition as before. + // To be compatible with old implementations, if the partition column is not a string type, + // still use range partition. + // TODO: remove this compatibility code in the future, use list partition directly later. + if (partitionExprType.isStringType() && (partitionByExpr instanceof SlotRef) && + !(partitionRefTableExpr instanceof FunctionCallExpr)) { + statement.setPartitionType(PartitionType.LIST); + } else { + statement.setPartitionType(PartitionType.RANGE); + } + } + } + private void checkPartitionColumnWithBaseOlapTable(SlotRef slotRef, OlapTable table) { PartitionInfo partitionInfo = table.getPartitionInfo(); if (partitionInfo.isUnPartitioned()) { @@ -1007,7 +1068,7 @@ boolean replacePaimonTableAlias(SlotRef slotRef, PaimonTable paimonTable, BaseTa private void checkPartitionColumnType(Column partitionColumn) { PrimitiveType type = partitionColumn.getPrimitiveType(); - if (!type.isFixedPointType() && !type.isDateType() && type != PrimitiveType.CHAR && type != PrimitiveType.VARCHAR) { + if (!type.isFixedPointType() && !type.isDateType() && !type.isStringType()) { throw new SemanticException("Materialized view partition exp column:" + partitionColumn.getName() + " with type " + type + " not supported"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateMaterializedViewStatement.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateMaterializedViewStatement.java index affd513963fb75..5b06ff3dfe6bf5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateMaterializedViewStatement.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateMaterializedViewStatement.java @@ -22,6 +22,7 @@ import com.starrocks.catalog.Column; import com.starrocks.catalog.Index; import com.starrocks.catalog.KeysType; +import com.starrocks.catalog.PartitionType; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.parser.NodePosition; import com.starrocks.sql.plan.ExecPlan; @@ -49,7 +50,12 @@ public class CreateMaterializedViewStatement extends DdlStmt { private boolean ifNotExists; private String comment; private RefreshSchemeClause refreshSchemeDesc; - private ExpressionPartitionDesc expressionPartitionDesc; + + // partition by clause which may be list or range partition expr. + private final Expr partitionByExpr; + // partition type of the mv which is deduced by its referred base table. + private PartitionType partitionType; + private Map properties; private QueryStatement queryStatement; private DistributionDesc distributionDesc; @@ -88,7 +94,7 @@ public CreateMaterializedViewStatement(TableName tableName, boolean ifNotExists, List indexDefs, String comment, RefreshSchemeClause refreshSchemeDesc, - ExpressionPartitionDesc expressionPartitionDesc, + Expr partitionByExpr, DistributionDesc distributionDesc, List sortKeys, Map properties, QueryStatement queryStatement, @@ -101,7 +107,7 @@ public CreateMaterializedViewStatement(TableName tableName, boolean ifNotExists, this.ifNotExists = ifNotExists; this.comment = comment; this.refreshSchemeDesc = refreshSchemeDesc; - this.expressionPartitionDesc = expressionPartitionDesc; + this.partitionByExpr = partitionByExpr; this.distributionDesc = distributionDesc; this.sortKeys = sortKeys; this.properties = properties; @@ -149,12 +155,23 @@ public void setRefreshSchemeDesc(RefreshSchemeClause refreshSchemeDesc) { this.refreshSchemeDesc = refreshSchemeDesc; } - public ExpressionPartitionDesc getPartitionExpDesc() { - return expressionPartitionDesc; + /** + * Get partition by expr of the mv + */ + public Expr getPartitionByExpr() { + return partitionByExpr; + } + + /** + * Get partition type of the mv + * @return + */ + public PartitionType getPartitionType() { + return partitionType; } - public void setPartitionExpDesc(ExpressionPartitionDesc expressionPartitionDesc) { - this.expressionPartitionDesc = expressionPartitionDesc; + public void setPartitionType(PartitionType partitionType) { + this.partitionType = partitionType; } public void setKeysType(KeysType keysType) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/common/ListPartitionDiffResult.java b/fe/fe-core/src/main/java/com/starrocks/sql/common/ListPartitionDiffResult.java index bb59c654cba001..56f5fa84a40b5b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/common/ListPartitionDiffResult.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/common/ListPartitionDiffResult.java @@ -18,11 +18,12 @@ import java.util.List; import java.util.Map; +import java.util.Set; /** * The result of diffing between materialized view and the base tables. */ -public class ListPartitionDiffResult { +public class ListPartitionDiffResult extends PartitionDiffResult { // The partition range of the materialized view: public final Map mvListPartitionMap; // The partition range of the base tables: > @@ -36,7 +37,9 @@ public class ListPartitionDiffResult { public ListPartitionDiffResult(Map mvListPartitionMap, Map> refBaseTablePartitionMap, ListPartitionDiff listPartitionDiff, - Map> refBaseTableRefIdxMap) { + Map> refBaseTableRefIdxMap, + Map>> refBaseTableMVPartitionMap) { + super(refBaseTableMVPartitionMap); this.mvListPartitionMap = mvListPartitionMap; this.refBaseTablePartitionMap = refBaseTablePartitionMap; this.listPartitionDiff = listPartitionDiff; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/common/ListPartitionDiffer.java b/fe/fe-core/src/main/java/com/starrocks/sql/common/ListPartitionDiffer.java index 395d55f963cb41..ef6da3f007be7f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/common/ListPartitionDiffer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/common/ListPartitionDiffer.java @@ -255,7 +255,8 @@ public static boolean syncBaseTablePartitionInfos(MaterializedView mv, return true; } - public static ListPartitionDiffResult computeListPartitionDiff(MaterializedView mv) { + public static ListPartitionDiffResult computeListPartitionDiff(MaterializedView mv, + boolean isQueryRewrite) { // table -> map partition cell> Map> refBaseTablePartitionMap = Maps.newHashMap(); // merge all base table partition cells @@ -265,19 +266,33 @@ public static ListPartitionDiffResult computeListPartitionDiff(MaterializedView logMVPrepare(mv, "Partitioned mv collect base table infos failed"); return null; } - return computeListPartitionDiff(mv, refBaseTablePartitionMap, allBasePartitionItems, tableRefIdxes); + return computeListPartitionDiff(mv, refBaseTablePartitionMap, allBasePartitionItems, tableRefIdxes, isQueryRewrite); } public static ListPartitionDiffResult computeListPartitionDiff( MaterializedView mv, Map> refBaseTablePartitionMap, Map allBasePartitionItems, - Map> tableRefIdxes) { + Map> tableRefIdxes, + boolean isQueryRewrite) { + // generate the reference map between the base table and the mv // TODO: prune the partitions based on ttl Map mvPartitionNameToListMap = mv.getListPartitionItems(); ListPartitionDiff diff = ListPartitionDiffer.getListPartitionDiff( allBasePartitionItems, mvPartitionNameToListMap); - return new ListPartitionDiffResult(mvPartitionNameToListMap, refBaseTablePartitionMap, diff, tableRefIdxes); + + // collect external partition column mapping + Map>> externalPartitionMaps = Maps.newHashMap(); + if (!isQueryRewrite) { + try { + collectExternalPartitionNameMapping(mv.getRefBaseTablePartitionColumns(), externalPartitionMaps); + } catch (Exception e) { + LOG.warn("Get external partition column mapping failed.", DebugUtil.getStackTrace(e)); + return null; + } + } + return new ListPartitionDiffResult(mvPartitionNameToListMap, refBaseTablePartitionMap, diff, tableRefIdxes, + externalPartitionMaps); } /** diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/common/PartitionDiffResult.java b/fe/fe-core/src/main/java/com/starrocks/sql/common/PartitionDiffResult.java new file mode 100644 index 00000000000000..d32a3a5e101592 --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/common/PartitionDiffResult.java @@ -0,0 +1,34 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.common; + +import com.starrocks.catalog.Table; + +import java.util.Map; +import java.util.Set; + +public abstract class PartitionDiffResult { + // For external table, the mapping of base table partition to mv partition: + // > + public final Map>> refBaseTableMVPartitionMap; + + public PartitionDiffResult(Map>> refBaseTableMVPartitionMap) { + this.refBaseTableMVPartitionMap = refBaseTableMVPartitionMap; + } + + public Map>> getRefBaseTableMVPartitionMap() { + return refBaseTableMVPartitionMap; + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/common/PartitionDiffer.java b/fe/fe-core/src/main/java/com/starrocks/sql/common/PartitionDiffer.java index c32a957a4770b8..9ca3d4bc6ba06c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/common/PartitionDiffer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/common/PartitionDiffer.java @@ -14,9 +14,50 @@ package com.starrocks.sql.common; +import com.starrocks.catalog.Column; +import com.starrocks.catalog.Table; +import com.starrocks.common.AnalysisException; +import com.starrocks.connector.PartitionUtil; + +import java.util.Map; +import java.util.Set; + /** * {@link PartitionDiffer} is used to compare the difference between two partitions which can be range * partition or list partition. */ public abstract class PartitionDiffer { + /** + * To solve multi partition columns' problem of external table, record the mv partition name to all the same + * partition names map here. + * @param partitionTableAndColumns the partition table and its partition column + * @param result the result map + */ + public static void collectExternalPartitionNameMapping(Map partitionTableAndColumns, + Map>> result) throws AnalysisException { + for (Map.Entry e1 : partitionTableAndColumns.entrySet()) { + Table refBaseTable = e1.getKey(); + Column refPartitionColumn = e1.getValue(); + collectExternalBaseTablePartitionMapping(refBaseTable, refPartitionColumn, result); + } + } + + /** + * Collect the external base table's partition name to its intersected materialized view names. + * @param refBaseTable the base table + * @param refTablePartitionColumn the partition column of the base table + * @param result the result map + * @throws AnalysisException + */ + private static void collectExternalBaseTablePartitionMapping( + Table refBaseTable, + Column refTablePartitionColumn, + Map>> result) throws AnalysisException { + if (refBaseTable.isNativeTableOrMaterializedView()) { + return; + } + Map> mvPartitionNameMap = PartitionUtil.getMVPartitionNameMapOfExternalTable(refBaseTable, + refTablePartitionColumn, PartitionUtil.getPartitionNames(refBaseTable)); + result.put(refBaseTable, mvPartitionNameMap); + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/common/RangePartitionDiffResult.java b/fe/fe-core/src/main/java/com/starrocks/sql/common/RangePartitionDiffResult.java index 2ef73ee3063be7..b0188dd75f4dc4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/common/RangePartitionDiffResult.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/common/RangePartitionDiffResult.java @@ -24,14 +24,11 @@ /** * The result of diffing between materialized view and the base tables. */ -public class RangePartitionDiffResult { +public class RangePartitionDiffResult extends PartitionDiffResult { // The partition range of the materialized view: public final Map> mvRangePartitionMap; // The partition range of the base tables: > public final Map>> refBaseTablePartitionMap; - // For external table, the mapping of base table partition to mv partition: - // > - public final Map>> refBaseTableMVPartitionMap; // The diff result of partition range between materialized view and base tables public final RangePartitionDiff rangePartitionDiff; @@ -39,9 +36,9 @@ public RangePartitionDiffResult(Map> mvRangePartitio Map>> refBaseTablePartitionMap, Map>> refBaseTableMVPartitionMap, RangePartitionDiff rangePartitionDiff) { + super(refBaseTableMVPartitionMap); this.mvRangePartitionMap = mvRangePartitionMap; this.refBaseTablePartitionMap = refBaseTablePartitionMap; - this.refBaseTableMVPartitionMap = refBaseTableMVPartitionMap; this.rangePartitionDiff = rangePartitionDiff; } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/common/RangePartitionDiffer.java b/fe/fe-core/src/main/java/com/starrocks/sql/common/RangePartitionDiffer.java index fc6f3820cd4a0d..05e2584f4d5113 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/common/RangePartitionDiffer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/common/RangePartitionDiffer.java @@ -221,10 +221,8 @@ private static boolean isRangeIncluded(Range rangeToCheck, Range

> */ - public static Map>> syncBaseTablePartitionInfos( - MaterializedView mv, - Expr mvPartitionExpr, - Map>> basePartitionMaps) { + public static Map>> syncBaseTablePartitionInfos(MaterializedView mv, + Expr mvPartitionExpr) { Map partitionTableAndColumn = mv.getRefBaseTablePartitionColumns(); if (partitionTableAndColumn.isEmpty()) { return Maps.newHashMap(); @@ -240,14 +238,6 @@ public static Map>> syncBaseTablePartitio PartitionUtil.getPartitionKeyRange(refBT, refBTPartitionColumn, mvPartitionExpr); refBaseTablePartitionMap.put(refBT, refTablePartitionKeyMap); - - // To solve multi partition columns' problem of external table, record the mv partition name to all the same - // partition names map here. - if (!refBT.isNativeTableOrMaterializedView()) { - basePartitionMaps.put(refBT, - PartitionUtil.getMVPartitionNameMapOfExternalTable(refBT, - refBTPartitionColumn, PartitionUtil.getPartitionNames(refBT))); - } } } catch (UserException | SemanticException e) { LOG.warn("Partition differ collects ref base table partition failed.", e); @@ -336,17 +326,13 @@ public static RangePartitionDiffResult computeRangePartitionDiff(MaterializedVie Range rangeToInclude, boolean isQueryRewrite) { Expr mvPartitionExpr = mv.getPartitionExpr(); - // collect all ref base table's partition range map - Map>> extRBTMVPartitionNameMap = Maps.newHashMap(); - Map>> rBTPartitionMap = syncBaseTablePartitionInfos(mv, mvPartitionExpr, - extRBTMVPartitionNameMap); - return computeRangePartitionDiff(mv, rangeToInclude, extRBTMVPartitionNameMap, rBTPartitionMap, isQueryRewrite); + Map>> rBTPartitionMap = syncBaseTablePartitionInfos(mv, mvPartitionExpr); + return computeRangePartitionDiff(mv, rangeToInclude, rBTPartitionMap, isQueryRewrite); } public static RangePartitionDiffResult computeRangePartitionDiff( MaterializedView mv, Range rangeToInclude, - Map>> extRBTMVPartitionNameMap, Map>> rBTPartitionMap, boolean isQueryRewrite) { Expr mvPartitionExpr = mv.getPartitionExpr(); @@ -394,6 +380,12 @@ public static RangePartitionDiffResult computeRangePartitionDiff( LOG.warn("Materialized view compute partition difference with base table failed: rangePartitionDiff is null."); return null; } + Map>> extRBTMVPartitionNameMap = Maps.newHashMap(); + if (!isQueryRewrite) { + // To solve multi partition columns' problem of external table, record the mv partition name to all the same + // partition names map here. + collectExternalPartitionNameMapping(refBaseTableAndColumns, extRBTMVPartitionNameMap); + } return new RangePartitionDiffResult(mvRangePartitionMap, rBTPartitionMap, extRBTMVPartitionNameMap, rangePartitionDiff); } catch (AnalysisException e) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java index 8733fba14fe24c..c0a5357350b71f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java @@ -1853,7 +1853,7 @@ public ParseNode visitCreateMaterializedViewStatement( RefreshSchemeClause refreshSchemeDesc = null; Map properties = new HashMap<>(); - ExpressionPartitionDesc expressionPartitionDesc = null; + Expr partitionByExpr = null; DistributionDesc distributionDesc = null; List sortKeys = null; @@ -1881,17 +1881,17 @@ public ParseNode visitCreateMaterializedViewStatement( // process partition by if (desc.primaryExpression() != null) { - if (expressionPartitionDesc != null) { + if (partitionByExpr != null) { throw new ParsingException(PARSER_ERROR_MSG.duplicatedClause("PARTITION", "building materialized view"), clausePos); } Expr expr = (Expr) visit(desc.primaryExpression()); if (expr instanceof SlotRef) { - expressionPartitionDesc = new ExpressionPartitionDesc(expr); + partitionByExpr = expr; } else if (expr instanceof FunctionCallExpr) { AnalyzerUtils.checkAndExtractPartitionCol((FunctionCallExpr) expr, null, AnalyzerUtils.MV_DATE_TRUNC_SUPPORTED_PARTITION_FORMAT); - expressionPartitionDesc = new ExpressionPartitionDesc(expr); + partitionByExpr = expr; } else { throw new ParsingException(PARSER_ERROR_MSG.unsupportedExprWithInfo(expr.toSql(), "PARTITION BY"), expr.getPos()); @@ -1925,9 +1925,9 @@ public ParseNode visitCreateMaterializedViewStatement( } } if (refreshSchemeDesc instanceof SyncRefreshSchemeDesc) { - if (expressionPartitionDesc != null) { + if (partitionByExpr != null) { throw new ParsingException(PARSER_ERROR_MSG.forbidClauseInMV("SYNC refresh type", "PARTITION BY"), - expressionPartitionDesc.getPos()); + partitionByExpr.getPos()); } if (distributionDesc != null) { throw new ParsingException(PARSER_ERROR_MSG.forbidClauseInMV("SYNC refresh type", "DISTRIBUTION BY"), @@ -1948,7 +1948,7 @@ public ParseNode visitCreateMaterializedViewStatement( context.indexDesc() == null ? null : getIndexDefs(context.indexDesc()), comment, refreshSchemeDesc, - expressionPartitionDesc, distributionDesc, sortKeys, properties, queryStatement, queryStartIndex, + partitionByExpr, distributionDesc, sortKeys, properties, queryStatement, queryStartIndex, createPos(context)); } diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java index b990fdace4b53f..97092522b9a3d0 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java @@ -58,7 +58,6 @@ import com.starrocks.sql.ast.CreateMaterializedViewStatement; import com.starrocks.sql.ast.CreateMaterializedViewStmt; import com.starrocks.sql.ast.DmlStmt; -import com.starrocks.sql.ast.ExpressionPartitionDesc; import com.starrocks.sql.ast.RefreshSchemeClause; import com.starrocks.sql.ast.StatementBase; import com.starrocks.sql.optimizer.MvRewritePreprocessor; @@ -854,10 +853,10 @@ public void testPartitionWithFunctionIn() { try { CreateMaterializedViewStatement createMaterializedViewStatement = (CreateMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); - ExpressionPartitionDesc partitionExpDesc = createMaterializedViewStatement.getPartitionExpDesc(); - Assert.assertFalse(partitionExpDesc.isFunction()); - Assert.assertTrue(partitionExpDesc.getExpr() instanceof SlotRef); - Assert.assertEquals("ss", partitionExpDesc.getSlotRef().getColumnName()); + Expr partitionByExpr = createMaterializedViewStatement.getPartitionByExpr(); + Assert.assertTrue(partitionByExpr instanceof SlotRef); + SlotRef slotRef = (SlotRef) partitionByExpr; + Assert.assertEquals("ss", slotRef.getColumnName()); } catch (Exception e) { Assert.fail(e.getMessage()); } @@ -876,10 +875,10 @@ public void testPartitionWithFunctionInUseStr2Date() { try { CreateMaterializedViewStatement createMaterializedViewStatement = (CreateMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); - ExpressionPartitionDesc partitionExpDesc = createMaterializedViewStatement.getPartitionExpDesc(); - Assert.assertFalse(partitionExpDesc.isFunction()); - Assert.assertTrue(partitionExpDesc.getExpr() instanceof SlotRef); - Assert.assertEquals("ss", partitionExpDesc.getSlotRef().getColumnName()); + Expr partitionByExpr = createMaterializedViewStatement.getPartitionByExpr(); + Assert.assertTrue(partitionByExpr instanceof SlotRef); + SlotRef slotRef = (SlotRef) partitionByExpr; + Assert.assertEquals("ss", slotRef.getColumnName()); } catch (Exception e) { Assert.fail(e.getMessage()); } @@ -915,11 +914,12 @@ public void testPartitionWithFunction() { try { CreateMaterializedViewStatement createMaterializedViewStatement = (CreateMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); - ExpressionPartitionDesc partitionExpDesc = createMaterializedViewStatement.getPartitionExpDesc(); - Assert.assertTrue(partitionExpDesc.isFunction()); - Assert.assertTrue(partitionExpDesc.getExpr() instanceof FunctionCallExpr); - Assert.assertEquals(partitionExpDesc.getExpr().getChild(1), partitionExpDesc.getSlotRef()); - Assert.assertEquals("ss", partitionExpDesc.getSlotRef().getColumnName()); + Expr partitionByExpr = createMaterializedViewStatement.getPartitionByExpr(); + Assert.assertTrue(partitionByExpr instanceof FunctionCallExpr); + List slotRefs = Lists.newArrayList(); + partitionByExpr.collect(SlotRef.class, slotRefs); + Assert.assertEquals(partitionByExpr.getChild(1), slotRefs.get(0)); + Assert.assertEquals("ss", slotRefs.get(0).getColumnName()); } catch (Exception e) { Assert.fail(e.getMessage()); } @@ -939,11 +939,12 @@ public void testPartitionWithFunctionUseStr2Date() throws Exception { "as select a, b, c, d from jdbc0.partitioned_db0.tbl1;"; CreateMaterializedViewStatement createMaterializedViewStatement = (CreateMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); - ExpressionPartitionDesc partitionExpDesc = createMaterializedViewStatement.getPartitionExpDesc(); - Assert.assertTrue(partitionExpDesc.isFunction()); - Assert.assertTrue(partitionExpDesc.getExpr() instanceof FunctionCallExpr); - Assert.assertEquals(partitionExpDesc.getExpr().getChild(0), partitionExpDesc.getSlotRef()); - Assert.assertEquals("d", partitionExpDesc.getSlotRef().getColumnName()); + Expr partitionByExpr = createMaterializedViewStatement.getPartitionByExpr(); + Assert.assertTrue(partitionByExpr instanceof FunctionCallExpr); + List slotRefs = Lists.newArrayList(); + partitionByExpr.collect(SlotRef.class, slotRefs); + Assert.assertEquals(partitionByExpr.getChild(0), slotRefs.get(0)); + Assert.assertEquals("d", slotRefs.get(0).getColumnName()); } // slot @@ -986,11 +987,12 @@ public void testPartitionWithFunctionNoAlias() { try { CreateMaterializedViewStatement createMaterializedViewStatement = (CreateMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); - ExpressionPartitionDesc partitionExpDesc = createMaterializedViewStatement.getPartitionExpDesc(); - Assert.assertTrue(partitionExpDesc.isFunction()); - Assert.assertTrue(partitionExpDesc.getExpr() instanceof FunctionCallExpr); - Assert.assertEquals(partitionExpDesc.getExpr().getChild(1), partitionExpDesc.getSlotRef()); - Assert.assertEquals("k1", partitionExpDesc.getSlotRef().getColumnName()); + Expr partitionByExpr = createMaterializedViewStatement.getPartitionByExpr(); + Assert.assertTrue(partitionByExpr instanceof FunctionCallExpr); + List slotRefs = Lists.newArrayList(); + partitionByExpr.collect(SlotRef.class, slotRefs); + Assert.assertEquals(partitionByExpr.getChild(1), slotRefs.get(0)); + Assert.assertEquals("k1", slotRefs.get(0).getColumnName()); } catch (Exception e) { Assert.fail(e.getMessage()); } @@ -1009,11 +1011,12 @@ public void testPartitionWithoutFunction() { try { CreateMaterializedViewStatement createMaterializedViewStatement = (CreateMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); - ExpressionPartitionDesc partitionExpDesc = createMaterializedViewStatement.getPartitionExpDesc(); - Assert.assertFalse(partitionExpDesc.isFunction()); - Assert.assertTrue(partitionExpDesc.getExpr() instanceof SlotRef); - Assert.assertEquals(partitionExpDesc.getExpr(), partitionExpDesc.getSlotRef()); - Assert.assertEquals("k1", partitionExpDesc.getSlotRef().getColumnName()); + Expr partitionByExpr = createMaterializedViewStatement.getPartitionByExpr(); + Assert.assertTrue(partitionByExpr instanceof SlotRef); + List slotRefs = Lists.newArrayList(); + partitionByExpr.collect(SlotRef.class, slotRefs); + Assert.assertEquals(partitionByExpr, slotRefs.get(0)); + Assert.assertEquals("k1", slotRefs.get(0).getColumnName()); } catch (Exception e) { Assert.fail(e.getMessage()); } @@ -4776,4 +4779,58 @@ public void testCreateMVWithLocationAndPersist() throws Exception { backend = systemInfoService.getBackend(12011); systemInfoService.dropBackend(backend); } + + @Test + public void testCreateListPartitionedMVOfOlap() throws Exception { + String sql = "CREATE TABLE `s1` (\n" + + " `id` varchar(36),\n" + + " `location_id` varchar(36),\n" + + " `location_id_hash` int,\n" + + " `source_id` varchar(36),\n" + + " `person_id` varchar(36)\n" + + ") ENGINE=OLAP\n" + + "PRIMARY KEY(`id`,`location_id`,`location_id_hash`)\n" + + "PARTITION BY (`location_id_hash`)\n" + + "DISTRIBUTED BY HASH(`id`) BUCKETS 3\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ");"; + starRocksAssert.withTable(sql); + String mvSql = "create materialized view test_mv1\n" + + "PARTITION BY `location_id_hash`\n" + + "DISTRIBUTED BY HASH(`id`) BUCKETS 3\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ") \n" + + "as select `id`, `location_id`, `location_id_hash` from `s1`;"; + starRocksAssert.withMaterializedView(mvSql, () -> { + MaterializedView mv = starRocksAssert.getMv("test", "test_mv1"); + Assert.assertTrue(mv.getPartitionInfo().isListPartition()); + }); + starRocksAssert.dropTable("s1"); + } + + @Test + public void testCreateListPartitionedMVOfExternal() throws Exception { + String sql = "create materialized view mv1 " + + "partition by (d)" + + "distributed by hash(a) buckets 10 " + + "REFRESH DEFERRED MANUAL " + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ") " + + "as select a, b, c, d from jdbc0.partitioned_db0.tbl1;"; + CreateMaterializedViewStatement stmt = + (CreateMaterializedViewStatement) UtFrameUtils.parseStmtWithNewParser(sql, connectContext); + Expr partitionByExpr = stmt.getPartitionByExpr(); + Assert.assertTrue(partitionByExpr instanceof SlotRef); + List slotRefs = Lists.newArrayList(); + partitionByExpr.collect(SlotRef.class, slotRefs); + Assert.assertEquals(partitionByExpr, slotRefs.get(0)); + Assert.assertEquals("d", slotRefs.get(0).getColumnName()); + starRocksAssert.withMaterializedView(sql, () -> { + MaterializedView mv = starRocksAssert.getMv("test", "mv1"); + Assert.assertTrue(mv.getPartitionInfo().isListPartition()); + }); + } } \ No newline at end of file diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java index 155028ba6701ad..843c00fd00dfc1 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java @@ -429,7 +429,6 @@ public void testChooseBestRelatedMVs2() { connectContext.getSessionVariable().setCboMaterializedViewRewriteRelatedMVsLimit(1); validMVs = preprocessor.chooseBestRelatedMVs(queryTables, relatedMVs, logicalTree); Assert.assertEquals(1, validMVs.size()); - Assert.assertTrue(containsMV(validMVs, "mv_3")); connectContext.getSessionVariable().setEnableMaterializedViewPlanCache(true); // if mv_3 is in the plan cache diff --git a/test/sql/test_materialized_view_refresh/R/test_mv_with_list_partitions_hive b/test/sql/test_materialized_view_refresh/R/test_mv_with_list_partitions_hive new file mode 100644 index 00000000000000..e11ea54c786335 --- /dev/null +++ b/test/sql/test_materialized_view_refresh/R/test_mv_with_list_partitions_hive @@ -0,0 +1,128 @@ +-- name: test_mv_with_list_partitions_hive +create external catalog mv_hive_${uuid0} +properties +( + "type" = "hive", + "hive.catalog.type" = "hive", + "hive.metastore.uris" = "${hive_metastore_uris}" +); +-- result: +-- !result +set catalog mv_hive_${uuid0}; +-- result: +-- !result +create database mv_hive_db_${uuid0}; +-- result: +-- !result +use mv_hive_db_${uuid0}; +-- result: +-- !result +CREATE TABLE t1 ( + `id` varchar(36), + `location_id` varchar(36), + `location_id_hash` int, + `source_id` varchar(36), + `person_id` varchar(36) +) +PARTITION BY (person_id); +-- result: +-- !result +INSERT INTO t1 VALUES (1, 'beijing', 20, 'a', 'a1'), (2, 'guangdong', 30, 'b', 'b1'), (3, 'guangdong', 20, 'c', 'c1'); +-- result: +-- !result +set catalog default_catalog; +-- result: +-- !result +create database db_${uuid0}; +-- result: +-- !result +use db_${uuid0}; +-- result: +-- !result +create materialized view test_mv1 +PARTITION BY `person_id` +DISTRIBUTED BY HASH(`id`) BUCKETS 3 +PROPERTIES ( + "replication_num" = "1" +) +as select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1; +-- result: +-- !result +refresh materialized view test_mv1 with sync mode; +select * from test_mv1 order by 1, 2, 3; +-- result: +1 beijing a1 +2 guangdong b1 +3 guangdong c1 +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1", "test_mv1") +-- result: +True +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where location_id='guangdong'", "test_mv1") +-- result: +True +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where location_id='beijing'", "test_mv1") +-- result: +True +-- !result +select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 order by 1, 2, 3; +-- result: +1 beijing a1 +2 guangdong b1 +3 guangdong c1 +-- !result +select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where location_id='guangdong' order by 1, 2, 3; +-- result: +2 guangdong b1 +3 guangdong c1 +-- !result +select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where location_id='beijing' order by 1, 2, 3; +-- result: +1 beijing a1 +-- !result +INSERT INTO mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 VALUES (3, 'guangdong', 30, 'c', 'c1'); +-- result: +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1", "test_mv1") +-- result: +True +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where location_id='guangdong'", "test_mv1") +-- result: +True +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where location_id='beijing'", "test_mv1") +-- result: +True +-- !result +select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 order by 1, 2, 3; +-- result: +1 beijing a1 +2 guangdong b1 +3 guangdong c1 +3 guangdong c1 +-- !result +select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where location_id='guangdong' order by 1, 2, 3; +-- result: +2 guangdong b1 +3 guangdong c1 +3 guangdong c1 +-- !result +select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where location_id='beijing' order by 1, 2, 3; +-- result: +1 beijing a1 +-- !result +drop materialized view test_mv1; +-- result: +-- !result +drop database default_catalog.db_${uuid0} force; +-- result: +-- !result +drop table mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 force; +-- result: +-- !result +drop database mv_hive_${uuid0}.mv_hive_db_${uuid0} force; +-- result: +-- !result \ No newline at end of file diff --git a/test/sql/test_materialized_view_refresh/R/test_mv_with_list_partitions_iceberg b/test/sql/test_materialized_view_refresh/R/test_mv_with_list_partitions_iceberg new file mode 100644 index 00000000000000..acefbb4e0fd038 --- /dev/null +++ b/test/sql/test_materialized_view_refresh/R/test_mv_with_list_partitions_iceberg @@ -0,0 +1,128 @@ +-- name: test_mv_with_list_partitions_iceberg +create external catalog mv_iceberg_${uuid0} +properties +( + "type" = "iceberg", + "iceberg.catalog.type" = "hive", + "hive.metastore.uris" = "${iceberg_catalog_hive_metastore_uris}" +); +-- result: +-- !result +set catalog mv_iceberg_${uuid0}; +-- result: +-- !result +create database mv_iceberg_db_${uuid0}; +-- result: +-- !result +use mv_iceberg_db_${uuid0}; +-- result: +-- !result +CREATE TABLE t1 ( + `id` varchar(36), + `location_id` varchar(36), + `location_id_hash` int, + `source_id` varchar(36), + `person_id` varchar(36) +) +PARTITION BY (person_id); +-- result: +-- !result +INSERT INTO t1 VALUES (1, 'beijing', 20, 'a', 'a1'), (2, 'guangdong', 30, 'b', 'b1'), (3, 'guangdong', 20, 'c', 'c1'); +-- result: +-- !result +set catalog default_catalog; +-- result: +-- !result +create database db_${uuid0}; +-- result: +-- !result +use db_${uuid0}; +-- result: +-- !result +create materialized view test_mv1 +PARTITION BY `person_id` +DISTRIBUTED BY HASH(`id`) BUCKETS 3 +PROPERTIES ( + "replication_num" = "1" +) +as select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1; +-- result: +-- !result +refresh materialized view test_mv1 with sync mode; +select * from test_mv1 order by 1, 2, 3; +-- result: +1 beijing a1 +2 guangdong b1 +3 guangdong c1 +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1", "test_mv1") +-- result: +True +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 where location_id='guangdong'", "test_mv1") +-- result: +True +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 where location_id='beijing'", "test_mv1") +-- result: +True +-- !result +select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 order by 1, 2, 3; +-- result: +1 beijing a1 +2 guangdong b1 +3 guangdong c1 +-- !result +select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 where location_id='guangdong' order by 1, 2, 3; +-- result: +2 guangdong b1 +3 guangdong c1 +-- !result +select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 where location_id='beijing' order by 1, 2, 3; +-- result: +1 beijing a1 +-- !result +INSERT INTO mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 VALUES (3, 'guangdong', 30, 'c', 'c1'); +-- result: +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1", "test_mv1") +-- result: +True +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 where location_id='guangdong'", "test_mv1") +-- result: +True +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 where location_id='beijing'", "test_mv1") +-- result: +True +-- !result +select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 order by 1, 2, 3; +-- result: +1 beijing a1 +2 guangdong b1 +3 guangdong c1 +3 guangdong c1 +-- !result +select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 where location_id='guangdong' order by 1, 2, 3; +-- result: +2 guangdong b1 +3 guangdong c1 +3 guangdong c1 +-- !result +select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 where location_id='beijing' order by 1, 2, 3; +-- result: +1 beijing a1 +-- !result +drop materialized view test_mv1; +-- result: +-- !result +drop database default_catalog.db_${uuid0} force; +-- result: +-- !result +drop table mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 force; +-- result: +-- !result +drop database mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0} force; +-- result: +-- !result \ No newline at end of file diff --git a/test/sql/test_materialized_view_refresh/R/test_mv_with_list_partitions_olap b/test/sql/test_materialized_view_refresh/R/test_mv_with_list_partitions_olap new file mode 100644 index 00000000000000..2da9a597c91262 --- /dev/null +++ b/test/sql/test_materialized_view_refresh/R/test_mv_with_list_partitions_olap @@ -0,0 +1,94 @@ +-- name: test_mv_with_list_partitions_olap +CREATE TABLE `t1` ( + `id` varchar(36), + `location_id` varchar(36), + `location_id_hash` int, + `source_id` varchar(36), + `person_id` varchar(36) +) ENGINE=OLAP +PRIMARY KEY(`id`,`location_id`,`location_id_hash`) +PARTITION BY (`location_id_hash`) +DISTRIBUTED BY HASH(`id`) BUCKETS 3 +PROPERTIES ( + "replication_num" = "1" +); +-- result: +-- !result +INSERT INTO t1 VALUES (1, 'beijing', 20, 'a', 'a1'), (2, 'guangdong', 30, 'b', 'b1'), (3, 'guangdong', 20, 'c', 'c1'); +-- result: +-- !result +create materialized view test_mv1 +PARTITION BY `location_id_hash` +DISTRIBUTED BY HASH(`id`) BUCKETS 3 +PROPERTIES ( + "replication_num" = "1" +) +as select `id`, `location_id`, `location_id_hash` from `t1`; +-- result: +-- !result +refresh materialized view test_mv1 with sync mode; +select * from test_mv1 order by 1, 2, 3; +-- result: +1 beijing 20 +2 guangdong 30 +3 guangdong 20 +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `location_id_hash` from `t1`", "test_mv1") +-- result: +True +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `location_id_hash` from `t1` where location_id='guangdong'", "test_mv1") +-- result: +True +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `location_id_hash` from `t1` where location_id='beijing'", "test_mv1") +-- result: +True +-- !result +select `id`, `location_id`, `location_id_hash` from `t1` order by 1, 2, 3; +-- result: +1 beijing 20 +2 guangdong 30 +3 guangdong 20 +-- !result +select `id`, `location_id`, `location_id_hash` from `t1` where location_id='guangdong' order by 1, 2, 3; +-- result: +2 guangdong 30 +3 guangdong 20 +-- !result +select `id`, `location_id`, `location_id_hash` from `t1` where location_id='beijing' order by 1, 2, 3; +-- result: +1 beijing 20 +-- !result +INSERT INTO t1 VALUES (3, 'guangdong', 30, 'c', 'c1'); +-- result: +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `location_id_hash` from `t1`", "test_mv1") +-- result: +True +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `location_id_hash` from `t1` where location_id='guangdong'", "test_mv1") +-- result: +True +-- !result +function: print_hit_materialized_view("select `id`, `location_id`, `location_id_hash` from `t1` where location_id='beijing'", "test_mv1") +-- result: +True +-- !result +select `id`, `location_id`, `location_id_hash` from `t1` order by 1, 2, 3; +-- result: +1 beijing 20 +2 guangdong 30 +3 guangdong 20 +3 guangdong 30 +-- !result +select `id`, `location_id`, `location_id_hash` from `t1` where location_id='guangdong' order by 1, 2, 3; +-- result: +2 guangdong 30 +3 guangdong 20 +3 guangdong 30 +-- !result +select `id`, `location_id`, `location_id_hash` from `t1` where location_id='beijing' order by 1, 2, 3; +-- result: +1 beijing 20 +-- !result \ No newline at end of file diff --git a/test/sql/test_materialized_view_refresh/T/test_mv_with_list_partitions_hive b/test/sql/test_materialized_view_refresh/T/test_mv_with_list_partitions_hive new file mode 100644 index 00000000000000..bbbcf17fe0e76e --- /dev/null +++ b/test/sql/test_materialized_view_refresh/T/test_mv_with_list_partitions_hive @@ -0,0 +1,62 @@ +-- name: test_mv_with_list_partitions_hive + +create external catalog mv_hive_${uuid0} +properties +( + "type" = "hive", + "hive.catalog.type" = "hive", + "hive.metastore.uris" = "${hive_metastore_uris}" +); + + +-- create hive table +set catalog mv_hive_${uuid0}; +create database mv_hive_db_${uuid0}; +use mv_hive_db_${uuid0}; + +CREATE TABLE t1 ( + `id` varchar(36), + `location_id` varchar(36), + `location_id_hash` int, + `source_id` varchar(36), + `person_id` varchar(36) +) +PARTITION BY (person_id); +INSERT INTO t1 VALUES (1, 'beijing', 20, 'a', 'a1'), (2, 'guangdong', 30, 'b', 'b1'), (3, 'guangdong', 20, 'c', 'c1'); + +set catalog default_catalog; +create database db_${uuid0}; +use db_${uuid0}; + +create materialized view test_mv1 +PARTITION BY `person_id` +DISTRIBUTED BY HASH(`id`) BUCKETS 3 +PROPERTIES ( + "replication_num" = "1" +) +as select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1; + +refresh materialized view test_mv1 with sync mode; + +select * from test_mv1 order by 1, 2, 3; + +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1", "test_mv1") +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where location_id='guangdong'", "test_mv1") +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where location_id='beijing'", "test_mv1") +select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 order by 1, 2, 3; +select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where location_id='guangdong' order by 1, 2, 3; +select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where location_id='beijing' order by 1, 2, 3; + +INSERT INTO mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 VALUES (3, 'guangdong', 30, 'c', 'c1'); +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1", "test_mv1") +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where location_id='guangdong'", "test_mv1") +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where location_id='beijing'", "test_mv1") +select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 order by 1, 2, 3; +select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where location_id='guangdong' order by 1, 2, 3; +select `id`, `location_id`, `person_id` from mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where location_id='beijing' order by 1, 2, 3; + + +drop materialized view test_mv1; +drop database default_catalog.db_${uuid0} force; +drop table mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 force; +drop database mv_hive_${uuid0}.mv_hive_db_${uuid0} force; \ No newline at end of file diff --git a/test/sql/test_materialized_view_refresh/T/test_mv_with_list_partitions_iceberg b/test/sql/test_materialized_view_refresh/T/test_mv_with_list_partitions_iceberg new file mode 100644 index 00000000000000..e8061883f89978 --- /dev/null +++ b/test/sql/test_materialized_view_refresh/T/test_mv_with_list_partitions_iceberg @@ -0,0 +1,61 @@ +-- name: test_mv_with_list_partitions_iceberg + +create external catalog mv_iceberg_${uuid0} +properties +( + "type" = "iceberg", + "iceberg.catalog.type" = "hive", + "hive.metastore.uris" = "${iceberg_catalog_hive_metastore_uris}" +); + +-- create iceberg table +set catalog mv_iceberg_${uuid0}; +create database mv_iceberg_db_${uuid0}; +use mv_iceberg_db_${uuid0}; + +CREATE TABLE t1 ( + `id` varchar(36), + `location_id` varchar(36), + `location_id_hash` int, + `source_id` varchar(36), + `person_id` varchar(36) +) +PARTITION BY (person_id); +INSERT INTO t1 VALUES (1, 'beijing', 20, 'a', 'a1'), (2, 'guangdong', 30, 'b', 'b1'), (3, 'guangdong', 20, 'c', 'c1'); + +set catalog default_catalog; +create database db_${uuid0}; +use db_${uuid0}; + +create materialized view test_mv1 +PARTITION BY `person_id` +DISTRIBUTED BY HASH(`id`) BUCKETS 3 +PROPERTIES ( + "replication_num" = "1" +) +as select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1; + +refresh materialized view test_mv1 with sync mode; + +select * from test_mv1 order by 1, 2, 3; + +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1", "test_mv1") +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 where location_id='guangdong'", "test_mv1") +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 where location_id='beijing'", "test_mv1") +select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 order by 1, 2, 3; +select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 where location_id='guangdong' order by 1, 2, 3; +select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 where location_id='beijing' order by 1, 2, 3; + +INSERT INTO mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 VALUES (3, 'guangdong', 30, 'c', 'c1'); +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1", "test_mv1") +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 where location_id='guangdong'", "test_mv1") +function: print_hit_materialized_view("select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 where location_id='beijing'", "test_mv1") +select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 order by 1, 2, 3; +select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 where location_id='guangdong' order by 1, 2, 3; +select `id`, `location_id`, `person_id` from mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 where location_id='beijing' order by 1, 2, 3; + + +drop materialized view test_mv1; +drop database default_catalog.db_${uuid0} force; +drop table mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0}.t1 force; +drop database mv_iceberg_${uuid0}.mv_iceberg_db_${uuid0} force; \ No newline at end of file diff --git a/test/sql/test_materialized_view_refresh/T/test_mv_with_list_partitions_olap b/test/sql/test_materialized_view_refresh/T/test_mv_with_list_partitions_olap new file mode 100644 index 00000000000000..8e7a588cef7998 --- /dev/null +++ b/test/sql/test_materialized_view_refresh/T/test_mv_with_list_partitions_olap @@ -0,0 +1,44 @@ +-- name: test_mv_with_list_partitions_olap + +CREATE TABLE `t1` ( + `id` varchar(36), + `location_id` varchar(36), + `location_id_hash` int, + `source_id` varchar(36), + `person_id` varchar(36) +) ENGINE=OLAP +PRIMARY KEY(`id`,`location_id`,`location_id_hash`) +PARTITION BY (`location_id_hash`) +DISTRIBUTED BY HASH(`id`) BUCKETS 3 +PROPERTIES ( + "replication_num" = "1" +); +INSERT INTO t1 VALUES (1, 'beijing', 20, 'a', 'a1'), (2, 'guangdong', 30, 'b', 'b1'), (3, 'guangdong', 20, 'c', 'c1'); + +create materialized view test_mv1 +PARTITION BY `location_id_hash` +DISTRIBUTED BY HASH(`id`) BUCKETS 3 +PROPERTIES ( + "replication_num" = "1" +) +as select `id`, `location_id`, `location_id_hash` from `t1`; + +refresh materialized view test_mv1 with sync mode; + +select * from test_mv1 order by 1, 2, 3; + +function: print_hit_materialized_view("select `id`, `location_id`, `location_id_hash` from `t1`", "test_mv1") +function: print_hit_materialized_view("select `id`, `location_id`, `location_id_hash` from `t1` where location_id='guangdong'", "test_mv1") +function: print_hit_materialized_view("select `id`, `location_id`, `location_id_hash` from `t1` where location_id='beijing'", "test_mv1") +select `id`, `location_id`, `location_id_hash` from `t1` order by 1, 2, 3; +select `id`, `location_id`, `location_id_hash` from `t1` where location_id='guangdong' order by 1, 2, 3; +select `id`, `location_id`, `location_id_hash` from `t1` where location_id='beijing' order by 1, 2, 3; + +INSERT INTO t1 VALUES (3, 'guangdong', 30, 'c', 'c1'); +function: print_hit_materialized_view("select `id`, `location_id`, `location_id_hash` from `t1`", "test_mv1") +function: print_hit_materialized_view("select `id`, `location_id`, `location_id_hash` from `t1` where location_id='guangdong'", "test_mv1") +function: print_hit_materialized_view("select `id`, `location_id`, `location_id_hash` from `t1` where location_id='beijing'", "test_mv1") +select `id`, `location_id`, `location_id_hash` from `t1` order by 1, 2, 3; +select `id`, `location_id`, `location_id_hash` from `t1` where location_id='guangdong' order by 1, 2, 3; +select `id`, `location_id`, `location_id_hash` from `t1` where location_id='beijing' order by 1, 2, 3; +