@@ -108,11 +114,11 @@ protected TableScan(RelOptCluster cluster, RelTraitSet traitSet,
protected TableScan(RelOptCluster cluster, RelTraitSet traitSet,
RelOptTable table, SqlNodeList hints, SqlNode indexNode) {
- this(cluster, traitSet, table, hints, indexNode, null, null);
+ this(cluster, traitSet, table, hints, indexNode, null, null, null);
}
protected TableScan(RelOptCluster cluster, RelTraitSet traitSet, RelOptTable table, SqlNodeList hints,
- SqlNode indexNode, RexNode flashback, SqlNode partitions) {
+ SqlNode indexNode, RexNode flashback, SqlOperator flashbackOperator, SqlNode partitions) {
super(cluster, traitSet);
this.table = table;
this.hints = hints;
@@ -121,6 +127,7 @@ protected TableScan(RelOptCluster cluster, RelTraitSet traitSet, RelOptTable tab
cluster.getPlanner().registerSchema(table.getRelOptSchema());
}
this.flashback = flashback;
+ this.flashbackOperator = flashbackOperator;
this.partitions = partitions;
}
/**
@@ -265,6 +272,14 @@ public void setFlashback(RexNode flashback) {
this.flashback = flashback;
}
+ public SqlOperator getFlashbackOperator() {
+ return flashbackOperator;
+ }
+
+ public void setFlashbackOperator(SqlOperator flashbackOperator) {
+ this.flashbackOperator = flashbackOperator;
+ }
+
public SqlNode getPartitions() {
return partitions;
}
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/rel/core/WindowAggregateCall.java b/polardbx-calcite/src/main/java/org/apache/calcite/rel/core/WindowAggregateCall.java
index 2f056fc9d..78456a56b 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/rel/core/WindowAggregateCall.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/rel/core/WindowAggregateCall.java
@@ -67,6 +67,25 @@ public WindowAggregateCall copy(List args, int filterArg, List args, int filterArg) {
+ return new WindowAggregateCall(aggFunction, distinct, approximate, args,
+ filterArg, type, name, constants, offset);
+ }
+
+ @Override
+ public AggregateCall copy(List args, int filterArg, boolean isDistinct, String newName) {
+ return new WindowAggregateCall(aggFunction, isDistinct, approximate, args,
+ filterArg, type, newName, constants, offset);
+ }
+
+ @Override
+ public AggregateCall withDistinct(boolean distinct) {
+ return distinct == this.distinct ? this
+ : new WindowAggregateCall(aggFunction, distinct, approximate, argList, filterArg, type, name,
+ constants, offset);
+ }
+
public AggregateCall create(SqlAggFunction aggFunction,
boolean distinct, boolean approximate, List argList,
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/rel/logical/LogicalTableScan.java b/polardbx-calcite/src/main/java/org/apache/calcite/rel/logical/LogicalTableScan.java
index 763733402..52eed454c 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/rel/logical/LogicalTableScan.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/rel/logical/LogicalTableScan.java
@@ -31,6 +31,7 @@
import org.apache.calcite.schema.Table;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.parser.SqlParserPos;
import java.util.List;
@@ -81,8 +82,9 @@ public LogicalTableScan(RelOptCluster cluster, RelTraitSet traitSet,
public LogicalTableScan(RelOptCluster cluster, RelTraitSet traitSet,
RelOptTable table, SqlNodeList hints, SqlNode indexNode, RexNode flashback,
+ SqlOperator flashbackOperator,
SqlNode partitions) {
- super(cluster, traitSet, table, hints, indexNode, flashback, partitions);
+ super(cluster, traitSet, table, hints, indexNode, flashback, flashbackOperator, partitions);
}
@Deprecated // to be removed before 2.0
@@ -110,12 +112,12 @@ public LogicalTableScan(RelInput input) {
*/
public static LogicalTableScan create(RelOptCluster cluster,
final RelOptTable relOptTable) {
- return create(cluster, relOptTable, new SqlNodeList(SqlParserPos.ZERO), null, null, null);
+ return create(cluster, relOptTable, new SqlNodeList(SqlParserPos.ZERO), null, null, null, null);
}
public static LogicalTableScan create(RelOptCluster cluster,
final RelOptTable relOptTable, SqlNodeList hints, SqlNode indexNode,
- RexNode flashback, SqlNode partitions) {
+ RexNode flashback, SqlOperator flashbackOperator, SqlNode partitions) {
final Table table = relOptTable.unwrap(Table.class);
final RelTraitSet traitSet =
@@ -127,7 +129,8 @@ public static LogicalTableScan create(RelOptCluster cluster,
}
return ImmutableList.of();
}).simplify();
- return new LogicalTableScan(cluster, traitSet, relOptTable, hints, indexNode, flashback, partitions);
+ return new LogicalTableScan(cluster, traitSet, relOptTable, hints, indexNode, flashback, flashbackOperator,
+ partitions);
}
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/polardbx-calcite/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
index 076e55f15..b46e8b9f9 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
@@ -1441,7 +1441,8 @@ public SqlNode asFrom() {
final SqlIdentifier identifier = (SqlIdentifier) node;
final SqlIdentifier newIdentifier =
new SqlIdentifier(((SqlIdentifier) node).names, ((SqlIdentifier) node).getCollation(),
- node.getParserPosition(), null, null, identifier.partitions, identifier.flashback);
+ node.getParserPosition(), null, null, identifier.partitions, identifier.flashback,
+ identifier.flashbackOperator);
final SqlIdentifier alias =
new SqlIdentifier(ImmutableList.of(neededAlias), null, POS, null, identifier.indexNode);
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlAggFunction.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlAggFunction.java
index 6c1f6481a..c78b4372c 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlAggFunction.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlAggFunction.java
@@ -24,7 +24,9 @@
import org.apache.calcite.sql.type.SqlReturnTypeInference;
import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql.validate.SqlValidatorScope;
+import org.apache.calcite.util.Optionality;
+import javax.annotation.Nonnull;
import java.util.List;
/**
@@ -131,6 +133,23 @@ public RelDataType getReturnType(RelDataTypeFactory typeFactory) {
public boolean allowsFilter() {
return true;
}
+
+ /** Returns whether this aggregate function allows the {@code DISTINCT}
+ * keyword.
+ *
+ * The default implementation returns {@link Optionality#OPTIONAL},
+ * which is appropriate for most aggregate functions, including {@code SUM}
+ * and {@code COUNT}.
+ *
+ *
Some aggregate functions, for example {@code MIN}, produce the same
+ * result with or without {@code DISTINCT}, and therefore return
+ * {@link Optionality#IGNORED} to indicate this. For such functions,
+ * Calcite will probably remove {@code DISTINCT} while optimizing the query.
+ */
+ public @Nonnull
+ Optionality getDistinctOptionality() {
+ return Optionality.OPTIONAL;
+ }
}
// End SqlAggFunction.java
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlAsOf57Operator.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlAsOf57Operator.java
new file mode 100644
index 000000000..0f2737765
--- /dev/null
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlAsOf57Operator.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright [2013-2021], Alibaba Group Holding Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.calcite.sql;
+
+import org.apache.calcite.sql.type.InferTypes;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlOperandTypeChecker;
+import org.apache.calcite.sql.type.SqlOperandTypeInference;
+import org.apache.calcite.sql.type.SqlReturnTypeInference;
+
+public class SqlAsOf57Operator extends SqlSpecialOperator {
+ public SqlAsOf57Operator() {
+ this(
+ "AS OF TSO",
+ SqlKind.AS_OF,
+ 20,
+ true,
+ ReturnTypes.ARG0,
+ InferTypes.RETURN_TYPE,
+ OperandTypes.ANY_ANY);
+ }
+
+ protected SqlAsOf57Operator(String name, SqlKind kind, int prec,
+ boolean leftAssoc, SqlReturnTypeInference returnTypeInference,
+ SqlOperandTypeInference operandTypeInference,
+ SqlOperandTypeChecker operandTypeChecker) {
+ super(name, kind, prec, leftAssoc, returnTypeInference,
+ operandTypeInference, operandTypeChecker);
+ }
+
+ public void unparse(
+ SqlWriter writer,
+ SqlCall call,
+ int leftPrec,
+ int rightPrec) {
+ assert call.operandCount() >= 2;
+ final SqlWriter.Frame frame =
+ writer.startList(
+ SqlWriter.FrameTypeEnum.SIMPLE);
+ call.operand(0).unparse(writer, leftPrec, getLeftPrec());
+ final boolean needsSpace = true;
+ writer.setNeedWhitespace(needsSpace);
+ if (writer.getDialect().allowsAsOf()) {
+ writer.sep("AS OF TSO");
+ writer.setNeedWhitespace(needsSpace);
+ }
+ call.operand(1).unparse(writer, getRightPrec(), rightPrec);
+ writer.endList(frame);
+ }
+}
\ No newline at end of file
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlCreateTable.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlCreateTable.java
index 03efd077c..ca20cf251 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlCreateTable.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlCreateTable.java
@@ -521,6 +521,8 @@ public boolean shouldLoad() {
|| trimmedComment.equalsIgnoreCase("load_s3")
|| trimmedComment.equalsIgnoreCase("load_local_disk")
|| trimmedComment.equalsIgnoreCase("load_nfs")
+ || trimmedComment.equalsIgnoreCase("load_s3")
+ || trimmedComment.equalsIgnoreCase("load_abs")
|| trimmedComment.equalsIgnoreCase("load_external_disk");
}
@@ -2738,8 +2740,7 @@ public static void addCompositeIndex(Map indexColumn
Set orderedIndexColumnNames = new LinkedHashSet<>(shardingKey);
final String suffix = buildUnifyIndexName(orderedIndexColumnNames, 45);
- final String indexName = addFkIndex ? (foreignKeyIndexName == null ?
- buildForeignKeyIndexName(existingIndexNames, suffix) : foreignKeyIndexName) :
+ final String indexName = addFkIndex ? buildForeignKeyName(foreignKeyIndexName, existingIndexNames, suffix) :
buildIndexName(existingIndexNames, suffix);
final MySqlTableIndex mySqlTableIndex = new MySqlTableIndex();
@@ -2765,6 +2766,13 @@ public static void addCompositeIndex(Map indexColumn
}
}
+ public static String buildForeignKeyName(String foreignKeyIndexName, Set existingIndexNames,
+ String suffix) {
+ return foreignKeyIndexName == null ?
+ buildForeignKeyIndexName(existingIndexNames, suffix) :
+ SqlIdentifier.surroundWithBacktick(foreignKeyIndexName);
+ }
+
private static List preparAutoCompositeIndexs(List shardKeys,
Map columnDefMap,
int maxLen) {
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlIdentifier.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlIdentifier.java
index 0d3773357..79666c431 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlIdentifier.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlIdentifier.java
@@ -100,6 +100,11 @@ public String apply(String s) {
*/
public SqlNode flashback;
+ /**
+ * 记录AS OF 种类：AS_OF/AS_OF_80/AS_OF_57
+ */
+ public SqlOperator flashbackOperator;
+
/**
* This identifier's collation (if any).
*/
@@ -153,7 +158,7 @@ public SqlIdentifier(
// for (String name : names) {
// assert name != null;
// }
- this(names, collation, pos, componentPositions, null, null, null);
+ this(names, collation, pos, componentPositions, null, null, null, null);
}
public SqlIdentifier(
@@ -161,14 +166,14 @@ public SqlIdentifier(
SqlCollation collation,
SqlParserPos pos,
List componentPositions, SqlNode indexNode) {
- this(names, collation, pos, componentPositions, indexNode, null, null);
+ this(names, collation, pos, componentPositions, indexNode, null, null, null);
}
public SqlIdentifier(
String name,
SqlParserPos pos,
SqlNode indexNode) {
- this(ImmutableList.of(name), null, pos, null, indexNode, null, null);
+ this(ImmutableList.of(name), null, pos, null, indexNode, null, null, null);
}
/**
@@ -181,6 +186,15 @@ public SqlIdentifier(
SqlCollation collation,
SqlParserPos pos,
List componentPositions, SqlNode indexNode, SqlNode partitions, SqlNode flashback) {
+ this(ImmutableList.copyOf(names), collation, pos, componentPositions, indexNode, partitions, flashback, null);
+ }
+
+ public SqlIdentifier(
+ List names,
+ SqlCollation collation,
+ SqlParserPos pos,
+ List componentPositions, SqlNode indexNode, SqlNode partitions, SqlNode flashback,
+ SqlOperator flashbackOperator) {
super(pos);
this.names = ImmutableList.copyOf(names);
this.collation = collation;
@@ -192,16 +206,21 @@ public SqlIdentifier(
this.indexNode = indexNode;
this.partitions = partitions;
this.flashback = flashback;
+ this.flashbackOperator = flashbackOperator;
}
- /** Creates an identifier that is a singleton wildcard star. */
+ /**
+ * Creates an identifier that is a singleton wildcard star.
+ */
public static SqlIdentifier star(SqlParserPos pos) {
return star(ImmutableList.of(""), pos, ImmutableList.of(pos));
}
- /** Creates an identifier that ends in a wildcard star. */
+ /**
+ * Creates an identifier that ends in a wildcard star.
+ */
public static SqlIdentifier star(List names, SqlParserPos pos,
- List componentPositions) {
+ List componentPositions) {
return new SqlIdentifier(Lists.transform(names, STAR_TO_EMPTY), null, pos,
componentPositions);
}
@@ -213,7 +232,7 @@ public SqlKind getKind() {
}
@Override public SqlNode clone(SqlParserPos pos) {
- return new SqlIdentifier(names, collation, pos, componentPositions, indexNode, partitions, flashback);
+ return new SqlIdentifier(names, collation, pos, componentPositions, indexNode, partitions, flashback, flashbackOperator);
}
public String toStringWithBacktick() {
@@ -498,6 +517,10 @@ public SqlMonotonicity getMonotonicity(SqlValidatorScope scope) {
final SqlIdentifier fqId = qualified.identifier;
return qualified.namespace.resolve().getMonotonicity(Util.last(fqId.names));
}
+
+ public SqlOperator getFlashbackOperator() {
+ return flashbackOperator;
+ }
}
// End SqlIdentifier.java
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlMoveDatabase.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlMoveDatabase.java
index 2309d33f7..03a8844fa 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlMoveDatabase.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlMoveDatabase.java
@@ -57,7 +57,7 @@ public SqlMoveDatabase(SqlParserPos pos, Map> storageGroups
this.isCleanUpCommand = isCleanUpCommand;
if (!isCleanUpCommand) {
this.currentSourceGroupKey = firstGroup;
- this.currentTargetGroupKey = GroupInfoUtil.buildScaloutGroupName(firstGroup);
+ this.currentTargetGroupKey = GroupInfoUtil.buildScaleOutGroupName(firstGroup);
}
}
@@ -75,7 +75,7 @@ public void setCurrentStorageInstId(String currentStorageInstId) {
public void setCleanUpGroups(String groupName) {
Map> toBeCleanGroups = new HashMap<>();
- toBeCleanGroups.put(VIRTUAL_STORAGE_ID, ImmutableList.of(GroupInfoUtil.buildScaloutGroupName(groupName)));
+ toBeCleanGroups.put(VIRTUAL_STORAGE_ID, ImmutableList.of(GroupInfoUtil.buildScaleOutGroupName(groupName)));
this.storageGroups = toBeCleanGroups;
}
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlPartition.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlPartition.java
index aecea5263..78e2d5604 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlPartition.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlPartition.java
@@ -16,16 +16,9 @@
package org.apache.calcite.sql;
-import com.alibaba.polardbx.common.exception.TddlRuntimeException;
-import com.alibaba.polardbx.common.exception.code.ErrorCode;
-import com.alibaba.polardbx.common.utils.GeneralUtil;
import com.alibaba.polardbx.common.utils.TStringUtil;
import com.alibaba.polardbx.druid.util.StringUtils;
-import com.google.common.base.Preconditions;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.sql.parser.SqlParserPos;
-import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.util.SqlVisitor;
import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql.validate.SqlValidatorScope;
@@ -76,18 +69,37 @@ public String toString() {
StringBuilder sb = new StringBuilder("");
sb.append("PARTITION ");
sb.append(name);
- sb.append(" ");
- sb.append(values.toString());
+ if (values != null) {
+ sb.append(" ");
+ sb.append(values.toString());
+ }
if (TStringUtil.isNotEmpty(locality)) {
sb.append(" LOCALITY=");
sb.append(TStringUtil.quoteString(locality));
}
+
+ if (subPartitionCount != null) {
+ sb.append(" ");
+ sb.append("SUBPARTITIONS ");
+ sb.append(subPartitionCount.toString());
+ }
+ if (subPartitions != null && !subPartitions.isEmpty()) {
+ sb.append(" ");
+ sb.append("(");
+ for (int i = 0; i < subPartitions.size(); i++) {
+ if (i > 0) {
+ sb.append(",");
+ }
+ SqlNode subPart = subPartitions.get(i);
+ sb.append(subPart.toString());
+ }
+ sb.append(")");
+ }
return sb.toString();
}
@Override
public void unparse(SqlWriter writer, int leftPrec, int rightPrec) {
-
}
@Override
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlSubPartition.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlSubPartition.java
index 1adebaab7..9953a844d 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlSubPartition.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlSubPartition.java
@@ -70,8 +70,10 @@ public String toString() {
StringBuilder sb = new StringBuilder("");
sb.append("SUBPARTITION ");
sb.append(name);
- sb.append(" ");
- sb.append(values.toString());
+ if (values != null) {
+ sb.append(" ");
+ sb.append(values.toString());
+ }
if (TStringUtil.isNotEmpty(locality)) {
sb.append(" LOCALITY=");
sb.append(TStringUtil.quoteString(locality));
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlTableOptions.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlTableOptions.java
index cf07383d3..c25005ba2 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlTableOptions.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql/SqlTableOptions.java
@@ -50,7 +50,7 @@ public class SqlTableOptions extends SqlCall {
private SqlCharStringLiteral indexDir;
private SqlLiteral delayKeyWrite;
private InsertMethod insertMethod;
- private SqlCall keyBlockSize;
+ private SqlNumericLiteral keyBlockSize;
private SqlNumericLiteral maxRows;
private SqlNumericLiteral minRows;
private PackKeys packKeys;
@@ -304,11 +304,11 @@ public void setInsertMethod(InsertMethod insertMethod) {
this.insertMethod = insertMethod;
}
- public SqlCall getKeyBlockSize() {
+ public SqlNumericLiteral getKeyBlockSize() {
return keyBlockSize;
}
- public void setKeyBlockSize(SqlCall keyBlockSize) {
+ public void setKeyBlockSize(SqlNumericLiteral keyBlockSize) {
this.keyBlockSize = keyBlockSize;
}
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql/fun/SqlMinMaxAggFunction.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql/fun/SqlMinMaxAggFunction.java
index 94f855bec..78010e75d 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/sql/fun/SqlMinMaxAggFunction.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql/fun/SqlMinMaxAggFunction.java
@@ -16,6 +16,8 @@
*/
package org.apache.calcite.sql.fun;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.sql.SqlAggFunction;
@@ -24,10 +26,8 @@
import org.apache.calcite.sql.SqlSplittableAggFunction;
import org.apache.calcite.sql.type.OperandTypes;
import org.apache.calcite.sql.type.ReturnTypes;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
import org.apache.calcite.sql.type.SqlReturnTypeInference;
+import org.apache.calcite.util.Optionality;
import java.util.List;
@@ -142,6 +142,10 @@ public RelDataType getReturnType(RelDataTypeFactory typeFactory) {
}
return super.unwrap(clazz);
}
+
+ @Override public Optionality getDistinctOptionality() {
+ return Optionality.IGNORED;
+ }
}
// End SqlMinMaxAggFunction.java
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql/fun/SqlSingleValueAggFunction.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql/fun/SqlSingleValueAggFunction.java
index 6e745df1a..258cd3783 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/sql/fun/SqlSingleValueAggFunction.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql/fun/SqlSingleValueAggFunction.java
@@ -16,6 +16,7 @@
*/
package org.apache.calcite.sql.fun;
+import com.google.common.collect.ImmutableList;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.sql.SqlAggFunction;
@@ -23,8 +24,7 @@
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.type.OperandTypes;
import org.apache.calcite.sql.type.ReturnTypes;
-
-import com.google.common.collect.ImmutableList;
+import org.apache.calcite.util.Optionality;
import java.util.List;
@@ -71,6 +71,10 @@ public RelDataType getReturnType(RelDataTypeFactory typeFactory) {
public RelDataType getType() {
return type;
}
+
+ @Override public Optionality getDistinctOptionality() {
+ return Optionality.IGNORED;
+ }
}
// End SqlSingleValueAggFunction.java
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java
index aab31c07a..bd9d15224 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java
@@ -18,6 +18,7 @@
import com.google.common.collect.ImmutableList;
import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.SqlAsOf57Operator;
import org.apache.calcite.sql.SqlAsOf80Operator;
import org.apache.calcite.sql.SqlAsOfOperator;
import org.apache.calcite.sql.SqlAsOperator;
@@ -173,9 +174,20 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable {
* with an alias.
*/
public static final SqlAsOperator AS = new SqlAsOperator();
+ /**
+ * as of timestamp; 5.7 物理sql使用 as of timestamp; 8.0 需转成 AS OF GCN
+ */
public static final SqlAsOfOperator AS_OF = new SqlAsOfOperator();
+ /**
+ * 8.0版本: 用户sql为as of tso，物理sql使用 AS OF GCN
+ */
public static final SqlAsOf80Operator AS_OF_80 = new SqlAsOf80Operator();
+ /**
+ * 5.7 版本: 用户sql为as of tso，物理sql使用 as of tso
+ */
+ public static final SqlAsOf57Operator AS_OF_57 = new SqlAsOf57Operator();
+
/**
* ARGUMENT_ASSIGNMENT operator (=<)
* assigns an argument to a function call to a particular named parameter.
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql/type/SqlTypeName.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql/type/SqlTypeName.java
index 3d75d4e9e..8271c5fbf 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/sql/type/SqlTypeName.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql/type/SqlTypeName.java
@@ -73,8 +73,6 @@ public enum SqlTypeName {
REAL(PrecScale.NO_NO, false, Types.REAL, SqlTypeFamily.NUMERIC),
DOUBLE(PrecScale.NO_NO, false, Types.DOUBLE, SqlTypeFamily.NUMERIC),
- // TODO: remove UNSIGNED/SIGNED
- // add by xiaoying
UNSIGNED(PrecScale.NO_NO, false, Types.BIGINT, SqlTypeFamily.NUMERIC),
SIGNED(PrecScale.NO_NO, false, Types.BIGINT, SqlTypeFamily.NUMERIC),
@@ -170,7 +168,16 @@ public enum SqlTypeName {
public static final int MAX_TIME_FRACTIONAL_SECOND_SCALE = 6;
// Cached map of enum values
- private static final Map VALUES_MAP = Util.enumConstants(SqlTypeName.class);
+ private static final Map VALUES_MAP = ImmutableMap.builder()
+ .putAll(Util.enumConstants(SqlTypeName.class))
+ // For Alias in MySQL https://dev.mysql.com/doc/refman/5.7/en/cast-functions.html#function_cast
+ // SIGNED [INTEGER]
+ // Produces a signed BIGINT value.
+ // UNSIGNED [INTEGER]
+ // Produces an unsigned BIGINT value.
+ .put("SIGNED INTEGER", SIGNED)
+ .put("UNSIGNED INTEGER", UNSIGNED)
+ .build();
// categorizations used by SqlTypeFamily definitions
@@ -397,15 +404,6 @@ public static long getSize() {
* @return Type name, or null if not found
*/
public static SqlTypeName get(String name) {
- if (false) {
- // The following code works OK, but the spurious exceptions are
- // annoying.
- try {
- return SqlTypeName.valueOf(name);
- } catch (IllegalArgumentException e) {
- return null;
- }
- }
return VALUES_MAP.get(name);
}
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
index 75c80c82b..bc3fcc42f 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
@@ -5606,7 +5606,17 @@ public void validateGsiColumn(List> glob
}
}
- public static void validateUnsupportedColumTypeWithCci(MySqlCreateTableStatement stmt, List primaryKeys,
+ public static void validateUnsupportedTypeWithCciWhenModifyColumn(SqlColumnDeclaration columnDeclaration) {
+ final List deniedTypes = Arrays.asList(
+ "text", "binary", "varbinary", "blob", "timestamp", "time", "year", "json", "enum", "set", "point", "geometry");
+ String columnType = columnDeclaration.getDataType().getTypeName().getLastName().toLowerCase();
+ if (deniedTypes.contains(columnType)) {
+ throw new TddlRuntimeException(ErrorCode.ERR_UNSUPPORTED_COLUMN_TYPE_WITH_CCI,
+ "MODIFY/CHANGE COLUMN", columnType);
+ }
+ }
+
+ public static void validateUnsupportedColumnTypeWithCci(MySqlCreateTableStatement stmt, List primaryKeys,
List sortKeys, List shardingKeys) {
final String[] deniedTypes = {
"float", "double", "decimal", "numeric", "json", "enum", "set", "point", "geometry"};
@@ -6900,6 +6910,10 @@ public void validateInsert(SqlInsert insert) {
insert.getTargetColumnList(),
false, null);
+ // for INSERT t() values(), in which no source or target column list is specified,
+ // rewrite SqlNode with INSERT t() values(DEFAULT, DEFAULT, ...)
+ checkAndRewriteEmptySource(insert, targetRowType);
+
final SqlNode source = insert.getSource();
if (source instanceof SqlSelect) {
final SqlSelect sqlSelect = (SqlSelect) source;
@@ -6948,6 +6962,29 @@ public void validateInsert(SqlInsert insert) {
}
}
+ public void checkAndRewriteEmptySource(SqlInsert insert, RelDataType targetRowType) {
+ if (insert.getTargetColumnList() == null) {
+ final SqlNode originSource = insert.getSource();
+ if (originSource instanceof SqlBasicCall && originSource.getKind() == SqlKind.VALUES) {
+ final SqlBasicCall values = (SqlBasicCall) originSource;
+ final SqlBasicCall row = (SqlBasicCall) values.getOperands()[0];
+ // If target column list and source value list is all empty,
+ // rewrite source to VALUES(DEFAULT, DEFAULT, ...)
+ if (row.getOperands() == null || row.getOperands().length == 0) {
+ final List fieldNames = targetRowType.getFieldNames();
+ final long fieldCount = fieldNames.stream().filter(f -> !isImplicitKey(f)).count();
+ final SqlNode[] defaults = new SqlNode[(int) fieldCount];
+ for (int i = 0; i < defaults.length; i++) {
+ // Add DEFAULT to VALUES for each target column
+ defaults[i] =
+ new SqlBasicCall(SqlStdOperatorTable.DEFAULT, SqlNode.EMPTY_ARRAY, SqlParserPos.ZERO);
+ }
+ values.setOperand(0, new SqlBasicCall(row.getOperator(), defaults, row.getParserPosition()));
+ }
+ }
+ }
+ }
+
/**
* Validates insert values against the constraint of a modifiable view.
*
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/polardbx-calcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index cbca1b158..a4f7a5e07 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -108,8 +108,8 @@
import org.apache.calcite.rel.ddl.AlterTableSetTableGroup;
import org.apache.calcite.rel.ddl.AnalyzeTable;
import org.apache.calcite.rel.ddl.ChangeConsensusRole;
-import org.apache.calcite.rel.ddl.ConvertAllSequences;
import org.apache.calcite.rel.ddl.ClearFileStorage;
+import org.apache.calcite.rel.ddl.ConvertAllSequences;
import org.apache.calcite.rel.ddl.CreateDatabase;
import org.apache.calcite.rel.ddl.CreateFileStorage;
import org.apache.calcite.rel.ddl.CreateFunction;
@@ -178,6 +178,7 @@
import org.apache.calcite.rel.logical.LogicalUnion;
import org.apache.calcite.rel.logical.LogicalValues;
import org.apache.calcite.rel.metadata.RelColumnMapping;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.stream.Delta;
import org.apache.calcite.rel.stream.LogicalDelta;
import org.apache.calcite.rel.type.RelDataType;
@@ -260,8 +261,8 @@
import org.apache.calcite.sql.SqlChangeConsensusRole;
import org.apache.calcite.sql.SqlCharStringLiteral;
import org.apache.calcite.sql.SqlCheckColumnarIndex;
-import org.apache.calcite.sql.SqlConvertAllSequences;
import org.apache.calcite.sql.SqlClearFileStorage;
+import org.apache.calcite.sql.SqlConvertAllSequences;
import org.apache.calcite.sql.SqlCreate;
import org.apache.calcite.sql.SqlCreateDatabase;
import org.apache.calcite.sql.SqlCreateFileStorage;
@@ -347,8 +348,8 @@
import org.apache.calcite.sql.SqlUpdate;
import org.apache.calcite.sql.SqlUtil;
import org.apache.calcite.sql.SqlUtil.SpecialIdentiFiers;
-import org.apache.calcite.sql.SqlValuesTableSource;
import org.apache.calcite.sql.SqlValuesOperator;
+import org.apache.calcite.sql.SqlValuesTableSource;
import org.apache.calcite.sql.SqlWindow;
import org.apache.calcite.sql.SqlWith;
import org.apache.calcite.sql.SqlWithItem;
@@ -513,7 +514,7 @@ public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator val
Prepare.CatalogReader catalogReader, RelOptPlanner planner, RexBuilder rexBuilder,
SqlRexConvertletTable convertletTable) {
this(viewExpander, validator, catalogReader, RelOptCluster.create(planner, rexBuilder), convertletTable,
- Config.DEFAULT, Integer.MAX_VALUE);
+ Config.DEFAULT, Integer.MAX_VALUE, false);
}
@Deprecated // to be removed before 2.0
@@ -526,13 +527,15 @@ public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator val
public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator,
Prepare.CatalogReader catalogReader, RelOptCluster cluster,
SqlRexConvertletTable convertletTable, Config config) {
- this(viewExpander, validator, catalogReader, cluster, convertletTable, config, config.getInSubQueryThreshold());
+ this(viewExpander, validator, catalogReader, cluster, convertletTable, config, config.getInSubQueryThreshold(),
+ config.isRemoveDistinct());
}
/* Creates a converter. */
public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator,
Prepare.CatalogReader catalogReader, RelOptCluster cluster,
- SqlRexConvertletTable convertletTable, Config config, int inSubQueryThreshold) {
+ SqlRexConvertletTable convertletTable, Config config, int inSubQueryThreshold,
+ boolean isRemoveDistinct) {
this.viewExpander = viewExpander;
this.opTab = (validator == null) ? SqlStdOperatorTable.instance() : validator.getOperatorTable();
this.validator = validator;
@@ -543,7 +546,9 @@ public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator val
this.cluster = Preconditions.checkNotNull(cluster);
this.exprConverter = new SqlNodeToRexConverterImpl(convertletTable);
this.explainParamCount = 0;
- this.config = new ConfigBuilder().withConfig(config).withInSubQueryThreshold(inSubQueryThreshold).build();
+ this.config = new ConfigBuilder().withConfig(config)
+ .withInSubQueryThreshold(inSubQueryThreshold)
+ .withRemoveDistinct(isRemoveDistinct).build();
this.relBuilder = RelFactories.LOGICAL_BUILDER.create(cluster, null);
}
@@ -2170,7 +2175,7 @@ protected void convertFrom(Blackboard bb, SqlNode from) {
SqlNode tableName = call.operand(0);
SqlNode timeStamp = call.operand(1);
final RexNode timestampExpr = bb.convertExpression(timeStamp);
- this.hintBlackboard.beginAsOf(timestampExpr);
+ this.hintBlackboard.beginAsOf(Pair.of(timestampExpr, call.getOperator()));
SqlIdentifier identifier = (SqlIdentifier) tableName;
convertIdentifier(bb, identifier, null, identifier.indexNode, identifier.partitions);
this.hintBlackboard.endAsOf();
@@ -2606,10 +2611,11 @@ private void convertIdentifier(Blackboard bb, SqlIdentifier id, SqlNodeList exte
} else {
SqlNodeList hint = this.hintBlackboard.currentHints(Util.last(table.getQualifiedName()));
if (this.hintBlackboard.hasAsOf()) {
- tableRel = LogicalTableScan.create(cluster, table, hint, indexNode, this.hintBlackboard.peekAsOf(),
+ Pair asOf = this.hintBlackboard.peekAsOf();
+ tableRel = LogicalTableScan.create(cluster, table, hint, indexNode, asOf.getKey(), asOf.getValue(),
partitions);
} else {
- tableRel = LogicalTableScan.create(cluster, table, hint, indexNode, null, partitions);
+ tableRel = LogicalTableScan.create(cluster, table, hint, indexNode, null, null, partitions);
}
}
@@ -3305,7 +3311,61 @@ protected final void createAggImpl(Blackboard bb, final AggConverter aggConverte
*/
protected RelNode createAggregate(Blackboard bb, ImmutableBitSet groupSet, ImmutableList groupSets,
List aggCalls) {
- return LogicalAggregate.create(bb.root, groupSet, groupSets, aggCalls);
+
+ final List aggregateCalls = new ArrayList<>();
+ Set nullRefs = new HashSet<>();
+ for (AggregateCall aggregateCall : aggCalls) {
+ if (groupSets.size() <= 1 && config.isRemoveDistinct()) {
+ aggregateCall = removeRedundantAggregateDistinct(
+ aggregateCall, groupSet, bb.root, nullRefs);
+ }
+ aggregateCalls.add(aggregateCall);
+ }
+
+ RelNode relNode = bb.root;
+ if (nullRefs.size() > 0) {
+ final List fields = relNode.getRowType().getFieldList();
+ final List isNotNullOperands = new ArrayList<>();
+ for (Integer nullRef : nullRefs) {
+ RelDataType dataType = fields.get(nullRef).getType();
+ if (dataType.isNullable()) {
+ isNotNullOperands.add(relBuilder.isNotNull(rexBuilder.makeInputRef(relNode, nullRef)));
+ }
+ }
+ if (isNotNullOperands.size() > 0) {
+ relNode = LogicalFilter.create(
+ relNode, RexUtil.composeConjunction(rexBuilder, isNotNullOperands, true));
+ }
+ }
+ return LogicalAggregate.create(relNode, groupSet, groupSets, aggregateCalls);
+ }
+
+ private AggregateCall removeRedundantAggregateDistinct(
+ AggregateCall aggregateCall,
+ ImmutableBitSet groupSet,
+ RelNode relNode,
+ Set nullRefs) {
+ final List argList = aggregateCall.getArgList();
+ if (aggregateCall.isDistinct()) {
+ final RelMetadataQuery mq = relNode.getCluster().getMetadataQuery();
+ final ImmutableBitSet distinctArg = ImmutableBitSet.builder()
+ .addAll(argList)
+ .build();
+ final ImmutableBitSet columns = groupSet.union(distinctArg);
+ final Boolean alreadyUnique =
+ mq.areColumnsUnique(relNode, columns);
+ if (alreadyUnique != null && alreadyUnique) {
+ // columns have been distinct or columns are primary keys
+ if (aggregateCall.getAggregation().getKind() == SqlKind.COUNT) {
+ //in order to avoid generate the physical sql "count(a, b, c)" for the singleTable,
+ //and mysql don't support count multi-column.
+ return aggregateCall;
+ } else {
+ return aggregateCall.withDistinct(false);
+ }
+ }
+ }
+ return aggregateCall;
}
public RexDynamicParam convertDynamicParam(final SqlDynamicParam dynamicParam) {
@@ -5063,7 +5123,7 @@ protected List getUpdateSrcTables(SqlUpdate sqlUpdate) {
if (srcNode instanceof SqlIdentifier) {
// Table name
result.add(new TableInfoNode(srcNode, srcWithAlias, ImmutableList.of(getTargetTable(srcNode))));
- } else if (srcNode instanceof SqlSelect) {
+ } else if (srcNode instanceof SqlSelect || RelOptUtil.isUnion(srcNode)) {
// SubQuery
final List tableNames = sqlUpdate.subQueryTables(srcNode);
result.add(new TableInfoNode(srcNode, srcWithAlias,
@@ -5110,7 +5170,7 @@ protected TableInfo getDeleteTargetTables(SqlDelete sqlDelete, Blackboard bb, Re
// Table name
srcTableInfos.add(
new TableModify.TableInfoNode(srcNode, srcWithAlias, ImmutableList.of(getTargetTable(srcNode))));
- } else if (srcNode instanceof SqlSelect) {
+ } else if (srcNode instanceof SqlSelect || RelOptUtil.isUnion(srcNode)) {
// SubQuery
final List tableNames = sqlDelete.subQueryTables(srcNode);
srcTableInfos.add(new TableModify.TableInfoNode(srcNode, srcWithAlias,
@@ -5441,7 +5501,7 @@ private List buildUpdateSourceInfo(SqlUpdate call, Map t instanceof SqlDynamicParam);
RelNode valuesRel;
- if (values instanceof SqlValuesTableSource){
+ if (values instanceof SqlValuesTableSource) {
valuesRel = convertValues(bb, (SqlValuesTableSource) values, targetRowType);
- }else if (allIsDynamic) {
+ } else if (allIsDynamic) {
valuesRel = convertDynamicRowValues(bb, values, values.getOperandList(), true, targetRowType);
- }else {
+ } else {
valuesRel = convertRowValues(bb, values, values.getOperandList(), true, targetRowType);
}
@@ -6011,7 +6071,7 @@ private void convertValuesImpl(Blackboard bb, SqlCall values, RelDataType target
// ?
}
- private DynamicValues convertValues(Blackboard bb, SqlValuesTableSource values, RelDataType targetRowType){
+ private DynamicValues convertValues(Blackboard bb, SqlValuesTableSource values, RelDataType targetRowType) {
if (targetRowType == null) {
targetRowType = SqlTypeUtil.promoteToRowType(typeFactory, validator.getValidatedNodeType(values), null);
}
@@ -6024,14 +6084,13 @@ private DynamicValues convertValues(Blackboard bb, SqlValuesTableSource values,
}
tupleList.add(tuple.build());
}
- DynamicValues dynamicValues = DynamicValues.create(
+ DynamicValues dynamicValues = DynamicValues.create(
cluster,
cluster.traitSet().replace(RelDistributions.SINGLETON), //for mpp
targetRowType, tupleList.build());
return dynamicValues;
}
-
// ~ Inner Classes
// ----------------------------------------------------------
@@ -7739,6 +7798,8 @@ public interface Config {
* usage of OR in all cases.
*/
int getInSubQueryThreshold();
+
+ boolean isRemoveDistinct();
}
/**
@@ -7753,6 +7814,7 @@ public static class ConfigBuilder {
private boolean explain;
private boolean expand = true;
private int inSubQueryThreshold = DEFAULT_IN_SUB_QUERY_THRESHOLD;
+ private boolean removeDistinct = true;
private ConfigBuilder() {
}
@@ -7768,6 +7830,7 @@ public ConfigBuilder withConfig(Config config) {
this.explain = config.isExplain();
this.expand = config.isExpand();
this.inSubQueryThreshold = config.getInSubQueryThreshold();
+ this.removeDistinct = config.isRemoveDistinct();
return this;
}
@@ -7811,12 +7874,17 @@ public ConfigBuilder withInSubQueryThreshold(int inSubQueryThreshold) {
return this;
}
+ public ConfigBuilder withRemoveDistinct(boolean removeDistinct) {
+ this.removeDistinct = removeDistinct;
+ return this;
+ }
+
/**
* Builds a {@link Config}.
*/
public Config build() {
return new ConfigImpl(convertTableAccess, decorrelationEnabled, trimUnusedFields, createValuesRel, explain,
- expand, inSubQueryThreshold);
+ expand, inSubQueryThreshold, removeDistinct);
}
}
@@ -7833,9 +7901,11 @@ private static class ConfigImpl implements Config {
private final boolean explain;
private final int inSubQueryThreshold;
private final boolean expand;
+ private final boolean removeDistinct;
private ConfigImpl(boolean convertTableAccess, boolean decorrelationEnabled, boolean trimUnusedFields,
- boolean createValuesRel, boolean explain, boolean expand, int inSubQueryThreshold) {
+ boolean createValuesRel, boolean explain, boolean expand, int inSubQueryThreshold,
+ boolean removeDistinct) {
this.convertTableAccess = convertTableAccess;
this.decorrelationEnabled = decorrelationEnabled;
this.trimUnusedFields = trimUnusedFields;
@@ -7843,6 +7913,7 @@ private ConfigImpl(boolean convertTableAccess, boolean decorrelationEnabled, boo
this.explain = explain;
this.expand = expand;
this.inSubQueryThreshold = inSubQueryThreshold;
+ this.removeDistinct = removeDistinct;
}
public boolean isConvertTableAccess() {
@@ -7872,6 +7943,10 @@ public boolean isExpand() {
public int getInSubQueryThreshold() {
return inSubQueryThreshold;
}
+
+ public boolean isRemoveDistinct() {
+ return removeDistinct;
+ }
}
private static class AliasContext {
@@ -7903,7 +7978,7 @@ public static class HintBlackboard {
private final Deque aliasStack = new ArrayDeque<>();
- private final Deque asOfStack = new ArrayDeque<>();
+ private final Deque> asOfStack = new ArrayDeque<>();
public void beginSelect() {
hintStack.push(new HashMap<>(2));
@@ -7925,11 +8000,11 @@ public Map currentGroups() {
return hintStack.peek();
}
- public void beginAsOf(RexNode flashback) {
+ public void beginAsOf(Pair flashback) {
asOfStack.push(flashback);
}
- public RexNode endAsOf() {
+ public Pair endAsOf() {
return this.asOfStack.pop();
}
@@ -7937,7 +8012,7 @@ public boolean hasAsOf() {
return !asOfStack.isEmpty();
}
- public RexNode peekAsOf() {
+ public Pair peekAsOf() {
return this.asOfStack.peek();
}
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/tools/RelBuilder.java b/polardbx-calcite/src/main/java/org/apache/calcite/tools/RelBuilder.java
index ae8f5aab1..6a8bcb165 100644
--- a/polardbx-calcite/src/main/java/org/apache/calcite/tools/RelBuilder.java
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/tools/RelBuilder.java
@@ -16,6 +16,14 @@
*/
package org.apache.calcite.tools;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
import org.apache.calcite.linq4j.Ord;
import org.apache.calcite.plan.Context;
import org.apache.calcite.plan.Contexts;
@@ -28,14 +36,33 @@
import org.apache.calcite.rel.RelCollations;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.*;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.CorrelationId;
+import org.apache.calcite.rel.core.JoinInfo;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.core.TableModify;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.core.Values;
import org.apache.calcite.rel.logical.LogicalSemiJoin;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rel.type.RelDataTypeFieldImpl;
-import org.apache.calcite.rex.*;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexCorrelVariable;
+import org.apache.calcite.rex.RexExecutor;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexSimplify;
+import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.runtime.Hook;
import org.apache.calcite.schema.SchemaPlus;
import org.apache.calcite.server.CalciteServerStatement;
@@ -52,20 +79,12 @@
import org.apache.calcite.util.ImmutableIntList;
import org.apache.calcite.util.Litmus;
import org.apache.calcite.util.NlsString;
+import org.apache.calcite.util.Optionality;
import org.apache.calcite.util.Pair;
import org.apache.calcite.util.Util;
import org.apache.calcite.util.mapping.Mapping;
import org.apache.calcite.util.mapping.Mappings;
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-
import javax.annotation.Nonnull;
import java.math.BigDecimal;
import java.util.AbstractList;
@@ -910,13 +929,14 @@ public RelBuilder scan(Iterable tableNames) {
return this;
}
- public RelBuilder flashback(RexNode flashback) {
+ public RelBuilder flashback(RexNode flashback, SqlOperator flashbackOperator) {
if (stack.peek() == null) {
return this;
}
RelNode last = stack.peek().rel;
- if (last instanceof TableScan){
- ((TableScan)last).setFlashback(flashback);
+ if (last instanceof TableScan) {
+ ((TableScan) last).setFlashback(flashback);
+ ((TableScan) last).setFlashbackOperator(flashbackOperator);
}
return this;
}
@@ -2146,7 +2166,10 @@ private static class AggCallImpl implements AggCall {
boolean approximate, RexNode filter,
String alias, ImmutableList operands) {
this.aggFunction = aggFunction;
- this.distinct = distinct;
+ // If the aggregate function ignores DISTINCT,
+ // make the DISTINCT flag FALSE.
+ this.distinct = distinct
+ && aggFunction.getDistinctOptionality() != Optionality.IGNORED;
this.approximate = approximate;
this.filter = filter;
this.alias = alias;
diff --git a/polardbx-calcite/src/main/java/org/apache/calcite/util/Optionality.java b/polardbx-calcite/src/main/java/org/apache/calcite/util/Optionality.java
new file mode 100644
index 000000000..2e34dbd2e
--- /dev/null
+++ b/polardbx-calcite/src/main/java/org/apache/calcite/util/Optionality.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.util;
+
+/**
+ * Four states that describe whether a particular behavior or
+ * property is allowed and/or not allowed.
+ */
+public enum Optionality {
+ /** A property is mandatory if an instance must possess it;
+ * it is an error if it does not. */
+ MANDATORY,
+
+ /** A property is optional if an instance may or may not possess it;
+ * neither state is an error. */
+ OPTIONAL,
+
+ /** A property is ignored if an instance may or may not possess it;
+ * if it possesses the property, the effect is as if it does not. */
+ IGNORED,
+
+ /** A property is forbidden if an instance must not possess it;
+ * it is an error if the instance has the property. */
+ FORBIDDEN
+}
diff --git a/polardbx-calcite/src/test/java/org/apache/calcite/sql/test/BuildForeignKeyNameTest.java b/polardbx-calcite/src/test/java/org/apache/calcite/sql/test/BuildForeignKeyNameTest.java
new file mode 100644
index 000000000..ea4dffee5
--- /dev/null
+++ b/polardbx-calcite/src/test/java/org/apache/calcite/sql/test/BuildForeignKeyNameTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the License);
+ * you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.calcite.sql.test;
+
+import org.apache.calcite.sql.SqlCreateTable;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashSet;
+
+public class BuildForeignKeyNameTest {
+
+ @Test
+ public void testBuildForeignKeyName() {
+ String fkName1 = "tab_ibfk_1";
+ String fkName2 = " tab_ibfk_2";
+ String fkName3 = " a tab_ibfk_3";
+
+ Assert.assertEquals("`tab_ibfk_1`", SqlCreateTable.buildForeignKeyName(fkName1, new HashSet<>(),""));
+ Assert.assertEquals("` tab_ibfk_2`", SqlCreateTable.buildForeignKeyName(fkName2, new HashSet<>(),""));
+ Assert.assertEquals("` a tab_ibfk_3`", SqlCreateTable.buildForeignKeyName(fkName3, new HashSet<>(),""));
+ }
+}
diff --git a/polardbx-calcite/src/test/java/org/apache/calcite/sql/test/SqlTypeNameTest.java b/polardbx-calcite/src/test/java/org/apache/calcite/sql/test/SqlTypeNameTest.java
index b8f624390..262012405 100644
--- a/polardbx-calcite/src/test/java/org/apache/calcite/sql/test/SqlTypeNameTest.java
+++ b/polardbx-calcite/src/test/java/org/apache/calcite/sql/test/SqlTypeNameTest.java
@@ -364,6 +364,20 @@ public class SqlTypeNameTest {
null,
tn);
}
+
+ @Test public void testSigned() {
+ SqlTypeName tn = SqlTypeName.get("SIGNED");
+ assertEquals(SqlTypeName.SIGNED, tn);
+
+ tn = SqlTypeName.get("SIGNED INTEGER");
+ assertEquals(SqlTypeName.SIGNED, tn);
+
+ tn = SqlTypeName.get("UNSIGNED");
+ assertEquals(SqlTypeName.UNSIGNED, tn);
+
+ tn = SqlTypeName.get("UNSIGNED INTEGER");
+ assertEquals(SqlTypeName.UNSIGNED, tn);
+ }
}
// End SqlTypeNameTest.java
diff --git a/polardbx-calcite/src/test/java/org/apache/calcite/sql/validate/ColumnarForbidTest.java b/polardbx-calcite/src/test/java/org/apache/calcite/sql/validate/ColumnarForbidTest.java
new file mode 100644
index 000000000..53a12bbc1
--- /dev/null
+++ b/polardbx-calcite/src/test/java/org/apache/calcite/sql/validate/ColumnarForbidTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the License);
+ * you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.calcite.sql.validate;
+
+import com.alibaba.polardbx.common.exception.TddlRuntimeException;
+import org.apache.calcite.sql.SqlColumnDeclaration;
+import org.apache.calcite.sql.SqlDataTypeSpec;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.junit.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ColumnarForbidTest {
+
+ @Test(expected = TddlRuntimeException.class)
+ public void ForbidCitableModifyUnsupportedType() {
+ SqlIdentifier sqlIdentifier = mock(SqlIdentifier.class);
+ SqlColumnDeclaration sqlColumnDeclaration = mock(SqlColumnDeclaration.class);
+ SqlDataTypeSpec sqlDataTypeSpec = mock(SqlDataTypeSpec.class);
+
+ when(sqlColumnDeclaration.getDataType()).thenReturn(sqlDataTypeSpec);
+ when(sqlDataTypeSpec.getTypeName()).thenReturn(sqlIdentifier);
+ when(sqlIdentifier.getLastName()).thenReturn("binary");
+
+ SqlValidatorImpl.validateUnsupportedTypeWithCciWhenModifyColumn(sqlColumnDeclaration);
+ }
+
+ @Test
+ public void ForbidCitableModifySupportedType() {
+ SqlIdentifier sqlIdentifier = mock(SqlIdentifier.class);
+ SqlColumnDeclaration sqlColumnDeclaration = mock(SqlColumnDeclaration.class);
+ SqlDataTypeSpec sqlDataTypeSpec = mock(SqlDataTypeSpec.class);
+
+ when(sqlColumnDeclaration.getDataType()).thenReturn(sqlDataTypeSpec);
+ when(sqlDataTypeSpec.getTypeName()).thenReturn(sqlIdentifier);
+ when(sqlIdentifier.getLastName()).thenReturn("char");
+
+ SqlValidatorImpl.validateUnsupportedTypeWithCciWhenModifyColumn(sqlColumnDeclaration);
+ }
+}
diff --git a/polardbx-common/pom.xml b/polardbx-common/pom.xml
index aafe2b2f4..9910129de 100644
--- a/polardbx-common/pom.xml
+++ b/polardbx-common/pom.xml
@@ -135,7 +135,7 @@
org.apache.hadoop
hadoop-client
- ${hadoop-client.version}
+ ${hadoop.version}
commons-logging
@@ -187,6 +187,11 @@
com.github.ben-manes.caffeine
caffeine
+
+ org.mockito
+ mockito-inline
+ test
+
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/Engine.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/Engine.java
index 972bf6d67..21639977a 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/Engine.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/Engine.java
@@ -32,6 +32,7 @@ public enum Engine {
LOCAL_DISK,
EXTERNAL_DISK,
S3,
+ ABS,
OSS,
NFS,
MEMORY;
@@ -57,6 +58,8 @@ public static boolean hasCache(Engine engine) {
case OSS:
case EXTERNAL_DISK:
case NFS:
+ case S3:
+ case ABS:
return true;
default:
return false;
@@ -73,6 +76,7 @@ public static boolean isFileStore(Engine engine) {
case EXTERNAL_DISK:
case S3:
case NFS:
+ case ABS:
return true;
default:
return false;
@@ -88,6 +92,8 @@ public static boolean supportColumnar(Engine engine) {
case LOCAL_DISK:
case EXTERNAL_DISK:
case NFS:
+ case S3:
+ case ABS:
return true;
default:
return false;
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/TddlConstants.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/TddlConstants.java
index 15a6d982f..99ca31c9a 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/TddlConstants.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/TddlConstants.java
@@ -84,4 +84,8 @@ public class TddlConstants {
public static final String FOREIGN_KEY_PREFIX = "";
public static final int LONG_ENOUGH_TIMEOUT_FOR_DDL_ON_XPROTO_CONN = 7 * 24 * 60 * 60 * 1000;
+
+ public static final String BLACK_LIST_CONF = "BLACK_LIST_CONF";
+
+ public static final String ENABLE_JAVA_UDF = "ENABLE_JAVA_UDF";
}
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/charset/CollationName.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/charset/CollationName.java
index 5cafe0c41..ae9e8574d 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/charset/CollationName.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/charset/CollationName.java
@@ -317,7 +317,10 @@ public enum CollationName {
// for utf8
UTF8_GENERAL_CI, UTF8_BIN, UTF8_UNICODE_CI, UTF8_GENERAL_MYSQL500_CI,
- // for utf8mb
+ // for utf8mb4 in MySQL 8.0
+ UTF8MB4_ZH_0900_AS_CS,
+
+ // for utf8mb4
UTF8MB4_GENERAL_CI, UTF8MB4_BIN, UTF8MB4_UNICODE_CI,
// for utf16
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/charset/MySQLUnicodeUtils.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/charset/MySQLUnicodeUtils.java
index 69122884e..13955b5f5 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/charset/MySQLUnicodeUtils.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/charset/MySQLUnicodeUtils.java
@@ -16,14 +16,10 @@
package com.alibaba.polardbx.common.charset;
-import com.alibaba.polardbx.common.utils.GeneralUtil;
import io.airlift.slice.DynamicSliceOutput;
import io.airlift.slice.Slice;
import io.airlift.slice.SliceOutput;
-import java.nio.charset.Charset;
-import java.util.Optional;
-
public class MySQLUnicodeUtils {
public static byte[][] LATIN1_TO_UTF8_BYTES = {
{(byte) 0x00}, {(byte) 0x01}, {(byte) 0x02}, {(byte) 0x03}, {(byte) 0x04}, {(byte) 0x05}, {(byte) 0x06},
@@ -114,4 +110,25 @@ public static boolean utf8ToLatin1(byte[] buff, int begin, int end, byte[] res)
}
return isUtf8FromLatin1;
}
+
+ public static boolean utf8ToLatin1(Slice slice, byte[] res, int len) {
+ int pos = 0, begin = 0, end = slice.length();
+ boolean isUtf8FromLatin1 = true;
+ while (begin < end && pos < len) {
+ int uc1 = ((int) slice.getByte(begin++)) & 0xFF;
+ // 0xxxxxxx
+ if (uc1 < 0x80) {
+ res[pos++] = (byte) uc1;
+ } else if (begin < end) {
+ if (uc1 != 0xC2 && uc1 != 0xC3) {
+ isUtf8FromLatin1 = false;
+ }
+ int uc2 = ((int) slice.getByte(begin++)) & 0xFF;
+ res[pos++] = (byte) (((uc1 & 0x1f) << 6) | (uc2 ^ 0x80));
+ } else {
+ res[pos++] = (byte) 0xFF;
+ }
+ }
+ return isUtf8FromLatin1;
+ }
}
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/constants/ServerVariables.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/constants/ServerVariables.java
index 1472b9098..426149b6b 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/constants/ServerVariables.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/constants/ServerVariables.java
@@ -18,6 +18,7 @@
import com.alibaba.polardbx.common.cdc.CdcConstants;
import com.alibaba.polardbx.common.properties.ConnectionProperties;
+import com.alibaba.polardbx.common.properties.DynamicConfig;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.lang.StringUtils;
@@ -3667,6 +3668,10 @@ public static boolean isGlobalBanned(String variable) {
return globalBannedVariables.contains(variable.toLowerCase());
}
+ public static boolean isGlobalBlackList(String variable) {
+ return DynamicConfig.getInstance().getBlacklistConf().contains(variable.toLowerCase());
+ }
+
public static boolean isCdcGlobal(String variable) {
return StringUtils.startsWith(variable.toLowerCase(), CdcConstants.CONFIG_KEY_PREFIX);
}
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/ddl/newengine/DdlPlanState.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/ddl/newengine/DdlPlanState.java
index 0c2ff7381..125b822fb 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/ddl/newengine/DdlPlanState.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/ddl/newengine/DdlPlanState.java
@@ -21,7 +21,7 @@ public enum DdlPlanState {
INIT,
EXECUTING,
SUCCESS,
- TERMINATED
- ;
+ TERMINATED,
+ PAUSE_ON_NON_MAINTENANCE_WINDOW;
}
\ No newline at end of file
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/exception/code/ErrorCode.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/exception/code/ErrorCode.java
index aeaef6c29..d3512268e 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/exception/code/ErrorCode.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/exception/code/ErrorCode.java
@@ -708,6 +708,8 @@ public enum ErrorCode {
ERR_STATISTIC_JOB_INTERRUPTED(ErrorType.Optimizer, 4531),
+ ERR_IN_PRUNING(ErrorType.Optimizer, 4532),
+
// ============= executor 从4600下标开始================
//
ERR_FUNCTION(ErrorType.Executor, 4600),
@@ -1244,6 +1246,7 @@ public enum ErrorCode {
// ================= concurrency control Related Exceptions ===================
ERR_CCL(ErrorType.CCL, 9201),
+ ERR_CCL_RESCHEDULE(ErrorType.CCL, 9202),
ERR_LOGICAL_TABLE_UNSUPPORTED(ErrorType.Executor, 9203),
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/jdbc/ITransactionPolicy.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/jdbc/ITransactionPolicy.java
index 2b45c4b54..efa20b327 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/jdbc/ITransactionPolicy.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/jdbc/ITransactionPolicy.java
@@ -68,7 +68,9 @@ enum TransactionClass {
AUTO_COMMIT,
- AUTO_COMMIT_TSO;
+ AUTO_COMMIT_TSO,
+
+ ARCHIVE;
public boolean isA(EnumSet set) {
return set.contains(this);
@@ -80,14 +82,17 @@ public boolean isA(EnumSet set) {
TransactionClass.TSO,
TransactionClass.TSO_READONLY,
TransactionClass.AUTO_COMMIT_SINGLE_SHARD,
- TSO_2PC_OPT);
+ TSO_2PC_OPT,
+ TransactionClass.ARCHIVE);
public static final EnumSet EXPLICIT_TRANSACTION = EnumSet
.of(TransactionClass.XA,
TransactionClass.XA_TSO,
TransactionClass.TSO,
TransactionClass.ALLOW_READ_CROSS_DB,
- TSO_2PC_OPT);
+ TransactionClass.COBAR_STYLE,
+ TSO_2PC_OPT,
+ TransactionClass.ARCHIVE);
public static final EnumSet TSO_TRANSACTION = EnumSet
.of(TransactionClass.TSO,
@@ -110,7 +115,8 @@ public boolean isA(EnumSet set) {
.of(TransactionClass.XA,
TransactionClass.XA_TSO,
TransactionClass.TSO,
- TSO_2PC_OPT);
+ TSO_2PC_OPT,
+ TransactionClass.ARCHIVE);
public static final EnumSet SUPPORT_PARALLEL_GET_CONNECTION_TRANSACTION = EnumSet
.of(TransactionClass.XA,
@@ -119,7 +125,8 @@ public boolean isA(EnumSet set) {
TransactionClass.AUTO_COMMIT,
TransactionClass.AUTO_COMMIT_SINGLE_SHARD,
TransactionClass.AUTO_COMMIT_TSO,
- TransactionClass.TSO_READONLY);
+ TransactionClass.TSO_READONLY,
+ TransactionClass.ARCHIVE);
public static final EnumSet ALLOW_GROUP_PARALLELISM_WITHOUT_SHARE_READVIEW_TRANSACTION =
EnumSet.of(TransactionClass.AUTO_COMMIT,
@@ -131,6 +138,7 @@ public boolean isA(EnumSet set) {
NoTransaction NO_TRANSACTION = new NoTransaction();
DefaultPolicy XA = new DefaultPolicy(TransactionClass.XA);
Tso TSO = new Tso();
+ DefaultPolicy ARCHIVE = new DefaultPolicy(TransactionClass.ARCHIVE);
TransactionClass getTransactionType(boolean isAutoCommit, boolean isReadOnly);
@@ -259,6 +267,8 @@ static ITransactionPolicy of(String name) {
return ITransactionPolicy.ALLOW_READ_CROSS_DB;
case "NO_TRANSACTION":
return ITransactionPolicy.NO_TRANSACTION;
+ case "ARCHIVE":
+ return ITransactionPolicy.ARCHIVE;
default:
throw new TddlRuntimeException(ErrorCode.ERR_CONFIG, "Unknown transaction policy: " + name);
}
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/jdbc/PruneRawString.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/jdbc/PruneRawString.java
index 57b08b7ae..60264b93a 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/jdbc/PruneRawString.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/jdbc/PruneRawString.java
@@ -17,13 +17,10 @@
package com.alibaba.polardbx.common.jdbc;
import com.alibaba.polardbx.common.utils.GeneralUtil;
-import com.alibaba.polardbx.common.utils.TStringUtil;
import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
import java.util.BitSet;
import java.util.List;
-import java.util.Set;
/**
* @author fangwu
@@ -88,11 +85,16 @@ public Object getObj(int index, int skIndex) {
@Override
public String display() {
- String rs = "PruneRaw(" + buildRawString() + ")";
- if (rs.length() > 4096) {
- return rs.substring(0, 4096) + "...";
+ StringBuilder stringBuilder = new StringBuilder();
+ if (size() == super.getObjList().size()) {
+ stringBuilder.append("NonPruneRaw(" + buildRawString() + ")");
+ } else {
+ stringBuilder.append("PruneRaw(" + buildRawString() + ")");
}
- return rs;
+ if (stringBuilder.length() > 4096) {
+ return stringBuilder.substring(0, 4096) + "...";
+ }
+ return stringBuilder.toString();
}
@Override
@@ -195,12 +197,16 @@ private void paramCheck() {
if (indexes == null || indexes.isEmpty()) {
GeneralUtil.nestedException("RawString init ERROR MUITI_INDEX mode with invalid indexes:" + indexes);
}
+ return;
default:
GeneralUtil.nestedException("RawString init ERROR mode invalid :" + pruneMode);
}
}
public void merge(PruneRawString pruneRawString) {
+ if (size() == super.getObjList().size()) {
+ return;
+ }
if (pruneMode == PRUNE_MODE.RANGE) {
transformModeToMultiIndex();
}
@@ -208,6 +214,11 @@ public void merge(PruneRawString pruneRawString) {
pruneRawString.transformModeToMultiIndex();
}
indexes.or(pruneRawString.indexes);
+ if (indexes.cardinality() == super.getObjList().size()) {
+ pruneMode = PRUNE_MODE.RANGE;
+ startIndex = 0;
+ endIndex = super.getObjList().size();
+ }
}
private void transformModeToMultiIndex() {
@@ -230,4 +241,29 @@ private void transformModeToMultiIndex() {
public enum PRUNE_MODE {
RANGE, MULTI_INDEX;
}
+
+ /**
+ * do the same thing as RawString.pruneStep, but return itself.
+ * WARNING: this method will change this PruneRawString itself.
+ */
+ @Override
+ public PruneRawString pruneStep(int curIndex) {
+ pruneMode = PruneRawString.PRUNE_MODE.RANGE;
+ startIndex = curIndex;
+ endIndex = curIndex + 1;
+ return this;
+ }
+
+ @Override
+ public PruneRawString clone() {
+ if (pruneMode == PRUNE_MODE.RANGE) {
+ return new PruneRawString(super.getObjList(), pruneMode, startIndex, endIndex, null);
+ } else {
+ return new PruneRawString(super.getObjList(), pruneMode, -1, -1, (BitSet) indexes.clone());
+ }
+ }
+
+ public int getSourceSize() {
+ return super.size();
+ }
}
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/jdbc/RawString.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/jdbc/RawString.java
index 86db3f3a2..3887908ca 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/jdbc/RawString.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/jdbc/RawString.java
@@ -210,4 +210,15 @@ public Object acquireObject(int subIndex, int skIndex) {
return getObj(subIndex, skIndex);
}
}
+
+ /**
+ * build one PruneRawString with one object(target by curIndex) inside this RawString
+ */
+ public PruneRawString pruneStep(int curIndex) {
+ return new PruneRawString(this.getObjList(), PruneRawString.PRUNE_MODE.RANGE, curIndex, curIndex + 1, null);
+ }
+
+ public RawString clone() {
+ return new RawString(this.getObjList());
+ }
}
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/jdbc/TableName.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/jdbc/TableName.java
index 314f9a7f3..d0f8c5165 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/jdbc/TableName.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/jdbc/TableName.java
@@ -36,11 +36,6 @@ public String getTableName() {
return this.tableName;
}
- public void setTableName(String tableName) throws SQLException {
- this.checkName(tableName);
- this.tableName = tableName.trim();
- }
-
private void checkName(String tName) throws SQLException {
if (StringUtils.isEmptyOrWhitespaceOnly(tName)) {
throw new SQLException("tableName should not be empty", "S1009");
@@ -63,9 +58,6 @@ private void checkName(String tName) throws SQLException {
case '\u001a':
needsHexEscape = true;
break;
- case ' ':
- needsHexEscape = true;
- break;
case '"':
needsHexEscape = true;
break;
@@ -80,7 +72,6 @@ private void checkName(String tName) throws SQLException {
throw new SQLException("tableName format error: " + this.tableName, "S1009");
}
}
-
}
}
@@ -89,4 +80,3 @@ public String toString() {
return getTableName();
}
}
-
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/mock/MockUtils.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/mock/MockUtils.java
new file mode 100644
index 000000000..f62186de7
--- /dev/null
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/mock/MockUtils.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright [2013-2021], Alibaba Group Holding Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.polardbx.common.mock;
+
+import java.lang.reflect.Field;
+
+public class MockUtils {
+
+ public interface ThrowableRunnable {
+ void run() throws Throwable;
+ }
+
+ public interface ThrowableConsumer {
+ void accept(T t) throws Throwable;
+ }
+
+ public static T assertThrows(Class expectedType,
+ String expectedMessage,
+ ThrowableRunnable methodCall) {
+ try {
+ methodCall.run();
+ } catch (Throwable e) {
+ if (expectedType.isInstance(e)) {
+ if (expectedMessage != null) {
+ if (!expectedMessage.equals(e.getMessage())) {
+ throw new AssertionError(
+ "Expected message: " + expectedMessage + ", actual: " + e.getMessage()
+ );
+ }
+ }
+ return (T) e;
+ }
+ throw new AssertionError("Expected exception: " + expectedType.getName(), e);
+ }
+ throw new AssertionError("Expected exception: " + expectedType.getName());
+ }
+
+ public static void setInternalState(Object target, String fieldName, Object value) {
+ try {
+ Field field = target.getClass().getDeclaredField(fieldName);
+ field.setAccessible(true);
+ field.set(target, value);
+ } catch (NoSuchFieldException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/Constants.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/Constants.java
index ccf4358a4..df9921746 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/Constants.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/Constants.java
@@ -90,7 +90,9 @@ private Constants() {
public static final long MULTIPART_UPLOAD_PART_SIZE_DEFAULT =
104857600; // 100 MB
- /** The minimum multipart size which OSS supports. */
+ /**
+ * The minimum multipart size which OSS supports.
+ */
public static final int MULTIPART_MIN_SIZE = 100 * 1024;
public static final int MULTIPART_UPLOAD_PART_NUM_LIMIT = 10000;
@@ -155,4 +157,16 @@ private Constants() {
public static final String UPLOAD_ACTIVE_BLOCKS_KEY =
"fs.oss.upload.active.blocks";
public static final int UPLOAD_ACTIVE_BLOCKS_DEFAULT = 4;
+
+ // S3 access verification
+ public static final String S3_ACCESS_KEY = "fs.s3a.access.key";
+ public static final String S3_SECRET_KEY = "fs.s3a.secret.key";
+
+ // ABS access verification
+ // fs.azure.account.key..blob.
+ public static final String ABS_URI_SUFFIX_PATTERN = "%s.blob.%s";
+ public static final String ABS_ACCOUNT_KEY_PATTERN = "fs.azure.account.key." + ABS_URI_SUFFIX_PATTERN;
+
+ public static final String AZURE_WASBS_SCHEME = "wasbs";
+ public static final String AZURE_WASB_SCHEME = "wasb";
}
\ No newline at end of file
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/InputStreamWithRateLimiter.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/InputStreamWithRateLimiter.java
new file mode 100644
index 000000000..e03a8e772
--- /dev/null
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/InputStreamWithRateLimiter.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright [2013-2021], Alibaba Group Holding Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.polardbx.common.oss.filesystem;
+
+import org.apache.hadoop.fs.PositionedReadable;
+import org.apache.hadoop.fs.Seekable;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class InputStreamWithRateLimiter extends InputStream implements Seekable, PositionedReadable {
+ final private FileSystemRateLimiter rateLimiter;
+ final private InputStream in;
+
+ public InputStreamWithRateLimiter(InputStream in, FileSystemRateLimiter rateLimiter) {
+ if (!(in instanceof Seekable) || !(in instanceof PositionedReadable)) {
+ throw new IllegalArgumentException(
+ "In is not an instance of Seekable or PositionedReadable");
+ }
+ this.in = in;
+ this.rateLimiter = rateLimiter;
+ }
+
+ @Override
+ public synchronized int read() throws IOException {
+ rateLimiter.acquireRead(1);
+ return in.read();
+ }
+
+ @Override
+ public synchronized int read(byte @NotNull [] buf, int off, int len) throws IOException {
+ rateLimiter.acquireRead(len);
+ return in.read(buf, off, len);
+ }
+
+ @Override
+ public synchronized int read(byte @NotNull [] b) throws IOException {
+ rateLimiter.acquireRead(b.length);
+ return in.read(b);
+ }
+
+ @Override
+ public synchronized long skip(long n) throws IOException {
+ // TODO(siyun): should this be tracked by rate limiter?
+ return in.skip(n);
+ }
+
+ @Override
+ public synchronized int available() throws IOException {
+ return in.available();
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ in.close();
+ }
+
+ @Override
+ public synchronized void mark(int readlimit) {
+ in.mark(readlimit);
+ }
+
+ @Override
+ public synchronized void reset() throws IOException {
+ in.reset();
+ }
+
+ @Override
+ public synchronized boolean markSupported() {
+ return in.markSupported();
+ }
+
+ @Override
+ public int read(long position, byte[] buffer, int offset, int length) throws IOException {
+ rateLimiter.acquireRead(length);
+ return ((PositionedReadable) in).read(position, buffer, offset, length);
+ }
+
+ @Override
+ public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
+ rateLimiter.acquireRead(length);
+ ((PositionedReadable) in).readFully(position, buffer, offset, length);
+ }
+
+ @Override
+ public void readFully(long position, byte[] buffer) throws IOException {
+ rateLimiter.acquireRead(buffer.length);
+ ((PositionedReadable) in).readFully(position, buffer);
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ ((Seekable) in).seek(pos);
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return ((Seekable) in).getPos();
+ }
+
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ return ((Seekable) in).seekToNewSource(targetPos);
+ }
+
+ public InputStream getWrappedStream() {
+ return in;
+ }
+}
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/NFSFileSystem.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/NFSFileSystem.java
index 1a9af7af0..4f4a7cd3c 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/NFSFileSystem.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/NFSFileSystem.java
@@ -47,7 +47,7 @@
import static java.util.concurrent.TimeUnit.SECONDS;
-public class NFSFileSystem extends FileSystem {
+public class NFSFileSystem extends FileSystem implements RateLimitable {
private static final Logger LOG =
LoggerFactory.getLogger(NFSFileSystem.class);
@@ -380,6 +380,7 @@ public FileStatus getFileStatusImpl(Path path) throws IOException {
}
}
+ @Override
public FileSystemRateLimiter getRateLimiter() {
return this.rateLimiter;
}
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/OSSFileSystem.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/OSSFileSystem.java
index f250c08fb..3289c7e40 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/OSSFileSystem.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/OSSFileSystem.java
@@ -74,7 +74,7 @@
/**
* To access OSS blob system in a filesystem style.
*/
-public class OSSFileSystem extends FileSystem {
+public class OSSFileSystem extends FileSystem implements RateLimitable {
private static final Logger LOG =
LoggerFactory.getLogger(OSSFileSystem.class);
private URI uri;
@@ -348,6 +348,7 @@ public FileStatus getFileStatusImpl(Path path) throws IOException {
}
}
+ @Override
public FileSystemRateLimiter getRateLimiter() {
return this.rateLimiter;
}
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/OutputStreamWithRateLimiter.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/OutputStreamWithRateLimiter.java
new file mode 100644
index 000000000..89f31b0c0
--- /dev/null
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/OutputStreamWithRateLimiter.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright [2013-2021], Alibaba Group Holding Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.polardbx.common.oss.filesystem;
+
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class OutputStreamWithRateLimiter extends OutputStream {
+ final private FileSystemRateLimiter rateLimiter;
+ final private OutputStream out;
+
+ public OutputStreamWithRateLimiter(OutputStream out, FileSystemRateLimiter rateLimiter) {
+ this.out = out;
+ this.rateLimiter = rateLimiter;
+ }
+
+ @Override
+ public synchronized void write(byte @NotNull [] b) throws IOException {
+ rateLimiter.acquireWrite(b.length);
+ out.write(b);
+ }
+
+ @Override
+ public synchronized void write(byte @NotNull [] b, int off, int len) throws IOException {
+ rateLimiter.acquireWrite(len);
+ out.write(b, off, len);
+ }
+
+ @Override
+ public synchronized void flush() throws IOException {
+ out.flush();
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ out.close();
+ }
+
+ @Override
+ public synchronized void write(int b) throws IOException {
+ rateLimiter.acquireWrite(1);
+ out.write(b);
+ }
+
+ public OutputStream getWrappedStream() {
+ return out;
+ }
+}
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/RateLimitable.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/RateLimitable.java
new file mode 100644
index 000000000..907c940f5
--- /dev/null
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/oss/filesystem/RateLimitable.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright [2013-2021], Alibaba Group Holding Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.polardbx.common.oss.filesystem;
+
+public interface RateLimitable {
+ FileSystemRateLimiter getRateLimiter();
+}
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/partition/MurmurHashUtils.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/partition/MurmurHashUtils.java
index ca0c5142e..9f8882e28 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/partition/MurmurHashUtils.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/partition/MurmurHashUtils.java
@@ -33,20 +33,32 @@ private static long murmurHash3_128(byte[] data, int seed) {
return longVal;
}
- private static final HashFunction zeroSeedMurmur3hashFunc = Hashing.murmur3_128(0);
+ private static int murmurHash3_32(byte[] data) {
+ HashCode hashCode = zeroSeedMurmur3hashFunc32.hashBytes(data);
+ int intVal = hashCode.asInt();
+ return intVal;
+ }
+
+ private static final HashFunction zeroSeedMurmur3hashFunc32 = Hashing.murmur3_32(0);
+
+ private static final HashFunction zeroSeedMurmur3hashFunc128 = Hashing.murmur3_128(0);
private static long murmurHash3_128(long data) {
- HashCode hashCode = zeroSeedMurmur3hashFunc.hashLong(data);
+ HashCode hashCode = zeroSeedMurmur3hashFunc128.hashLong(data);
long longVal = hashCode.asLong();
return longVal;
}
- public static long murmurHashWithZeroSeed(byte[] data) {
+ public static long murmurHash128WithZeroSeed(long data) {
+ return murmurHash3_128(data);
+ }
+
+ public static long murmurHash128WithZeroSeed(byte[] data) {
return murmurHash3_128(data, 0);
}
- public static long murmurHashWithZeroSeed(long data) {
- return murmurHash3_128(data);
+ public static int murmurHash32WithZeroSeed(byte[] data) {
+ return murmurHash3_32(data);
}
}
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/properties/ConnectionParams.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/properties/ConnectionParams.java
index ca33c6622..64ad01c15 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/properties/ConnectionParams.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/properties/ConnectionParams.java
@@ -360,7 +360,7 @@ public class ConnectionParams {
new IntConfigParam(ConnectionProperties.PHYSICAL_DDL_MDL_WAITING_TIMEOUT,
-1, //5
Attribute.MAX_PHYSICAL_DDL_MDL_WAITING_TIMEOUT, //Integer.MAX_VALUE
- Integer.valueOf(Attribute.PHYSICAL_DDL_MDL_WAITING_TIMEOUT), //15
+ -1,
false);
/**
@@ -1253,6 +1253,11 @@ public static void addSupportedParam(ConfigParam param) {
true,
false);
+ public final static BooleanConfigParam BACKFILL_USE_RETURNING = new BooleanConfigParam(
+ ConnectionProperties.BACKFILL_USE_RETURNING,
+ true,
+ false);
+
/**
* enable parallel physical table backfill
*/
@@ -2036,6 +2041,9 @@ public static void addSupportedParam(ConfigParam param) {
public static final BooleanConfigParam ENABLE_LV_SUBQUERY_UNWRAP = new BooleanConfigParam(
ConnectionProperties.ENABLE_LV_SUBQUERY_UNWRAP, true, true);
+ public static final BooleanConfigParam EXPLAIN_PRUNING_DETAIL = new BooleanConfigParam(
+ ConnectionProperties.EXPLAIN_PRUNING_DETAIL, false, true);
+
public static final BooleanConfigParam ENABLE_FILTER_REORDER = new BooleanConfigParam(
ConnectionProperties.ENABLE_FILTER_REORDER, true, true);
@@ -2626,6 +2634,11 @@ public static void addSupportedParam(ConfigParam param) {
true,
true);
+ public static final BooleanConfigParam ENABLE_PUSHDOWN_DISTINCT = new BooleanConfigParam(
+ ConnectionProperties.ENABLE_PUSHDOWN_DISTINCT,
+ true,
+ true);
+
public static final BooleanConfigParam ENABLE_INDEX_SKYLINE = new BooleanConfigParam(
ConnectionProperties.ENABLE_INDEX_SKYLINE,
false,
@@ -2692,18 +2705,6 @@ public static void addSupportedParam(ConfigParam param) {
true
);
- public static final BooleanConfigParam ENABLE_JAVA_UDF = new BooleanConfigParam(
- ConnectionProperties.ENABLE_JAVA_UDF,
- true,
- true
- );
-
- public static final BooleanConfigParam CHECK_INVALID_JAVA_UDF = new BooleanConfigParam(
- ConnectionProperties.CHECK_INVALID_JAVA_UDF,
- true,
- true
- );
-
public static final LongConfigParam MAX_JAVA_UDF_NUM = new LongConfigParam(
ConnectionProperties.MAX_JAVA_UDF_NUM,
0L,
@@ -3074,7 +3075,7 @@ public static class ConnectionParamValues {
public static final IntConfigParam IN_SUB_QUERY_THRESHOLD = new IntConfigParam(
ConnectionProperties.IN_SUB_QUERY_THRESHOLD, 2, Integer.MAX_VALUE,
- 8, true);
+ 100000, true);
public static final BooleanConfigParam ENABLE_IN_SUB_QUERY_FOR_DML = new BooleanConfigParam(
ConnectionProperties.ENABLE_IN_SUB_QUERY_FOR_DML, false, Boolean.TRUE);
@@ -3255,6 +3256,14 @@ public static class ConnectionParamValues {
false,
true);
+ /**
+ * Label if auto use key syntax for all local index on show create table
+ */
+ public static final BooleanConfigParam ENABLE_USE_KEY_FOR_ALL_LOCAL_INDEX = new BooleanConfigParam(
+ ConnectionProperties.ENABLE_USE_KEY_FOR_ALL_LOCAL_INDEX,
+ false,
+ true);
+
/**
* Label if auto use range/list columns partitions for "part by range/list", default is true
*/
@@ -3503,6 +3512,9 @@ public static class ConnectionParamValues {
public static final BooleanConfigParam ENABLE_NDV_USE_COLUMNAR = new BooleanConfigParam(
ConnectionProperties.ENABLE_NDV_USE_COLUMNAR, false, true);
+ public static final BooleanConfigParam ENABLE_MPP_NDV_USE_COLUMNAR = new BooleanConfigParam(
+ ConnectionProperties.ENABLE_MPP_NDV_USE_COLUMNAR, false, true);
+
/**
* expire time(sec) for ndv sketch info
*/
@@ -3660,7 +3672,7 @@ public static class ConnectionParamValues {
new IntConfigParam(ConnectionProperties.CHANGE_SET_APPLY_BATCH, 1, 10 * 1024, 128, false);
public static final LongConfigParam CHANGE_SET_MEMORY_LIMIT = new LongConfigParam(
- ConnectionProperties.CHANGE_SET_MEMORY_LIMIT, 1024L, 1024 * 1024 * 1024L, 8 * 1024 * 1024L, false);
+ ConnectionProperties.CHANGE_SET_MEMORY_LIMIT, 1024L, 16 * 1024 * 1024L, 1024 * 1024L, false);
public static final BooleanConfigParam CN_ENABLE_CHANGESET =
new BooleanConfigParam(ConnectionProperties.CN_ENABLE_CHANGESET, true, true);
@@ -3886,19 +3898,10 @@ public static class ConnectionParamValues {
true);
/**
- * the min size of IN expr that would be pruned
- */
- public static final IntConfigParam IN_PRUNE_SIZE = new IntConfigParam(
- ConnectionProperties.IN_PRUNE_SIZE, 0, Integer.MAX_VALUE, 150, true);
-
- /**
- * the batch size of IN expr being pruned
+ * the MAX size of IN expr being pruned
*/
- public static final IntConfigParam IN_PRUNE_STEP_SIZE = new IntConfigParam(
- ConnectionProperties.IN_PRUNE_STEP_SIZE, 1, Integer.MAX_VALUE, 10, true);
-
public static final IntConfigParam IN_PRUNE_MAX_TIME = new IntConfigParam(
- ConnectionProperties.IN_PRUNE_MAX_TIME, 1, Integer.MAX_VALUE, 100, true);
+ ConnectionProperties.IN_PRUNE_MAX_TIME, 1, Integer.MAX_VALUE, 100000, true);
public static final IntConfigParam MAX_IN_PRUNE_CACHE_SIZE = new IntConfigParam(
ConnectionProperties.MAX_IN_PRUNE_CACHE_SIZE, 0, Integer.MAX_VALUE, 200, true);
@@ -3940,6 +3943,16 @@ public static class ConnectionParamValues {
false,
false);
+ public static final BooleanConfigParam OUTPUT_MYSQL_ERROR_CODE =
+ new BooleanConfigParam(ConnectionProperties.OUTPUT_MYSQL_ERROR_CODE,
+ false,
+ false);
+
+ public static final StringConfigParam MAPPING_TO_MYSQL_ERROR_CODE =
+ new StringConfigParam(ConnectionProperties.MAPPING_TO_MYSQL_ERROR_CODE,
+ "",
+ false);
+
public static final StringConfigParam PURGE_OSS_FILE_CRON_EXPR = new StringConfigParam(
ConnectionProperties.PURGE_OSS_FILE_CRON_EXPR, "0 0 1 ? * WED", true);
@@ -4571,9 +4584,35 @@ public static class ConnectionParamValues {
public static final BooleanConfigParam PHYSICAL_BACKFILL_SPEED_TEST = new BooleanConfigParam(
ConnectionProperties.PHYSICAL_BACKFILL_SPEED_TEST,
+ false,
+ true);
+
+ /**
+ * rebalance start point
+ */
+ public static final BooleanConfigParam REBALANCE_MAINTENANCE_ENABLE = new BooleanConfigParam(
+ ConnectionProperties.REBALANCE_MAINTENANCE_ENABLE,
true,
true);
+ /**
+ * rebalance start point
+ */
+ public static final StringConfigParam REBALANCE_MAINTENANCE_TIME_START =
+ new StringConfigParam(ConnectionProperties.REBALANCE_MAINTENANCE_TIME_START, "00:00", true);
+
+ /**
+ * rebalance stop point
+ */
+ public static final StringConfigParam REBALANCE_MAINTENANCE_TIME_END =
+ new StringConfigParam(ConnectionProperties.REBALANCE_MAINTENANCE_TIME_END, "00:00", true);
+
+ public static final BooleanConfigParam CANCEL_REBALANCE_JOB_DUE_MAINTENANCE = new BooleanConfigParam(
+ ConnectionProperties.CANCEL_REBALANCE_JOB_DUE_MAINTENANCE,
+ false,
+ true
+ );
+
public static final BooleanConfigParam ENABLE_DEADLOCK_DETECTION_80 = new BooleanConfigParam(
ConnectionProperties.ENABLE_DEADLOCK_DETECTION_80,
false,
@@ -4818,6 +4857,12 @@ public static class ConnectionParamValues {
true
);
+ public static final BooleanConfigParam ENABLE_1PC_OPT = new BooleanConfigParam(
+ ConnectionProperties.ENABLE_1PC_OPT,
+ true,
+ true
+ );
+
public static final BooleanConfigParam ENABLE_SINGLE_SHARD_WRITE = new BooleanConfigParam(
ConnectionProperties.ENABLE_SINGLE_SHARD_WRITE, true, true);
}
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/properties/ConnectionProperties.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/properties/ConnectionProperties.java
index 09b0ad82a..5c89c64dd 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/properties/ConnectionProperties.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/properties/ConnectionProperties.java
@@ -684,10 +684,6 @@ public class ConnectionProperties {
public static final String ENABLE_UDF = "ENABLE_UDF";
- public static final String ENABLE_JAVA_UDF = "ENABLE_JAVA_UDF";
-
- public static final String CHECK_INVALID_JAVA_UDF = "CHECK_INVALID_JAVA_UDF";
-
public static final String MAX_JAVA_UDF_NUM = "MAX_JAVA_UDF_NUM";
public static final String FORCE_DROP_JAVA_UDF = "FORCE_DROP_JAVA_UDF";
@@ -865,6 +861,8 @@ public class ConnectionProperties {
public static final String ENABLE_LV_SUBQUERY_UNWRAP = "ENABLE_LV_SUBQUERY_UNWRAP";
+ public static final String EXPLAIN_PRUNING_DETAIL = "EXPLAIN_PRUNING_DETAIL";
+
public static final String ENABLE_FILTER_REORDER = "ENABLE_FILTER_REORDER";
public static final String ENABLE_CONSTANT_FOLD = "ENABLE_CONSTANT_FOLD";
@@ -1095,6 +1093,8 @@ public class ConnectionProperties {
*/
public static final String BACKFILL_MAX_SAMPLE_SIZE = "BACKFILL_MAX_SAMPLE_SIZE";
+ public static final String BACKFILL_USE_RETURNING = "BACKFILL_USE_RETURNING";
+
/**
* enable split physical table for backfill
*/
@@ -1802,6 +1802,11 @@ public class ConnectionProperties {
*/
public static final String ENABLE_COMPLEX_DML_CROSS_DB = "ENABLE_COMPLEX_DML_CROSS_DB";
public static final String COMPLEX_DML_WITH_TRX = "COMPLEX_DML_WITH_TRX";
+
+ public static final String ENABLE_PUSHDOWN_DISTINCT = "ENABLE_PUSHDOWN_DISTINCT";
+ /**
+ * Enable index selection
+ */
public static final String ENABLE_INDEX_SELECTION = "ENABLE_INDEX_SELECTION";
public static final String ENABLE_INDEX_SELECTION_PRUNE = "ENABLE_INDEX_SELECTION_PRUNE";
@@ -2055,6 +2060,11 @@ public class ConnectionProperties {
* Label if auto use range-key subpart for index of auto-part table, default is true
*/
public static final String ENABLE_AUTO_USE_RANGE_FOR_TIME_INDEX = "ENABLE_AUTO_USE_RANGE_FOR_TIME_INDEX";
+ /**
+ * Label if auto use key syntax for all local index on show create table
+ */
+ public static final String ENABLE_USE_KEY_FOR_ALL_LOCAL_INDEX = "ENABLE_USE_KEY_FOR_ALL_LOCAL_INDEX";
+
/**
* Label if auto use range/list columns partitions for "part by range/list", default is true
*/
@@ -2158,6 +2168,8 @@ public class ConnectionProperties {
public static final String ENABLE_HA_CHECK_TASK_LOG = "ENABLE_HA_CHECK_TASK_LOG";
public static final String ENABLE_NDV_USE_COLUMNAR = "ENABLE_NDV_USE_COLUMNAR";
+
+ public static final String ENABLE_MPP_NDV_USE_COLUMNAR = "ENABLE_MPP_NDV_USE_COLUMNAR";
/**
* ndv sketch expire time
*/
@@ -2328,6 +2340,12 @@ public class ConnectionProperties {
*/
public static final String IN_PRUNE_STEP_SIZE = "IN_PRUNE_STEP_SIZE";
public static final String IN_PRUNE_MAX_TIME = "IN_PRUNE_MAX_TIME";
+ public static final String PRUNING_TIME_WARNING_THRESHOLD = "PRUNING_TIME_WARNING_THRESHOLD";
+
+ public static final String ENABLE_PRUNING_IN = "ENABLE_PRUNING_IN";
+
+ public static final String ENABLE_PRUNING_IN_DML = "ENABLE_PRUNING_IN_DML";
+
/**
* the max num of pruning info cache by logical view
*/
@@ -2377,6 +2395,10 @@ public class ConnectionProperties {
public static final String ENABLE_STANDBY_BACKFILL = "ENABLE_STANDBY_BACKFILL";
public static final String PHYSICAL_DDL_IGNORED_ERROR_CODE = "PHYSICAL_DDL_IGNORED_ERROR_CODE";
public static final String DDL_PAUSE_DURING_EXCEPTION = "DDL_PAUSE_DURING_EXCEPTION";
+ public static final String OUTPUT_MYSQL_ERROR_CODE = "OUTPUT_MYSQL_ERROR_CODE";
+
+ public static final String MAPPING_TO_MYSQL_ERROR_CODE = "MAPPING_TO_MYSQL_ERROR_CODE";
+
public static final String CHANGE_SET_REPLAY_TIMES = "CHANGE_SET_REPLAY_TIMES";
public static final String CHANGE_SET_APPLY_BATCH = "CHANGE_SET_APPLY_BATCH";
public static final String CHANGE_SET_MEMORY_LIMIT = "CHANGE_SET_MEMORY_LIMIT";
@@ -2601,6 +2623,13 @@ public class ConnectionProperties {
public static final String PHYSICAL_BACKFILL_SPEED_TEST =
"PHYSICAL_BACKFILL_SPEED_TEST";
+ public static final String REBALANCE_MAINTENANCE_ENABLE = "REBALANCE_MAINTENANCE_ENABLE";
+ public static final String REBALANCE_MAINTENANCE_TIME_START = "REBALANCE_MAINTENANCE_TIME_START";
+
+ public static final String REBALANCE_MAINTENANCE_TIME_END = "REBALANCE_MAINTENANCE_TIME_END";
+
+ public static final String CANCEL_REBALANCE_JOB_DUE_MAINTENANCE = "CANCEL_REBALANCE_JOB_DUE_MAINTENANCE";
+
public static final String ENABLE_DEADLOCK_DETECTION_80 = "ENABLE_DEADLOCK_DETECTION_80";
public static final String MOCK_COLUMNAR_INDEX = "MOCK_COLUMNAR_INDEX";
public static final String MCI_FORMAT = "MCI_FORMAT";
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/properties/DynamicConfig.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/properties/DynamicConfig.java
index c6fc8786f..2c170fbd7 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/properties/DynamicConfig.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/properties/DynamicConfig.java
@@ -16,13 +16,23 @@
package com.alibaba.polardbx.common.properties;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.TypeReference;
+import com.alibaba.fastjson.parser.Feature;
+import com.alibaba.polardbx.common.TddlConstants;
import com.alibaba.polardbx.common.constants.IsolationLevel;
import com.alibaba.polardbx.common.statementsummary.StatementSummaryManager;
import com.alibaba.polardbx.common.utils.version.InstanceVersion;
+import com.alibaba.polardbx.common.utils.TStringUtil;
import com.alibaba.polardbx.config.ConfigDataMode;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
import java.util.regex.Pattern;
/**
@@ -336,21 +346,47 @@ public void loadValue(Logger logger, String key, String value) {
case ConnectionProperties.ENABLE_TRX_DEBUG_MODE:
enableTrxDebugMode = parseValue(value, Boolean.class, false);
break;
- case ConnectionProperties.ENABLE_MQ_CACHE_COST_BY_THREAD:
- enableMQCacheByThread = parseValue(value, Boolean.class, true);
- break;
case ConnectionProperties.INSTANCE_READ_ONLY:
instanceReadOnly = parseValue(value, Boolean.class, false);
break;
- case ConnectionProperties.ENABLE_1PC_OPT:
- enable1PcOpt = parseValue(value, Boolean.class, true);
- break;
case ConnectionProperties.MIN_SNAPSHOT_KEEP_TIME:
minSnapshotKeepTime = parseValue(value, Integer.class, 5 * 60 * 1000);
break;
case ConnectionProperties.FORBID_AUTO_COMMIT_TRX:
forbidAutoCommitTrx = parseValue(value, Boolean.class, false);
break;
+ case ConnectionProperties.MAPPING_TO_MYSQL_ERROR_CODE:
+ errorCodeMapping = initErrorCodeMapping(value);
+ break;
+ case ConnectionProperties.PRUNING_TIME_WARNING_THRESHOLD:
+ pruningTimeWarningThreshold = parseValue(value, Long.class, 500L);
+ break;
+ case ConnectionProperties.ENABLE_PRUNING_IN:
+ enablePruningIn = parseValue(value, Boolean.class, true);
+ break;
+ case ConnectionProperties.ENABLE_PRUNING_IN_DML:
+ enablePruningInDml = parseValue(value, Boolean.class, true);
+ break;
+ case ConnectionProperties.ENABLE_MQ_CACHE_COST_BY_THREAD:
+ enableMQCacheByThread = parseValue(value, Boolean.class, true);
+ break;
+ case ConnectionProperties.ENABLE_USE_KEY_FOR_ALL_LOCAL_INDEX:
+ enableUseKeyForAllLocalIndex = parseValue(value, Boolean.class, false);
+ break;
+ case TddlConstants.BLACK_LIST_CONF:
+ String blockLists = parseValue(value, String.class, "");
+ List tempBlackList = new ArrayList<>();
+ if (StringUtils.isNotBlank(blockLists)) {
+ String[] blockListArr = blockLists.split(",");
+ for (String blockList : blockListArr) {
+ if (StringUtils.isNotBlank(blockList)) {
+ tempBlackList.add(blockList.toLowerCase(Locale.ROOT));
+ }
+ }
+ }
+ blackListConf = tempBlackList;
+ break;
+
default:
FileConfig.getInstance().loadValue(logger, key, value);
break;
@@ -644,7 +680,8 @@ public boolean isEnableAutoUseRangeForTimeIndex() {
}
private static final String DEFAULT_PASSWORD_CHECK_PATTERN_STR = "^[0-9A-Za-z!@#$%^&*()_+=-]{6,32}$";
- private static final Pattern DEFAULT_PASSWORD_CHECK_PATTERN = Pattern.compile(DEFAULT_PASSWORD_CHECK_PATTERN_STR);
+ private static final Pattern DEFAULT_PASSWORD_CHECK_PATTERN =
+ Pattern.compile(DEFAULT_PASSWORD_CHECK_PATTERN_STR);
private volatile Pattern passwordCheckPattern = DEFAULT_PASSWORD_CHECK_PATTERN;
@@ -881,12 +918,6 @@ public boolean isInstanceReadOnly() {
return instanceReadOnly;
}
- private volatile boolean enable1PcOpt = true;
-
- public boolean isEnable1PcOpt() {
- return enable1PcOpt;
- }
-
// 5 min.
private volatile long minSnapshotKeepTime = 5 * 60 * 1000;
@@ -896,10 +927,59 @@ public long getMinSnapshotKeepTime() {
private volatile boolean forbidAutoCommitTrx = false;
+ private volatile Map errorCodeMapping = new HashMap<>();
+
public boolean isForbidAutoCommitTrx() {
return forbidAutoCommitTrx;
}
+ public Map getErrorCodeMapping() {
+ return errorCodeMapping;
+ }
+
+ private Map initErrorCodeMapping(String mapping) {
+ if (TStringUtil.isNotBlank(mapping)) {
+ try {
+ return JSON.parseObject(mapping, new TypeReference>() {
+ }, Feature.IgnoreAutoType);
+ } catch (Exception ignored) {
+ }
+ }
+ return new HashMap<>();
+ }
+
+ private boolean enableUseKeyForAllLocalIndex =
+ Boolean.valueOf(ConnectionParams.ENABLE_USE_KEY_FOR_ALL_LOCAL_INDEX.getDefault());
+
+ public boolean isEnableUseKeyForAllLocalIndex() {
+ return enableUseKeyForAllLocalIndex;
+ }
+
+ // pruning warning threshold in microsecond
+ private volatile long pruningTimeWarningThreshold = 500;
+
+ public long getPruningTimeWarningThreshold() {
+ return pruningTimeWarningThreshold;
+ }
+
+ private volatile boolean enablePruningIn = true;
+
+ private volatile boolean enablePruningInDml = true;
+
+ public boolean isEnablePruningIn() {
+ return enablePruningIn;
+ }
+
+ public boolean isEnablePruningInDml() {
+ return enablePruningInDml;
+ }
+
+ private volatile List blackListConf = new ArrayList<>();
+
+ public List getBlacklistConf() {
+ return blackListConf;
+ }
+
public static T parseValue(String value, Class type, T defaultValue) {
if (value == null) {
return defaultValue;
diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/utils/GeneralUtil.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/utils/GeneralUtil.java
index b959ec015..39f122297 100644
--- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/utils/GeneralUtil.java
+++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/utils/GeneralUtil.java
@@ -40,6 +40,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.function.Supplier;
public class GeneralUtil {
@@ -583,4 +584,16 @@ public static String removeIdxSuffix(String source) {
return source;
}
}
+
+ /**
+ * return target length random string
+ */
+ public static String randomString(int length) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < length; i++) {
+ char ch = (char) (ThreadLocalRandom.current().nextInt('x' - 'a') + 'a');
+ sb.append(ch);
+ }
+ return sb.toString();
+ }
}
diff --git a/polardbx-common/src/test/java/com/alibaba/polardbx/common/EngineTest.java b/polardbx-common/src/test/java/com/alibaba/polardbx/common/EngineTest.java
new file mode 100644
index 000000000..5558bdfce
--- /dev/null
+++ b/polardbx-common/src/test/java/com/alibaba/polardbx/common/EngineTest.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright [2013-2021], Alibaba Group Holding Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.polardbx.common;
+
+import junit.framework.TestCase;
+
+public class EngineTest extends TestCase {
+
+ public void testOf() {
+ assertEquals(Engine.OSS, Engine.of("OSS"));
+ assertEquals(Engine.INNODB, Engine.of(""));
+ boolean catchException = false;
+ try {
+ Engine.of("NON_EXIST_ENGINE");
+ } catch (Throwable t) {
+ catchException = true;
+ assertTrue(t.getMessage().equalsIgnoreCase("Unknown engine name:NON_EXIST_ENGINE"));
+ }
+ if (!catchException) {
+ throw new AssertionError("Engine.of() method should fail");
+ }
+ }
+
+ public void testHasCache() {
+ assertTrue(Engine.hasCache(Engine.OSS));
+ assertTrue(Engine.hasCache(Engine.EXTERNAL_DISK));
+ assertTrue(Engine.hasCache(Engine.NFS));
+ assertTrue(Engine.hasCache(Engine.S3));
+ assertTrue(Engine.hasCache(Engine.ABS));
+ assertFalse(Engine.hasCache(Engine.INNODB));
+ assertFalse(Engine.hasCache(null));
+ }
+
+ public void testIsFileStore() {
+ assertTrue(Engine.isFileStore(Engine.OSS));
+ assertTrue(Engine.isFileStore(Engine.EXTERNAL_DISK));
+ assertTrue(Engine.isFileStore(Engine.LOCAL_DISK));
+ assertTrue(Engine.isFileStore(Engine.NFS));
+ assertTrue(Engine.isFileStore(Engine.S3));
+ assertTrue(Engine.isFileStore(Engine.ABS));
+ assertFalse(Engine.isFileStore(Engine.INNODB));
+ assertFalse(Engine.isFileStore((Engine) null));
+ }
+
+ public void testSupportColumnar() {
+ assertTrue(Engine.supportColumnar(Engine.OSS));
+ assertTrue(Engine.supportColumnar(Engine.EXTERNAL_DISK));
+ assertTrue(Engine.supportColumnar(Engine.LOCAL_DISK));
+ assertTrue(Engine.supportColumnar(Engine.NFS));
+ assertTrue(Engine.supportColumnar(Engine.S3));
+ assertTrue(Engine.supportColumnar(Engine.ABS));
+ assertFalse(Engine.supportColumnar(Engine.INNODB));
+ assertFalse(Engine.supportColumnar((Engine) null));
+ }
+
+ public void testTestIsFileStore() {
+ assertTrue(Engine.isFileStore("OSS"));
+ }
+}
\ No newline at end of file
diff --git a/polardbx-common/src/test/java/com/alibaba/polardbx/common/jdbc/PruneRawStringTest.java b/polardbx-common/src/test/java/com/alibaba/polardbx/common/jdbc/PruneRawStringTest.java
new file mode 100644
index 000000000..31b364db3
--- /dev/null
+++ b/polardbx-common/src/test/java/com/alibaba/polardbx/common/jdbc/PruneRawStringTest.java
@@ -0,0 +1,59 @@
+package com.alibaba.polardbx.common.jdbc;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Test;
+
+import java.util.stream.IntStream;
+
+/**
+ * @author fangwu
+ */
+public class PruneRawStringTest {
+
+ @Test
+ public void testGetObj() {
+ PruneRawString pruneRawString = new PruneRawString(ImmutableList.copyOf(IntStream.range(0, 1000).iterator()),
+ PruneRawString.PRUNE_MODE.RANGE, 0, 1000, null);
+ assert pruneRawString.display().contains("NonPruneRaw");
+ try {
+ pruneRawString.getObj(1000, -1);
+ } catch (Exception e) {
+ System.out.println(e.getMessage());
+ assert e.getMessage().contains("PruneRawString error array index out of bounds");
+ }
+
+ pruneRawString = new PruneRawString(ImmutableList.copyOf(IntStream.range(0, 1000).iterator()),
+ PruneRawString.PRUNE_MODE.RANGE, 0, 999, null);
+ assert !pruneRawString.display().contains("NonPruneRaw");
+
+ pruneRawString = new PruneRawString(ImmutableList.copyOf(IntStream.range(0, 4096).iterator()),
+ PruneRawString.PRUNE_MODE.RANGE, 0, 4096, null);
+ assert pruneRawString.display().contains("NonPruneRaw");
+ assert pruneRawString.display().endsWith("...");
+ }
+
+ @Test
+ public void testMerge() {
+ PruneRawString pruneRawString = new PruneRawString(ImmutableList.copyOf(IntStream.range(0, 1000).iterator()),
+ PruneRawString.PRUNE_MODE.RANGE, 0, 300, null);
+ PruneRawString pruneRawString2 = new PruneRawString(ImmutableList.copyOf(IntStream.range(0, 1000).iterator()),
+ PruneRawString.PRUNE_MODE.RANGE, 100, 500, null);
+ pruneRawString.merge(pruneRawString2);
+
+ assert pruneRawString.pruneMode == PruneRawString.PRUNE_MODE.MULTI_INDEX;
+ assert pruneRawString2.pruneMode == PruneRawString.PRUNE_MODE.MULTI_INDEX;
+ assert pruneRawString.size() == 500;
+ PruneRawString pruneRawString3 = new PruneRawString(ImmutableList.copyOf(IntStream.range(0, 1000).iterator()),
+ PruneRawString.PRUNE_MODE.RANGE, 500, 1000, null);
+ pruneRawString.merge(pruneRawString3);
+ assert pruneRawString.pruneMode == PruneRawString.PRUNE_MODE.RANGE;
+ assert pruneRawString3.pruneMode == PruneRawString.PRUNE_MODE.MULTI_INDEX;
+
+ PruneRawString pruneRawString4 = new PruneRawString(ImmutableList.copyOf(IntStream.range(0, 1000).iterator()),
+ PruneRawString.PRUNE_MODE.RANGE, 0, 1000, null);
+ pruneRawString4.merge(pruneRawString);
+
+ assert pruneRawString4.pruneMode == PruneRawString.PRUNE_MODE.RANGE;
+ assert pruneRawString.pruneMode == PruneRawString.PRUNE_MODE.RANGE;
+ }
+}
diff --git a/polardbx-common/src/test/java/com/alibaba/polardbx/common/jdbc/RawStringTest.java b/polardbx-common/src/test/java/com/alibaba/polardbx/common/jdbc/RawStringTest.java
index b0a01d23d..ac8323fdb 100644
--- a/polardbx-common/src/test/java/com/alibaba/polardbx/common/jdbc/RawStringTest.java
+++ b/polardbx-common/src/test/java/com/alibaba/polardbx/common/jdbc/RawStringTest.java
@@ -17,12 +17,14 @@
package com.alibaba.polardbx.common.jdbc;
import com.alibaba.polardbx.common.utils.Assert;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import junit.framework.TestCase;
import org.junit.Test;
import java.util.BitSet;
import java.util.List;
+import java.util.stream.IntStream;
/**
* @author fangwu
@@ -114,4 +116,15 @@ public void testPruneRawStringWithTypeConvert() {
RawString newPr = pr.convertType(o -> o, -1);
Assert.assertTrue("3,'s\\n'".equals(newPr.buildRawString()));
}
+
+ @Test
+ public void testPruneStep() {
+ RawString rawString = new RawString(ImmutableList.copyOf(IntStream.range(0, 1000).iterator()));
+
+ for (int i = 0; i < 1000; i++) {
+ PruneRawString pruneRawString = rawString.pruneStep(i);
+ assert pruneRawString.size() == 1;
+ assert pruneRawString.getObj(0, -1) == rawString.getObj(i, -1);
+ }
+ }
}
\ No newline at end of file
diff --git a/polardbx-common/src/test/java/com/alibaba/polardbx/common/jdbc/TableNameTest.java b/polardbx-common/src/test/java/com/alibaba/polardbx/common/jdbc/TableNameTest.java
new file mode 100644
index 000000000..a9d95936a
--- /dev/null
+++ b/polardbx-common/src/test/java/com/alibaba/polardbx/common/jdbc/TableNameTest.java
@@ -0,0 +1,72 @@
+package com.alibaba.polardbx.common.jdbc;
+
+import com.alibaba.polardbx.common.utils.Assert;
+import org.junit.Test;
+
+import java.sql.SQLException;
+
+public class TableNameTest {
+
+ @Test
+ public void testTableName() throws SQLException {
+ try {
+ TableName tableName = new TableName("t1");
+ } catch (SQLException e) {
+ throw e;
+ }
+ }
+
+ @Test
+ public void testTableNameWithBlank() throws SQLException {
+ try {
+ TableName tableName = new TableName("t1 t2");
+ } catch (SQLException e) {
+ throw e;
+ }
+ }
+
+ @Test
+ public void testTableNameErr() throws SQLException {
+ try {
+ TableName tableName = new TableName("t'1");
+ } catch (SQLException e) {
+ Assert.assertTrue(e.getMessage().contains("tableName format error"));
+ }
+ }
+
+ @Test
+ public void testTableNameErr2() throws SQLException {
+ try {
+ TableName tableName = new TableName("t\"1");
+ } catch (SQLException e) {
+ Assert.assertTrue(e.getMessage().contains("tableName format error"));
+ }
+ }
+
+ @Test
+ public void testTableNameErr3() throws SQLException {
+ try {
+ TableName tableName = new TableName("t\\1");
+ } catch (SQLException e) {
+ Assert.assertTrue(e.getMessage().contains("tableName format error"));
+ }
+ }
+
+ @Test
+ public void testTableNameErr4() throws SQLException {
+ try {
+ TableName tableName = new TableName("t\n1");
+ } catch (SQLException e) {
+ Assert.assertTrue(e.getMessage().contains("tableName format error"));
+ }
+ }
+
+ @Test
+ public void testTableNameErr5() throws SQLException {
+ try {
+ TableName tableName = new TableName("");
+ } catch (SQLException e) {
+ Assert.assertTrue(e.getMessage().contains("tableName should not be empty"));
+ }
+ }
+}
diff --git a/polardbx-common/src/test/java/com/alibaba/polardbx/common/oss/filesystem/InputStreamWithRateLimiterTest.java b/polardbx-common/src/test/java/com/alibaba/polardbx/common/oss/filesystem/InputStreamWithRateLimiterTest.java
new file mode 100644
index 000000000..9e32e65a1
--- /dev/null
+++ b/polardbx-common/src/test/java/com/alibaba/polardbx/common/oss/filesystem/InputStreamWithRateLimiterTest.java
@@ -0,0 +1,189 @@
+/*
+ * Copyright [2013-2021], Alibaba Group Holding Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.polardbx.common.oss.filesystem;
+
+import com.alibaba.polardbx.common.mock.MockUtils;
+import org.apache.hadoop.fs.PositionedReadable;
+import org.apache.hadoop.fs.Seekable;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+@RunWith(MockitoJUnitRunner.class)
+public class InputStreamWithRateLimiterTest {
+
+ private FileSystemRateLimiter rateLimiter;
+ private InputStreamWithRateLimiter inputStreamWithRateLimiter;
+ private InputStream inputStream;
+
+ @Before
+ public void setUp() {
+ rateLimiter = Mockito.mock(FileSystemRateLimiter.class);
+ inputStream = Mockito.mock(SeekableByteArrayInputStream.class);
+ inputStreamWithRateLimiter = new InputStreamWithRateLimiter(inputStream, rateLimiter);
+ }
+
+ @Test
+ public void testNewStream() {
+ MockUtils.assertThrows(
+ IllegalArgumentException.class,
+ "In is not an instance of Seekable or PositionedReadable",
+ () -> {
+ new InputStreamWithRateLimiter(new ByteArrayInputStream(new byte[0]), rateLimiter);
+ }
+ );
+ }
+
+ @Test
+ public void testRead() throws IOException {
+ // Arrange
+ Mockito.when(inputStream.read()).thenReturn(1);
+ Mockito.when(inputStream.read(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt()))
+ .thenReturn(2);
+ Mockito.when(inputStream.read(Mockito.any(byte[].class))).thenReturn(3);
+ Mockito.when(
+ ((PositionedReadable) inputStream).read(Mockito.anyLong(), Mockito.any(byte[].class), Mockito.anyInt(),
+ Mockito.anyInt())).thenReturn(4);
+
+ int byteRead = inputStreamWithRateLimiter.read();
+ Assert.assertEquals(1, byteRead);
+ Mockito.verify(rateLimiter).acquireRead(1);
+ Mockito.verify(inputStream).read();
+
+ byte[] buf = new byte[3];
+ byte[] buf2 = new byte[6];
+ byteRead = inputStreamWithRateLimiter.read(buf, 0, 2);
+ Assert.assertEquals(2, byteRead);
+ Mockito.verify(rateLimiter).acquireRead(2);
+ Mockito.verify(inputStream).read(buf, 0, 2);
+
+ byteRead = inputStreamWithRateLimiter.read(buf);
+ Assert.assertEquals(3, byteRead);
+ Mockito.verify(rateLimiter).acquireRead(3);
+ Mockito.verify(inputStream).read(buf);
+
+ byteRead = inputStreamWithRateLimiter.read(0, buf2, 0, 4);
+ Assert.assertEquals(4, byteRead);
+ Mockito.verify(rateLimiter).acquireRead(4);
+ Mockito.verify((PositionedReadable) inputStream).read(0, buf2, 0, 4);
+
+ inputStreamWithRateLimiter.readFully(0, buf2, 0, 5);
+ Mockito.verify(rateLimiter).acquireRead(5);
+ Mockito.verify((PositionedReadable) inputStream).readFully(0, buf2, 0, 5);
+
+ inputStreamWithRateLimiter.readFully(0, buf2);
+ Mockito.verify(rateLimiter).acquireRead(6);
+ Mockito.verify((PositionedReadable) inputStream).readFully(0, buf2);
+ }
+
+ @Test
+ public void testForwardingMethods() throws IOException {
+ inputStreamWithRateLimiter.seek(1);
+ Mockito.verify(((Seekable) inputStream)).seek(1);
+ inputStreamWithRateLimiter.seekToNewSource(1);
+ Mockito.verify(((Seekable) inputStream)).seekToNewSource(1);
+ inputStreamWithRateLimiter.getPos();
+ Mockito.verify(((Seekable) inputStream)).getPos();
+
+ inputStreamWithRateLimiter.skip(1);
+ Mockito.verify(inputStream).skip(1);
+
+ inputStreamWithRateLimiter.available();
+ Mockito.verify(inputStream).available();
+
+ inputStreamWithRateLimiter.mark(1);
+ Mockito.verify(inputStream).mark(1);
+
+ inputStreamWithRateLimiter.reset();
+ Mockito.verify(inputStream).reset();
+
+ inputStreamWithRateLimiter.markSupported();
+ Mockito.verify(inputStream).markSupported();
+
+ inputStreamWithRateLimiter.close();
+ Mockito.verify(inputStream).close();
+ }
+
+ @Test
+ public void testReadWhenRateLimiterThrowsIOException() throws IOException {
+ // Arrange
+ Mockito.doThrow(new IOException("Rate limit exceeded")).when(rateLimiter).acquireRead(1);
+
+ // Act & Assert
+ MockUtils.assertThrows(IOException.class, "Rate limit exceeded", () -> inputStreamWithRateLimiter.read());
+
+ Mockito.verify(rateLimiter).acquireRead(1);
+ Mockito.verify(inputStream, Mockito.never()).read();
+ }
+
+ public class SeekableByteArrayInputStream extends ByteArrayInputStream implements Seekable, PositionedReadable {
+
+ public SeekableByteArrayInputStream(byte[] buf) {
+ super(buf);
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ if (pos < 0 || pos > count) {
+ throw new IOException("Position out of bounds");
+ }
+ this.pos = (int) pos;
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return pos;
+ }
+
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ return false;
+ }
+
+ @Override
+ public int read(long position, byte[] buffer, int offset, int length) throws IOException {
+ if (position >= count) {
+ return -1;
+ }
+ if (position + length > count) {
+ length = count - (int) position;
+ }
+ System.arraycopy(buf, (int) position, buffer, offset, length);
+ return length;
+ }
+
+ @Override
+ public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
+ int nread = read(position, buffer, offset, length);
+ if (nread < length) {
+ throw new IOException("Reached end of stream");
+ }
+ }
+
+ @Override
+ public void readFully(long position, byte[] buffer) throws IOException {
+ readFully(position, buffer, 0, buffer.length);
+ }
+ }
+}
\ No newline at end of file
diff --git a/polardbx-common/src/test/java/com/alibaba/polardbx/common/oss/filesystem/OutputStreamWithRateLimiterTest.java b/polardbx-common/src/test/java/com/alibaba/polardbx/common/oss/filesystem/OutputStreamWithRateLimiterTest.java
new file mode 100644
index 000000000..5f77eb863
--- /dev/null
+++ b/polardbx-common/src/test/java/com/alibaba/polardbx/common/oss/filesystem/OutputStreamWithRateLimiterTest.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright [2013-2021], Alibaba Group Holding Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.polardbx.common.oss.filesystem;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+@RunWith(MockitoJUnitRunner.class)
+public class OutputStreamWithRateLimiterTest {
+ private OutputStream mockOutputStream;
+ private FileSystemRateLimiter mockRateLimiter;
+ private OutputStreamWithRateLimiter outputStreamWithRateLimiter;
+
+ @Before
+ public void setUp() {
+ mockOutputStream = Mockito.mock(OutputStream.class);
+ mockRateLimiter = Mockito.mock(FileSystemRateLimiter.class);
+ outputStreamWithRateLimiter = new OutputStreamWithRateLimiter(mockOutputStream, mockRateLimiter);
+ }
+
+ @Test
+ public void testWriteArray() throws IOException {
+ byte[] b = new byte[] {1, 2, 3, 4};
+ outputStreamWithRateLimiter.write(b);
+ Mockito.verify(mockRateLimiter).acquireWrite(b.length);
+ Mockito.verify(mockOutputStream).write(b);
+ }
+
+ @Test
+ public void testWriteArrayWithOffsetAndLength() throws IOException {
+ byte[] b = new byte[] {1, 2, 3, 4, 5, 6};
+ int off = 2;
+ int len = 4;
+ outputStreamWithRateLimiter.write(b, off, len);
+ Mockito.verify(mockRateLimiter).acquireWrite(len);
+ Mockito.verify(mockOutputStream).write(b, off, len);
+ }
+
+ @Test
+ public void testFlush() throws IOException {
+ outputStreamWithRateLimiter.flush();
+ Mockito.verify(mockOutputStream).flush();
+ }
+
+ @Test
+ public void testClose() throws IOException {
+ outputStreamWithRateLimiter.close();
+ Mockito.verify(mockOutputStream).close();
+ }
+
+ @Test
+ public void testWriteInt() throws IOException {
+ byte b = 1;
+ outputStreamWithRateLimiter.write(b);
+ Mockito.verify(mockRateLimiter).acquireWrite(1);
+ Mockito.verify(mockOutputStream).write(b);
+ }
+}
\ No newline at end of file
diff --git a/polardbx-common/src/test/java/com/alibaba/polardbx/common/properties/DynamicConfigTest.java b/polardbx-common/src/test/java/com/alibaba/polardbx/common/properties/DynamicConfigTest.java
index 7435424cd..09535f19b 100644
--- a/polardbx-common/src/test/java/com/alibaba/polardbx/common/properties/DynamicConfigTest.java
+++ b/polardbx-common/src/test/java/com/alibaba/polardbx/common/properties/DynamicConfigTest.java
@@ -1,15 +1,33 @@
package com.alibaba.polardbx.common.properties;
+import com.alibaba.polardbx.common.TddlConstants;
+import com.alibaba.polardbx.common.constants.ServerVariables;
import org.junit.Test;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
/**
* @author fangwu
*/
public class DynamicConfigTest {
@Test
public void testLoadInDegradationNum() {
- assert DynamicConfig.getInstance().getInDegradationNum() == 100L;
+ assertTrue(DynamicConfig.getInstance().getInDegradationNum() == 100L);
DynamicConfig.getInstance().loadValue(null, ConnectionProperties.STATISTIC_IN_DEGRADATION_NUMBER, "1357");
- assert DynamicConfig.getInstance().getInDegradationNum() == 1357L;
+ assertTrue(DynamicConfig.getInstance().getInDegradationNum() == 1357L);
+ }
+
+ @Test
+ public void testBlackListConf() {
+ assertTrue(DynamicConfig.getInstance().getBlacklistConf().size() == 0);
+ DynamicConfig.getInstance().loadValue(null, TddlConstants.BLACK_LIST_CONF, "");
+ assertTrue(DynamicConfig.getInstance().getBlacklistConf().size() == 0);
+
+ DynamicConfig.getInstance().loadValue(null, TddlConstants.BLACK_LIST_CONF, "x1,y1");
+ assertTrue(DynamicConfig.getInstance().getBlacklistConf().size() == 2);
+ assertTrue(ServerVariables.isGlobalBlackList("x1"));
+ assertTrue(ServerVariables.isGlobalBlackList("y1"));
+ assertFalse(ServerVariables.isGlobalBlackList("y1,x1"));
}
}
diff --git a/polardbx-common/src/test/java/com/alibaba/polardbx/common/utils/hash/Murmur3_128MethodTest.java b/polardbx-common/src/test/java/com/alibaba/polardbx/common/utils/hash/Murmur3_128MethodTest.java
index a7916d275..39f11d362 100644
--- a/polardbx-common/src/test/java/com/alibaba/polardbx/common/utils/hash/Murmur3_128MethodTest.java
+++ b/polardbx-common/src/test/java/com/alibaba/polardbx/common/utils/hash/Murmur3_128MethodTest.java
@@ -163,7 +163,7 @@ public void testAllWithMurmurHashUtils() {
private void equalWithGuavaHash(long data) {
long originResult = zeroSeedMurmur3hashFunc.hashLong(data).asLong();
- long curResult = MurmurHashUtils.murmurHashWithZeroSeed(data);
+ long curResult = MurmurHashUtils.murmurHash128WithZeroSeed(data);
Assert.assertEquals(originResult, curResult);
}
}
diff --git a/polardbx-executor/pom.xml b/polardbx-executor/pom.xml
index f23692506..c704e309f 100644
--- a/polardbx-executor/pom.xml
+++ b/polardbx-executor/pom.xml
@@ -14,10 +14,8 @@
206
- 3.2.2
1.6.9
3.10.2
- 3.2.2
5.9.0
43.0
4.12
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/atom/CacheVariables.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/atom/CacheVariables.java
index 241566080..3bd60f737 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/atom/CacheVariables.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/atom/CacheVariables.java
@@ -12,7 +12,7 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
-*/
+ */
package com.alibaba.polardbx.atom;
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/InsertSplitter.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/InsertSplitter.java
index b91747d67..3f8c53fb2 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/InsertSplitter.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/InsertSplitter.java
@@ -168,6 +168,9 @@ private ResultCursor executeSplit(ByteString sql, ExecutionContext executionCont
executionContext.getConnection().setLastInsertId(lastInsertId);
executionContext.getConnection().setReturnedLastInsertId(returnedLastInsertId);
+ //insert split没有走参数化，所以要在这添加MultiLine的判断
+ sql.setMultiLine(lexer.getLine() > 1);
+
AffectRowCursor arc = new AffectRowCursor(affectRows);
return new ResultCursor(arc);
}
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/archive/writer/OSSBackFillExtractor.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/archive/writer/OSSBackFillExtractor.java
index 7c28bc021..03d633b03 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/archive/writer/OSSBackFillExtractor.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/archive/writer/OSSBackFillExtractor.java
@@ -136,6 +136,7 @@ private Cursor doExtract(PhyTableOperation extractPlan, ExecutionContext extract
case LOCAL_DISK:
case EXTERNAL_DISK:
case NFS:
+ case ABS:
RelNode fileStorePlan =
OSSTableScan.fromPhysicalTableOperation(extractPlan, extractEc, this.sourceTableName, 1);
@@ -155,6 +156,7 @@ private List> doConsume(PhyTableOperation extract
case OSS:
case EXTERNAL_DISK:
case NFS:
+ case ABS:
return consumeFileStore(extractPlan, extractEc, batchConsumer, extractCursor);
case INNODB:
default:
@@ -242,6 +244,7 @@ protected List splitAndInitUpperBound(f
case LOCAL_DISK:
case EXTERNAL_DISK:
case NFS:
+ case ABS:
plan = OSSTableScan.fromPhysicalTableOperation(phyTableOperation, baseEc, this.sourceTableName, 1);
break;
case INNODB:
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/archive/writer/OSSBackFillLoader.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/archive/writer/OSSBackFillLoader.java
index 0a2000b1c..0e92570b7 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/archive/writer/OSSBackFillLoader.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/archive/writer/OSSBackFillLoader.java
@@ -58,7 +58,7 @@ protected OSSBackFillLoader(String schemaName, String tableName, SqlInsert inser
BiFunction, ExecutionContext, List> executeFunc,
Map sourceTargetGroupMap, String designateLogicalPart) {
super(schemaName, tableName, insert, insertIgnore, checkerPlan, checkerPkMapping, checkerParamMapping,
- executeFunc, false);
+ executeFunc, false, null);
this.sourceTargetGroupMap = sourceTargetGroupMap;
this.designateLogicalPart = designateLogicalPart;
}
@@ -167,6 +167,6 @@ public int executeInsert(SqlInsert sqlInsert, String schemaName, String tableNam
.insertIntoTable(null, sqlInsert, tableMeta, targetGroup, phyTableName, schemaName, executionContext,
executeFunc,
false,
- false, this.designateLogicalPart);
+ false, this.designateLogicalPart, false, null);
}
}
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/backfill/Loader.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/backfill/Loader.java
index 560704ed0..e853499bb 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/backfill/Loader.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/backfill/Loader.java
@@ -21,10 +21,12 @@
import com.alibaba.polardbx.common.exception.code.ErrorCode;
import com.alibaba.polardbx.common.jdbc.ParameterContext;
import com.alibaba.polardbx.common.jdbc.Parameters;
+import com.alibaba.polardbx.common.properties.ConnectionParams;
import com.alibaba.polardbx.common.utils.GeneralUtil;
import com.alibaba.polardbx.common.utils.Pair;
import com.alibaba.polardbx.executor.ExecutorHelper;
import com.alibaba.polardbx.executor.common.ExecutorContext;
+import com.alibaba.polardbx.executor.common.TopologyHandler;
import com.alibaba.polardbx.executor.cursor.Cursor;
import com.alibaba.polardbx.executor.gsi.GsiUtils;
import com.alibaba.polardbx.executor.spi.ITransactionManager;
@@ -35,16 +37,20 @@
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.sql.SqlInsert;
+import java.sql.Connection;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.Supplier;
import static com.alibaba.polardbx.common.exception.code.ErrorCode.ER_DUP_ENTRY;
+import static com.alibaba.polardbx.executor.columns.ColumnBackfillExecutor.isAllDnUseXDataSource;
import static com.alibaba.polardbx.executor.gsi.GsiUtils.SQLSTATE_DUP_ENTRY;
/**
@@ -71,12 +77,16 @@ public abstract class Loader extends PhyOperationBuilderCommon {
private final int[] checkerPkMapping;
private final ITransactionManager tm;
protected final boolean mirrorCopy;
+ protected final String backfillReturning;
+ protected boolean conflictDetection;
+ protected final boolean usingBackfillReturning;
protected final BiFunction, ExecutionContext, List> executeFunc;
protected Loader(String schemaName, String tableName, SqlInsert insert, SqlInsert insertIgnore,
ExecutionPlan checkerPlan, int[] checkerPkMapping, int[] checkerParamMapping,
- BiFunction, ExecutionContext, List> executeFunc, boolean mirrorCopy) {
+ BiFunction, ExecutionContext, List> executeFunc,
+ boolean mirrorCopy, String backfillReturning) {
this.schemaName = schemaName;
this.tableName = tableName;
this.sqlInsert = insert;
@@ -87,6 +97,9 @@ protected Loader(String schemaName, String tableName, SqlInsert insert, SqlInser
this.executeFunc = executeFunc;
this.tm = ExecutorContext.getContext(schemaName).getTransactionManager();
this.mirrorCopy = mirrorCopy;
+ this.backfillReturning = backfillReturning;
+ this.conflictDetection = !mirrorCopy;
+ this.usingBackfillReturning = backfillReturning != null && conflictDetection;
}
/**
@@ -95,6 +108,48 @@ protected Loader(String schemaName, String tableName, SqlInsert insert, SqlInser
public int fillIntoIndex(List> batchParams,
Pair> baseEcAndIndexPair,
Supplier checker) {
+ if (usingBackfillReturning) {
+ return fillIntoIndexWithReturning(batchParams, baseEcAndIndexPair, checker);
+ } else {
+ return fillIntoIndexWithInsert(batchParams, baseEcAndIndexPair, checker);
+ }
+ }
+
+ public int fillIntoIndexWithReturning(List> batchParams,
+ Pair> baseEcAndIndexPair,
+ Supplier checker) {
+ if (batchParams.isEmpty()) {
+ return 0;
+ }
+
+ baseEcAndIndexPair.getKey().setTxIsolation(Connection.TRANSACTION_READ_COMMITTED);
+
+ return GsiUtils.wrapWithDistributedXATrx(tm, baseEcAndIndexPair.getKey(), (insertEc) -> {
+ int result = 0;
+ try {
+ // Batch insert
+ result = applyBatchWithReturning(batchParams, insertEc.copy(), baseEcAndIndexPair.getValue().getKey(),
+ baseEcAndIndexPair.getValue().getValue());
+
+ // Batch insert success, check lock exists
+ return checkBeforeCommit(checker, insertEc, result);
+ } catch (TddlNestableRuntimeException e) {
+ // Batch insert failed
+ SQLRecorderLogger.ddlLogger
+ .warn(MessageFormat.format(
+ "[{0}] Batch insert(returning) failed first row: {1} cause: {2}, phyTableName: {3}",
+ baseEcAndIndexPair.getKey().getTraceId(),
+ GsiUtils.rowToString(batchParams.get(0)),
+ e.getMessage(), baseEcAndIndexPair.getValue().getValue()));
+
+ throw e;
+ }
+ });
+ }
+
+ public int fillIntoIndexWithInsert(List> batchParams,
+ Pair> baseEcAndIndexPair,
+ Supplier checker) {
if (batchParams.isEmpty()) {
return 0;
}
@@ -139,7 +194,7 @@ public int fillIntoIndex(List> batchParams,
for (Map param : batchParams) {
int single = applyRow(param, insertEc.copy(), baseEcAndIndexPair.getValue().getKey(),
- baseEcAndIndexPair.getValue().getValue(), true);
+ baseEcAndIndexPair.getValue().getValue());
if (single < 1) {
// Compare row
@@ -240,7 +295,27 @@ private int applyBatch(List> batchParams, Executi
newEc.setParams(parameters);
- return executeInsert(sqlInsert, schemaName, tableName, newEc, sourceDbIndex, phyTableName);
+ SqlInsert insert = conflictDetection ? sqlInsert : sqlInsertIgnore;
+
+ return executeInsert(insert, schemaName, tableName, newEc, sourceDbIndex, phyTableName);
+ }
+
+ private int applyBatchWithReturning(List> batchParams, ExecutionContext newEc,
+ String sourceDbIndex, String phyTableName) {
+ // Construct params for each batch
+ String orgBackfillReturning = newEc.getBackfillReturning();
+ try {
+ Parameters parameters = new Parameters();
+ parameters.setBatchParams(batchParams);
+
+ newEc.setParams(parameters);
+
+ newEc.setBackfillReturning(backfillReturning);
+
+ return executeInsert(sqlInsertIgnore, schemaName, tableName, newEc, sourceDbIndex, phyTableName);
+ } finally {
+ newEc.setBackfillReturning(orgBackfillReturning);
+ }
}
/**
@@ -249,21 +324,54 @@ private int applyBatch(List> batchParams, Executi
* @param param Parameter
* @param newEc Copied ExecutionContext
* @param sourceDbIndex the rows extract from which physicalDb
- * @param ignore Use insert ignore
* @return Affected rows
*/
private int applyRow(Map param, ExecutionContext newEc, String sourceDbIndex,
- String phyTableName,
- boolean ignore) {
+ String phyTableName) {
Parameters parameters = new Parameters();
parameters.setParams(param);
newEc.setParams(parameters);
- return executeInsert(ignore ? sqlInsertIgnore : sqlInsert, schemaName, tableName, newEc, sourceDbIndex,
- phyTableName);
+ return executeInsert(sqlInsertIgnore, schemaName, tableName, newEc, sourceDbIndex, phyTableName);
}
public abstract int executeInsert(SqlInsert sqlInsert, String schemaName, String tableName,
ExecutionContext executionContext, String sourceDbIndex, String phyTableName);
+
+ public static boolean canUseBackfillReturning(ExecutionContext ec, String schemaName) {
+ final ExecutorContext executorContext = ExecutorContext.getContext(schemaName);
+ final TopologyHandler topologyHandler = executorContext.getTopologyHandler();
+ final boolean allDnUseXDataSource = isAllDnUseXDataSource(topologyHandler);
+
+ return executorContext.getStorageInfoManager().supportsBackfillReturning()
+ && ec.getParamManager()
+ .getBoolean(ConnectionParams.BACKFILL_USE_RETURNING) && allDnUseXDataSource;
+ }
+
+ public static int getReturningAffectRows(List> returningResult,
+ ExecutionContext newEc) {
+ List> orgParams = newEc.getParams().getBatchParameters();
+
+ if (returningResult.isEmpty()) {
+ return orgParams.size();
+ }
+
+ // 判断
+ for (Map baseParam : returningResult) {
+ final List pkParams = new ArrayList<>();
+
+ for (int i = 0; i < baseParam.size(); i++) {
+ final ParameterContext pkParam = baseParam.get(i + 1);
+ pkParams.add(
+ Optional.ofNullable(pkParam.getArgs()[1]).map(e -> e.toString().toLowerCase()).orElse("NULL"));
+ }
+
+ throw new TddlRuntimeException(ErrorCode.ERR_GLOBAL_SECONDARY_INDEX_BACKFILL_DUPLICATE_ENTRY,
+ String.join("-", pkParams),
+ "PRIMARY");
+ }
+
+ return orgParams.size();
+ }
}
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/balancer/policy/PolicyUtils.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/balancer/policy/PolicyUtils.java
index 323b78af8..aa9621be7 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/balancer/policy/PolicyUtils.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/balancer/policy/PolicyUtils.java
@@ -16,6 +16,8 @@
package com.alibaba.polardbx.executor.balancer.policy;
+import com.alibaba.polardbx.common.exception.TddlRuntimeException;
+import com.alibaba.polardbx.common.exception.code.ErrorCode;
import com.alibaba.polardbx.common.utils.GeneralUtil;
import com.alibaba.polardbx.gms.locality.LocalityDetailInfoRecord;
import com.alibaba.polardbx.gms.metadb.MetaDbDataSource;
@@ -31,14 +33,16 @@
import com.alibaba.polardbx.optimizer.OptimizerContext;
import com.alibaba.polardbx.optimizer.OptimizerContext;
import com.alibaba.polardbx.optimizer.locality.LocalityManager;
+import com.alibaba.polardbx.optimizer.partition.PartitionByDefinition;
import com.alibaba.polardbx.optimizer.partition.PartitionInfoManager;
-import com.alibaba.polardbx.optimizer.partition.PartitionInfoManager;
+import com.alibaba.polardbx.optimizer.partition.PartitionSpec;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.stream.Collectors;
/**
@@ -140,12 +144,34 @@ public static List getLocalityDetails(String schema, S
TableGroupConfig tableGroupConfig = TableGroupUtils.getTableGroupInfoByGroupName(schema, tableGroup);
List localityDetailInfoRecords = new ArrayList<>();
int rowNum = 0;
- PartitionGroupRecord partitionGroupRecord = tableGroupConfig.getPartitionGroupByName(partitionGroup);
- String locality = (partitionGroupRecord == null)?"":partitionGroupRecord.getLocality();
+ String originalLocality = "";
+ PartitionSpec partitionSpec = null;
+ if (tableGroupConfig.getTables().size() > 0) {
+ String tableName = tableGroupConfig.getTables().get(0);
+ PartitionByDefinition subPartitionByDefinition =
+ OptimizerContext.getContext(schema).getPartitionInfoManager().getPartitionInfo(tableName)
+ .getPartitionBy().getSubPartitionBy();
+ if (subPartitionByDefinition != null) {
+ throw new TddlRuntimeException(ErrorCode.ERR_INVALID_DDL_PARAMS,
+ String.format(
+ "invalid alter locality operation on partition! we don't support alter table group [%s] locality with secondary partition",
+ tableGroup));
+ }
+ partitionSpec =
+ OptimizerContext.getContext(schema).getPartitionInfoManager().getPartitionInfo(tableName)
+ .getPartitionBy().getPartitionByPartName(partitionGroup);
+ } else {
+ throw new TddlRuntimeException(ErrorCode.ERR_INVALID_DDL_PARAMS,
+ String.format(
+ "invalid alter locality operation on partition! table group [%s] contains no table",
+ tableGroup));
+ }
+ String locality = Optional.ofNullable(partitionSpec.getLocality()).orElse("");
localityDetailInfoRecords.add(new LocalityDetailInfoRecord(rowNum++,
LocalityDetailInfoRecord.LOCALITY_TYPE_PARTITIONGROUP,
- partitionGroupRecord.id,
- partitionGroupRecord.partition_name,
+ // If with secondary partition, we must change it ourselvers.
+ tableGroupConfig.getPartitionGroupByName(partitionGroup).id,
+ partitionSpec.getName(),
locality));
return localityDetailInfoRecords;
} catch (SQLException e) {
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/chunk/EnumBlock.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/chunk/EnumBlock.java
index 5233f431f..9f680990a 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/chunk/EnumBlock.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/chunk/EnumBlock.java
@@ -144,7 +144,7 @@ public long hashCodeUseXxhash(int pos) {
String val = getString(pos);
Integer index = enumValues.get(val);
if (index == null) {
- return NULL_HASH_CODE;
+ return XxhashUtils.finalShuffle(0);
} else {
return XxhashUtils.finalShuffle(index);
}
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/columns/ColumnBackfillExecutor.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/columns/ColumnBackfillExecutor.java
index 45d442b5c..b2e9f4511 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/columns/ColumnBackfillExecutor.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/columns/ColumnBackfillExecutor.java
@@ -435,7 +435,7 @@ public Map> getSourcePhyTables() {
/**
* Check whether all dn using XProtocol
*/
- private static boolean isAllDnUseXDataSource(TopologyHandler topologyHandler) {
+ public static boolean isAllDnUseXDataSource(TopologyHandler topologyHandler) {
return topologyHandler.getGroupNames().stream()
.allMatch(groupName -> Optional.ofNullable(topologyHandler.get(groupName))
.map((Function) IGroupExecutor::getDataSource)
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/common/StorageInfoManager.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/common/StorageInfoManager.java
index 894ef9990..67c21f133 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/common/StorageInfoManager.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/common/StorageInfoManager.java
@@ -76,6 +76,7 @@ public class StorageInfoManager extends AbstractLifecycle {
private volatile boolean supportOpenSSL;
private volatile boolean supportSharedReadView;
private volatile boolean supportsReturning;
+ private volatile boolean supportsBackfillReturning;
private volatile boolean supportsAlterType;
private boolean readOnly;
private boolean lowerCaseTableNames;
@@ -124,6 +125,7 @@ public StorageInfoManager(TopologyHandler topologyHandler) {
supportXA = false;
supportsBloomFilter = false;
supportsReturning = false;
+ supportsBackfillReturning = false;
Preconditions.checkNotNull(topologyHandler);
this.topologyHandler = topologyHandler;
@@ -316,6 +318,45 @@ public static boolean checkSupportReturning(DataSource dataSource) {
}
}
+ public static boolean checkSupportBackfillReturning(DataSource dataSource) {
+ if (!ConfigDataMode.isPolarDbX() || XConfig.GALAXY_X_PROTOCOL) {
+ return false;
+ }
+ try (Connection conn = dataSource.getConnection();
+ Statement stmt = conn.createStatement();
+ ResultSet rs = stmt.executeQuery("call dbms_admin.show_native_procedure()")) {
+ boolean supportReturning = false;
+ while (rs.next()) {
+ final String schemaName = rs.getString(1);
+ final String procName = rs.getString(2);
+ supportReturning |= "dbms_trans".equalsIgnoreCase(schemaName) && "backfill".equalsIgnoreCase(procName);
+ if (supportReturning) {
+ break;
+ }
+ }
+ return supportReturning;
+ } catch (SQLException ex) {
+ final boolean ER_SP_DOES_NOT_EXIST =
+ "42000".equalsIgnoreCase(ex.getSQLState()) && 1305 == ex.getErrorCode() && ex.getMessage()
+ .contains("does not exist");
+ if (ER_SP_DOES_NOT_EXIST) {
+ logger.warn("PROCEDURE dbms_admin.show_native_procedure does not exist");
+ return false;
+ }
+
+ final boolean ER_PLUGGABLE_PROTOCOL_COMMAND_NOT_SUPPORTED =
+ "HY000".equalsIgnoreCase(ex.getSQLState()) && 3130 == ex.getErrorCode() && ex.getMessage()
+ .contains("Command not supported by pluggable protocols");
+ if (ER_PLUGGABLE_PROTOCOL_COMMAND_NOT_SUPPORTED) {
+ logger.warn("Do not support call dbms_amdin procedures within XPotocol");
+ return false;
+ }
+
+ throw new TddlRuntimeException(ErrorCode.ERR_OTHER, ex,
+ "Failed to check returning support: " + ex.getMessage());
+ }
+ }
+
public static boolean checkSupportAlterType(DataSource dataSource) {
try (Connection conn = dataSource.getConnection();
Statement stmt = conn.createStatement();
@@ -502,6 +543,7 @@ protected void doInit() {
boolean tmpSupportMdlDeadlockDetection = true;
boolean tmpSupportsBloomFilter = true;
boolean tmpSupportsReturning = true;
+ boolean tmpSupportsBackfillReturning = true;
boolean tmpSupportsAlterType = true;
boolean tmpLowerCaseTableNames = true;
boolean tmpSupportOpenSSL = true;
@@ -546,6 +588,7 @@ protected void doInit() {
tmpSupportOpenSSL &= storageInfo.supportOpenSSL;
tmpSupportHyperLogLog &= storageInfo.supportHyperLogLog;
tmpSupportsReturning &= storageInfo.supportsReturning;
+ tmpSupportsBackfillReturning &= storageInfo.supportsBackfillReturning;
tmpSupportsAlterType &= storageInfo.supportsAlterType;
tmpLowerCaseTableNames &= enableLowerCaseTableNames(storageInfo);
tmpSupportSharedReadView &= storageInfo.supportSharedReadView;
@@ -567,6 +610,7 @@ protected void doInit() {
this.supportXA = tmpSupportXA && !readOnly;
this.supportsBloomFilter = tmpSupportsBloomFilter;
this.supportsReturning = tmpSupportsReturning;
+ this.supportsBackfillReturning = tmpSupportsBackfillReturning;
this.supportsAlterType = tmpSupportsAlterType;
this.supportTso = tmpSupportTso && (metaDbUsesXProtocol() || tmpRDS80);
this.supportTsoHeartbeat = tmpSupportTsoHeartbeat && metaDbUsesXProtocol();
@@ -640,6 +684,7 @@ protected void doDestroy() {
supportXA = false;
supportsBloomFilter = false;
supportsReturning = false;
+ supportsBackfillReturning = false;
}
private StorageInfo initStorageInfo(Group group, IDataSource dataSource) {
@@ -805,6 +850,14 @@ public boolean supportsReturning() {
return supportsReturning;
}
+ public boolean supportsBackfillReturning() {
+ if (!isInited()) {
+ init();
+ }
+
+ return supportsBackfillReturning;
+ }
+
public boolean supportsAlterType() {
if (!isInited()) {
init();
@@ -881,6 +934,7 @@ public static class StorageInfo {
public final boolean supportLizard1PCTransaction;
public final boolean supportsBloomFilter;
public final boolean supportsReturning;
+ public final boolean supportsBackfillReturning;
public final boolean supportsAlterType;
public final int lowerCaseTableNames;
public final boolean supportPerformanceSchema;
@@ -909,6 +963,7 @@ public StorageInfo(
boolean supportLizard1PCTransaction,
boolean supportsBloomFilter,
boolean supportsReturning,
+ boolean supportsBackfillReturning,
boolean supportsAlterType,
int lowerCaseTableNames,
boolean supportPerformanceSchema,
@@ -936,6 +991,7 @@ public StorageInfo(
this.supportLizard1PCTransaction = supportLizard1PCTransaction;
this.supportsBloomFilter = supportsBloomFilter;
this.supportsReturning = supportsReturning;
+ this.supportsBackfillReturning = supportsBackfillReturning;
this.supportsAlterType = supportsAlterType;
this.lowerCaseTableNames = lowerCaseTableNames;
this.supportPerformanceSchema = supportPerformanceSchema;
@@ -969,6 +1025,7 @@ public static StorageInfo create(IDataSource dataSource) {
false,
false,
false,
+ false,
1,
false,
false,
@@ -998,6 +1055,7 @@ public static StorageInfo create(IDataSource dataSource) {
Optional polarxUDFInfo = PolarxUDFInfo.build(dataSource);
boolean supportsBloomFilter = polarxUDFInfo.map(PolarxUDFInfo::supportsBloomFilter).orElse(false);
boolean supportsReturning = checkSupportReturning(dataSource);
+ boolean supportsBackfillReturning = checkSupportBackfillReturning(dataSource);
boolean supportsAlterType = checkSupportAlterType(dataSource);
boolean supportCtsTransaction = checkSupportCtsTransaction(dataSource);
boolean supportAsyncCommit = checkSupportAsyncCommit(dataSource);
@@ -1028,6 +1086,7 @@ public static StorageInfo create(IDataSource dataSource) {
supportLizard1PCTransaction,
supportsBloomFilter,
supportsReturning,
+ supportsBackfillReturning,
supportsAlterType,
lowerCaseTableNames,
supportPerformanceSchema,
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/common/TopologyHandler.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/common/TopologyHandler.java
index 122f2ec06..667cdec86 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/common/TopologyHandler.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/common/TopologyHandler.java
@@ -37,6 +37,7 @@
import com.alibaba.polardbx.gms.topology.DbGroupInfoAccessor;
import com.alibaba.polardbx.gms.topology.DbGroupInfoManager;
import com.alibaba.polardbx.gms.topology.DbGroupInfoRecord;
+import com.alibaba.polardbx.gms.topology.DbTopologyManager;
import com.alibaba.polardbx.gms.util.GroupInfoUtil;
import com.alibaba.polardbx.group.config.Weight;
import com.alibaba.polardbx.group.jdbc.TGroupDataSource;
@@ -230,11 +231,13 @@ private void mergeMatrix(Matrix matrix) {
}
List grpListToClose = new ArrayList<>();
+ List grpNameListToClose = new ArrayList<>();
for (Group oldGroup : oldGroups) {
boolean found = newGrpNameSet.contains(oldGroup.getName().toUpperCase());
if (!found) {
// 关闭老的group
grpListToClose.add(oldGroup);
+ grpNameListToClose.add(oldGroup.getName());
}
}
for (Group oldGroup : oldScaleOutGroups) {
@@ -242,6 +245,7 @@ private void mergeMatrix(Matrix matrix) {
if (!found) {
// 关闭老的scale out group
grpListToClose.add(oldGroup);
+ grpNameListToClose.add(oldGroup.getName());
}
}
loadAllTransGroupList();
@@ -259,6 +263,8 @@ private void mergeMatrix(Matrix matrix) {
logger.error(e);
}
}
+ DbTopologyManager.refreshGroupKeysIntoTopologyMapping(this.allTransGroupList, grpNameListToClose,
+ matrix.getSchemaName());
}
}
@@ -569,7 +575,6 @@ protected void loadAllTransGroupList() {
LoggerInit.TDDL_DYNAMIC_CONFIG.info(logMsg);
this.allTransGroupList = groupNames;
-
if (topologyChanger != null) {
topologyChanger.onTopology(this);
}
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/builder/gsi/CreateGlobalIndexBuilder.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/builder/gsi/CreateGlobalIndexBuilder.java
index 19819aead..d871e6261 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/builder/gsi/CreateGlobalIndexBuilder.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/builder/gsi/CreateGlobalIndexBuilder.java
@@ -1059,7 +1059,7 @@ protected SqlNode createClusteredIndexTable(SqlIdentifier indexTableName,
// validate unsupported columnar columns
if (isColumnar) {
- SqlValidatorImpl.validateUnsupportedColumTypeWithCci(
+ SqlValidatorImpl.validateUnsupportedColumnTypeWithCci(
indexTableStmt, indexTableStmt.getPrimaryKeyNames(),
gsiPreparedData.getColumns().stream()
.map(SqlIndexColumnName::getColumnNameStr)
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/AlterTableGroupChangeSetJobFactory.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/AlterTableGroupChangeSetJobFactory.java
index 5fa0f60c9..5ca81fc57 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/AlterTableGroupChangeSetJobFactory.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/AlterTableGroupChangeSetJobFactory.java
@@ -295,7 +295,8 @@ protected ExecutableDdlJob doCreate() {
phyTb);
CloneTableDataFileTask cloneTableDataFileTask =
new CloneTableDataFileTask(schemaName, tableName, srcDbAndGroup, tarDbAndGroup, phyTb,
- phyPartNames, sourceStorageId, sourceHostIpAndPort, targetHostsIpAndPort, batchSize);
+ phyPartNames, sourceStorageId, sourceHostIpAndPort, targetHostsIpAndPort, batchSize,
+ tableMeta.isEncryption());
cloneTableDataFileTask.setTaskId(ID_GENERATOR.nextId());
List importTableSpaceTasks = new ArrayList<>();
@@ -306,7 +307,8 @@ protected ExecutableDdlJob doCreate() {
srcTarGroup,
Pair.of(sourceStorageId, targetStorageId), storageInstAndUserInfos, batchSize, parallelism,
minUpdateBatch,
- waitLsn);
+ waitLsn,
+ tableMeta.isEncryption());
storageInstAndUserInfos.computeIfAbsent(sourceStorageId,
key -> PhysicalBackfillUtils.getUserPasswd(sourceStorageId));
Pair userAndPasswd = storageInstAndUserInfos.computeIfAbsent(targetStorageId,
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/AlterTableGroupMovePartitionJobFactory.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/AlterTableGroupMovePartitionJobFactory.java
index ef13f5460..8ebf0b1dd 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/AlterTableGroupMovePartitionJobFactory.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/AlterTableGroupMovePartitionJobFactory.java
@@ -296,7 +296,7 @@ public void constructSubTasks(String schemaName, ExecutableDdlJob executableDdlJ
}
if (preparedData.isUsePhysicalBackfill()) {
- for (List pipeLine : subTaskJobFactory.getPhysicalyTaskPipeLine()) {
+ for (List pipeLine : GeneralUtil.emptyIfNull(subTaskJobFactory.getPhysicalyTaskPipeLine())) {
DdlTask parentLeaveNode;
if (leavePipeLineQueue.size() < parallelism) {
parentLeaveNode = syncLsnTask;
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/AlterTableMovePartitionJobFactory.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/AlterTableMovePartitionJobFactory.java
index abc7acb7f..2d34988cc 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/AlterTableMovePartitionJobFactory.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/AlterTableMovePartitionJobFactory.java
@@ -421,7 +421,7 @@ public void constructSubTasks(String schemaName, ExecutableDdlJob executableDdlJ
new SyncLsnTask(schemaName, sourceGroupAndStorageIdMap, targetGroupAndStorageIdMap);
executableDdlJob.addTask(syncLsnTask);
- for (List pipeLine : subTaskJobFactory.getPhysicalyTaskPipeLine()) {
+ for (List pipeLine : GeneralUtil.emptyIfNull(subTaskJobFactory.getPhysicalyTaskPipeLine())) {
DdlTask parentLeaveNode;
if (leavePipeLineQueue.size() < parallelism) {
parentLeaveNode = syncLsnTask;
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/CreateFileStorageJobFactory.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/CreateFileStorageJobFactory.java
index 1e1390e47..aeced71ff 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/CreateFileStorageJobFactory.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/CreateFileStorageJobFactory.java
@@ -31,27 +31,35 @@
import java.util.Map;
import java.util.Set;
+import static com.alibaba.polardbx.gms.topology.SystemDbHelper.DEFAULT_DB_NAME;
+
public class CreateFileStorageJobFactory extends DdlJobFactory {
private static final Logger logger = LoggerFactory.getLogger("oss");
private ExecutionContext executionContext;
private Engine engine;
private Map items;
+ private Map azureItems;
public CreateFileStorageJobFactory(
Engine engine, Map items,
+ Map azureItems,
ExecutionContext executionContext) {
this.executionContext = executionContext;
this.engine = engine;
this.items = items;
+ this.azureItems = azureItems;
}
@Override
protected void validate() {
if (!items.containsKey(FileStorageInfoKey.FILE_URI)) {
- throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS, "Should contain FILE_URI in with!");
+ if (Engine.ABS != engine) {
+ throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS, "Should contain FILE_URI in with!");
+ }
}
- if (engine.name().equalsIgnoreCase("OSS")) {
+ switch (engine) {
+ case OSS: {
if (!items.containsKey(FileStorageInfoKey.ENDPOINT)) {
throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS, "Should contain ENDPOINT in with!");
}
@@ -69,13 +77,58 @@ protected void validate() {
throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS,
"Should contain ACCESS_KEY_SECRET in with!");
}
+ break;
+ }
+ case ABS: {
+ if (!items.containsKey(FileStorageInfoKey.AZURE_CONNECTION_STRING)) {
+ throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS,
+ "Should contain AZURE_CONNECTION_STRING in with!");
+ }
+ if (!items.containsKey(FileStorageInfoKey.AZURE_CONTAINER_NAME)) {
+ throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS,
+ "Should contain AZURE_CONTAINER_NAME in with!");
+ }
+
+ // check connection string
+ if (!azureItems.containsKey(FileStorageInfoKey.AzureConnectionStringKey.DefaultEndpointsProtocol)) {
+ throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS,
+ "Should contain DefaultEndpointsProtocol in connection string!");
+ }
+ if (!azureItems.containsKey(FileStorageInfoKey.AzureConnectionStringKey.AccountName)) {
+ throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS,
+ "Should contain AccountName in connection string!");
+ }
+ if (!azureItems.containsKey(FileStorageInfoKey.AzureConnectionStringKey.AccountKey)) {
+ throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS,
+ "Should contain AccountKey in connection string!");
+ }
+ if (!azureItems.containsKey(FileStorageInfoKey.AzureConnectionStringKey.EndpointSuffix)) {
+ throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS,
+ "Should contain EndpointSuffix in connection string!");
+ }
+ break;
+ }
+ case S3: {
+ if (!items.containsKey(FileStorageInfoKey.ACCESS_KEY_ID)) {
+ throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS, "Should contain ACCESS_KEY_ID in with!");
+ }
+ if (!items.containsKey(FileStorageInfoKey.ACCESS_KEY_SECRET)) {
+ throw new TddlRuntimeException(ErrorCode.ERR_EXECUTE_ON_OSS,
+ "Should contain ACCESS_KEY_SECRET in with!");
+ }
+ break;
+ }
+ default:
+ break;
}
}
@Override
protected ExecutableDdlJob doCreate() {
ExecutableDdlJob executableDdlJob = new ExecutableDdlJob();
- executableDdlJob.addTask(new CreateFileStorageTask(executionContext.getSchemaName(), engine.name(), items));
+ executableDdlJob.addTask(
+ new CreateFileStorageTask(DEFAULT_DB_NAME, engine.name(), items, azureItems)
+ );
return executableDdlJob;
}
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/CreateJavaFunctionJobFactory.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/CreateJavaFunctionJobFactory.java
index 9020f5544..61a59869b 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/CreateJavaFunctionJobFactory.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/CreateJavaFunctionJobFactory.java
@@ -83,9 +83,7 @@ private void checkJavaCodeValid() {
String javaCode = sqlCreateFunction.getJavaCode();
String functionName = sqlCreateFunction.getFuncName();
String className = StringUtils.funcNameToClassName(functionName);
- if (InstConfUtil.getBool(ConnectionParams.CHECK_INVALID_JAVA_UDF)) {
- CompileUtils.checkInvalidJavaCode(javaCode, className);
- }
+ CompileUtils.checkInvalidJavaCode(javaCode, className);
}
private void checkDataTypeValid() {
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/MoveDatabaseChangeSetJobFactory.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/MoveDatabaseChangeSetJobFactory.java
index 6e2d01c7f..5518f8e0c 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/MoveDatabaseChangeSetJobFactory.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/MoveDatabaseChangeSetJobFactory.java
@@ -19,7 +19,6 @@
import com.alibaba.polardbx.common.properties.ConnectionParams;
import com.alibaba.polardbx.common.utils.Pair;
import com.alibaba.polardbx.executor.changeset.ChangeSetManager;
-import com.alibaba.polardbx.executor.common.ExecutorContext;
import com.alibaba.polardbx.executor.ddl.job.converter.DdlJobDataConverter;
import com.alibaba.polardbx.executor.ddl.job.converter.PhysicalPlanData;
import com.alibaba.polardbx.executor.ddl.job.task.backfill.MoveTableBackFillTask;
@@ -31,17 +30,14 @@
import com.alibaba.polardbx.executor.ddl.job.task.basic.PhysicalBackfillTask;
import com.alibaba.polardbx.executor.ddl.job.task.changset.ChangeSetApplyExecutorInitTask;
import com.alibaba.polardbx.executor.ddl.job.task.changset.ChangeSetApplyFinishTask;
-import com.alibaba.polardbx.executor.ddl.job.task.changset.MoveTableCheckTask;
import com.alibaba.polardbx.executor.ddl.job.task.changset.ChangeSetCatchUpTask;
import com.alibaba.polardbx.executor.ddl.job.task.changset.ChangeSetStartTask;
import com.alibaba.polardbx.executor.ddl.job.task.changset.MoveTableCheckTask;
-import com.alibaba.polardbx.executor.ddl.job.task.shared.EmptyTask;
import com.alibaba.polardbx.executor.ddl.newengine.job.DdlTask;
import com.alibaba.polardbx.executor.ddl.newengine.job.ExecutableDdlJob;
import com.alibaba.polardbx.executor.ddl.util.ChangeSetUtils;
import com.alibaba.polardbx.executor.physicalbackfill.PhysicalBackfillUtils;
import com.alibaba.polardbx.gms.topology.DbTopologyManager;
-import com.alibaba.polardbx.gms.topology.GroupDetailInfoExRecord;
import com.alibaba.polardbx.gms.util.GroupInfoUtil;
import com.alibaba.polardbx.optimizer.config.table.ComplexTaskMetaManager;
import com.alibaba.polardbx.optimizer.config.table.TableMeta;
@@ -52,14 +48,13 @@
import org.apache.calcite.rel.core.DDL;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
-import static com.alibaba.polardbx.executor.ddl.newengine.meta.DdlJobManager.ID_GENERATOR;
import static com.alibaba.polardbx.common.properties.ConnectionParams.CHANGE_SET_APPLY_OPTIMIZATION;
+import static com.alibaba.polardbx.executor.ddl.newengine.meta.DdlJobManager.ID_GENERATOR;
import static com.alibaba.polardbx.executor.ddl.util.ChangeSetUtils.genChangeSetCatchUpTasks;
public class MoveDatabaseChangeSetJobFactory extends MoveDatabaseSubTaskJobFactory {
@@ -195,14 +190,14 @@ protected ExecutableDdlJob doCreate() {
if (usePhysicalBackfill) {
for (Map.Entry> entry : sourceTableTopology.entrySet()) {
String srcGroupName = entry.getKey();
- String tarGroupName = GroupInfoUtil.buildScaloutGroupName(srcGroupName);
+ String tarGroupName = GroupInfoUtil.buildScaleOutGroupName(srcGroupName);
Pair srcTarGroup = Pair.of(srcGroupName, tarGroupName);
String sourceStorageId = sourceAndTarDnMap.computeIfAbsent(srcTarGroup.getKey(),
key -> DbTopologyManager.getStorageInstIdByGroupName(schemaName, srcTarGroup.getKey()));
String targetStorageId = tarGroupAndStorageIds.get(srcTarGroup.getValue());
String srcDbName = groupAndDbMap.computeIfAbsent(srcTarGroup.getKey(),
- key -> DbTopologyManager.getPhysicalDbNameByGroupKey(schemaName, srcTarGroup.getKey()));
+ key -> DbTopologyManager.getPhysicalDbNameByGroupKeyFromMetaDb(schemaName, srcTarGroup.getKey()));
Pair srcDbAndGroup = Pair.of(srcDbName.toLowerCase(), srcTarGroup.getKey());
String tarDbName;
@@ -233,7 +228,8 @@ protected ExecutableDdlJob doCreate() {
phyTb);
CloneTableDataFileTask cloneTableDataFileTask =
new CloneTableDataFileTask(schemaName, tableName, srcDbAndGroup, tarDbAndGroup, phyTb,
- phyPartNames, sourceStorageId, sourceHostIpAndPort, targetHostsIpAndPort, batchSize);
+ phyPartNames, sourceStorageId, sourceHostIpAndPort, targetHostsIpAndPort, batchSize,
+ tableMeta.isEncryption());
cloneTableDataFileTask.setTaskId(ID_GENERATOR.nextId());
List importTableSpaceTasks = new ArrayList<>();
@@ -244,7 +240,8 @@ protected ExecutableDdlJob doCreate() {
srcTarGroup,
Pair.of(sourceStorageId, targetStorageId), storageInstAndUserInfos, batchSize, parallelism,
minUpdateBatch,
- waitLsn);
+ waitLsn,
+ tableMeta.isEncryption());
storageInstAndUserInfos.computeIfAbsent(sourceStorageId,
key -> PhysicalBackfillUtils.getUserPasswd(sourceStorageId));
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/MoveDatabaseJobFactory.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/MoveDatabaseJobFactory.java
index cde7690b4..d301c79b6 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/MoveDatabaseJobFactory.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/MoveDatabaseJobFactory.java
@@ -24,7 +24,6 @@
import com.alibaba.polardbx.executor.balancer.Balancer;
import com.alibaba.polardbx.executor.balancer.stats.BalanceStats;
import com.alibaba.polardbx.executor.balancer.stats.GroupStats;
-import com.alibaba.polardbx.executor.balancer.stats.PartitionGroupStat;
import com.alibaba.polardbx.executor.ddl.job.builder.MoveDatabaseBuilder;
import com.alibaba.polardbx.executor.ddl.job.task.BaseValidateTask;
import com.alibaba.polardbx.executor.ddl.job.task.CostEstimableDdlTask;
@@ -36,9 +35,6 @@
import com.alibaba.polardbx.executor.ddl.job.task.basic.PauseCurrentJobTask;
import com.alibaba.polardbx.executor.ddl.job.task.basic.PhysicalBackfillTask;
import com.alibaba.polardbx.executor.ddl.job.task.basic.SyncLsnTask;
-import com.alibaba.polardbx.executor.ddl.job.task.shared.EmptyTask;
-import com.alibaba.polardbx.executor.ddl.job.task.shared.EmptyLogTask;
-import com.alibaba.polardbx.executor.ddl.job.task.shared.EmptyTask;
import com.alibaba.polardbx.executor.ddl.job.task.changset.ChangeSetApplyExecutorInitTask;
import com.alibaba.polardbx.executor.ddl.job.task.changset.ChangeSetApplyFinishTask;
import com.alibaba.polardbx.executor.ddl.newengine.job.DdlJobFactory;
@@ -72,8 +68,6 @@
import java.util.Set;
import java.util.TreeMap;
-import static com.alibaba.polardbx.common.properties.ConnectionParams.CHANGE_SET_APPLY_OPTIMIZATION;
-
/**
* Created by luoyanxin.
*
@@ -197,7 +191,7 @@ public void constructSubTasks(ExecutableDdlJob executableDdlJob,
executableDdlJob.addTask(syncLsnTask);
syncLsnTaskAdded = true;
}
- for (List pipeLine : subTaskJobFactory.getPhysicalyTaskPipeLine()) {
+ for (List pipeLine : GeneralUtil.emptyIfNull(subTaskJobFactory.getPhysicalyTaskPipeLine())) {
DdlTask parentLeaveNode;
if (leavePipeLineQueue.size() < parallelism) {
parentLeaveNode = syncLsnTask;
@@ -255,12 +249,12 @@ protected ExecutableDdlJob doCreate() {
for (String sourceGroup : entry.getValue()) {
if (!shareStorageMode) {
instGroupDbInfos.computeIfAbsent(entry.getKey(), o -> new ArrayList<>())
- .add(Pair.of(GroupInfoUtil.buildScaloutGroupName(sourceGroup),
+ .add(Pair.of(GroupInfoUtil.buildScaleOutGroupName(sourceGroup),
GroupInfoUtil.buildPhysicalDbNameFromGroupName(sourceGroup)));
} else {
String targetPhyDb = GroupInfoUtil.buildScaleOutPhyDbName(schemaName, sourceGroup);
instGroupDbInfos.computeIfAbsent(entry.getKey(), o -> new ArrayList<>())
- .add(Pair.of(GroupInfoUtil.buildScaloutGroupName(sourceGroup), targetPhyDb));
+ .add(Pair.of(GroupInfoUtil.buildScaleOutGroupName(sourceGroup), targetPhyDb));
}
}
}
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/gsi/CreateGsiJobFactory.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/gsi/CreateGsiJobFactory.java
index e15b6b4ae..f502f0857 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/gsi/CreateGsiJobFactory.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/gsi/CreateGsiJobFactory.java
@@ -236,6 +236,7 @@ protected ExecutableDdlJob doCreate() {
bringUpGsi = GsiTaskFactory.addGlobalIndexTasks(
schemaName,
primaryTableName,
+ oldIndexName,
indexTableName,
stayAtDeleteOnly,
stayAtWriteOnly,
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/gsi/CreatePartitionGsiJobFactory.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/gsi/CreatePartitionGsiJobFactory.java
index 9d9698c84..f79ffb459 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/gsi/CreatePartitionGsiJobFactory.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/factory/gsi/CreatePartitionGsiJobFactory.java
@@ -224,6 +224,7 @@ protected ExecutableDdlJob doCreate() {
bringUpGsi = GsiTaskFactory.addGlobalIndexTasks(
schemaName,
primaryTableName,
+ oldIndexName,
indexTableName,
stayAtDeleteOnly,
stayAtWriteOnly,
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/CloneTableDataFileTask.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/CloneTableDataFileTask.java
index cd0281998..b5b410fd1 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/CloneTableDataFileTask.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/CloneTableDataFileTask.java
@@ -64,6 +64,7 @@ public class CloneTableDataFileTask extends BaseDdlTask {
final List> targetHostsIpAndPort;
final String sourceStorageInstId;
final Long batchSize;
+ final boolean encrypted;
//don't serialize this parameter
private transient Map> storageInstAndUserInfos = new ConcurrentHashMap<>();
@@ -71,7 +72,7 @@ public class CloneTableDataFileTask extends BaseDdlTask {
public CloneTableDataFileTask(String schemaName, String logicalTableName, Pair srcDbAndGroup,
Pair tarDbAndGroup, String phyTableName, List phyPartNames,
String sourceStorageInstId, Pair sourceHostIpAndPort,
- List> targetHostsIpAndPort, Long batchSize) {
+ List> targetHostsIpAndPort, Long batchSize, Boolean encrypted) {
super(schemaName);
this.srcDbAndGroup = srcDbAndGroup;
this.tarDbAndGroup = tarDbAndGroup;
@@ -82,6 +83,7 @@ public CloneTableDataFileTask(String schemaName, String logicalTableName, Pair ph
PhysicalBackfillManager.BackfillObjectBean bean = backfillBean.backfillObject;
try {
PhysicalBackfillDetailInfoFieldJSON detailInfoFieldJSON = bean.detailInfo;
- PhysicalBackfillUtils.deleteInnodbDataFile(schemaName, bean.sourceGroupName, bean.physicalDb,
- detailInfoFieldJSON.getSourceHostAndPort().getKey(),
- detailInfoFieldJSON.getSourceHostAndPort().getValue(),
- PhysicalBackfillUtils.convertToCfgFileName(bean.sourceDirName), true, ec);
- PhysicalBackfillUtils.deleteInnodbDataFile(schemaName, bean.sourceGroupName, bean.physicalDb,
- detailInfoFieldJSON.getSourceHostAndPort().getKey(),
- detailInfoFieldJSON.getSourceHostAndPort().getValue(), bean.sourceDirName, true, ec);
+ PhysicalBackfillUtils.deleteInnodbDataFiles(schemaName, detailInfoFieldJSON.getSourceHostAndPort(),
+ bean.sourceDirName, bean.sourceGroupName, bean.physicalDb, true, ec);
} catch (Exception ex) {
//ignore
try {
@@ -219,7 +216,9 @@ private void cloneInnodbDataFile(boolean hasNoPhyPart, Map> entry : srcFileAndDirs.entrySet()) {
Pair srcFileAndDir = entry.getValue();
- boolean handlerIbdFile = false;
+ boolean handleCfgFile = false;
+ boolean handleIbdFile = false;
+ boolean needHandleCfpFile = encrypted;
do {
PhysicalBackfillUtils.checkInterrupted(ec, null);
PolarxPhysicalBackfill.FileInfo.Builder srcFileInfoBuilder =
@@ -227,8 +226,16 @@ private void cloneInnodbDataFile(boolean hasNoPhyPart, Map> storageInstAndUserInfos;
private final boolean waitLsn;
+ private final boolean encrypted;
//don't serialize those parameters
private transient long lastUpdateTime = 0l;
@@ -104,7 +105,8 @@ public PhysicalBackfillTask(String schemaName,
long batchSize,
long parallelism,
long minUpdateBatch,
- boolean waitLsn) {
+ boolean waitLsn,
+ boolean encrypted) {
super(schemaName);
this.schemaName = schemaName;
this.backfillId = backfillId;
@@ -118,6 +120,7 @@ public PhysicalBackfillTask(String schemaName,
this.parallelism = Math.max(parallelism, 1);
this.minUpdateBatch = minUpdateBatch;
this.waitLsn = waitLsn;
+ this.encrypted = encrypted;
this.curSpeedLimit = OptimizerContext.getContext(schemaName).getParamManager()
.getLong(ConnectionParams.PHYSICAL_BACKFILL_SPEED_LIMIT);
@@ -340,20 +343,10 @@ public void foreachPhysicalFile(final Pair srcDbAndGroup,
final Pair sourceHostIpAndPort = detailInfo.getSourceHostAndPort();
final AtomicReference interrupted = new AtomicReference<>(false);
- // copy the .cfg file before .ibd file
- String srcFileName = srcFileAndDir.getKey();
- String srcDir;
- if (initBean.isEmpty()) {
- srcDir = PhysicalBackfillUtils.convertToCfgFileName(
- srcFileAndDir.getValue() + PhysicalBackfillUtils.TEMP_FILE_POSTFIX);
- } else {
- srcDir = PhysicalBackfillUtils.convertToCfgFileName(srcFileAndDir.getValue());
- }
+ // copy the .cfg/.cfp file before .ibd file
- String tarFileName = targetFileAndDir.getKey();
- String tarDir = PhysicalBackfillUtils.convertToCfgFileName(targetFileAndDir.getValue());
- copyCfgFile(Pair.of(srcFileName, srcDir), srcDbAndGroup, sourceHostIpAndPort,
- Pair.of(tarFileName, tarDir), targetDbAndGroup, targetHost, consumer, ec);
+ copyCfgFile(srcFileAndDir, srcDbAndGroup, sourceHostIpAndPort,
+ targetFileAndDir, targetDbAndGroup, targetHost, consumer, !initBean.isEmpty(), ec);
if (bitSetPosMark == null || bitSetPosMark.length == 0) {
bitSet = new BitSet(offsetAndSize.size());
@@ -450,12 +443,9 @@ public void foreachPhysicalFile(final Pair srcDbAndGroup,
Pair ipPortPair = bfb.backfillObject.detailInfo.getSourceHostAndPort();
- PhysicalBackfillUtils.deleteInnodbDataFile(schemaName, srcDbAndGroup.getValue(), srcDbAndGroup.getKey(),
- ipPortPair.getKey(), ipPortPair.getValue(),
- PhysicalBackfillUtils.convertToCfgFileName(tempFileAndDir.getValue()), false, ec);
+ PhysicalBackfillUtils.deleteInnodbDataFiles(schemaName, ipPortPair,
+ tempFileAndDir.getValue(), srcDbAndGroup.getValue(), srcDbAndGroup.getKey(), false, ec);
- PhysicalBackfillUtils.deleteInnodbDataFile(schemaName, srcDbAndGroup.getValue(), srcDbAndGroup.getKey(),
- ipPortPair.getKey(), ipPortPair.getValue(), tempFileAndDir.getValue(), false, ec);
// After all physical table finished
backfillManager.updateBackfillObject(ImmutableList.of(bor));
@@ -670,62 +660,93 @@ private void fallocateIbdFile(final ExecutionContext ec, final Pair srcFileAndDir, final Pair srcDbAndGroup,
+ private void copyCfgFile(final Pair srcDataFileAndDir,
+ final Pair srcDbAndGroup,
final Pair sourceHostIpAndPort,
- final Pair targetFileAndDir, final Pair tarDbAndGroup,
+ final Pair targetDataFileAndDir, final Pair tarDbAndGroup,
final List> targetHosts, BatchConsumer consumer,
+ boolean isInit,
ExecutionContext ec) {
- //delete first before copy,because do not have backfillMeta for cfg file
+ //delete first before copy,because do not have backfillMeta for cfg/cfp file
+ String srcCfgDir;
+ String srcCfpDir;
+ List> srcTargetFilePair = new ArrayList<>();
+ if (!isInit) {
+ srcCfgDir = PhysicalBackfillUtils.convertToCfgFileName(
+ srcDataFileAndDir.getValue() + PhysicalBackfillUtils.TEMP_FILE_POSTFIX, PhysicalBackfillUtils.CFG);
+ srcCfpDir = PhysicalBackfillUtils.convertToCfgFileName(
+ srcDataFileAndDir.getValue() + PhysicalBackfillUtils.TEMP_FILE_POSTFIX, PhysicalBackfillUtils.CFP);
+ } else {
+ srcCfgDir =
+ PhysicalBackfillUtils.convertToCfgFileName(srcDataFileAndDir.getValue(), PhysicalBackfillUtils.CFG);
+ srcCfpDir =
+ PhysicalBackfillUtils.convertToCfgFileName(srcDataFileAndDir.getValue(), PhysicalBackfillUtils.CFP);
+ }
+
+ //PhysicalBackfillUtils.
+ //String tarFileName = targetFileAndDir.getKey();
+ String tarCfgDir =
+ PhysicalBackfillUtils.convertToCfgFileName(targetDataFileAndDir.getValue(), PhysicalBackfillUtils.CFG);
+ String tarCfpDir =
+ PhysicalBackfillUtils.convertToCfgFileName(targetDataFileAndDir.getValue(), PhysicalBackfillUtils.CFP);
+ srcTargetFilePair.add(Pair.of(srcCfgDir, tarCfgDir));
+ if (encrypted) {
+ srcTargetFilePair.add(Pair.of(srcCfpDir, tarCfpDir));
+ }
for (Pair pair : GeneralUtil.emptyIfNull(
targetHosts)) {
PhysicalBackfillUtils.deleteInnodbDataFile(schemaName, tarDbAndGroup.getValue(), tarDbAndGroup.getKey(),
- pair.getKey(), pair.getValue(), targetFileAndDir.getValue(), true, ec);
+ pair.getKey(), pair.getValue(), targetDataFileAndDir.getValue(), true, ec);
}
PolarxPhysicalBackfill.TransferFileDataOperator transferFileData = null;
Pair srcUserInfo = storageInstAndUserInfos.get(sourceTargetDnId.getKey());
Pair tarUserInfo = storageInstAndUserInfos.get(sourceTargetDnId.getValue());
- long offset = 0l;
- do {
- boolean success = false;
- int tryTime = 0;
+ for (Pair srcTarDir : srcTargetFilePair) {
+ long offset = 0l;
do {
- try (XConnection conn = (XConnection) (PhysicalBackfillUtils.getXConnectionForStorage(
- srcDbAndGroup.getKey(),
- sourceHostIpAndPort.getKey(), sourceHostIpAndPort.getValue(), srcUserInfo.getKey(),
- srcUserInfo.getValue(), -1))) {
-
- PolarxPhysicalBackfill.TransferFileDataOperator.Builder builder =
- PolarxPhysicalBackfill.TransferFileDataOperator.newBuilder();
-
- builder.setOperatorType(PolarxPhysicalBackfill.TransferFileDataOperator.Type.GET_DATA_FROM_SRC_IBD);
- PolarxPhysicalBackfill.FileInfo.Builder fileInfoBuilder =
- PolarxPhysicalBackfill.FileInfo.newBuilder();
- fileInfoBuilder.setFileName(srcFileAndDir.getKey());
- fileInfoBuilder.setTempFile(false);
- fileInfoBuilder.setDirectory(srcFileAndDir.getValue());
- fileInfoBuilder.setPartitionName("");
- builder.setFileInfo(fileInfoBuilder.build());
- builder.setBufferLen(batchSize);
- builder.setOffset(offset);
- transferFileData = conn.execReadBufferFromFile(builder);
- success = true;
- } catch (Exception ex) {
- if (tryTime >= PhysicalBackfillUtils.MAX_RETRY) {
- throw new TddlRuntimeException(ErrorCode.ERR_SCALEOUT_EXECUTE, ex);
+ boolean success = false;
+ int tryTime = 0;
+ do {
+ try (XConnection conn = (XConnection) (PhysicalBackfillUtils.getXConnectionForStorage(
+ srcDbAndGroup.getKey(),
+ sourceHostIpAndPort.getKey(), sourceHostIpAndPort.getValue(), srcUserInfo.getKey(),
+ srcUserInfo.getValue(), -1))) {
+
+ PolarxPhysicalBackfill.TransferFileDataOperator.Builder builder =
+ PolarxPhysicalBackfill.TransferFileDataOperator.newBuilder();
+
+ builder.setOperatorType(
+ PolarxPhysicalBackfill.TransferFileDataOperator.Type.GET_DATA_FROM_SRC_IBD);
+ PolarxPhysicalBackfill.FileInfo.Builder fileInfoBuilder =
+ PolarxPhysicalBackfill.FileInfo.newBuilder();
+ fileInfoBuilder.setFileName(srcDataFileAndDir.getKey());
+ fileInfoBuilder.setTempFile(false);
+ fileInfoBuilder.setDirectory(srcTarDir.getKey());
+ fileInfoBuilder.setPartitionName("");
+ builder.setFileInfo(fileInfoBuilder.build());
+ builder.setBufferLen(batchSize);
+ builder.setOffset(offset);
+ transferFileData = conn.execReadBufferFromFile(builder);
+ success = true;
+ } catch (Exception ex) {
+ if (tryTime >= PhysicalBackfillUtils.MAX_RETRY) {
+ throw new TddlRuntimeException(ErrorCode.ERR_SCALEOUT_EXECUTE, ex);
+ }
+ tryTime++;
}
- tryTime++;
+ } while (!success);
+ consumer.consume(tarDbAndGroup, Pair.of(srcDataFileAndDir.getKey(), srcTarDir.getValue()), targetHosts,
+ tarUserInfo,
+ transferFileData);
+ if (transferFileData.getBufferLen() < batchSize) {
+ break;
}
- } while (!success);
- consumer.consume(tarDbAndGroup, targetFileAndDir, targetHosts, tarUserInfo,
- transferFileData);
- if (transferFileData.getBufferLen() < batchSize) {
- return;
- }
- offset += transferFileData.getBufferLen();
- } while (true);
+ offset += transferFileData.getBufferLen();
+ } while (true);
+ }
}
}
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/TruncateTableValidateTask.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/TruncateTableValidateTask.java
index f3e015931..026d6505b 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/TruncateTableValidateTask.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/TruncateTableValidateTask.java
@@ -23,6 +23,7 @@
import com.alibaba.polardbx.executor.ddl.job.validator.TableValidator;
import com.alibaba.polardbx.gms.tablegroup.TableGroupConfig;
import com.alibaba.polardbx.optimizer.context.ExecutionContext;
+import org.apache.calcite.sql.SqlKind;
import lombok.Getter;
@Getter
@@ -43,6 +44,7 @@ public TruncateTableValidateTask(String schemaName, String logicalTableName, Tab
public void executeImpl(ExecutionContext executionContext) {
TableValidator.validateTableExistence(schemaName, logicalTableName, executionContext);
TableValidator.validateTableNotReferenceFk(schemaName, logicalTableName, executionContext);
+ TableValidator.validateTableWithCCI(schemaName, logicalTableName, executionContext, SqlKind.TRUNCATE_TABLE);
GsiValidator.validateAllowTruncateOnTable(schemaName, logicalTableName, executionContext);
if (tableGroupConfig != null) {
TableValidator.validateTableGroupChange(schemaName, tableGroupConfig);
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/oss/CreateFileStorageTask.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/oss/CreateFileStorageTask.java
index 25d78e3dc..dce217320 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/oss/CreateFileStorageTask.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/oss/CreateFileStorageTask.java
@@ -40,6 +40,7 @@
import java.io.IOException;
import java.sql.Connection;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
@@ -49,17 +50,24 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import static com.alibaba.polardbx.common.oss.filesystem.Constants.ABS_URI_SUFFIX_PATTERN;
+import static com.alibaba.polardbx.common.oss.filesystem.Constants.AZURE_WASBS_SCHEME;
+import static com.alibaba.polardbx.common.oss.filesystem.Constants.AZURE_WASB_SCHEME;
+
@Getter
@TaskName(name = "CreateFileStorageTask")
public class CreateFileStorageTask extends BaseGmsTask {
private String engineName;
private Map items;
+ private Map azureItems;
@JSONCreator
- public CreateFileStorageTask(String schemaName, String engineName, Map items) {
+ public CreateFileStorageTask(String schemaName, String engineName, Map items,
+ Map azureItems) {
super(schemaName, null);
this.engineName = engineName;
this.items = items;
+ this.azureItems = azureItems;
onExceptionTryRollback();
}
@@ -76,10 +84,10 @@ protected void executeImpl(Connection metaDbConnection, ExecutionContext executi
record1.priority = 1;
record1.regionId = "";
record1.availableZoneId = "";
- record1.cachePolicy = CachePolicy.META_AND_DATA_CACHE.getValue();
+ record1.cachePolicy = CachePolicy.DATA_CACHE.getValue();
record1.deletePolicy = DeletePolicy.MASTER_ONLY.getValue();
record1.status = 1;
- String uri = items.get(FileStorageInfoKey.FILE_URI).trim();
+ String uri = items.getOrDefault(FileStorageInfoKey.FILE_URI, "").trim();
if (!uri.endsWith("/")) {
uri = uri + "/";
}
@@ -90,6 +98,23 @@ protected void executeImpl(Connection metaDbConnection, ExecutionContext executi
record1.internalVpcEndpoint = items.get(FileStorageInfoKey.ENDPOINT);
record1.accessKeyId = items.get(FileStorageInfoKey.ACCESS_KEY_ID);
record1.accessKeySecret = PasswdUtil.encrypt(items.get(FileStorageInfoKey.ACCESS_KEY_SECRET));
+ } else if (Engine.S3.name().equalsIgnoreCase(record1.engine)) {
+ record1.accessKeyId = items.get(FileStorageInfoKey.ACCESS_KEY_ID);
+ record1.accessKeySecret = PasswdUtil.encrypt(items.get(FileStorageInfoKey.ACCESS_KEY_SECRET));
+ } else if (Engine.ABS.name().equalsIgnoreCase(record1.engine)) {
+ record1.accessKeyId = azureItems.get(FileStorageInfoKey.AzureConnectionStringKey.AccountName);
+ record1.accessKeySecret =
+ PasswdUtil.encrypt(azureItems.get(FileStorageInfoKey.AzureConnectionStringKey.AccountKey));
+ record1.externalEndpoint = azureItems.get(FileStorageInfoKey.AzureConnectionStringKey.EndpointSuffix);
+ record1.internalClassicEndpoint =
+ azureItems.get(FileStorageInfoKey.AzureConnectionStringKey.EndpointSuffix);
+ record1.internalVpcEndpoint = azureItems.get(FileStorageInfoKey.AzureConnectionStringKey.EndpointSuffix);
+ String endpointsProtocol =
+ azureItems.get(FileStorageInfoKey.AzureConnectionStringKey.DefaultEndpointsProtocol);
+ record1.fileUri =
+ ("https".equalsIgnoreCase(endpointsProtocol) ? AZURE_WASBS_SCHEME : AZURE_WASB_SCHEME) + "://"
+ + items.get(FileStorageInfoKey.AZURE_CONTAINER_NAME)
+ + "@" + String.format(ABS_URI_SUFFIX_PATTERN, record1.accessKeyId, record1.externalEndpoint) + "/";
}
if (fileStorageInfoAccessor.query(Engine.of(engineName)).size() != 0) {
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/oss/OSSTaskUtils.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/oss/OSSTaskUtils.java
index ac6100875..a7e5a1ee7 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/oss/OSSTaskUtils.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/basic/oss/OSSTaskUtils.java
@@ -182,7 +182,7 @@ public static Optional chooseRemoteNode(Long taskId) {
}
List candidates = new ArrayList<>(remoteNodeList);
candidates.add(null);
- int mod = (int) (MurmurHashUtils.murmurHashWithZeroSeed(taskId) % candidates.size());
+ int mod = (int) (MurmurHashUtils.murmurHash128WithZeroSeed(taskId) % candidates.size());
if (mod < 0) {
mod = mod + candidates.size();
}
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/factory/GsiTaskFactory.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/factory/GsiTaskFactory.java
index 984a0f6e7..0f7047b54 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/factory/GsiTaskFactory.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/factory/GsiTaskFactory.java
@@ -101,6 +101,7 @@ public static List createGlobalIndexTasks(String schemaName,
*/
public static List addGlobalIndexTasks(String schemaName,
String primaryTableName,
+ String oldIndexName,
String indexName,
boolean stayAtDeleteOnly,
boolean stayAtWriteOnly,
@@ -159,9 +160,10 @@ public static List addGlobalIndexTasks(String schemaName,
if (stayAtWriteOnly) {
return taskList;
}
+ String backFillSourceTableName = mirrorCopy ? oldIndexName : primaryTableName;
taskList.add(
- new LogicalTableBackFillTask(schemaName, primaryTableName, indexName, virtualColumns, backfillColumnMap,
- modifyStringColumns, false, mirrorCopy, modifyColumn));
+ new LogicalTableBackFillTask(schemaName, backFillSourceTableName, indexName, virtualColumns,
+ backfillColumnMap, modifyStringColumns, false, mirrorCopy, modifyColumn));
if (stayAtBackFill) {
return taskList;
}
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/tablegroup/TableGroupSyncTask.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/tablegroup/TableGroupSyncTask.java
index 2f3e672c5..1e23041c6 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/tablegroup/TableGroupSyncTask.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/tablegroup/TableGroupSyncTask.java
@@ -25,6 +25,8 @@
import com.alibaba.polardbx.optimizer.context.ExecutionContext;
import lombok.Getter;
+import java.sql.Connection;
+
@Getter
@TaskName(name = "TableGroupSyncTask")
public class TableGroupSyncTask extends BaseSyncTask {
@@ -42,7 +44,12 @@ public void executeImpl(ExecutionContext executionContext) {
syncTableGroup();
}
- private void syncTableGroup() {
+ @Override
+ protected void duringRollbackTransaction(Connection metaDbConnection, ExecutionContext executionContext) {
+ syncTableGroup();
+ }
+
+ protected void syncTableGroup() {
try {
SyncManagerHelper.sync(new TableGroupSyncAction(schemaName, tableGroupName), SyncScope.ALL, true);
} catch (Throwable t) {
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/tablegroup/TableGroupsSyncTask.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/tablegroup/TableGroupsSyncTask.java
index 58fc76c35..d56386e84 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/tablegroup/TableGroupsSyncTask.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/task/tablegroup/TableGroupsSyncTask.java
@@ -26,6 +26,7 @@
import com.alibaba.polardbx.optimizer.context.ExecutionContext;
import lombok.Getter;
+import java.sql.Connection;
import java.util.List;
@Getter
@@ -45,7 +46,12 @@ public void executeImpl(ExecutionContext executionContext) {
syncTableGroup();
}
- private void syncTableGroup() {
+ @Override
+ protected void duringRollbackTransaction(Connection metaDbConnection, ExecutionContext executionContext) {
+ syncTableGroup();
+ }
+
+ protected void syncTableGroup() {
try {
SyncManagerHelper.sync(new TableGroupsSyncAction(schemaName, tableGroupNameList), SyncScope.ALL);
} catch (Throwable t) {
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/validator/ForeignKeyValidator.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/validator/ForeignKeyValidator.java
index 077e65d3e..8bdc5be6a 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/validator/ForeignKeyValidator.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/job/validator/ForeignKeyValidator.java
@@ -254,21 +254,6 @@ public static void validateFkConstraints(SqlCreateTable sqlCreateTableOrigin, St
def.getDataType().getTypeName().getLastName()
.toUpperCase());
- String charSetName = def.getDataType().getCharSetName();
- String collationName =
- def.getDataType().getCollationName();
-
- RelDataTypeFactory factory = new TddlTypeFactoryImpl(TddlRelDataTypeSystemImpl.getInstance());
- boolean nullable =
- Optional.ofNullable(def.getNotNull()).map(cn -> SqlColumnDeclaration.ColumnNull.NULL == cn)
- .orElse(true);
- RelDataType type = def.getDataType().deriveType(factory, nullable);
-
- if (charSetName == null && SqlTypeUtil.inCharFamily(type)) {
- charSetName = sqlCreateTable.getDefaultCharset();
- collationName = sqlCreateTable.getDefaultCollation();
- }
-
if (!columnTypeName.equals(
SqlDataTypeSpec.DrdsTypeName.from(column.getDataType().getStringSqlType().toUpperCase()))) {
throw new TddlRuntimeException(ErrorCode.ERR_CHANGE_COLUMN_FK_CONSTRAINT,
@@ -276,16 +261,15 @@ public static void validateFkConstraints(SqlCreateTable sqlCreateTableOrigin, St
data.refTableName, data.constraint);
}
- if (charSetName != null && !StringUtils.equalsIgnoreCase(charSetName,
- column.getDataType().getCharsetName().name())) {
- throw new TddlRuntimeException(ErrorCode.ERR_ADD_FK_CHARSET_COLLATION,
- schemaName, tableName, data.refSchema, data.refTableName);
- }
+ Pair charsetCollationName = getCharsetCollationName(def, sqlCreateTable);
+ if (charsetCollationName != null) {
+ String charSetName = charsetCollationName.getKey();
+ String collationName = charsetCollationName.getValue();
- if (collationName != null && !StringUtils.equalsIgnoreCase(collationName,
- column.getDataType().getCollationName().name())) {
- throw new TddlRuntimeException(ErrorCode.ERR_ADD_FK_CHARSET_COLLATION,
- schemaName, tableName, data.refSchema, data.refTableName);
+ validateCharset(schemaName, tableName, data.refSchema, data.refTableName, charSetName,
+ column.getDataType().getCharsetName().name());
+ validateCollate(schemaName, tableName, data.refSchema, data.refTableName, collationName,
+ column.getDataType().getCollationName().name());
}
// can not add fk on generated column
@@ -315,6 +299,34 @@ public static void validateFkConstraints(SqlCreateTable sqlCreateTableOrigin, St
}
}
+ public static Pair getCharsetCollationName(SqlColumnDeclaration def,
+ SqlCreateTable sqlCreateTable) {
+ String charSetName;
+ String collationName;
+
+ CollationName collationNameOfColSpec =
+ CollationName.findCollationName(def.getDataType().getCollationName());
+ CharsetName charsetNameOfColSpec = CollationName.getCharsetOf(collationNameOfColSpec);
+
+ RelDataTypeFactory factory = new TddlTypeFactoryImpl(TddlRelDataTypeSystemImpl.getInstance());
+ boolean nullable =
+ Optional.ofNullable(def.getNotNull()).map(cn -> SqlColumnDeclaration.ColumnNull.NULL == cn)
+ .orElse(true);
+ RelDataType type = def.getDataType().deriveType(factory, nullable);
+
+ if (charsetNameOfColSpec != null) {
+ charSetName = charsetNameOfColSpec.name();
+ collationName = collationNameOfColSpec.name();
+ } else if (SqlTypeUtil.inCharFamily(type)) {
+ charSetName = sqlCreateTable.getDefaultCharset();
+ collationName = sqlCreateTable.getDefaultCollation();
+ } else {
+ return null;
+ }
+
+ return new Pair<>(charSetName, collationName);
+ }
+
public static void validateCharset(String srcSchemaName, String srcTableName, String refSchemaName,
String refTableName, String srcCharSet, String refTableCharset) {
CharsetName srcTableCharsetName = CharsetName.of(srcCharSet);
@@ -589,14 +601,29 @@ public static void validateAddReferredTableFkIndex(ForeignKeyData data, Executio
// table referenced itself
if (sqlCreateTable != null && data.refTableName.equalsIgnoreCase(tableName)) {
+ List> keys = new ArrayList<>();
if (sqlCreateTable.getKeys() != null) {
- for (Pair key : sqlCreateTable.getKeys()) {
- List indexColumnList = new ArrayList<>();
- key.getValue().getColumns().stream().map(c -> c.getColumnName().getLastName())
- .forEach(indexColumnList::add);
- if (hasFkColumnIndex(indexColumnList, columnsHash)) {
- return;
- }
+ keys.addAll(sqlCreateTable.getKeys());
+ }
+ if (sqlCreateTable.getUniqueKeys() != null) {
+ keys.addAll(sqlCreateTable.getUniqueKeys());
+ }
+ if (sqlCreateTable.getClusteredKeys() != null) {
+ keys.addAll(sqlCreateTable.getClusteredKeys());
+ }
+ if (sqlCreateTable.getGlobalKeys() != null) {
+ keys.addAll(sqlCreateTable.getGlobalKeys());
+ }
+ if (sqlCreateTable.getGlobalUniqueKeys() != null) {
+ keys.addAll(sqlCreateTable.getGlobalUniqueKeys());
+ }
+
+ for (Pair key : keys) {
+ List indexColumnList = new ArrayList<>();
+ key.getValue().getColumns().stream().map(c -> c.getColumnName().getLastName())
+ .forEach(indexColumnList::add);
+ if (hasFkColumnIndex(indexColumnList, columnsHash)) {
+ return;
}
}
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/newengine/DdlEngineRequester.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/newengine/DdlEngineRequester.java
index e902ab2e1..dce98748d 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/newengine/DdlEngineRequester.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/ddl/newengine/DdlEngineRequester.java
@@ -41,6 +41,7 @@
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
import java.time.Duration;
import java.util.ArrayList;
@@ -49,6 +50,7 @@
import java.util.Map;
import java.util.stream.Collectors;
+import static com.alibaba.polardbx.common.TddlConstants.INFORMATION_SCHEMA;
import static com.alibaba.polardbx.common.ddl.newengine.DdlConstants.LESS_WAITING_TIME;
import static com.alibaba.polardbx.common.ddl.newengine.DdlConstants.MORE_WAITING_TIME;
import static com.alibaba.polardbx.common.ddl.newengine.DdlType.ALTER_TABLEGROUP;
@@ -102,6 +104,11 @@ public long executeSubJob(long parentJobId, long parentTaskId, boolean forRollba
}
public void execute() {
+ if (StringUtils.equalsIgnoreCase(ddlContext.getSchemaName(), INFORMATION_SCHEMA)) {
+ throw DdlHelper.logAndThrowError(LOGGER,
+ "The DDL job can not be executed under the database 'information_schema'");
+ }
+
ddlContext.setResources(ddlJob.getExcludeResources());
// Create a new job and put it in the queue.
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gms/GmsTableMetaManager.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gms/GmsTableMetaManager.java
index 3eccd7961..f2a868413 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gms/GmsTableMetaManager.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gms/GmsTableMetaManager.java
@@ -62,6 +62,7 @@
import com.alibaba.polardbx.gms.partition.TablePartitionRecord;
import com.alibaba.polardbx.gms.tablegroup.ComplexTaskOutlineRecord;
import com.alibaba.polardbx.gms.topology.DbInfoManager;
+import com.alibaba.polardbx.gms.topology.DbInfoRecord;
import com.alibaba.polardbx.gms.util.MetaDbUtil;
import com.alibaba.polardbx.optimizer.OptimizerContext;
import com.alibaba.polardbx.optimizer.config.table.ColumnMeta;
@@ -721,6 +722,8 @@ public static TableMeta buildTableMeta(String schemaName, TablesRecord tableReco
hasPrimaryKey, TableStatus.convert(tableRecord.status), tableRecord.version, tableRecord.flag);
res.setId(tableRecord.id);
res.setEngine(Engine.of(tableRecord.engine));
+
+ res.setEncryption(TableMetaParser.parseEncryption(schemaName, tableRecord.createOptions));
if (Engine.isFileStore(res.getEngine())) {
res.buildFileStoreMeta(columnMappingMap, columnMetaMap);
}
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gms/MultiVersionColumnarSchema.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gms/MultiVersionColumnarSchema.java
index b280de9d8..a41a3083e 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gms/MultiVersionColumnarSchema.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gms/MultiVersionColumnarSchema.java
@@ -29,12 +29,9 @@
import com.alibaba.polardbx.gms.metadb.table.TableInfoManager;
import com.alibaba.polardbx.gms.util.MetaDbUtil;
import com.alibaba.polardbx.optimizer.config.table.ColumnMeta;
-import com.alibaba.polardbx.optimizer.core.TddlRelDataTypeSystemImpl;
-import com.alibaba.polardbx.optimizer.core.TddlTypeFactoryImpl;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.jetbrains.annotations.NotNull;
import java.sql.Connection;
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gms/util/AlterRepartitionUtils.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gms/util/AlterRepartitionUtils.java
index de832bfee..27d1df8d7 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gms/util/AlterRepartitionUtils.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gms/util/AlterRepartitionUtils.java
@@ -773,20 +773,27 @@ public static SqlAlterTablePartitionKey generateSqlPartitionKey(String schemaNam
StringBuilder sb = new StringBuilder();
sb.append("alter table ");
- sb.append(tableName);
-
- if (dbPartitionBy != null) {
- sb.append(" dbpartition by ").append(dbPartitionBy);
- if (dbPartitions != null) {
- sb.append(" dbpartitions ").append(dbPartitions);
- }
+ sb.append(SqlIdentifier.surroundWithBacktick(tableName));
+
+ if (tableRecord.getTableType() == GsiMetaManager.TableType.SHARDING.getValue()
+ || tableRecord.getTableType() == GsiMetaManager.TableType.GSI.getValue()) {
+ if (dbPartitionBy != null) {
+ sb.append(" dbpartition by ").append(dbPartitionBy);
+ if (dbPartitions != null) {
+ sb.append(" dbpartitions ").append(dbPartitions);
+ }
- if (tbPartitionBy != null) {
- sb.append(" tbpartition by ").append(tbPartitionBy);
- if (tbPartitions != null) {
- sb.append(" tbpartitions ").append(tbPartitions);
+ if (tbPartitionBy != null) {
+ sb.append(" tbpartition by ").append(tbPartitionBy);
+ if (tbPartitions != null) {
+ sb.append(" tbpartitions ").append(tbPartitions);
+ }
}
}
+ } else if (tableRecord.getTableType() == GsiMetaManager.TableType.SINGLE.getValue()) {
+ sb.append(" single");
+ } else if (tableRecord.getTableType() == GsiMetaManager.TableType.BROADCAST.getValue()) {
+ sb.append(" broadcast");
}
String sql = sb.toString();
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gms/util/StatisticUtils.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gms/util/StatisticUtils.java
index 5da1d5df3..b15341981 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gms/util/StatisticUtils.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gms/util/StatisticUtils.java
@@ -109,7 +109,6 @@
import java.util.Arrays;
import java.util.BitSet;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -723,7 +722,6 @@ public static void sampleTableDdl(String schemaName, String logicalTableName, Ex
h.buildFromData(objs.stream().filter(d -> isReady ? topN.get(d) == 0 : true).toArray());
cacheLine.setHistogram(colName, h);
}
- cacheLine.setRowCount((long) (rows.size() / sampleRate / sampleRateUp));
cacheLine.setSampleRate(sampleRate);
cacheLine.setLastModifyTime(unixTimeStamp());
@@ -1024,7 +1022,7 @@ private static double scanAnalyze(String schema, String logicalTableName, List());
}
if (trx != null) {
trx.close();
@@ -1035,14 +1033,17 @@ private static double scanAnalyze(String schema, String logicalTableName, List columnMetaList,
- float sampleRate) {
+ protected static String constructScanSamplingSql(String logicalTableName, List columnMetaList,
+ float sampleRate) {
StringBuilder sql = new StringBuilder();
String cmdExtraSamplePercentage = "";
cmdExtraSamplePercentage = ",sample_percentage=" + sampleRate * 100;
- sql.append("/*+TDDL:cmd_extra(merge_union=false,ENABLE_DIRECT_PLAN=false" + cmdExtraSamplePercentage + ") */ "
- + "select ");
+ sql.append(
+ "/*+TDDL:cmd_extra("
+ + "enable_post_planner=false,enable_index_selection=false,merge_union=false,enable_direct_plan=false"
+ + cmdExtraSamplePercentage + ") */ "
+ + "select ");
boolean first = true;
for (ColumnMeta columnMeta : columnMetaList) {
if (first) {
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/BackfillExecutor.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/BackfillExecutor.java
index 9a07d7fa6..4322fa9b3 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/BackfillExecutor.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/BackfillExecutor.java
@@ -61,7 +61,8 @@ public BackfillExecutor(BiFunction, ExecutionContext, List
}
public int backfill(String schemaName, String primaryTable, String indexName, boolean useBinary,
- boolean useChangeSet, List modifyStringColumns, ExecutionContext baseEc) {
+ boolean useChangeSet, boolean canUseReturning, List modifyStringColumns,
+ ExecutionContext baseEc) {
final long batchSize = baseEc.getParamManager().getLong(ConnectionParams.GSI_BACKFILL_BATCH_SIZE);
final long speedLimit = baseEc.getParamManager().getLong(ConnectionParams.GSI_BACKFILL_SPEED_LIMITATION);
final long speedMin = baseEc.getParamManager().getLong(ConnectionParams.GSI_BACKFILL_SPEED_MIN);
@@ -92,7 +93,8 @@ public int backfill(String schemaName, String primaryTable, String indexName, bo
GsiExtractor.create(schemaName, primaryTable, indexName, batchSize, speedMin, speedLimit, parallelism,
useBinary, modifyStringColumns, baseEc);
loader =
- GsiLoader.create(schemaName, primaryTable, indexName, this.executeFunc, baseEc.isUseHint(), baseEc);
+ GsiLoader.create(schemaName, primaryTable, indexName, this.executeFunc, baseEc.isUseHint(),
+ canUseReturning, baseEc);
}
boolean finished;
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/GsiUtils.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/GsiUtils.java
index b5acc071c..45109a78d 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/GsiUtils.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/GsiUtils.java
@@ -1043,6 +1043,11 @@ public static BackfillObjectRecord buildBackfillObjectRecord(long jobId, String
extra);
}
+ public static R wrapWithDistributedXATrx(ITransactionManager tm, ExecutionContext baseEc,
+ Function call) {
+ return wrapWithTransaction(tm, ITransactionPolicy.XA, baseEc, call);
+ }
+
public static R wrapWithDistributedTrx(ITransactionManager tm, ExecutionContext baseEc,
Function call) {
return wrapWithTransaction(tm, tm.getDefaultDistributedTrxPolicy(baseEc), baseEc, call);
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/InsertIndexExecutor.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/InsertIndexExecutor.java
index d90b11abe..d101c7a4e 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/InsertIndexExecutor.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/InsertIndexExecutor.java
@@ -205,11 +205,9 @@ public static int backfillIntoPartitionedTable(RelNode logicalInsert,
String schemaName,
ExecutionContext executionContext,
BiFunction, ExecutionContext, List> executeFunc,
- boolean mayInsertDuplicate,
- PartitionInfo newPartition,
- String targetGroup,
- String phyTableName,
- boolean mirrorCopy) {
+ boolean mayInsertDuplicate, PartitionInfo newPartition,
+ String targetGroup, String phyTableName, boolean mirrorCopy,
+ boolean returning, List> result) {
Map>> shardResults;
boolean forceReshard = executionContext.getParamManager().getBoolean(ConnectionParams.FORCE_RESHARD);
boolean skipShard = mirrorCopy && StringUtils.isNotEmpty(targetGroup) && !forceReshard;
@@ -255,7 +253,7 @@ public static int backfillIntoPartitionedTable(RelNode logicalInsert,
}
}
return insertIntoTable(logicalInsert, sqlInsert, tableMeta, "", schemaName, executionContext, executeFunc,
- mayInsertDuplicate, shardResults);
+ mayInsertDuplicate, shardResults, returning, result);
}
public static int insertIntoTable(RelNode logicalInsert, SqlInsert sqlInsert, TableMeta tableMeta,
@@ -263,7 +261,8 @@ public static int insertIntoTable(RelNode logicalInsert, SqlInsert sqlInsert, Ta
ExecutionContext executionContext,
BiFunction, ExecutionContext, List> executeFunc,
boolean mayInsertDuplicate,
- boolean mayForGsi, String partName) {
+ boolean mayForGsi, String partName,
+ boolean returning, List> result) {
Map>> shardResults;
boolean forceReshard = executionContext.getParamManager().getBoolean(ConnectionParams.FORCE_RESHARD);
if (StringUtils.isBlank(targetGroup) || StringUtils.isBlank(phyTableName) || forceReshard) {
@@ -316,7 +315,7 @@ public static int insertIntoTable(RelNode logicalInsert, SqlInsert sqlInsert, Ta
return insertIntoTable(logicalInsert, sqlInsert, tableMeta, targetGroup, schemaName, executionContext,
executeFunc,
- mayInsertDuplicate, shardResults);
+ mayInsertDuplicate, shardResults, returning, result);
}
public static int insertIntoTable(RelNode logicalInsert, SqlInsert sqlInsert, TableMeta tableMeta,
@@ -328,7 +327,7 @@ public static int insertIntoTable(RelNode logicalInsert, SqlInsert sqlInsert, Ta
return insertIntoTable(logicalInsert, sqlInsert, tableMeta, targetGroup, phyTableName, schemaName,
executionContext,
executeFunc,
- mayInsertDuplicate, mayForGsi, "");
+ mayInsertDuplicate, mayForGsi, "", false, null);
}
public static int insertIntoTable(RelNode logicalInsert, SqlInsert sqlInsert, TableMeta tableMeta,
@@ -336,7 +335,8 @@ public static int insertIntoTable(RelNode logicalInsert, SqlInsert sqlInsert, Ta
ExecutionContext executionContext,
BiFunction, ExecutionContext, List> executeFunc,
boolean mayInsertDuplicate,
- Map>> shardResults) {
+ Map>> shardResults,
+ boolean returning, List> result) {
final RelOptCluster cluster = SqlConverter.getInstance(schemaName, executionContext).createRelOptCluster();
RelTraitSet traitSet = RelTraitSet.createEmpty();
@@ -359,16 +359,6 @@ public static int insertIntoTable(RelNode logicalInsert, SqlInsert sqlInsert, Ta
SqlInsert newSqlInsert = visitor.visit(sqlInsert);
BytesSql sql = RelUtils.toNativeBytesSql(newSqlInsert, DbType.MYSQL);
-// PhyTableOperation phyTableModify =
-// new PhyTableOperation(cluster, traitSet, rowType, null, logicalInsert);
-// phyTableModify.setKind(newSqlInsert.getKind());
-// phyTableModify.setDbIndex(targetDb);
-// phyTableModify.setTableNames(ImmutableList.of(ImmutableList.of(targetTb)));
-// phyTableModify.setBytesSql(sql);
-// phyTableModify.setNativeSqlNode(newSqlInsert);
-// phyTableModify.setParam(outputParams);
-// phyTableModify.setBatchParameters(null);
-
PhyTableOpBuildParams buildParams = new PhyTableOpBuildParams();
buildParams.setSchemaName(schemaName);
buildParams.setLogTables(ImmutableList.of(tableMeta.getTableName()));
@@ -412,6 +402,11 @@ public static int insertIntoTable(RelNode logicalInsert, SqlInsert sqlInsert, Ta
}
}
+ if (returning && result != null) {
+ result.addAll(ExecUtils.getReturningResultByCursors(cursors, false));
+ return 0;
+ }
+
return ExecUtils.getAffectRowsByCursors(cursors, false);
}
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/PhysicalPlanBuilder.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/PhysicalPlanBuilder.java
index 992700ce3..5b270ffd4 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/PhysicalPlanBuilder.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/PhysicalPlanBuilder.java
@@ -80,8 +80,8 @@
import org.apache.calcite.sql.SqlSelect.LockMode;
import org.apache.calcite.sql.SqlSelectWithPartition;
import org.apache.calcite.sql.SqlUpdate;
-import org.apache.calcite.sql.fun.SqlHashCheckAggFunction;
import org.apache.calcite.sql.fun.SqlBinaryFunction;
+import org.apache.calcite.sql.fun.SqlHashCheckAggFunction;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.type.SqlTypeFactoryImpl;
@@ -745,7 +745,7 @@ public PhyTableOperation buildInsertSelectForOMCBackfill(TableMeta tableMeta,
boolean withLowerBound, boolean withUpperBound,
LockMode lockMode, boolean isInsertIgnore) {
SqlInsert sqlInsert = buildSqlInsertSelectForOMCBackfill(targetColumnNames, sourceColumnNames,
- primaryKeys, withLowerBound, withUpperBound, lockMode, isInsertIgnore);
+ primaryKeys, withLowerBound, withUpperBound, lockMode, isInsertIgnore, tableMeta.isHasPrimaryKey());
return buildDmlPhyTblOpTemplate(tableMeta.getSchemaName(), sqlInsert, tableMeta);
}
@@ -753,7 +753,8 @@ public SqlInsert buildSqlInsertSelectForOMCBackfill(List targetColumnNam
List sourceColumnNames,
List primaryKeys,
boolean withLowerBound, boolean withUpperBound,
- LockMode lockMode, boolean isInsertIgnore) {
+ LockMode lockMode, boolean isInsertIgnore,
+ boolean forceIndex) {
initParams(0);
// build select list
@@ -788,8 +789,9 @@ public SqlInsert buildSqlInsertSelectForOMCBackfill(List targetColumnNam
null == condition ? upperBound : PlannerUtils.buildAndTree(ImmutableList.of(condition, upperBound));
}
+ SqlNode target = forceIndex ? from : targetTableNode;
final SqlSelect sqlSelect =
- new SqlSelect(SqlParserPos.ZERO, null, selectList, from, condition, null, null, null, null,
+ new SqlSelect(SqlParserPos.ZERO, null, selectList, target, condition, null, null, null, null,
null, null);
sqlSelect.setLockMode(lockMode);
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/backfill/CdasLoader.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/backfill/CdasLoader.java
index a1181de70..875168777 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/backfill/CdasLoader.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/backfill/CdasLoader.java
@@ -54,6 +54,8 @@
import java.util.function.Supplier;
import java.util.stream.Collectors;
+import static org.apache.calcite.sql.fun.SqlStdOperatorTable.EQUALS;
+
/**
* Created by zhuqiwei.
*
@@ -66,9 +68,9 @@ public class CdasLoader extends Loader {
private CdasLoader(String schemaName, String tableName, SqlInsert insert, SqlInsert insertIgnore,
ExecutionPlan checkerPlan, int[] checkerPkMapping, int[] checkerParamMapping,
BiFunction, ExecutionContext, List> executeFunc,
- int[] insertedParamMapping, boolean needParamProject) {
+ int[] insertedParamMapping, boolean needParamProject, String backfillReturning) {
super(schemaName, tableName, insert, insertIgnore, checkerPlan, checkerPkMapping, checkerParamMapping,
- executeFunc, false);
+ executeFunc, false, backfillReturning);
this.insertedParamMapping = insertedParamMapping;
this.needParamProjection = needParamProject;
}
@@ -80,6 +82,9 @@ public static CdasLoader create(String srcSchemaName, String dstSchemaName, Stri
final OptimizerContext optimizerContextDst = OptimizerContext.getContext(dstSchemaName);
final OptimizerContext optimizerContextSrc = OptimizerContext.getContext(srcSchemaName);
+ boolean canUseReturning =
+ canUseBackfillReturning(ec, dstSchemaName) && canUseBackfillReturning(ec, srcSchemaName);
+
// Construct target table
final SqlNode targetTableParam = BuildPlanUtils.buildTargetTable();
// Construct targetColumnList
@@ -150,7 +155,8 @@ public static CdasLoader create(String srcSchemaName, String dstSchemaName, Stri
final TddlRuleManager tddlRuleManagerDst = optimizerContextDst.getRuleManager();
final Set filterColumns = Sets.newTreeSet(String::compareToIgnoreCase);
final Set primaryKeys = Sets.newTreeSet(String::compareToIgnoreCase);
- primaryKeys.addAll(GlobalIndexMeta.getPrimaryKeys(srcTableMeta));
+ final List pkList = GlobalIndexMeta.getPrimaryKeys(srcTableMeta);
+ primaryKeys.addAll(pkList);
filterColumns.addAll(primaryKeys);
filterColumns.addAll(tddlRuleManagerSrc.getSharedColumns(srcTableName));
filterColumns.addAll(tddlRuleManagerDst.getSharedColumns(dstTableName));
@@ -179,7 +185,8 @@ public static CdasLoader create(String srcSchemaName, String dstSchemaName, Stri
return new CdasLoader(dstSchemaName, dstTableName, sqlInsert, sqlInsertIgnore, checkerPlan, checkerPkMapping,
checkerParamMapping,
- executeFunc, insertMappings, needParamProject);
+ executeFunc, insertMappings, needParamProject,
+ canUseReturning ? String.join(",", pkList) : null);
}
@@ -187,8 +194,13 @@ public static CdasLoader create(String srcSchemaName, String dstSchemaName, Stri
public int executeInsert(SqlInsert sqlInsert, String schemaName, String tableName,
ExecutionContext executionContext, String sourceDbIndex, String phyTableName) {
TableMeta tableMeta = executionContext.getSchemaManager(schemaName).getTable(tableName);
- return InsertIndexExecutor
- .insertIntoTable(null, sqlInsert, tableMeta, schemaName, executionContext, executeFunc, false);
+ List> returningRes = new ArrayList<>();
+ int affectRows = InsertIndexExecutor
+ .insertIntoTable(null, sqlInsert, tableMeta, "", "",
+ schemaName, executionContext, executeFunc, false, true,
+ "", usingBackfillReturning, returningRes);
+
+ return usingBackfillReturning ? getReturningAffectRows(returningRes, executionContext) : affectRows;
}
@Override
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/backfill/GsiChangeSetLoader.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/backfill/GsiChangeSetLoader.java
index 80bab52c0..d76c39bdd 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/backfill/GsiChangeSetLoader.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/backfill/GsiChangeSetLoader.java
@@ -66,7 +66,8 @@ protected GsiChangeSetLoader(String schemaName, String tableName, SqlInsert inse
BiFunction, ExecutionContext, List> executeFunc) {
super(schemaName, tableName, insert, insertIgnore, checkerPlan, checkerPkMapping, checkerParamMapping,
executeFunc,
- true);
+ true, null);
+ this.conflictDetection = true;
this.tableNameMapping = tableNameMapping;
}
@@ -174,7 +175,8 @@ public int executeInsert(SqlInsert sqlInsert, String schemaName, String tableNam
PartitionInfo newPartInfo = tableMeta.getNewPartitionInfo();
return InsertIndexExecutor
.backfillIntoPartitionedTable(null, sqlInsert, tableMeta, schemaName, executionContext, executeFunc,
- false, newPartInfo, sourceDbIndex, tableNameMapping.get(phyTableName), this.mirrorCopy);
+ false, newPartInfo, sourceDbIndex, tableNameMapping.get(phyTableName), this.mirrorCopy, false,
+ null);
} else {
return InsertIndexExecutor
.insertIntoTable(null, sqlInsert, tableMeta, sourceDbIndex, tableNameMapping.get(phyTableName),
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/backfill/GsiLoader.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/backfill/GsiLoader.java
index cbf06933d..ab526dd88 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/backfill/GsiLoader.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/backfill/GsiLoader.java
@@ -17,7 +17,11 @@
package com.alibaba.polardbx.executor.gsi.backfill;
import com.alibaba.polardbx.executor.backfill.Extractor;
+import com.alibaba.polardbx.common.jdbc.ParameterContext;
+import com.alibaba.polardbx.common.properties.ConnectionParams;
import com.alibaba.polardbx.executor.backfill.Loader;
+import com.alibaba.polardbx.executor.common.ExecutorContext;
+import com.alibaba.polardbx.executor.common.TopologyHandler;
import com.alibaba.polardbx.executor.cursor.Cursor;
import com.alibaba.polardbx.executor.gsi.InsertIndexExecutor;
import com.alibaba.polardbx.executor.gsi.PhysicalPlanBuilder;
@@ -42,6 +46,7 @@
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParserPos;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -49,6 +54,8 @@
import java.util.stream.Collectors;
import static com.alibaba.polardbx.common.TddlConstants.IMPLICIT_COL_NAME;
+import static com.alibaba.polardbx.executor.columns.ColumnBackfillExecutor.isAllDnUseXDataSource;
+import static org.apache.calcite.sql.fun.SqlStdOperatorTable.EQUALS;
/**
* Fill batch data into index table with duplication check
@@ -57,14 +64,14 @@ public class GsiLoader extends Loader {
private GsiLoader(String schemaName, String tableName, SqlInsert insert, SqlInsert insertIgnore,
ExecutionPlan checkerPlan, int[] checkerPkMapping, int[] checkerParamMapping,
- BiFunction, ExecutionContext, List> executeFunc) {
+ BiFunction, ExecutionContext, List> executeFunc, String backfillReturning) {
super(schemaName, tableName, insert, insertIgnore, checkerPlan, checkerPkMapping, checkerParamMapping,
- executeFunc, false);
+ executeFunc, false, backfillReturning);
}
public static Loader create(String schemaName, String primaryTable, String indexTable,
BiFunction, ExecutionContext, List> executeFunc,
- boolean useHint, ExecutionContext ec) {
+ boolean useHint, boolean canUseReturning, ExecutionContext ec) {
final OptimizerContext optimizerContext = OptimizerContext.getContext(schemaName);
// Construct target table
@@ -120,7 +127,8 @@ public static Loader create(String schemaName, String primaryTable, String index
final TddlRuleManager tddlRuleManager = optimizerContext.getRuleManager();
final Set filterColumns = Sets.newTreeSet(String::compareToIgnoreCase);
final Set primaryKeys = Sets.newTreeSet(String::compareToIgnoreCase);
- primaryKeys.addAll(Extractor.getPrimaryKeys(primaryTableMeta, ec));
+ final List pkList = Extractor.getPrimaryKeys(primaryTableMeta, ec);
+ primaryKeys.addAll(pkList);
filterColumns.addAll(primaryKeys);
filterColumns.addAll(tddlRuleManager.getSharedColumns(primaryTable));
filterColumns.addAll(tddlRuleManager.getSharedColumns(indexTable));
@@ -158,14 +166,20 @@ public static Loader create(String schemaName, String primaryTable, String index
checkerPlan,
checkerPkMapping,
checkerParamMapping,
- executeFunc);
+ executeFunc,
+ canUseReturning ? String.join(",", pkList) : null);
}
@Override
public int executeInsert(SqlInsert sqlInsert, String schemaName, String tableName,
ExecutionContext executionContext, String sourceDbIndex, String phyTableName) {
TableMeta tableMeta = executionContext.getSchemaManager(schemaName).getTable(tableName);
- return InsertIndexExecutor
- .insertIntoTable(null, sqlInsert, tableMeta, schemaName, executionContext, executeFunc, false);
+ List> returningRes = new ArrayList<>();
+ int affectRows = InsertIndexExecutor
+ .insertIntoTable(null, sqlInsert, tableMeta, "", "",
+ schemaName, executionContext, executeFunc, false, true,
+ "", usingBackfillReturning, returningRes);
+
+ return usingBackfillReturning ? getReturningAffectRows(returningRes, executionContext) : affectRows;
}
}
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/utils/Transformer.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/utils/Transformer.java
index 3cb7af189..40c3b3295 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/utils/Transformer.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/gsi/utils/Transformer.java
@@ -151,6 +151,10 @@ public static List> convertUpperBoundWithDefaultF
* @param cursor result cursor of select
* @return batch parameters for insert
*/
+ public static List> buildBatchParam(Cursor cursor, boolean useBinary) {
+ return buildBatchParam(cursor, useBinary, null);
+ }
+
public static List> buildBatchParam(Cursor cursor, boolean useBinary,
Set notConvertColumns) {
final List> batchParams = new ArrayList<>();
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/handler/LogicalShowCreateTableHandler.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/handler/LogicalShowCreateTableHandler.java
index 23aef2d26..1b67ec98f 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/handler/LogicalShowCreateTableHandler.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/handler/LogicalShowCreateTableHandler.java
@@ -115,7 +115,7 @@ public static String reorgLogicalColumnOrder(String schemaName, String logicalTa
for (SQLTableElement tableElement : createTableStmt.getTableElementList()) {
if (tableElement instanceof SQLColumnDefinition) {
String physicalColumnName =
- SQLUtils.normalize(((SQLColumnDefinition) tableElement).getColumnName());
+ SQLUtils.normalizeNoTrim(((SQLColumnDefinition) tableElement).getColumnName());
if (TStringUtil.equalsIgnoreCase(physicalColumnName, logicalColumn.columnName)) {
newTableElements.add(tableElement);
break;
diff --git a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/handler/LogicalShowCreateTablesForPartitionDatabaseHandler.java b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/handler/LogicalShowCreateTablesForPartitionDatabaseHandler.java
index 2c2de2aa2..86d8536f3 100644
--- a/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/handler/LogicalShowCreateTablesForPartitionDatabaseHandler.java
+++ b/polardbx-executor/src/main/java/com/alibaba/polardbx/executor/handler/LogicalShowCreateTablesForPartitionDatabaseHandler.java
@@ -26,10 +26,11 @@
import com.alibaba.polardbx.common.properties.ConnectionParams;
import com.alibaba.polardbx.common.properties.ConnectionProperties;
import com.alibaba.polardbx.common.utils.TStringUtil;
-import com.alibaba.polardbx.config.ConfigDataMode;
import com.alibaba.polardbx.common.utils.logger.Logger;
import com.alibaba.polardbx.common.utils.logger.LoggerFactory;
+import com.alibaba.polardbx.config.ConfigDataMode;
import com.alibaba.polardbx.druid.sql.SQLUtils;
+import com.alibaba.polardbx.druid.sql.ast.SQLExpr;
import com.alibaba.polardbx.druid.sql.ast.SQLName;
import com.alibaba.polardbx.druid.sql.ast.SQLOrderingSpecification;
import com.alibaba.polardbx.druid.sql.ast.SQLPartitionBy;
@@ -66,6 +67,8 @@
import com.alibaba.polardbx.gms.metadb.table.ColumnarTableEvolutionRecord;
import com.alibaba.polardbx.gms.metadb.table.IndexStatus;
import com.alibaba.polardbx.gms.metadb.table.IndexVisibility;
+import com.alibaba.polardbx.gms.metadb.table.TablesAccessor;
+import com.alibaba.polardbx.gms.metadb.table.TablesRecord;
import com.alibaba.polardbx.gms.tablegroup.TableGroupConfig;
import com.alibaba.polardbx.gms.topology.DbInfoManager;
import com.alibaba.polardbx.optimizer.OptimizerContext;
@@ -324,7 +327,7 @@ public Cursor handle(RelNode logicalPlan, ExecutionContext executionContext) {
List indexDefs =
buildIndexDefs(schemaName, gsiMeta, tableName, tableMeta, localIndexes, showCreateTable.isFull(),
- needShowHashByRange);
+ needShowHashByRange, executionContext);
createTable.getTableElementList().addAll(indexDefs);
if (tableMeta.isAutoPartition() && showCreateTable.isFull()) {
@@ -572,7 +575,10 @@ public List buildIndexDefs(String schemaName,
TableMeta meta,
List localIndexes,
boolean full,
- boolean needShowHashByRange) {
+ boolean needShowHashByRange,
+ ExecutionContext executionContext) {
+ boolean enableUseKeyForAllLocalIndex =
+ executionContext.getParamManager().getBoolean(ConnectionParams.ENABLE_USE_KEY_FOR_ALL_LOCAL_INDEX);
Set ignoredLocalIndexNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
List indexDefs = new ArrayList<>();
if (meta.withGsi() && gsiMeta.getTableMeta() != null && !gsiMeta.getTableMeta().isEmpty()) {
@@ -638,6 +644,8 @@ public List buildIndexDefs(String schemaName,
final MySqlUnique indeDef = new MySqlUnique();
indeDef.getIndexDefinition().setIndex(true);
+ boolean isAutoPartTbl = meta.isAutoPartition();
+ useKeyInsteadOfIndexIfNeed(full, enableUseKeyForAllLocalIndex, isAutoPartTbl, indeDef);
if (!indexMeta.nonUnique) {
indeDef.getIndexDefinition().setType("UNIQUE");
}
@@ -665,6 +673,11 @@ public List buildIndexDefs(String schemaName,
if (options != null) {
indeDef.setDictionaryColumns(options.get(ColumnarTableOptions.DICTIONARY_COLUMNS));
}
+ TablesAccessor tablesAccessor = new TablesAccessor();
+ String engine = getColumnarIndexEngine(schemaName, indexMeta.indexName, tablesAccessor);
+ if (engine != null) {
+ indeDef.setEngineName(new SQLIdentifierExpr(engine));
+ }
}
if (!coveringColumns.isEmpty() || full || !meta.isAutoPartition()) {
@@ -682,12 +695,14 @@ public List buildIndexDefs(String schemaName,
indexName = SQLUtils.normalizeNoTrim(key.getName().getSimpleName());
if (meta.isAutoPartition() || full) {
key.getIndexDefinition().setLocal(true);
+ removeLocalKeyWordOnShowCreateTableIfNeed(full, enableUseKeyForAllLocalIndex, key);
}
} else if (localIndexElement instanceof MySqlTableIndex) {
MySqlTableIndex key = (MySqlTableIndex) localIndexElement;
indexName = SQLUtils.normalizeNoTrim(key.getName().getSimpleName());
if (meta.isAutoPartition() || full) {
key.setLocal(true);
+ removeLocalKeyWordOnShowCreateTableIfNeed(full, enableUseKeyForAllLocalIndex, key);
}
} else {
continue;
@@ -721,6 +736,20 @@ private Map