feat(dbm-services): 补充tendbcluster 语义检验规则

dbm-services/mysql/db-simulation/app/service/simulation_task.go

@@ -296,8 +296,8 @@ func (t *SimulationTask) SimulationRun(containerName string, xlogger *logger.Log
 				xlogger.Error("when execute %s at %s, failed  %s\n", e.SQLFile, realexcutedbs[idx-1], err.Error())
 				xlogger.Error("stderr:\n	%s", stderr.String())
 				xlogger.Error("stdout:\n	%s", stdout.String())
-				return sstdout, sstderr, fmt.Errorf("exec %s in %s failed:%s", e.SQLFile, realexcutedbs[idx-1],
-					err.Error())
+				return sstdout, sstderr, fmt.Errorf("\nexec %s in %s failed:%s\n %s", e.SQLFile, realexcutedbs[idx-1],
+					err.Error(), stderr.String())
 			}
 			xlogger.Info("%s \n %s", stdout.String(), stderr.String())
 		}

dbm-services/mysql/db-simulation/app/syntax/rule.go

@@ -106,9 +106,9 @@ func (c *CheckerResult) Trigger(rule *BoolRuleItem, additionalMsg string) {
 		return
 	}
 	if rule.Ban {
-		c.BanWarns = append(c.BanWarns, fmt.Sprintf("%s:%s", rule.Desc, additionalMsg))
+		c.BanWarns = append(c.BanWarns, fmt.Sprintf("%s:%s\n%s", rule.Desc, additionalMsg, rule.Suggestion))
 	} else {
-		c.RiskWarns = append(c.RiskWarns, fmt.Sprintf("%s:%s", rule.Desc, additionalMsg))
+		c.RiskWarns = append(c.RiskWarns, fmt.Sprintf("%s:%s\n%s", rule.Desc, additionalMsg, rule.Suggestion))
 	}
 }
 
@@ -128,13 +128,15 @@ type RuleItem struct {
 	Expr        string `yaml:"expr"`
 	Desc        string `yaml:"desc"`
 	Ban         bool   `yaml:"ban"`
+	Suggestion  string `yaml:"suggestion"`
 }
 
 // BoolRuleItem 开关型规则，只需配置开启或者关闭即可
 type BoolRuleItem struct {
-	Desc   string `yaml:"desc"`
-	Ban    bool   `yaml:"ban"`
-	TurnOn bool   `yaml:"turnOn"`
+	Desc       string `yaml:"desc"`
+	Ban        bool   `yaml:"ban"`
+	TurnOn     bool   `yaml:"turnOn"`
+	Suggestion string `yaml:"suggestion"`
 }
 
 // Rules TODO

dbm-services/mysql/db-simulation/app/syntax/spider_create_table_rule.go

@@ -46,83 +46,102 @@ func (c CreateTableResult) SpiderChecker(spiderVersion string) (r *CheckerResult
 }
 
 func (c CreateTableResult) shardKeyChecker(r *CheckerResult) {
+	var has_shard_key bool
+	var shardKeyCol string
+	var err error
 	logger.Info("start shardKeyChecker...")
 	// 如果沒有任何索引,则直接返回错误
 	if len(c.CreateDefinitions.KeyDefs) <= 0 {
 		r.Trigger(SR.SpiderCreateTableRule.NoIndexExists, "")
 		return
 	}
-	uniqueKeys, commonKeys := c.findTablesIndex()
+	pk, uks, keys := c.findTablesIndex()
 	tableComment := c.GetComment()
-	logger.Info("tableComment is %s", tableComment)
-	// 如果table comment 为空,表示没有指定shard key
-	// 或者table comnent 没有指定shardkey
-	// 由中控自主选择
-	if cmutil.IsEmpty(tableComment) || !strings.Contains(tableComment, "shard_key") {
-		// 如果没有唯一索引，如果包含多个普通索引，则必须指定shard_key,否则需要报错
-		if len(commonKeys) > 1 {
-			r.Trigger(SR.SpiderCreateTableRule.MustSpecialShardKeyOnlyHaveCommonIndex, "")
-			return
-		}
-	}
 	// 如果存在多个唯一健（含主键),多个唯一键都没有包含相同的字段也是不允许的
 	var pubCols []string
-	logger.Info("uniqueKeys is %v,len is %d", uniqueKeys, len(uniqueKeys))
-	if len(uniqueKeys) > 1 {
-		pubCols = findCommonColByKeys(uniqueKeys)
+	logger.Info("uniqueKeys is %v,len is %d", uks, len(uks))
+	if len(uks) > 1 {
+		pubCols = findCommonColByKeys(uks)
 		if len(pubCols) < 1 {
 			r.Trigger(SR.SpiderCreateTableRule.NoPubColAtMultUniqueIndex, "")
 			return
 		}
 	}
-	// table comment 不为空的时候 先校验comment 格式是否合法
-	legal, msg := c.validateSpiderComment(tableComment)
-	if !legal {
-		r.Trigger(SR.SpiderCreateTableRule.IllegalComment, msg)
-		return
-	}
-	shardKeyCol, err := util.ParseGetShardKeyForSpider(tableComment)
-	if err != nil {
-		// Todo 错误处理
-		logger.Error("parse %s comment %s shard key failed %s", c.TableName, tableComment, err.Error())
-		return
-	}
-	// 如果存在索引,但是shard key不属于任何索引
-	if !c.shardKeyIsIndex(shardKeyCol) {
-		r.Trigger(SR.SpiderCreateTableRule.ShardKeyNotIndex, "")
-		return
-	}
-	switch {
-	case len(uniqueKeys) == 1:
-		if !c.shardKeyExistInKeys(shardKeyCol, uniqueKeys) {
-			r.Trigger(SR.SpiderCreateTableRule.NoPubColAtMultUniqueIndex, shardKeyCol)
+	logger.Info("tableComment is %s", tableComment)
+
+	if cmutil.IsNotEmpty(tableComment) {
+		// table comment 不为空的时候 先校验comment 格式是否合法
+		legal, msg := c.validateSpiderComment(tableComment)
+		if !legal {
+			r.Trigger(SR.SpiderCreateTableRule.IllegalComment, msg)
+			return
+		}
+		shardKeyCol, err = util.ParseGetShardKeyForSpider(tableComment)
+		if err != nil {
+			// Todo 错误处理
+			logger.Error("parse %s comment %s shard key failed %s", c.TableName, tableComment, err.Error())
 			return
 		}
-	//  如果存在 一个或者多个唯一索引(包含主键)
-	case len(uniqueKeys) > 1:
-		// shard_key只能是其中的共同部分；否则无法建表；
-		if !slices.Contains(pubCols, shardKeyCol) {
-			r.Trigger(SR.SpiderCreateTableRule.NoPubColAtMultUniqueIndex, shardKeyCol)
+		has_shard_key = true
+	}
+	// 如果table comment 为空,表示没有指定shard key,或table comnent 没有指定shardkey 由中控自主选择
+	if !has_shard_key {
+		switch {
+		case pk != nil:
+			return
+		case len(uks) > 1:
+			return
+		case len(keys) > 1:
+			// 如果没有唯一索引，如果包含多个普通索引，则必须指定shard_key,否则需要报错
+			r.Trigger(SR.SpiderCreateTableRule.MustSpecialShardKeyOnlyHaveCommonIndex, "")
 			return
 		}
-	// 如果只存在多个普通索引，shard_key只能是其中任意一个的一部分
-	case len(uniqueKeys) < 1 && len(commonKeys) > 1:
-		if !c.shardKeyExistInKeys(shardKeyCol, commonKeys) {
-			r.Trigger(SR.SpiderCreateTableRule.MustSpecialShardKeyOnlyHaveCommonIndex, shardKeyCol)
+	} else {
+		// 如果存在索引,但是shard key不属于任何索引
+		if !c.shardKeyIsIndex(shardKeyCol) {
+			r.Trigger(SR.SpiderCreateTableRule.ShardKeyNotIndex, "")
 			return
 		}
+		switch {
+		case len(uks) == 1:
+			if !c.shardKeyExistInKeys(shardKeyCol, uks) {
+				r.Trigger(SR.SpiderCreateTableRule.NoPubColAtMultUniqueIndex, shardKeyCol)
+				return
+			}
+		//  如果存在 一个或者多个唯一索引(包含主键)
+		case len(uks) > 1:
+			// shard_key只能是其中的共同部分；否则无法建表；
+			if !slices.Contains(pubCols, shardKeyCol) {
+				r.Trigger(SR.SpiderCreateTableRule.NoPubColAtMultUniqueIndex, shardKeyCol)
+				return
+			}
+		// 如果只存在多个普通索引，shard_key只能是其中任意一个的一部分
+		case len(uks) < 1 && len(uks) > 1:
+			if !c.shardKeyExistInKeys(shardKeyCol, uks) {
+				r.Trigger(SR.SpiderCreateTableRule.MustSpecialShardKeyOnlyHaveCommonIndex, shardKeyCol)
+				return
+			}
+		}
+		shardKeyColDef := c.getColDef(shardKeyCol)
+		// 如果shard key 列允许为null
+		if shardKeyColDef.Nullable {
+			r.Trigger(SR.SpiderCreateTableRule.ShardKeyNotNull, shardKeyColDef.ColName)
+		}
 	}
 }
 
-func (c CreateTableResult) findTablesIndex() (uniqueKeys []KeyDef, commonKeys []KeyDef) {
+func (c CreateTableResult) findTablesIndex() (pk *KeyDef, uks []KeyDef, keys []KeyDef) {
 	for _, key := range c.CreateDefinitions.KeyDefs {
-		if key.PrimaryKey || key.UniqueKey {
-			uniqueKeys = append(uniqueKeys, key)
-		} else {
-			commonKeys = append(commonKeys, key)
+		switch {
+		case key.PrimaryKey:
+			pk = &key
+		case key.UniqueKey:
+			uks = append(uks, key)
+		default:
+			keys = append(keys, key)
 		}
 	}
-	return uniqueKeys, commonKeys
+	return pk, uks, keys
 }
 
 // findCommonColByKeys 寻找多个唯一键中的公共列
@@ -177,3 +196,13 @@ func (c CreateTableResult) shardKeyExistInKeys(shardKeyCol string, keys []KeyDef
 	}
 	return false
 }
+
+func (c CreateTableResult) getColDef(colName string) (colDef ColDef) {
+	for _, col := range c.CreateDefinitions.ColDefs {
+		if strings.Compare(col.ColName, colName) == 0 {
+			colDef = col
+			break
+		}
+	}
+	return colDef
+}

dbm-services/mysql/db-simulation/app/syntax/spider_rule.go

@@ -35,6 +35,7 @@ type SpiderCreateTableRule struct {
 	NoIndexExists                          *BoolRuleItem `yaml:"NoIndexExists"`
 	NoPubColAtMultUniqueIndex              *BoolRuleItem `yaml:"NoPubColAtMultUniqueIndex"`
 	MustSpecialShardKeyOnlyHaveCommonIndex *BoolRuleItem `yaml:"MustSpecialShardKeyOnlyHaveCommonIndex"`
+	ShardKeyNotNull                        *BoolRuleItem `yaml:"ShardKeyNotNull"`
 }
 
 func init() {

dbm-services/mysql/db-simulation/spider_rule.yaml

@@ -51,6 +51,7 @@ SpiderCreateTableRule:
     turnOn: true
     ban: true
     desc: "create table 语句中列字符集定义与表字符集不一致"
+    suggestion: "去掉列字符集"
   CreateWithSelect:
     turnOn: true
     ban: true
@@ -75,11 +76,18 @@ SpiderCreateTableRule:
     turnOn: true
     ban: true
     desc: "没有任何索引存在"
+    suggestion: "请添加索引、唯一键或主键"
   NoPubColAtMultUniqueIndex:
     turnOn: true
     ban: true
-    desc: "存在多个唯一健（含主键),shard_key只能是其中的共同部分"
+    desc: "用户comment指定了shard_key,存在多个唯一健（含主键),shard_key只能是其中的共同部分"
   MustSpecialShardKeyOnlyHaveCommonIndex:
     turnOn: true
     ban: true
     desc: "不存在唯一索引,只包含包含多个普通索引,则必须指定shard_key"
+    suggestion: "请指定shard_key"
+  ShardKeyNotNull:
+    turnOn: true
+    ban: true
+    desc: "如果是shard key 不是主键,默认值就一定要指定成 not null"
+    suggestion: "请指定shard_key列 is not null"

helm-charts/bk-dbm/charts/db-simulation/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 0.0.1-alpha.38
+appVersion: 0.0.1-alpha.39
 description: A Helm chart for Kubernetes
 name: db-simulation
 type: application