fix: checking for split brain

yandex · Mar 19, 2024 · d0108bf · d0108bf
1 parent e4dc375
commit d0108bf
Show file tree

Hide file tree

Showing 6 changed files with 50 additions and 6 deletions.
diff --git a/internal/app/app.go b/internal/app/app.go
@@ -827,6 +827,12 @@ func (app *App) calcActiveNodes(clusterState, clusterStateDcs map[string]*NodeSt
 		app.logger.Warnf("failed to get master status %v", err)
 		return nil, err
 	}
+	muuid, err := masterNode.UUID()
+	if err != nil {
+		app.logger.Warnf("failed to get master uuid %v", err)
+		return nil, err
+	}
+
 	for host, node := range clusterState {
 		if host == master {
 			activeNodes = append(activeNodes, master)
@@ -869,7 +875,7 @@ func (app *App) calcActiveNodes(clusterState, clusterStateDcs map[string]*NodeSt
 			continue
 		}
 		sgtids := gtids.ParseGtidSet(sstatus.ExecutedGtidSet)
-		if !(sstatus.ReplicationState == mysql.ReplicationRunning && isGTIDLessOrEqual(sgtids, mgtids)) {
+		if sstatus.ReplicationState != mysql.ReplicationRunning || isSplitBrained(sgtids, mgtids, muuid) {
 			app.logger.Errorf("calc active nodes: %s is not replicating or splitbrained, deleting from active...", host)
 			continue
 		}

diff --git a/internal/app/util.go b/internal/app/util.go
@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"time"
 
+	gomysql "github.com/go-mysql-org/go-mysql/mysql"
+	"github.com/google/uuid"
 	"github.com/yandex/mysync/internal/log"
 	"github.com/yandex/mysync/internal/mysql"
 	"github.com/yandex/mysync/internal/mysql/gtids"
@@ -233,6 +235,25 @@ func isGTIDLessOrEqual(slaveGtidSet, masterGtidSet gtids.GTIDSet) bool {
 	return masterGtidSet.Contain(slaveGtidSet) || masterGtidSet.Equal(slaveGtidSet)
 }
 
+func isSplitBrained(slaveGtidSet, masterGtidSet *gomysql.MysqlGTIDSet, masterUUID uuid.UUID) bool {
+	for _, MasterSet := range masterGtidSet.Sets {
+		slaveSet, ok := slaveGtidSet.Sets[MasterSet.SID.String()]
+		if !ok {
+			continue
+		}
+		if MasterSet.Contain(slaveSet) || MasterSet == slaveSet {
+			continue
+		}
+
+		if MasterSet.SID == masterUUID {
+			continue
+		}
+
+		return false
+	}
+	return true
+}
+
 func validatePriority(priority *int64) error {
 	if priority == nil || *priority >= 0 {
 		return nil

diff --git a/internal/mysql/data.go b/internal/mysql/data.go
@@ -28,6 +28,10 @@ type readOnlyResult struct {
 	SuperReadOnly int `db:"SuperReadOnly"`
 }
 
+type ServerUUIDResult struct {
+	ServerUUID string `db:"server_uuid"`
+}
+
 // CascadeNodeConfiguration is a dcs node configuration for cascade mysql replica
 type CascadeNodeConfiguration struct {
 	// StreamFrom - is a host to stream from. Can be changed from CLI.

diff --git a/internal/mysql/gtids/wrapper.go b/internal/mysql/gtids/wrapper.go
@@ -6,10 +6,10 @@ import (
 
 type GTIDSet = mysql.GTIDSet
 
-func ParseGtidSet(gtidset string) mysql.GTIDSet {
-	parsed, err := mysql.ParseGTIDSet(mysql.MySQLFlavor, gtidset)
+func ParseGtidSet(gtidset string) *mysql.MysqlGTIDSet {
+	parsed, err := mysql.ParseMysqlGTIDSet(gtidset)
 	if err != nil {
 		panic(err)
 	}
-	return parsed
+	return parsed.(*mysql.MysqlGTIDSet)
 }
diff --git a/internal/mysql/node.go b/internal/mysql/node.go
@@ -17,10 +17,11 @@ import (
 	"syscall"
 	"time"
 
+	mysql2 "github.com/go-mysql-org/go-mysql/mysql"
 	"github.com/go-sql-driver/mysql"
+	"github.com/google/uuid"
 	"github.com/jmoiron/sqlx"
 	"github.com/shirou/gopsutil/v3/process"
-
 	"github.com/yandex/mysync/internal/config"
 	"github.com/yandex/mysync/internal/log"
 	"github.com/yandex/mysync/internal/mysql/gtids"
@@ -578,7 +579,7 @@ func (n *Node) GTIDExecuted() (*GTIDExecuted, error) {
 }
 
 // GTIDExecuted returns global transaction id executed
-func (n *Node) GTIDExecutedParsed() (gtids.GTIDSet, error) {
+func (n *Node) GTIDExecutedParsed() (*mysql2.MysqlGTIDSet, error) {
 	gtid, err := n.GTIDExecuted()
 	if err != nil {
 		return nil, err
@@ -602,6 +603,16 @@ func (n *Node) GetBinlogs() ([]Binlog, error) {
 	return binlogs, err
 }
 
+// UUID returns server_uuid
+func (n *Node) UUID() (uuid.UUID, error) {
+	var r ServerUUIDResult
+	err := n.queryRow(queryGetUUID, nil, &r)
+	if err != nil {
+		return uuid.UUID{}, err
+	}
+	return uuid.Parse(r.ServerUUID)
+}
+
 // IsReadOnly returns (true, true) if MySQL Node in (read-only, super-read-only) mode
 func (n *Node) IsReadOnly() (bool, bool, error) {
 	var ror readOnlyResult

diff --git a/internal/mysql/queries.go b/internal/mysql/queries.go
@@ -6,6 +6,7 @@ const (
 	queryReplicaStatus                  = "replica_status"
 	queryGetVersion                     = "get_version"
 	queryGTIDExecuted                   = "gtid_executed"
+	queryGetUUID                        = "get_uuid"
 	queryShowBinaryLogs                 = "binary_logs"
 	queryReplicationLag                 = "replication_lag"
 	querySlaveHosts                     = "slave_hosts"
@@ -53,6 +54,7 @@ var DefaultQueries = map[string]string{
 	queryReplicaStatus:       `SHOW REPLICA STATUS FOR CHANNEL :channel`,
 	queryGetVersion:          `SELECT sys.version_major() AS MajorVersion, sys.version_minor() AS MinorVersion, sys.version_patch() AS PatchVersion`,
 	queryGTIDExecuted:        `SELECT @@GLOBAL.gtid_executed  as Executed_Gtid_Set`,
+	queryGetUUID:             `SELECT @@server_uuid as server_uuid`,
 	queryShowBinaryLogs:      `SHOW BINARY LOGS`,
 	querySlaveHosts:          `SHOW SLAVE HOSTS`,
 	queryReplicationLag:      ``,