support async replication

internal/app/app.go

@@ -700,6 +700,11 @@
 		app.logger.Errorf("failed to update active nodes in dcs: %v", err)
 	}
 
+	err = app.updateMdbReplMonTs(master)
+	if err != nil {
+		app.logger.Errorf("failed to update mdb_repl_mon timestamp: %v", err)
+	}
+
 	return stateManager
 }
 
@@ -2118,9 +2123,28 @@
 		if gtidExecuted.Contain(gtidset) {
 			return true, nil
 		}
-		if app.dcs.Get(pathCurrentSwitch, new(Switchover)) == dcs.ErrNotFound {
+		switchover := new(Switchover)
+		if app.dcs.Get(pathCurrentSwitch, switchover) == dcs.ErrNotFound {
 			return false, nil
 		}
+		if app.config.ASync && switchover.Cause == CauseAuto {
+			app.logger.Infof("async mode is active and this is auto switch so we checking new master delay")
+			ts, err := app.GetMdbReplMonTs()
+			if err != nil {
+				app.logger.Errorf("failed to get mdb repl mon ts: %v", err)
+				continue
+			}
+			delay, err := node.CalcMdbReplMonTsDelay(ts)
+			if err != nil {
+				app.logger.Errorf("failed to calc mdb repl mon ts: %v", err)
+				continue
+			}
+			if delay < app.config.ASyncAllowedLag {
+				app.logger.Infof("async allowed lag is %d and current lag on host %s is %d, so we don't wait for catch up any more",
+					app.config.ASyncAllowedLag, node.Host(), delay)
+				return true, nil
+			}
+		}
 		time.Sleep(sleep)
 		if time.Now().After(deadline) {
 			break
@@ -2224,6 +2248,15 @@
 	return positions, util.CombineErrors(errs)
 }
 
+func (app *App) updateMdbReplMonTs(master string) error {
+	masterNode := app.cluster.Get(master)
+	ts, err := masterNode.GetMdbReplMonTs()
+	if err != nil {
+		return fmt.Errorf("failed to get master mdb_repl_mon timestamp: %v", err)
+	}
+	return app.SetMdbReplMonTs(ts)
+}
+
 /*
 Run enters the main application loop
 When Run exits mysync process is over

internal/app/app_dcs.go

@@ -228,3 +228,24 @@
 	}
 	return "", nil
 }
+
+func (app *App) SetMdbReplMonTs(ts string) error {
+	err := app.dcs.Create(pathMasterReplMonTs, ts)
+	if err != nil && err != dcs.ErrExists {
+		return err
+	}
+	err = app.dcs.Set(pathMasterReplMonTs, ts)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (app *App) GetMdbReplMonTs() (string, error) {
+	var ts string
+	err := app.dcs.Get(pathMasterReplMonTs, &ts)
+	if errors.Is(err, dcs.ErrNotFound) {
+		return "", nil
+	}
+	return ts, err
+}

internal/app/data.go

@@ -67,6 +67,9 @@ const (
 	pathResetupStatus = "resetup_status"
 
 	pathLastShutdownNodeTime = "last_shutdown_node_time"
+
+	// last known timestamp from mysql.mdb_repl_mon
+	pathMasterReplMonTs = "master_repl_mon_ts"
 )
 
 var (

internal/config/config.go

@@ -88,6 +88,8 @@
 	ReplicationChannel                      string                       `config:"replication_channel" yaml:"replication_channel"`
 	ExternalReplicationChannel              string                       `config:"external_replication_channel" yaml:"external_replication_channel"`
 	ExternalReplicationType                 util.ExternalReplicationType `config:"external_replication_type" yaml:"external_replication_type"`
+	ASync                                	bool                         `config:"async" yaml:"async"`
+	ASyncAllowedLag                       	int64                        `config:"async_allowed_lag" yaml:"async_allowed_lag"`
 }
 
 // DefaultConfig returns default configuration for MySync
@@ -164,6 +166,8 @@
 		ReplicationChannel:                      "",
 		ExternalReplicationChannel:              "external",
 		ExternalReplicationType:                 util.Disabled,
+		ASync:          						false,
+		ASyncAllowedLag: 0,
 	}
 	return config, nil
 }
@@ -205,5 +209,8 @@
 	if cfg.NotCriticalDiskUsage > cfg.CriticalDiskUsage {
 		return fmt.Errorf("not_critical_disk_usage should be <= critical_disk_usage")
 	}
+	if cfg.SemiSync && cfg.ASync {
+		return fmt.Errorf("can't run in both semisync and async mode")
+	}
 	return nil
 }

internal/mysql/data.go

@@ -268,6 +268,14 @@
 	PatchVersion int `db:"PatchVersion"`
 }
 
+type MdbReplMonTs struct {
+	Timestamp string `db:"ts"`
+}
+
+type MdbReplMonTsDelay struct {
+	Delay int64 `db:"delay"`
+}
+
 const (
 	Version80Major              = 8
 	Version80Minor              = 0

internal/mysql/node.go

@@ -985,3 +985,15 @@
 	}
 	return nil
 }
+
+func (n *Node) GetMdbReplMonTs() (string, error) {
+	result := new(MdbReplMonTs)
+	err := n.queryRow(queryGetMdbReplMonTs, nil, result)
+	return result.Timestamp, err
+}
+
+func (n *Node) CalcMdbReplMonTsDelay(ts string) (int64, error) {
+	result := new(MdbReplMonTsDelay)
+	err := n.queryRow(queryCalcMdbReplMonTsDelay, map[string]interface{}{"ts": ts}, result)
+	return result.Delay, err
+}

internal/mysql/queries.go

@@ -45,6 +45,8 @@
 	querySetInnodbFlushLogAtTrxCommit   = "set_innodb_flush_log_at_trx_commit"
 	querySetSyncBinlog                  = "set_sync_binlog"
 	queryGetReplicationSettings         = "get_replication_settings"
+	queryGetMdbReplMonTs				= "get_mdb_repl_mon_ts"
+	queryCalcMdbReplMonTsDelay			= "calc_mdb_repl_mon_ts_delay"
 )
 
 var DefaultQueries = map[string]string{
@@ -123,4 +125,6 @@
 	querySetInnodbFlushLogAtTrxCommit: `SET GLOBAL innodb_flush_log_at_trx_commit = :level`,
 	queryGetReplicationSettings:       `SELECT @@innodb_flush_log_at_trx_commit as InnodbFlushLogAtTrxCommit, @@sync_binlog as SyncBinlog`,
 	querySetSyncBinlog:                `SET GLOBAL sync_binlog = :sync_binlog`,
+	queryGetMdbReplMonTs: 				`SELECT UNIX_TIMESTAMP(ts) AS ts FROM mysql.mdb_repl_mon`,
+	queryCalcMdbReplMonTsDelay: 		`SELECT UNIX_TIMESTAMP(CURRENT_TIMESTAMP(0))  - CAST(:ts AS DECIMAL(20,0)) AS delay`,
 }

tests/images/docker-compose.yaml

@@ -86,6 +86,8 @@ services:
       MYSQL_PORT: 3306
       MYSYNC_DISABLE_REPLICATION_ON_MAINT: "true"
       MYSYNC_SEMISYNC:
+      MYSYNC_ASYNC:
+      ASYNC_ALLOWED_LAG:
       MYSYNC_CRITICAL_DISK_USAGE:
       MYSYNC_KEEP_SUPER_WRITABLE_ON_CRITICAL_DISK_USAGE:
       MYSYNC_WAIT_FOR_SLAVE_COUNT:
@@ -128,6 +130,8 @@ services:
       MYSQL_PORT: 3306
       MYSYNC_DISABLE_REPLICATION_ON_MAINT: "true"
       MYSYNC_SEMISYNC:
+      MYSYNC_ASYNC:
+      ASYNC_ALLOWED_LAG:
       MYSYNC_CRITICAL_DISK_USAGE:
       MYSYNC_KEEP_SUPER_WRITABLE_ON_CRITICAL_DISK_USAGE:
       MYSYNC_WAIT_FOR_SLAVE_COUNT:
@@ -164,6 +168,8 @@ services:
       MYSQL_PORT: 3306
       MYSYNC_DISABLE_REPLICATION_ON_MAINT: "true"
       MYSYNC_SEMISYNC:
+      MYSYNC_ASYNC:
+      ASYNC_ALLOWED_LAG:
       MYSYNC_CRITICAL_DISK_USAGE:
       MYSYNC_KEEP_SUPER_WRITABLE_ON_CRITICAL_DISK_USAGE:
       MYSYNC_WAIT_FOR_SLAVE_COUNT:

tests/images/mysql/mysync.yaml

@@ -10,6 +10,8 @@ failover_cooldown: ${MYSYNC_FAILOVER_COOLDOWN:-60m}
 failover_delay: ${MYSYNC_FAILOVER_DELAY:-0s}
 inactivation_delay: ${MYSYNC_INACTIVATION_DELAY:-5s}
 semi_sync: ${MYSYNC_SEMISYNC:-true}
+async: ${MYSYNC_ASYNC:-false}
+async_allowed_lag: ${ASYNC_ALLOWED_LAG:-0}
 resetupfile: /tmp/mysync.resetup
 resetup_crashed_hosts: ${MYSYNC_RESETUP_CRASHED_HOSTS:-false}
 zookeeper: