diff --git a/.github/workflows/docker-jepsen.yml b/.github/workflows/docker-jepsen.yml index 8d1ef38f..97b84f16 100644 --- a/.github/workflows/docker-jepsen.yml +++ b/.github/workflows/docker-jepsen.yml @@ -1,10 +1,12 @@ -name: Docker tests +name: Jepsen tests on: push: branches: [ master ] pull_request: branches: [ master ] + schedule: + - cron: '0 0 * * *' env: GO_VERSION: 1.19 diff --git a/.github/workflows/docker-tests-8.0.yml b/.github/workflows/docker-tests-8.0.yml index 2de153da..303954a6 100644 --- a/.github/workflows/docker-tests-8.0.yml +++ b/.github/workflows/docker-tests-8.0.yml @@ -1,4 +1,4 @@ -name: Docker tests +name: Docker tests (8.0) on: push: diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index 76d12cfb..06c321b1 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -1,4 +1,4 @@ -name: Docker tests +name: Docker tests (5.7) on: push: diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index ce83177e..57380596 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -2,9 +2,9 @@ name: Linters on: push: - branches: - - master + branches: [ master ] pull_request: + branches: [ master ] permissions: contents: read diff --git a/tests/features/recovery.feature b/tests/features/recovery.feature index fc195abd..dcad5ea6 100644 --- a/tests/features/recovery.feature +++ b/tests/features/recovery.feature @@ -38,15 +38,15 @@ Feature: hosts recovery Then mysql host "mysql2" should be master And mysql host "mysql2" should be writable And mysql host "mysql2" should have variable "rpl_semi_sync_master_enabled" set to "0" - # Commit normal transaction Then I run SQL on mysql host "mysql2" - """ - CREATE TABLE splitbrain(id int); - """ + """ + CREATE TABLE splitbrain(id int); + """ # Emulate lost transactions on old master - When I run command on host "mysql1" + # mysync may set super_read_only while we run query + When I run command on host "mysql1" until return code is "0" with timeout "5" seconds """ mysql -e ' SET GLOBAL rpl_semi_sync_master_enabled = 0; @@ -55,7 +55,6 @@ Feature: hosts recovery SET GLOBAL read_only = 1; ' """ - Then command return code should be "0" When host "mysql1" is attached to the network Then mysql host "mysql1" should become available within "10" seconds diff --git a/tests/mysync_test.go b/tests/mysync_test.go index 49bcdbd1..ca346cc5 100644 --- a/tests/mysync_test.go +++ b/tests/mysync_test.go @@ -33,6 +33,7 @@ const ( zkName = "zoo" zkPort = 2181 zkConnectTimeout = 5 * time.Second + commandExecutionTimeout = 10 * time.Second mysqlName = "mysql" mysqlPort = 3306 mysqlAdminUser = "admin" @@ -543,13 +544,13 @@ func (tctx *testContext) stepHostIsDeleted(host string) error { func (tctx *testContext) stepMysqlOnHostKilled(host string) error { cmd := "supervisorctl signal KILL mysqld" - _, _, err := tctx.composer.RunCommand(host, cmd, 10*time.Second) + _, _, err := tctx.composer.RunCommand(host, cmd, commandExecutionTimeout) return err } func (tctx *testContext) stepMysqlOnHostStarted(host string) error { cmd := "supervisorctl start mysqld" - _, _, err := tctx.composer.RunCommand(host, cmd, 10*time.Second) + _, _, err := tctx.composer.RunCommand(host, cmd, commandExecutionTimeout) return err } @@ -561,7 +562,7 @@ func (tctx *testContext) stepMysqlOnHostRestarted(host string) error { func (tctx *testContext) stepMysqlOnHostStopped(host string) error { cmd := "supervisorctl signal TERM mysqld" - _, _, err := tctx.composer.RunCommand(host, cmd, 10*time.Second) + _, _, err := tctx.composer.RunCommand(host, cmd, commandExecutionTimeout) return err } @@ -649,7 +650,7 @@ func (tctx *testContext) stepHostShouldHaveFileWithin(node string, path string, func (tctx *testContext) stepIRunCommandOnHost(host string, body *godog.DocString) error { cmd := strings.TrimSpace(body.Content) var err error - tctx.commandRetcode, tctx.commandOutput, err = tctx.composer.RunCommand(host, cmd, 10*time.Second) + tctx.commandRetcode, tctx.commandOutput, err = tctx.composer.RunCommand(host, cmd, commandExecutionTimeout) return err } @@ -658,7 +659,7 @@ func (tctx *testContext) stepSetUsedSpace(host string, percent int) error { return fmt.Errorf("incorrect percent value: %d", percent) } cmd := fmt.Sprintf("rm /tmp/usedspace && echo %d > /tmp/usedspace", percent) - _, _, err := tctx.composer.RunCommand(host, cmd, 10*time.Second) + _, _, err := tctx.composer.RunCommand(host, cmd, commandExecutionTimeout) return err } @@ -667,7 +668,7 @@ func (tctx *testContext) stepSetReadonlyStatus(host string, value string) error return fmt.Errorf("value must be true or false: %s", value) } cmd := fmt.Sprintf("rm /tmp/readonly && echo %s > /tmp/readonly", value) - code, output, err := tctx.composer.RunCommand(host, cmd, 10*time.Second) + code, output, err := tctx.composer.RunCommand(host, cmd, commandExecutionTimeout) if code != 0 { return fmt.Errorf("comand exit with code %d and output: %s", code, output) } @@ -686,7 +687,7 @@ func (tctx *testContext) stepIRunCommandOnHostWithTimeout(host string, timeout i return err } -func (tctx *testContext) stepIRunCommandOnHostUntilResultMatch(host string, pattern string, timeout int, body *godog.DocString) error { +func (tctx *testContext) stepIRunCommandOnHostUntilResultMatchWithTimeout(host string, pattern string, timeout int, body *godog.DocString) error { matcher, err := matchers.GetMatcher("regexp") if err != nil { return err @@ -706,6 +707,20 @@ func (tctx *testContext) stepIRunCommandOnHostUntilResultMatch(host string, patt return lastError } +func (tctx *testContext) stepIRunCommandOnHostUntilReturnCodeWithTimeout(host string, code int, timeout int, body *godog.DocString) error { + var lastError error + testutil.Retry(func() bool { + cmd := strings.TrimSpace(body.Content) + tctx.commandRetcode, tctx.commandOutput, lastError = tctx.composer.RunCommand(host, cmd, time.Duration(timeout)*time.Second) + if lastError != nil { + return false + } + return tctx.commandRetcode == code + }, time.Duration(timeout)*time.Second, time.Second) + + return lastError +} + // Make sure zk node is absent, then stop slave, change master & start slave again func (tctx *testContext) stepIChangeReplicationSource(host, replicationSource string) error { if err := tctx.stepIDeleteZookeeperNode(dcs.JoinPath("/test", dcs.PathHANodesPrefix, host)); err != nil { @@ -1330,7 +1345,8 @@ func InitializeScenario(s *godog.ScenarioContext) { s.Step(`^I run command on host "([^"]*)"$`, tctx.stepIRunCommandOnHost) s.Step(`^I run command on host "([^"]*)" with timeout "(\d+)" seconds$`, tctx.stepIRunCommandOnHostWithTimeout) s.Step(`^I run async command on host "([^"]*)"$`, tctx.stepIRunAsyncCommandOnHost) - s.Step(`^I run command on host "([^"]*)" until result match regexp "([^"]*)" with timeout "(\d+)" seconds$`, tctx.stepIRunCommandOnHostUntilResultMatch) + s.Step(`^I run command on host "([^"]*)" until result match regexp "([^"]*)" with timeout "(\d+)" seconds$`, tctx.stepIRunCommandOnHostUntilResultMatchWithTimeout) + s.Step(`^I run command on host "([^"]*)" until return code is "([^"]*)" with timeout "(\d+)" seconds$`, tctx.stepIRunCommandOnHostUntilReturnCodeWithTimeout) s.Step(`^I change replication source on host "([^"]*)" to "([^"]*)"$`, tctx.stepIChangeReplicationSource) s.Step(`^command return code should be "(\d+)"$`, tctx.stepCommandReturnCodeShouldBe) s.Step(`^command output should match (\w+)$`, tctx.stepCommandOutputShouldMatch)