From fab2765733ffb1e08f0185fc8e99066c9671eb47 Mon Sep 17 00:00:00 2001 From: Alvar Penning Date: Thu, 11 Apr 2024 10:27:19 +0200 Subject: [PATCH] icingadb.DB: Retry Schema Checks During testing, I just encountered a race condition where my Galera cluster was not yet ready, causing the initial schema check to fail. ``` 2024-04-11T08:13:40.401Z INFO icingadb Starting Icinga DB daemon (1.1.1) 2024-04-11T08:13:40.401Z INFO icingadb Connecting to database at 'mysql:3306' 2024-04-11T08:13:40.404Z FATAL icingadb Error 1047 (08S01): WSREP has not yet prepared node for application use can't check database schema version github.com/icinga/icingadb/pkg/icingadb.(*DB).CheckSchema /go/src/github.com/Icinga/icingadb/pkg/icingadb/db.go:115 main.run /go/src/github.com/Icinga/icingadb/cmd/icingadb/main.go:74 main.main /go/src/github.com/Icinga/icingadb/cmd/icingadb/main.go:37 runtime.main /usr/local/go/src/runtime/proc.go:271 runtime.goexit /usr/local/go/src/runtime/asm_amd64.s:1695 exit status 1 ``` This change now also retries the initial cluster check. References #698. --- pkg/icingadb/db.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/icingadb/db.go b/pkg/icingadb/db.go index 4c575e3ba..47940af9e 100644 --- a/pkg/icingadb/db.go +++ b/pkg/icingadb/db.go @@ -110,7 +110,19 @@ func (db *DB) CheckSchema(ctx context.Context) error { var version uint16 - err := db.QueryRowxContext(ctx, "SELECT version FROM icingadb_schema ORDER BY id DESC LIMIT 1").Scan(&version) + err := retry.WithBackoff( + ctx, + func(ctx context.Context) (err error) { + query := "SELECT version FROM icingadb_schema ORDER BY id DESC LIMIT 1" + err = db.QueryRowxContext(ctx, query).Scan(&version) + if err != nil { + err = internal.CantPerformQuery(err, query) + } + return + }, + retry.Retryable, + backoff.NewExponentialWithJitter(128*time.Millisecond, 1*time.Minute), + db.getDefaultRetrySettings()) if err != nil { return errors.Wrap(err, "can't check database schema version") }