From 3997feaf182514c9f56252747547cd5df5e25971 Mon Sep 17 00:00:00 2001 From: Matthieu` Date: Wed, 8 Nov 2023 14:27:07 +0900 Subject: [PATCH] Handle `wsrep_gtid_mode` enabled - Changed the parsing logic of the seqno field in the Galera state file - Added a struct to serialize/deserialize the GTID tied to the sequence number - Updated the unit tests --- go.mod | 2 +- go.sum | 4 +- pkg/galera/galera.go | 37 ++++++++++--- pkg/galera/galera_test.go | 105 ++++++++++++++++++++++++------------- pkg/galera/gtid.go | 46 ++++++++++++++++ pkg/galera/gtid_test.go | 92 ++++++++++++++++++++++++++++++++ pkg/handler/bootstrap.go | 4 +- pkg/handler/galerastate.go | 2 +- 8 files changed, 241 insertions(+), 51 deletions(-) create mode 100644 pkg/galera/gtid.go create mode 100644 pkg/galera/gtid_test.go diff --git a/go.mod b/go.mod index 0dc71f6..32eb4f3 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/go-chi/chi/v5 v5.0.8 github.com/go-chi/httprate v0.7.4 github.com/go-logr/logr v1.2.4 + github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.3.0 go.uber.org/zap v1.25.0 k8s.io/api v0.28.1 @@ -25,7 +26,6 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/go.sum b/go.sum index 8242777..fc4275a 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= diff --git a/pkg/galera/galera.go b/pkg/galera/galera.go index feb4ad5..b14507d 100644 --- a/pkg/galera/galera.go +++ b/pkg/galera/galera.go @@ -36,6 +36,7 @@ type GaleraState struct { Version string `json:"version"` UUID string `json:"uuid"` Seqno int `json:"seqno"` + GTID *GTID `json:"gtid"` SafeToBootstrap bool `json:"safeToBootstrap"` } @@ -60,7 +61,7 @@ func (g *GaleraState) Compare(other GaleraRecoverer) int { return 0 } -func (g *GaleraState) Marshal() ([]byte, error) { +func (g *GaleraState) MarshalText() ([]byte, error) { if _, err := guuid.Parse(g.UUID); err != nil { return nil, fmt.Errorf("invalid uuid: %v", err) } @@ -68,17 +69,19 @@ func (g *GaleraState) Marshal() ([]byte, error) { Version string UUID string Seqno int + GTID *GTID SafeToBootstrap int } tpl := createTpl("grastate.dat", `version: {{ .Version }} uuid: {{ .UUID }} -seqno: {{ .Seqno }} +seqno: {{ .Seqno }}{{ if .GTID }},{{ .GTID }}{{ end }} safe_to_bootstrap: {{ .SafeToBootstrap }}`) buf := new(bytes.Buffer) err := tpl.Execute(buf, tplOpts{ Version: g.Version, UUID: g.UUID, Seqno: g.Seqno, + GTID: g.GTID, SafeToBootstrap: func() int { if g.SafeToBootstrap { return 1 @@ -92,14 +95,17 @@ safe_to_bootstrap: {{ .SafeToBootstrap }}`) return buf.Bytes(), nil } -func (g *GaleraState) Unmarshal(text []byte) error { +func (g *GaleraState) UnmarshalText(text []byte) error { fileScanner := bufio.NewScanner(bytes.NewReader(text)) fileScanner.Split(bufio.ScanLines) - var version *string - var uuid *string - var seqno *int - var safeToBootstrap *bool + var ( + version *string + uuid *string + seqno *int + gtid *GTID + safeToBootstrap *bool + ) for fileScanner.Scan() { parts := strings.Split(fileScanner.Text(), ":") @@ -118,7 +124,18 @@ func (g *GaleraState) Unmarshal(text []byte) error { } uuid = &value case "seqno": - i, err := strconv.Atoi(value) + // When the `wsrep_gtid_mode` is set to `ON`, the `seqno` is + // actually a string of the form `seqno,gtid`. + seqnoStr, gtidStr, found := strings.Cut(value, ",") + if found { + gtid = >ID{} + err := gtid.UnmarshalText([]byte(gtidStr)) + if err != nil { + return fmt.Errorf("error parsing gtid: %v", err) + } + + } + i, err := strconv.Atoi(seqnoStr) if err != nil { return fmt.Errorf("error parsing seqno: %v", err) } @@ -141,6 +158,10 @@ func (g *GaleraState) Unmarshal(text []byte) error { g.Version = *version g.UUID = *uuid g.Seqno = *seqno + // Only set the GTID if it was found in the file. + if gtid != nil { + g.GTID = gtid + } g.SafeToBootstrap = *safeToBootstrap return nil } diff --git a/pkg/galera/galera_test.go b/pkg/galera/galera_test.go index 14ac335..e3c7011 100644 --- a/pkg/galera/galera_test.go +++ b/pkg/galera/galera_test.go @@ -65,11 +65,26 @@ seqno: -1 safe_to_bootstrap: 0`, wantErr: false, }, + { + name: "wsrep_gtid_mode enabled", + galeraState: &GaleraState{ + Version: "2.1", + UUID: "05f061bd-02a3-11ee-857c-aa370ff6666b", + Seqno: 1, + GTID: >ID{DomainID: 0, ServerID: 1, SequenceNumber: 2}, + SafeToBootstrap: true, + }, + want: `version: 2.1 +uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b +seqno: 1,0-1-2 +safe_to_bootstrap: 1`, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - bytes, err := tt.galeraState.Marshal() + bytes, err := tt.galeraState.MarshalText() if tt.wantErr && err == nil { t.Fatal("error expected, got nil") } @@ -93,17 +108,17 @@ func TestGaleraStateUnmarshal(t *testing.T) { { name: "empty", bytes: []byte(` -`), + `), want: GaleraState{}, wantErr: true, }, { name: "comment", bytes: []byte(`# GALERA saved state -version: 2.1 -uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b -seqno: 1 -safe_to_bootstrap: 1`), + version: 2.1 + uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b + seqno: 1 + safe_to_bootstrap: 1`), want: GaleraState{ Version: "2.1", UUID: "05f061bd-02a3-11ee-857c-aa370ff6666b", @@ -115,10 +130,10 @@ safe_to_bootstrap: 1`), { name: "indentation", bytes: []byte(`# GALERA saved state -version: 2.1 -uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b -seqno: 1 -safe_to_bootstrap: 1`), + version: 2.1 + uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b + seqno: 1 + safe_to_bootstrap: 1`), want: GaleraState{ Version: "2.1", UUID: "05f061bd-02a3-11ee-857c-aa370ff6666b", @@ -130,39 +145,39 @@ safe_to_bootstrap: 1`), { name: "invalid uuid", bytes: []byte(`# GALERA saved state -version: 2.1 -uuid: foo -seqno: -1 -safe_to_bootstrap: 1`), + version: 2.1 + uuid: foo + seqno: -1 + safe_to_bootstrap: 1`), want: GaleraState{}, wantErr: true, }, { name: "invalid seqno", bytes: []byte(`# GALERA saved state -version: 2.1 -uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b -seqno: foo -safe_to_bootstrap: 1`), + version: 2.1 + uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b + seqno: foo + safe_to_bootstrap: 1`), want: GaleraState{}, wantErr: true, }, { name: "invalid safe_to_bootstrap", bytes: []byte(`# GALERA saved state -version: 2.1 -uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b -seqno: 1 -safe_to_bootstrap: true`), + version: 2.1 + uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b + seqno: 1 + safe_to_bootstrap: true`), want: GaleraState{}, wantErr: true, }, { name: "safe_to_bootstrap true", bytes: []byte(`version: 2.1 -uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b -seqno: 1 -safe_to_bootstrap: 1`), + uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b + seqno: 1 + safe_to_bootstrap: 1`), want: GaleraState{ Version: "2.1", UUID: "05f061bd-02a3-11ee-857c-aa370ff6666b", @@ -174,9 +189,9 @@ safe_to_bootstrap: 1`), { name: "safe_to_bootstrap false", bytes: []byte(`version: 2.1 -uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b -seqno: 1 -safe_to_bootstrap: 0`), + uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b + seqno: 1 + safe_to_bootstrap: 0`), want: GaleraState{ Version: "2.1", UUID: "05f061bd-02a3-11ee-857c-aa370ff6666b", @@ -188,9 +203,9 @@ safe_to_bootstrap: 0`), { name: "negative seqno", bytes: []byte(`version: 2.1 -uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b -seqno: -1 -safe_to_bootstrap: 0`), + uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b + seqno: -1 + safe_to_bootstrap: 0`), want: GaleraState{ Version: "2.1", UUID: "05f061bd-02a3-11ee-857c-aa370ff6666b", @@ -202,25 +217,41 @@ safe_to_bootstrap: 0`), { name: "missing safe_to_bootstrap", bytes: []byte(`version: 2.1 -uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b -safe_to_bootstrap: 0`), + uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b + safe_to_bootstrap: 0`), want: GaleraState{}, wantErr: true, }, { name: "missing seqno", bytes: []byte(`version: 2.1 -uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b -safe_to_bootstrap: 0`), + uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b + safe_to_bootstrap: 0`), want: GaleraState{}, wantErr: true, }, + { + // See the following threadh: https://mariadb-operator.slack.com/archives/C056RAECH0W/p1699350363009529 + name: "wsrep_gtid_mode enabled", + bytes: []byte(`# GALERA saved state +version: 2.1 +uuid: 05f061bd-02a3-11ee-857c-aa370ff6666b +seqno: 1,0-1-2 +safe_to_bootstrap: 1`), + want: GaleraState{ + Version: "2.1", + UUID: "05f061bd-02a3-11ee-857c-aa370ff6666b", + Seqno: 1, + GTID: >ID{DomainID: 0, ServerID: 1, SequenceNumber: 2}, + SafeToBootstrap: true, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var galeraState GaleraState - err := galeraState.Unmarshal(tt.bytes) + err := galeraState.UnmarshalText(tt.bytes) if tt.wantErr && err == nil { t.Fatal("error expected, got nil") } @@ -412,7 +443,7 @@ Warning: Memory not freed: 280 2023-06-04 8:24:18 0 [Note] Plugin 'FEEDBACK' is disabled. 2023-06-04 8:24:18 0 [Note] Server socket created on IP: '0.0.0.0'. 2023-06-04 8:24:18 0 [Note] WSREP: Recovered position: 08dd3b99-ac6b-46f8-84bd-8cb8f9f949b0:3 -Warning: Memory not freed: 280 +Warning: Memory not freed: 280 `), want: Bootstrap{ UUID: "08dd3b99-ac6b-46f8-84bd-8cb8f9f949b0", diff --git a/pkg/galera/gtid.go b/pkg/galera/gtid.go new file mode 100644 index 0000000..b6013ce --- /dev/null +++ b/pkg/galera/gtid.go @@ -0,0 +1,46 @@ +package galera + +import ( + "fmt" + "strconv" + "strings" +) + +// GTID represents a MariaDB global transaction identifier. +// See: https://mariadb.com/kb/en/gtid/ +type GTID struct { + DomainID uint32 `json:"domainId"` + ServerID uint32 `json:"serverId"` + SequenceNumber uint64 `json:"sequenceNumber"` +} + +func (gtid *GTID) String() string { + return fmt.Sprintf("%d-%d-%d", gtid.DomainID, gtid.ServerID, gtid.SequenceNumber) +} + +func (gtid *GTID) MarshalText() ([]byte, error) { + return []byte(gtid.String()), nil +} + +func (gtid *GTID) UnmarshalText(text []byte) error { + parts := strings.Split(string(text), "-") + if len(parts) != 3 { + return fmt.Errorf("invalid gtid: %s", text) + } + domainID, err := strconv.ParseUint(parts[0], 10, 32) + if err != nil { + return fmt.Errorf("invalid domain id: %v", err) + } + serverID, err := strconv.ParseUint(parts[1], 10, 32) + if err != nil { + return fmt.Errorf("invalid server id: %v", err) + } + seqno, err := strconv.ParseUint(parts[2], 10, 64) + if err != nil { + return fmt.Errorf("invalid seqno: %v", err) + } + gtid.DomainID = uint32(domainID) + gtid.ServerID = uint32(serverID) + gtid.SequenceNumber = seqno + return nil +} diff --git a/pkg/galera/gtid_test.go b/pkg/galera/gtid_test.go new file mode 100644 index 0000000..ada6db2 --- /dev/null +++ b/pkg/galera/gtid_test.go @@ -0,0 +1,92 @@ +package galera_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/mariadb-operator/agent/pkg/galera" +) + +func TestGTIDUnmarshal(t *testing.T) { + var gtid galera.GTID + err := gtid.UnmarshalText([]byte("0-1`-2")) + if err != nil { + t.Fatal(err) + } + + want := galera.GTID{ + DomainID: 0, + ServerID: 1, + SequenceNumber: 2, + } + if diff := cmp.Diff(gtid, want); diff != "" { + t.Errorf("GTID.Unmarshal() mismatch (-want +got):\n%s", diff) + } +} + +func TestGTIDUnmarshalTextInvalidValues(t *testing.T) { + tt := []struct { + name string + in string + }{ + { + name: "invalid domain id", + in: "a-1-2", + }, + { + name: "invalid server id", + in: "0-a-2", + }, + { + name: "invalid seqno", + in: "0-1-a", + }, + { + name: "missing part", + in: "0-1", + }, + { + name: "too many parts", + in: "0-1-2-3", + }, + { + name: "out of range domain id", + in: "4294967296-1-2", + }, + { + name: "out of range server id", + in: "0-4294967296-2", + }, + { + name: "out of range seqno", + in: "0-1-18446744073709551616", + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + var gtid galera.GTID + err := gtid.UnmarshalText([]byte(tc.in)) + if err == nil { + t.Error("expected error, got nil") + } + }) + } +} + +func TestGTIDMarshalText(t *testing.T) { + gtid := galera.GTID{ + DomainID: 0, + ServerID: 1, + SequenceNumber: 2, + } + text, err := gtid.MarshalText() + if err != nil { + t.Fatalf("failed to marshal GTID to text: %v", err) + } + + want := "0-1-2" + if string(text) != want { + t.Errorf("GTID.MarshalText() = %q, want %q", text, want) + } +} diff --git a/pkg/handler/bootstrap.go b/pkg/handler/bootstrap.go index 05a47d7..f9baab2 100644 --- a/pkg/handler/bootstrap.go +++ b/pkg/handler/bootstrap.go @@ -89,14 +89,14 @@ func (b *Bootstrap) setSafeToBootstrap(bootstrap *galera.Bootstrap) error { } var galeraState galera.GaleraState - if err := galeraState.Unmarshal(bytes); err != nil { + if err := galeraState.UnmarshalText(bytes); err != nil { return fmt.Errorf("error unmarshaling galera state: %v", err) } galeraState.UUID = bootstrap.UUID galeraState.Seqno = bootstrap.Seqno galeraState.SafeToBootstrap = true - bytes, err = galeraState.Marshal() + bytes, err = galeraState.MarshalText() if err != nil { return fmt.Errorf("error marshaling galera state: %v", err) } diff --git a/pkg/handler/galerastate.go b/pkg/handler/galerastate.go index 8b8bf89..4153807 100644 --- a/pkg/handler/galerastate.go +++ b/pkg/handler/galerastate.go @@ -45,7 +45,7 @@ func (g *GaleraState) Get(w http.ResponseWriter, r *http.Request) { } var galeraState galera.GaleraState - if err := galeraState.Unmarshal(bytes); err != nil { + if err := galeraState.UnmarshalText(bytes); err != nil { g.responseWriter.WriteErrorf(w, "error unmarshaling galera state: %v", err) return }