From 2aa8c09ddd37b2f58eb55663ec0e0f8e843530b3 Mon Sep 17 00:00:00 2001 From: Marco Cadetg Date: Tue, 16 Apr 2024 11:47:03 +0200 Subject: [PATCH 01/11] implementing seed mechanism --- README.md | 3 +- .../controller/databaserequest_controller.go | 165 +++++++++++++----- test/e2e/e2e_test.go | 13 +- 3 files changed, 140 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index e6852e8..134c9d9 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ WIP - Work in Progress There is still a lot of work to be done on this project. The current status is that the controller is able to provision and deprovision MySQL databases. But there is still a lot of work to be done to make it production ready. - [x] Setup e2e tests -- [x] Provision MySQL databases (basic) - no support for additional users, seeding, etc. +- [x] Provision MySQL databases (basic + seeding) - no support for additional users etc. - [x] Deprovision MySQL databases - [ ] Provision PostgreSQL databases - [ ] Deprovision PostgreSQL databases @@ -53,6 +53,7 @@ Key Features: - Utilizes native Kubernetes secret resources for storing database credentials and connection details. - Pooling and Disabling Providers: Supports adding new providers to a pool and marking providers as disabled or unable to deprovision. - Migration Support: Offers mechanisms to migrate consumers between providers seamlessly. +- Able to reusing existing databases by using a seed secret to populate the database with data. ## Custom Resource Definitions diff --git a/internal/controller/databaserequest_controller.go b/internal/controller/databaserequest_controller.go index 02c42c9..4777063 100644 --- a/internal/controller/databaserequest_controller.go +++ b/internal/controller/databaserequest_controller.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "strconv" "strings" "sync" @@ -148,53 +149,61 @@ func (r *DatabaseRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } - if databaseRequest.Spec.DatabaseConnectionReference == nil { - if err := r.createDatabase(ctx, databaseRequest); err != nil { - return r.handleError(ctx, databaseRequest, "create-database", err) + var dbInfo *dbInfo + if databaseRequest.Spec.Seed != nil { + var err error + dbInfo, err = r.seedDatabase(ctx, databaseRequest) + if err != nil { + return r.handleError(ctx, databaseRequest, "seed-database", err) + } + if err := r.mysqlTestConnection(ctx, dbInfo); err != nil { + return r.handleError(ctx, databaseRequest, "seed-database-connection", err) } + } else { if databaseRequest.Spec.DatabaseConnectionReference == nil { - return r.handleError( - ctx, databaseRequest, "missing-connection-reference", errors.New("missing database connection reference")) + if err := r.createDatabase(ctx, databaseRequest); err != nil { + return r.handleError(ctx, databaseRequest, "create-database", err) + } + if databaseRequest.Spec.DatabaseConnectionReference == nil { + return r.handleError( + ctx, databaseRequest, "missing-connection-reference", errors.New("missing database connection reference")) + } } - } - if databaseRequest.Status.ObservedDatabaseConnectionReference != databaseRequest.Spec.DatabaseConnectionReference { - // update the database connection reference - // FIXME: maybe this needs some additional checks - // For example implement an update logic by checking: - // - the connection works to the potentially new database - // - might need to create user and password for the new connection? - // - updating the secret accordingly - // - anything else needed? - databaseRequest.Status.ObservedDatabaseConnectionReference = databaseRequest.Spec.DatabaseConnectionReference - } + if databaseRequest.Status.ObservedDatabaseConnectionReference != databaseRequest.Spec.DatabaseConnectionReference { + logger.Info("Database connection reference changed") + // This means that the database provider has changed and we need to test the connection. + // We will also update the service and secret BUT we do not create a new database, user or password. + // + databaseRequest.Status.ObservedDatabaseConnectionReference = databaseRequest.Spec.DatabaseConnectionReference + } - // Note at the moment we only have one "primary" connection per database request - // Implementing additional users would require to extend the logic here - // check if the database request is already created and the secret and service exist - var dbInfo dbInfo - switch databaseRequest.Spec.Type { - case mysqlType: - logger.Info("Get MySQL database information") - var err error - dbInfo, err = r.mysqlInfo(ctx, databaseRequest) - if err != nil { - return r.handleError(ctx, databaseRequest, "mysql-info", err) + // Note at the moment we only have one "primary" connection per database request + // Implementing additional users would require to extend the logic here + // check if the database request is already created and the secret and service exist + switch databaseRequest.Spec.Type { + case mysqlType: + logger.Info("Get MySQL database information") + var err error + dbInfo, err = r.mysqlInfo(ctx, databaseRequest) + if err != nil { + return r.handleError(ctx, databaseRequest, "mysql-info", err) + } + case postgresType: + logger.Info("Get PostgreSQL database information") + case mongodbType: + logger.Info("Get MongoDB database information") + default: + logger.Error(ErrInvalidDatabaseType, "Unsupported database type", "type", databaseRequest.Spec.Type) } - case postgresType: - logger.Info("Get PostgreSQL database information") - case mongodbType: - logger.Info("Get MongoDB database information") - default: - logger.Error(ErrInvalidDatabaseType, "Unsupported database type", "type", databaseRequest.Spec.Type) } - serviceChanged, err := r.handleService(ctx, &dbInfo, databaseRequest) + serviceChanged, err := r.handleService(ctx, dbInfo, databaseRequest) if err != nil { return r.handleError(ctx, databaseRequest, "handle-service", err) } - secretChanged, err := r.handleSecret(ctx, &dbInfo, databaseRequest) + secretChanged, err := r.handleSecret(ctx, dbInfo, databaseRequest) if err != nil { return r.handleError(ctx, databaseRequest, "handle-secret", err) } @@ -317,6 +326,7 @@ func (r *DatabaseRequestReconciler) handleService( return false, nil } +// handleSecret creates or updates the secret for the database request func (r *DatabaseRequestReconciler) handleSecret( ctx context.Context, dbInfo *dbInfo, databaseRequest *crdv1alpha1.DatabaseRequest) (bool, error) { // Note at the moment we only have one "primary" connection per database request @@ -362,6 +372,7 @@ func (r *DatabaseRequestReconciler) handleSecret( return false, nil } +// deleteDatabase deletes the database based on the database request func (r *DatabaseRequestReconciler) deleteDatabase( ctx context.Context, databaseRequest *crdv1alpha1.DatabaseRequest) (ctrl.Result, error) { // handle deletion logic @@ -418,6 +429,7 @@ func (r *DatabaseRequestReconciler) deleteDatabase( return ctrl.Result{}, nil } +// createDatabase creates the database based on the database request func (r *DatabaseRequestReconciler) createDatabase( ctx context.Context, databaseRequest *crdv1alpha1.DatabaseRequest) error { logger := log.FromContext(ctx) @@ -451,6 +463,21 @@ func (r *DatabaseRequestReconciler) createDatabase( return nil } +// seedDatabase returns the database information from the seed secret +func (r *DatabaseRequestReconciler) seedDatabase( + ctx context.Context, + databaseRequest *crdv1alpha1.DatabaseRequest, +) (*dbInfo, error) { + seed := &v1.Secret{} + if err := r.Get(ctx, types.NamespacedName{ + Name: databaseRequest.Spec.Seed.Name, + Namespace: databaseRequest.Spec.Seed.Namespace, + }, seed); err != nil { + return nil, fmt.Errorf("failed to get seed secret %s: %w", databaseRequest.Spec.Seed.Name, err) + } + return dbInfoFromSeed(seed) +} + // promLabels returns the prometheus labels for the database request func promLabels(databaseRequest *crdv1alpha1.DatabaseRequest, withError string) prometheus.Labels { var username, databaseName string @@ -506,6 +533,7 @@ type dbInfo struct { port int } +// getSecretData returns the secret data for the database func (m *dbInfo) getSecretData(name, serviceName string) map[string][]byte { name = strings.ToUpper(strings.Replace(name, "-", "_", -1)) return map[string][]byte{ @@ -517,6 +545,47 @@ func (m *dbInfo) getSecretData(name, serviceName string) map[string][]byte { } } +// dbInfoFromSeed returns a dbInfo struct from the seed secret +func dbInfoFromSeed(secret *v1.Secret) (*dbInfo, error) { + // check if the secret has all the required keys + info := &dbInfo{} + if val, ok := secret.Data["database"]; !ok { + return nil, errors.New("missing database key in seed secret") + } else { + info.database = string(val) + } + + if val, ok := secret.Data["hostname"]; !ok { + return nil, errors.New("missing hostname key in seed secret") + } else { + info.hostName = string(val) + } + + if val, ok := secret.Data["username"]; !ok { + return nil, errors.New("missing username key in seed secret") + } else { + info.userName = string(val) + } + + if val, ok := secret.Data["password"]; !ok { + return nil, errors.New("missing password key in seed secret") + } else { + info.password = string(val) + } + + if val, ok := secret.Data["port"]; !ok { + return nil, errors.New("missing port key in seed secret") + } else { + port, err := strconv.Atoi(string(val)) + if err != nil { + return nil, fmt.Errorf("failed to convert port to int: %w", err) + } + info.port = port + } + + return info, nil +} + // mysqlOperation performs the MySQL operations create and drop func (r *DatabaseRequestReconciler) mysqlOperation( ctx context.Context, @@ -744,16 +813,34 @@ func (r *DatabaseRequestReconciler) mysqlDeletion( func (r *DatabaseRequestReconciler) mysqlInfo( ctx context.Context, databaseRequest *crdv1alpha1.DatabaseRequest, -) (dbInfo, error) { +) (*dbInfo, error) { log.FromContext(ctx).Info("Retrieving MySQL database information") - dbInfo := dbInfo{} - if err := r.mysqlOperation(ctx, info, databaseRequest, &dbInfo); err != nil { - return dbInfo, fmt.Errorf("mysql db info failed: %w", err) + dbInfo := &dbInfo{} + if err := r.mysqlOperation(ctx, info, databaseRequest, dbInfo); err != nil { + return nil, fmt.Errorf("mysql db info failed: %w", err) } return dbInfo, nil } +// mysqlTestConnection tests the MySQL connection +func (r *DatabaseRequestReconciler) mysqlTestConnection( + ctx context.Context, + dbi *dbInfo, +) error { + log.FromContext(ctx).Info("Testing MySQL connection") + conn := mySQLConn{ + hostname: dbi.hostName, + username: dbi.userName, + password: dbi.password, + port: dbi.port, + } + if err := r.MySQLClient.Ping(ctx, conn.getDSN()); err != nil { + return fmt.Errorf("mysql test connection failed: %w", err) + } + return nil +} + // lock is a simple lock implementation to avoid creating the same database in parallel func (r *DatabaseRequestReconciler) lock(key string) { mu, _ := r.Locks.LoadOrStore(key, &sync.Mutex{}) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 7d80416..4c579e8 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -97,6 +97,9 @@ var _ = Describe("controller", Ordered, func() { cmd = exec.Command( "kubectl", "delete", "secret", "-n", "default", "-l", "app.kubernetes.io/instance=databaserequest-sample") _, _ = utils.Run(cmd) + cmd = exec.Command( + "kubectl", "delete", "secret", "-n", "default", "seed-mysql-secret") + _, _ = utils.Run(cmd) }) Context("Operator", func() { @@ -255,7 +258,15 @@ var _ = Describe("controller", Ordered, func() { secretNames = utils.GetNonEmptyLines(string(secretOutput)) ExpectWithOffset(1, secretNames).Should(HaveLen(1)) - // TODO(marco): maybe add a test connecting to the mysql database... + By("creating seed secret for the DatabaseRequest resource") + cmd = exec.Command("kubectl", "apply", "-f", "test/e2e/testdata/seed-secret.yaml") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("creating mysql pod to create seed credentials") + cmd = exec.Command("kubectl", "apply", "-f", "test/e2e/testdata/mysql-client-pod.yaml") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) // uncomment to debug ... //time.Sleep(15 * time.Minute) From a8ddf5696360ad3b9511abb9c397b537705c7277 Mon Sep 17 00:00:00 2001 From: Marco Cadetg Date: Thu, 16 May 2024 16:58:17 +0200 Subject: [PATCH 02/11] add seed-secret.yaml --- test/e2e/testdata/seed-secret.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 test/e2e/testdata/seed-secret.yaml diff --git a/test/e2e/testdata/seed-secret.yaml b/test/e2e/testdata/seed-secret.yaml new file mode 100644 index 0000000..8280836 --- /dev/null +++ b/test/e2e/testdata/seed-secret.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Secret +metadata: + name: mysql-seed-secret + namespace: default +type: Opaque +data: + #database: seed-database + database: c2VlZC1kYXRhYmFzZQ== + #password: seed-password + password: c2VlZC1wYXNzd29yZA== + #username: seed-username + username: c2VlZC11c2VybmFtZQ== + #host mysql-service.mysql + host: bXlzc2VydmljZS5teXNxbC5kYg== + #port 3306 + port: MzMwNg== + From d6a06c1e4a38d1cc34f7fe1c5ff75a638b34a213 Mon Sep 17 00:00:00 2001 From: Marco Cadetg Date: Thu, 16 May 2024 17:08:57 +0200 Subject: [PATCH 03/11] add missing e2e test file mysql-client-pod.yaml --- test/e2e/testdata/mysql-client-pod.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 test/e2e/testdata/mysql-client-pod.yaml diff --git a/test/e2e/testdata/mysql-client-pod.yaml b/test/e2e/testdata/mysql-client-pod.yaml new file mode 100644 index 0000000..0bb69e6 --- /dev/null +++ b/test/e2e/testdata/mysql-client-pod.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: mysql-init-pod + namespace: mysql +spec: + restartPolicy: Never + containers: + - name: mysql-client + image: mysql:8 # change to mysql:5.7 if you want to test with MySQL 5.7 but remember + # that the MySQL 5.7 image is is not multi-arch and will not work on ARM64 + # out of the box + command: ["sh", "-c"] + args: + - | + mysql -h mysql-service.mysql -uroot -pe2e-mysql-password -e "CREATE DATABASE IF NOT EXISTS seed-database;" \ No newline at end of file From 8327fd5fa606265cb841b71cefa6ac2b2283a050 Mon Sep 17 00:00:00 2001 From: Marco Cadetg Date: Mon, 27 May 2024 16:49:32 +0200 Subject: [PATCH 04/11] adding missing seed e2e test files --- .../crd_v1alpha1_seed_databaserequest.yaml | 17 ++++ test/e2e/e2e_test.go | 99 ++++++++++++++----- test/e2e/testdata/mysql-client-pod.yaml | 5 +- test/e2e/testdata/seed-secret.yaml | 4 +- 4 files changed, 98 insertions(+), 27 deletions(-) create mode 100644 config/samples/crd_v1alpha1_seed_databaserequest.yaml diff --git a/config/samples/crd_v1alpha1_seed_databaserequest.yaml b/config/samples/crd_v1alpha1_seed_databaserequest.yaml new file mode 100644 index 0000000..4270189 --- /dev/null +++ b/config/samples/crd_v1alpha1_seed_databaserequest.yaml @@ -0,0 +1,17 @@ +apiVersion: crd.lagoon.sh/v1alpha1 +kind: DatabaseRequest +metadata: + labels: + app.kubernetes.io/name: seed-databaserequest + app.kubernetes.io/instance: seed-databaserequest-sample + app.kubernetes.io/part-of: dbaas-controller + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: dbaas-controller + name: seed-databaserequest-sample +spec: + seed: + name: mysql-seed-secret + namespace: default + name: seed-mysql-db + scope: development + type: mysql \ No newline at end of file diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 4c579e8..65551e0 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -27,7 +27,10 @@ import ( "github.com/uselagoon/dbaas-controller/test/utils" ) -const namespace = "dbaas-controller-system" +const ( + namespace = "dbaas-controller-system" + timetout = "500s" +) var _ = Describe("controller", Ordered, func() { BeforeAll(func() { @@ -69,18 +72,31 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) By("removing the DatabaseRequest resource") - // we enforce the deletion by removing the finalizer + for _, name := range []string{"databaserequest-sample", "seed-databaserequest-sample"} { + // we enforce the deletion by removing the finalizer + cmd = exec.Command( + "kubectl", + "patch", + "databaserequest", + name, + "-p", + `{"metadata":{"finalizers":[]}}`, + "--type=merge", + ) + _, _ = utils.Run(cmd) + cmd = exec.Command("kubectl", "delete", "--force", "databaserequest", name) + _, _ = utils.Run(cmd) + + By("removing service and secret") + cmd = exec.Command( + "kubectl", "delete", "service", "-n", "default", "-l", fmt.Sprintf("app.kubernetes.io/instance=%s", name)) + _, _ = utils.Run(cmd) + cmd = exec.Command( + "kubectl", "delete", "secret", "-n", "default", "-l", fmt.Sprintf("app.kubernetes.io/instance=%s", name)) + _, _ = utils.Run(cmd) + } cmd = exec.Command( - "kubectl", - "patch", - "databaserequest", - "databaserequest-sample", - "-p", - `{"metadata":{"finalizers":[]}}`, - "--type=merge", - ) - _, _ = utils.Run(cmd) - cmd = exec.Command("kubectl", "delete", "--force", "databaserequest", "databaserequest-sample") + "kubectl", "delete", "secret", "-n", "default", "seed-mysql-secret") _, _ = utils.Run(cmd) By("removing manager namespace") @@ -90,16 +106,6 @@ var _ = Describe("controller", Ordered, func() { By("uninstalling MySQL pod") utils.UninstallMySQLPod() - By("removing service and secret") - cmd = exec.Command( - "kubectl", "delete", "service", "-n", "default", "-l", "app.kubernetes.io/instance=databaserequest-sample") - _, _ = utils.Run(cmd) - cmd = exec.Command( - "kubectl", "delete", "secret", "-n", "default", "-l", "app.kubernetes.io/instance=databaserequest-sample") - _, _ = utils.Run(cmd) - cmd = exec.Command( - "kubectl", "delete", "secret", "-n", "default", "seed-mysql-secret") - _, _ = utils.Run(cmd) }) Context("Operator", func() { @@ -177,7 +183,7 @@ var _ = Describe("controller", Ordered, func() { "--for=condition=Ready", "databasemysqlprovider", "databasemysqlprovider-sample", - "--timeout=60s", + fmt.Sprintf("--timeout=%s", timetout), ) _, err = utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -194,7 +200,7 @@ var _ = Describe("controller", Ordered, func() { "--for=condition=Ready", "databaserequest", "databaserequest-sample", - "--timeout=60s", + fmt.Sprintf("--timeout=%s", timetout), ) _, err = utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -268,6 +274,51 @@ var _ = Describe("controller", Ordered, func() { _, err = utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) + By("creating a seed DatabaseRequest resource") + cmd = exec.Command("kubectl", "apply", "-f", "config/samples/crd_v1alpha1_seed_databaserequest.yaml") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("validating that the DatabaseRequest resource is created") + cmd = exec.Command( + "kubectl", + "wait", + "--for=condition=Ready", + "databaserequest", + "seed-databaserequest-sample", + fmt.Sprintf("--timeout=%s", timetout), + ) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + // verify that the service and secret got created + By("validating that the service is created") + cmd = exec.Command( + "kubectl", + "get", + "service", + "-n", "default", + "-l", "app.kubernetes.io/instance=seed-databaserequest-sample", + ) + serviceOutput, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + serviceNames = utils.GetNonEmptyLines(string(serviceOutput)) + ExpectWithOffset(1, serviceNames).Should(HaveLen(2)) + ExpectWithOffset(1, serviceNames[1]).Should(ContainSubstring("seed-mysql-db")) + + By("validating that the secret is created") + cmd = exec.Command( + "kubectl", + "get", + "secret", + "-n", "default", + "-l", "app.kubernetes.io/instance=seed-databaserequest-sample", + ) + secretOutput, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + secretNames = utils.GetNonEmptyLines(string(secretOutput)) + ExpectWithOffset(1, secretNames).Should(HaveLen(2)) + // uncomment to debug ... //time.Sleep(15 * time.Minute) }) diff --git a/test/e2e/testdata/mysql-client-pod.yaml b/test/e2e/testdata/mysql-client-pod.yaml index 0bb69e6..2f15c8c 100644 --- a/test/e2e/testdata/mysql-client-pod.yaml +++ b/test/e2e/testdata/mysql-client-pod.yaml @@ -13,4 +13,7 @@ spec: command: ["sh", "-c"] args: - | - mysql -h mysql-service.mysql -uroot -pe2e-mysql-password -e "CREATE DATABASE IF NOT EXISTS seed-database;" \ No newline at end of file + mysql -h mysql-service.mysql -uroot -pe2e-test-password -e "CREATE DATABASE IF NOT EXISTS \`seed-database\`;" && + mysql -h mysql-service.mysql -uroot -pe2e-test-password -e "CREATE USER IF NOT EXISTS 'seed-username'@'%' IDENTIFIED BY 'seed-password';" && + mysql -h mysql-service.mysql -uroot -pe2e-test-password -e "GRANT ALL PRIVILEGES ON \`seed-database\`.* TO 'seed-username'@'%';" && + mysql -h mysql-service.mysql -uroot -pe2e-test-password -e "FLUSH PRIVILEGES;" \ No newline at end of file diff --git a/test/e2e/testdata/seed-secret.yaml b/test/e2e/testdata/seed-secret.yaml index 8280836..96734b9 100644 --- a/test/e2e/testdata/seed-secret.yaml +++ b/test/e2e/testdata/seed-secret.yaml @@ -11,8 +11,8 @@ data: password: c2VlZC1wYXNzd29yZA== #username: seed-username username: c2VlZC11c2VybmFtZQ== - #host mysql-service.mysql - host: bXlzc2VydmljZS5teXNxbC5kYg== + #hostname mysql-service.mysql + hostname: bXlzcWwtc2VydmljZS5teXNxbA== #port 3306 port: MzMwNg== From 9952e17670dbbbfa8840cd91d21ef4552c48acf8 Mon Sep 17 00:00:00 2001 From: Marco Cadetg Date: Thu, 30 May 2024 17:46:50 +0200 Subject: [PATCH 05/11] implement from seed to db reference in databasereqeust controller --- .../controller/databaserequest_controller.go | 161 ++++++++++++++++-- internal/database/database.go | 72 +++++++- internal/database/mock.go | 7 +- test/e2e/e2e_test.go | 19 +-- 4 files changed, 228 insertions(+), 31 deletions(-) diff --git a/internal/controller/databaserequest_controller.go b/internal/controller/databaserequest_controller.go index e8159eb..b1217f4 100644 --- a/internal/controller/databaserequest_controller.go +++ b/internal/controller/databaserequest_controller.go @@ -41,6 +41,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/prometheus/client_golang/prometheus" + "github.com/uselagoon/dbaas-controller/api/v1alpha1" crdv1alpha1 "github.com/uselagoon/dbaas-controller/api/v1alpha1" "github.com/uselagoon/dbaas-controller/internal/database" ) @@ -151,14 +152,16 @@ func (r *DatabaseRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ var dbInfo *dbInfo if databaseRequest.Spec.Seed != nil { - var err error - dbInfo, err = r.seedDatabase(ctx, databaseRequest) + seedInfo, err := r.handleSeed(ctx, databaseRequest) if err != nil { - return r.handleError(ctx, databaseRequest, "seed-database", err) - } - if err := r.relDBTestConnection(ctx, dbInfo, databaseRequest.Spec.Type); err != nil { - return r.handleError(ctx, databaseRequest, "seed-database-connection", err) - } + return r.handleError(ctx, databaseRequest, "handle-seed", err) + } + // now let's update the database request with the connection reference + databaseRequest.Spec.DatabaseConnectionReference = seedInfo.databaseProviderRef + databaseRequest.Status.ObservedDatabaseConnectionReference = databaseRequest.Spec.DatabaseConnectionReference + databaseRequest.Spec.Seed = nil + dbInfo = seedInfo.dbInfo + logger.Info("Seed database setup complete") } else { if databaseRequest.Spec.DatabaseConnectionReference == nil { if err := r.createDatabase(ctx, databaseRequest); err != nil { @@ -275,6 +278,49 @@ func (r *DatabaseRequestReconciler) handleError( return ctrl.Result{}, err } +func (r *DatabaseRequestReconciler) handleSeed(ctx context.Context, databaseRequest *crdv1alpha1.DatabaseRequest) (*seedDatabaseInfo, error) { + logger := log.FromContext(ctx) + logger.Info("Get seed database info") + seedInfo, err := r.relationalDatabaseInfoFromSeed( + ctx, + databaseRequest.Spec.Seed, + databaseRequest.Spec.Type, + databaseRequest.Spec.Scope, + ) + if err != nil { + return nil, err + } + logger.Info("Found seed connection", "connectionName", seedInfo.conn.name) + + // if we got a provider connection and db info we set the database info. + if err = r.RelationalDatabaseClient.SetDatabaseInfo( + ctx, + seedInfo.conn.getDSN(), + databaseRequest.Name, + databaseRequest.Namespace, + databaseRequest.Spec.Type, + database.RelationalDatabaseInfo{ + Username: seedInfo.dbInfo.userName, + Password: seedInfo.dbInfo.password, + Dbname: seedInfo.dbInfo.database, + }, + ); err != nil { + return nil, err + } + logger.Info("Set database info", "username", seedInfo.dbInfo.userName, "database", seedInfo.dbInfo.database) + // get rid of the seed secret + if err := r.Delete(ctx, &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: databaseRequest.Spec.Seed.Name, + Namespace: databaseRequest.Spec.Seed.Namespace, + }, + }); err != nil { + return nil, err + } + logger.Info("Deleted seed secret", "secret", databaseRequest.Spec.Seed.Name) + return seedInfo, nil +} + // handleService creates or updates the service for the database request // returns true if the service has been updated func (r *DatabaseRequestReconciler) handleService( @@ -456,16 +502,16 @@ func (r *DatabaseRequestReconciler) createDatabase( // seedDatabase returns the database information from the seed secret func (r *DatabaseRequestReconciler) seedDatabase( ctx context.Context, - databaseRequest *crdv1alpha1.DatabaseRequest, + seed *v1.SecretReference, ) (*dbInfo, error) { - seed := &v1.Secret{} + secret := &v1.Secret{} if err := r.Get(ctx, types.NamespacedName{ - Name: databaseRequest.Spec.Seed.Name, - Namespace: databaseRequest.Spec.Seed.Namespace, - }, seed); err != nil { - return nil, fmt.Errorf("failed to get seed secret %s: %w", databaseRequest.Spec.Seed.Name, err) + Name: seed.Name, + Namespace: seed.Namespace, + }, secret); err != nil { + return nil, fmt.Errorf("failed to get seed secret %s: %w", seed.Name, err) } - return dbInfoFromSeed(seed) + return dbInfoFromSeed(secret) } // promLabels returns the prometheus labels for the database request @@ -707,7 +753,7 @@ func (r *DatabaseRequestReconciler) relationalDatabaseOperation( return fmt.Errorf("%s db operation %s failed due to missing dbInfo", databaseRequest.Spec.Type, operation) } // get the database information - info, err := r.RelationalDatabaseClient.GetDatabase( + info, err := r.RelationalDatabaseClient.GetDatabaseInfo( ctx, conn.getDSN(), databaseRequest.Name, @@ -729,6 +775,91 @@ func (r *DatabaseRequestReconciler) relationalDatabaseOperation( } } +// seedDatabaseInfo is a struct to hold the seed database information +type seedDatabaseInfo struct { + dbInfo *dbInfo + conn *reldbConn + databaseProviderRef *v1alpha1.DatabaseConnectionReference +} + +// relationalDatabaseInfoFromSeed finds the relational database provider based on the seed secret +func (r *DatabaseRequestReconciler) relationalDatabaseInfoFromSeed( + ctx context.Context, + seed *v1.SecretReference, + dbType string, + scope string, +) (*seedDatabaseInfo, error) { + dbInfo, err := r.seedDatabase(ctx, seed) + if err != nil { + return nil, fmt.Errorf("%s db find connection from seed failed to get seed database: %w", dbType, err) + } + + // test if the connection works with the seed + if err := r.relDBTestConnection(ctx, dbInfo, dbType); err != nil { + return nil, fmt.Errorf("%s db find connection from seed failed to test connection: %w", dbType, err) + } + + dbProviders := &crdv1alpha1.RelationalDatabaseProviderList{} + if err := r.List(ctx, dbProviders); err != nil { + return nil, fmt.Errorf("%s db find connection from seed failed to list database providers: %w", + dbType, err, + ) + } + + var connection *crdv1alpha1.Connection + var databaseProviderRef *v1alpha1.DatabaseConnectionReference + for _, dbProvider := range dbProviders.Items { + if dbProvider.Spec.Scope == scope && dbProvider.Spec.Type == dbType { + for _, dbConnection := range dbProvider.Spec.Connections { + if dbConnection.Enabled && dbConnection.Hostname == dbInfo.hostName && + dbConnection.Port == dbInfo.port { + log.FromContext(ctx).Info("Found provider", "provider", dbProvider.Name) + connection = &dbConnection + databaseProviderRef = &crdv1alpha1.DatabaseConnectionReference{ + Name: connection.Name, + DatabaseObjectReference: v1.ObjectReference{ + Kind: dbProvider.Kind, + Name: dbProvider.Name, + UID: dbProvider.UID, + ResourceVersion: dbProvider.ResourceVersion, + }, + } + } + } + } + } + + if connection == nil { + return nil, fmt.Errorf("%s db find connection from seed failed due to provider not found", dbType) + } + + secret := &v1.Secret{} + if err := r.Get(ctx, types.NamespacedName{ + Name: connection.PasswordSecretRef.Name, + Namespace: connection.PasswordSecretRef.Namespace, + }, secret); err != nil { + return nil, fmt.Errorf("%s db find connection from seed failed to get connection password from secret: %w", + dbType, err, + ) + } + + password := string(secret.Data["password"]) + if password == "" { + return nil, fmt.Errorf("%s db find connection from seed failed due to empty password", dbType) + } + + conn := &reldbConn{ + dbType: dbType, + name: connection.Name, + hostname: connection.Hostname, + username: connection.Username, + password: password, + port: connection.Port, + } + + return &seedDatabaseInfo{dbInfo: dbInfo, conn: conn, databaseProviderRef: databaseProviderRef}, nil +} + // findRelationalDatabaseProvider finds the relational database provider with the same scope and the lower load // returns the provider, connection name and an error func (r *DatabaseRequestReconciler) findRelationalDatabaseProvider( diff --git a/internal/database/database.go b/internal/database/database.go index 2bcff05..ae5359b 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -68,8 +68,11 @@ type RelationalDatabaseInterface interface { // This function is idempotent and can be called multiple times without side effects. DropDatabase(ctx context.Context, dsn, name, namespace, dbType string) error - // GetDatabase returns the database name, username, and password for the given name and namespace. - GetDatabase(ctx context.Context, dsn, name, namespace, dbType string) (RelationalDatabaseInfo, error) + // GetDatabaseInfo returns the database name, username, and password for the given name and namespace. + GetDatabaseInfo(ctx context.Context, dsn, name, namespace, dbType string) (RelationalDatabaseInfo, error) + + // SetDatabaseInfo sets the database name, username, and password for the given name and namespace. + SetDatabaseInfo(ctx context.Context, dsn, name, namespace, dbType string, info RelationalDatabaseInfo) error } // RelationalDatabaseImpl is the implementation of the RelationalDatabaseInterface @@ -449,6 +452,31 @@ func (ri *RelationalDatabaseImpl) databaseInfoMySQL( return info, nil } +// insertUserinfoIntoMysql inserts the user into the users table +// This function is idempotent and can be called multiple times without side effects. +func (ri *RelationalDatabaseImpl) insertUserInfoIntoMysql( + ctx context.Context, + dsn, name, namespace string, + info RelationalDatabaseInfo, +) error { + db, err := ri.GetConnection(ctx, dsn, mysql) + if err != nil { + return fmt.Errorf("set database info failed to open %s database: %w", mysql, err) + } + _, err = db.ExecContext(ctx, "USE dbaas_controller") + if err != nil { + return fmt.Errorf("failed to select database: %w", err) + } + // Insert the user into the users table + _, err = db.ExecContext( + ctx, "INSERT IGNORE INTO users (name, namespace, username, password, dbname) VALUES (?, ?, ?, ?, ?)", + name, namespace, info.Username, info.Password, info.Dbname) + if err != nil { + return fmt.Errorf("failed to insert user into users table: %w", err) + } + return nil +} + // databaseInfoPostgreSQL returns the username, password, and database name for the given name and namespace. // It also creates the user and database if they do not exist. // This function is idempotent and can be called multiple times without side effects. @@ -489,8 +517,29 @@ func (ri *RelationalDatabaseImpl) databaseInfoPostgreSQL( return info, nil } -// GetDatabase returns the database name, username, and password for the given name and namespace. -func (ri *RelationalDatabaseImpl) GetDatabase( +// insertUserInfoIntoPostgreSQL inserts the user into the users table +// This function is idempotent and can be called multiple times without side effects. +func (ri *RelationalDatabaseImpl) insertUserInfoIntoPostgreSQL( + ctx context.Context, + dsn, name, namespace string, + info RelationalDatabaseInfo, +) error { + db, err := ri.GetConnection(ctx, dsn, postgres) + if err != nil { + return fmt.Errorf("set database info failed to open %s database: %w", postgres, err) + } + // Insert the user into the users table + _, err = db.ExecContext( + ctx, "INSERT INTO dbaas_controller.users (name, namespace, username, password, dbname) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", + name, namespace, info.Username, info.Password, info.Dbname) + if err != nil { + return fmt.Errorf("failed to insert user into users table: %w", err) + } + return nil +} + +// GetDatabaseInfo returns the database name, username, and password for the given name and namespace. +func (ri *RelationalDatabaseImpl) GetDatabaseInfo( ctx context.Context, dsn, name, namespace, dbType string, ) (RelationalDatabaseInfo, error) { @@ -502,3 +551,18 @@ func (ri *RelationalDatabaseImpl) GetDatabase( } return RelationalDatabaseInfo{}, fmt.Errorf("get database failed to get %s database: unsupported dbType", dbType) } + +// SetDatabaseInfo sets the database name, username, and password for the given name and namespace. +func (ri *RelationalDatabaseImpl) SetDatabaseInfo( + ctx context.Context, + dsn, name, namespace, dbType string, + info RelationalDatabaseInfo, +) error { + log.FromContext(ctx).Info("Setting database", "dbType", dbType, "name", name, "namespace", namespace) + if dbType == "mysql" { + return ri.insertUserInfoIntoMysql(ctx, dsn, name, namespace, info) + } else if dbType == "postgres" { + return ri.insertUserInfoIntoPostgreSQL(ctx, dsn, name, namespace, info) + } + return nil +} diff --git a/internal/database/mock.go b/internal/database/mock.go index 990fe4d..939e654 100644 --- a/internal/database/mock.go +++ b/internal/database/mock.go @@ -35,11 +35,16 @@ func (mi *RelationalDatabaseMock) DropDatabase(ctx context.Context, dsn, name, n return nil } -func (mi *RelationalDatabaseMock) GetDatabase( +func (mi *RelationalDatabaseMock) GetDatabaseInfo( ctx context.Context, dsn, name, namespace, kind string) (RelationalDatabaseInfo, error) { return RelationalDatabaseInfo{Username: "user", Password: "pass", Dbname: "db"}, nil } +func (mi *RelationalDatabaseMock) SetDatabaseInfo( + ctx context.Context, dsn, name, namespace, kind string, info RelationalDatabaseInfo) error { + return nil +} + func (mi *RelationalDatabaseMock) Load(ctx context.Context, dsn string, kind string) (int, error) { return 10, nil } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index b6caeb4..d95e0f7 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -261,6 +261,14 @@ var _ = Describe("controller", Ordered, func() { secretNames := utils.GetNonEmptyLines(string(secretOutput)) ExpectWithOffset(1, secretNames).Should(HaveLen(2)) + if name == "seed" { + By("checking that the seed secret is deleted") + cmd = exec.Command("kubectl", "get", "secret", "seed-mysql-secret") + _, err := utils.Run(cmd) + // expect error to occurred + ExpectWithOffset(1, err).To(HaveOccurred()) + } + By("deleting the DatabaseRequest resource the database is getting deprovisioned") cmd = exec.Command( "kubectl", @@ -296,17 +304,6 @@ var _ = Describe("controller", Ordered, func() { ExpectWithOffset(1, err).NotTo(HaveOccurred()) secretNames = utils.GetNonEmptyLines(string(secretOutput)) ExpectWithOffset(1, secretNames).Should(HaveLen(1)) - if name == "seed" { - By("deleting the seed secret") - cmd = exec.Command( - "kubectl", - "delete", - "secret", - "seed-mysql-secret", - ) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - } } // uncomment to debug ... From 6afdddcc21ae366de26bd311bf2e3c82cd138838 Mon Sep 17 00:00:00 2001 From: Marco Cadetg Date: Thu, 30 May 2024 17:53:09 +0200 Subject: [PATCH 06/11] minor lint issues --- internal/controller/databaserequest_controller.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/controller/databaserequest_controller.go b/internal/controller/databaserequest_controller.go index b1217f4..e64be93 100644 --- a/internal/controller/databaserequest_controller.go +++ b/internal/controller/databaserequest_controller.go @@ -278,7 +278,10 @@ func (r *DatabaseRequestReconciler) handleError( return ctrl.Result{}, err } -func (r *DatabaseRequestReconciler) handleSeed(ctx context.Context, databaseRequest *crdv1alpha1.DatabaseRequest) (*seedDatabaseInfo, error) { +func (r *DatabaseRequestReconciler) handleSeed( + ctx context.Context, + databaseRequest *crdv1alpha1.DatabaseRequest, +) (*seedDatabaseInfo, error) { logger := log.FromContext(ctx) logger.Info("Get seed database info") seedInfo, err := r.relationalDatabaseInfoFromSeed( @@ -814,7 +817,8 @@ func (r *DatabaseRequestReconciler) relationalDatabaseInfoFromSeed( if dbConnection.Enabled && dbConnection.Hostname == dbInfo.hostName && dbConnection.Port == dbInfo.port { log.FromContext(ctx).Info("Found provider", "provider", dbProvider.Name) - connection = &dbConnection + conn := dbConnection + connection = &conn databaseProviderRef = &crdv1alpha1.DatabaseConnectionReference{ Name: connection.Name, DatabaseObjectReference: v1.ObjectReference{ From 0ddf741e9d2f95d93193517d549691403aa83aa8 Mon Sep 17 00:00:00 2001 From: Marco Cadetg Date: Tue, 4 Jun 2024 18:56:22 +0200 Subject: [PATCH 07/11] adding failing seed test cases --- .../controller/databaserequest_controller.go | 29 +++--- .../relationaldatabaseprovider_controller.go | 22 +++-- internal/database/database.go | 28 ++++++ test/e2e/e2e_test.go | 92 ++++++++++++++++++- ...redential_broken_seed_databaserequest.yaml | 17 ++++ ...xisting_database_seed_databaserequest.yaml | 17 ++++ .../credential-broken-seed-secret.yaml | 18 ++++ .../non-existing-database-seed-secret.yaml | 18 ++++ 8 files changed, 219 insertions(+), 22 deletions(-) create mode 100644 test/e2e/testdata/crd_v1alpha1_credential_broken_seed_databaserequest.yaml create mode 100644 test/e2e/testdata/crd_v1alpha1_non_existing_database_seed_databaserequest.yaml create mode 100644 test/e2e/testdata/credential-broken-seed-secret.yaml create mode 100644 test/e2e/testdata/non-existing-database-seed-secret.yaml diff --git a/internal/controller/databaserequest_controller.go b/internal/controller/databaserequest_controller.go index e64be93..0fc2c9c 100644 --- a/internal/controller/databaserequest_controller.go +++ b/internal/controller/databaserequest_controller.go @@ -298,7 +298,7 @@ func (r *DatabaseRequestReconciler) handleSeed( // if we got a provider connection and db info we set the database info. if err = r.RelationalDatabaseClient.SetDatabaseInfo( ctx, - seedInfo.conn.getDSN(), + seedInfo.conn.getDSN(false), databaseRequest.Name, databaseRequest.Namespace, databaseRequest.Spec.Type, @@ -450,7 +450,9 @@ func (r *DatabaseRequestReconciler) deleteDatabase( Namespace: databaseRequest.Namespace, }, }); err != nil { - return r.handleError(ctx, databaseRequest, "delete-service", err) + if !apierrors.IsNotFound(err) { + return r.handleError(ctx, databaseRequest, "delete-service", err) + } } if err := r.Delete(ctx, &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -458,7 +460,9 @@ func (r *DatabaseRequestReconciler) deleteDatabase( Namespace: databaseRequest.Namespace, }, }); err != nil { - return r.handleError(ctx, databaseRequest, "delete-secret", err) + if !apierrors.IsNotFound(err) { + return r.handleError(ctx, databaseRequest, "delete-secret", err) + } } if controllerutil.RemoveFinalizer(databaseRequest, databaseRequestFinalizer) { if err := r.Update(ctx, databaseRequest); err != nil { @@ -705,7 +709,7 @@ func (r *DatabaseRequestReconciler) relationalDatabaseOperation( log.FromContext(ctx).Info("Creating relational database", "database", databaseRequest.Name) info, err := r.RelationalDatabaseClient.CreateDatabase( ctx, - conn.getDSN(), + conn.getDSN(false), databaseRequest.Name, databaseRequest.Namespace, databaseRequest.Spec.Type, @@ -736,7 +740,7 @@ func (r *DatabaseRequestReconciler) relationalDatabaseOperation( case drop: if err := r.RelationalDatabaseClient.DropDatabase( ctx, - conn.getDSN(), + conn.getDSN(false), databaseRequest.Name, databaseRequest.Namespace, databaseRequest.Spec.Type, @@ -758,7 +762,7 @@ func (r *DatabaseRequestReconciler) relationalDatabaseOperation( // get the database information info, err := r.RelationalDatabaseClient.GetDatabaseInfo( ctx, - conn.getDSN(), + conn.getDSN(false), databaseRequest.Name, databaseRequest.Namespace, databaseRequest.Spec.Type, @@ -798,7 +802,7 @@ func (r *DatabaseRequestReconciler) relationalDatabaseInfoFromSeed( } // test if the connection works with the seed - if err := r.relDBTestConnection(ctx, dbInfo, dbType); err != nil { + if err := r.relDBTestSeedConnection(ctx, dbInfo, dbType); err != nil { return nil, fmt.Errorf("%s db find connection from seed failed to test connection: %w", dbType, err) } @@ -915,7 +919,7 @@ func (r *DatabaseRequestReconciler) findRelationalDatabaseProvider( // check the load of the provider connection // we select the provider with the lower load log.FromContext(ctx).Info("Checking provider database connection", "connection", dbConnection.Name) - dbLoad, err := r.RelationalDatabaseClient.Load(ctx, conn.getDSN(), databaseRequest.Spec.Type) + dbLoad, err := r.RelationalDatabaseClient.Load(ctx, conn.getDSN(false), databaseRequest.Spec.Type) if err != nil { return nil, "", fmt.Errorf("%s db find provider failed to get load: %w", databaseRequest.Spec.Type, err) } @@ -966,8 +970,8 @@ func (r *DatabaseRequestReconciler) relDBInfo( return &dbInfo, nil } -// relDBTestConnection tests a mysql or postgres connection -func (r *DatabaseRequestReconciler) relDBTestConnection( +// relDBTestSeedConnection tests a mysql or postgres connection +func (r *DatabaseRequestReconciler) relDBTestSeedConnection( ctx context.Context, dbi *dbInfo, dbType string, @@ -979,9 +983,10 @@ func (r *DatabaseRequestReconciler) relDBTestConnection( username: dbi.userName, password: dbi.password, port: dbi.port, + name: dbi.database, } - if err := r.RelationalDatabaseClient.Ping(ctx, conn.getDSN(), dbType); err != nil { - return fmt.Errorf("relation database test connection failed: %w", err) + if err := r.RelationalDatabaseClient.Ping(ctx, conn.getDSN(true), dbType); err != nil { + return fmt.Errorf("relational database test connection failed: %w", err) } return nil } diff --git a/internal/controller/relationaldatabaseprovider_controller.go b/internal/controller/relationaldatabaseprovider_controller.go index c3034a3..773b0ee 100644 --- a/internal/controller/relationaldatabaseprovider_controller.go +++ b/internal/controller/relationaldatabaseprovider_controller.go @@ -184,7 +184,7 @@ func (r *RelationalDatabaseProviderReconciler) Reconcile(ctx context.Context, re // make a ping to the database to check if it's up and running and we can connect to it // if not, we should return an error and set the status to 0 // Note we could periodically check the status of the database and update the status accordingly... - if err := r.RelDBClient.Ping(ctx, conn.getDSN(), instance.Spec.Type); err != nil { + if err := r.RelDBClient.Ping(ctx, conn.getDSN(false), instance.Spec.Type); err != nil { errors = append(errors, err) dbStatus = append(dbStatus, crdv1alpha1.ConnectionStatus{ Name: conn.name, @@ -197,7 +197,7 @@ func (r *RelationalDatabaseProviderReconciler) Reconcile(ctx context.Context, re logger.Error(err, "Failed to ping the database", "hostname", conn.hostname) continue } - version, err := r.RelDBClient.Version(ctx, conn.getDSN(), instance.Spec.Type) + version, err := r.RelDBClient.Version(ctx, conn.getDSN(false), instance.Spec.Type) if err != nil { errors = append(errors, err) dbStatus = append(dbStatus, crdv1alpha1.ConnectionStatus{ @@ -213,7 +213,7 @@ func (r *RelationalDatabaseProviderReconciler) Reconcile(ctx context.Context, re } // check if the database is initialized - err = r.RelDBClient.Initialize(ctx, conn.getDSN(), instance.Spec.Type) + err = r.RelDBClient.Initialize(ctx, conn.getDSN(false), instance.Spec.Type) if err != nil { errors = append(errors, err) dbStatus = append(dbStatus, crdv1alpha1.ConnectionStatus{ @@ -327,17 +327,23 @@ type reldbConn struct { } // getDSN constructs the DSN string for the MySQL or PostgreSQL connection. -func (rc *reldbConn) getDSN() string { +func (rc *reldbConn) getDSN(useDatabase bool) string { + dsn := "" if rc.dbType == "mysql" { - return fmt.Sprintf("%s:%s@tcp(%s:%d)/", rc.username, rc.password, rc.hostname, rc.port) + dsn = fmt.Sprintf("%s:%s@tcp(%s:%d)/", rc.username, rc.password, rc.hostname, rc.port) + if useDatabase { + dsn += rc.name + } } else if rc.dbType == "postgres" { - return fmt.Sprintf( + dsn = fmt.Sprintf( "host=%s port=%d user=%s password=%s sslmode=disable", rc.hostname, rc.port, rc.username, rc.password, ) - } else { - return "" + if useDatabase { + dsn += fmt.Sprintf(" dbname=%s", rc.name) + } } + return dsn } // SetupWithManager sets up the controller with the Manager. diff --git a/internal/database/database.go b/internal/database/database.go index ae5359b..e49d846 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -3,10 +3,12 @@ package database import ( "context" "database/sql" + "errors" "fmt" "math/rand" _ "github.com/go-sql-driver/mysql" + md "github.com/go-sql-driver/mysql" "github.com/lib/pq" _ "github.com/lib/pq" @@ -115,6 +117,32 @@ func (ri *RelationalDatabaseImpl) Ping(ctx context.Context, dsn string, dbType s } if err := db.PingContext(ctx); err != nil { + if dbType == mysql { + log.FromContext(ctx).Error(err, "Failed to ping MMMMMySQL database") + var driverErr *md.MySQLError + if errors.As(err, &driverErr) { + switch driverErr.Number { + case 1044, 1045: + return fmt.Errorf("failed to ping %s database due to invalid credentials: %w", dbType, err) + case 1049: + return fmt.Errorf("failed to ping %s database due to database does not exist: %w", dbType, err) + default: + return fmt.Errorf("failed to ping driveErr %s database: %w", dbType, err) + } + } + } else if dbType == postgres { + var driverErr *pq.Error + if errors.As(err, &driverErr) { + switch driverErr.Code { + case "28P01": + return fmt.Errorf("failed to ping %s database due to invalid credentials: %w", dbType, err) + case "3D000": + return fmt.Errorf("failed to ping %s database due to database does not exist: %w", dbType, err) + default: + return fmt.Errorf("failed to ping %s database: %w", dbType, err) + } + } + } return fmt.Errorf("failed to ping %s database: %w", dbType, err) } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index d95e0f7..c93ec00 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -19,6 +19,7 @@ package e2e import ( "fmt" "os/exec" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -176,6 +177,7 @@ var _ = Describe("controller", Ordered, func() { } EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + By("validating that all database providers and database requests are working") for _, name := range []string{"mysql", "postgres", "seed"} { if name != "seed" { By("creating a RelationalDatabaseProvider resource") @@ -306,8 +308,94 @@ var _ = Describe("controller", Ordered, func() { ExpectWithOffset(1, secretNames).Should(HaveLen(1)) } - // uncomment to debug ... - //time.Sleep(15 * time.Minute) + By("validating that broken seed database request are failing in exptected way") + for _, name := range []string{"credential-broken-seed", "non-existing-database-seed"} { + By("creating seed secret for the DatabaseRequest resource") + cmd = exec.Command("kubectl", "apply", "-f", fmt.Sprintf("test/e2e/testdata/%s-secret.yaml", name)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("creating a DatabaseRequest resource") + // replace - with _ + dbrName := strings.ReplaceAll(name, "-", "_") + cmd = exec.Command( + "kubectl", + "apply", + "-f", + fmt.Sprintf("test/e2e/testdata/crd_v1alpha1_%s_databaserequest.yaml", dbrName), + ) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("validating that the DatabaseRequest resource is created but fails") + cmd = exec.Command( + "kubectl", + "get", + "databaserequest", + fmt.Sprintf("%s-databaserequest-sample", name), + "-o=jsonpath={.status.conditions[?(@.type=='Ready')].status}", + ) + for i := 0; i < 3; i++ { + output, err := utils.Run(cmd) + if err != nil { + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } + if strings.TrimSpace(string(output)) == "False" { + break + } else if i == 2 { + Expect(strings.TrimSpace(string(output))).To(Equal("False")) + } + // give it a bit of time to fail + time.Sleep(time.Second) + } + + // verify that the service and secret got created + By("validating that the service is not created") + cmd = exec.Command( + "kubectl", + "get", + "service", + "-n", "default", + "-l", fmt.Sprintf("app.kubernetes.io/instance=%s-databaserequest-sample", name), + ) + output, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + Expect(strings.TrimSpace(string(output))).To(Equal("No resources found in default namespace.")) + + By("validating that the secret is not created") + cmd = exec.Command( + "kubectl", + "get", + "secret", + "-n", "default", + "-l", fmt.Sprintf("app.kubernetes.io/instance=%s-databaserequest-sample", name), + ) + serviceOutput, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + Expect(strings.TrimSpace(string(serviceOutput))).To(Equal("No resources found in default namespace.")) + + By("validating that the seed secret is not deleted") + cmd = exec.Command("kubectl", "get", "secret", fmt.Sprintf("%s-secret", name)) + _, err = utils.Run(cmd) + // expect no error to have occurred because the secret should not get deleted + // if the database request is not successfully created + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("deleting the DatabaseRequest resource the database is getting deprovisioned") + cmd = exec.Command( + "kubectl", + "delete", + "databaserequest", + fmt.Sprintf("%s-databaserequest-sample", name), + ) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } }) + + // uncomment to debug ... + //time.Sleep(15 * time.Minute) + }) + }) diff --git a/test/e2e/testdata/crd_v1alpha1_credential_broken_seed_databaserequest.yaml b/test/e2e/testdata/crd_v1alpha1_credential_broken_seed_databaserequest.yaml new file mode 100644 index 0000000..959dec4 --- /dev/null +++ b/test/e2e/testdata/crd_v1alpha1_credential_broken_seed_databaserequest.yaml @@ -0,0 +1,17 @@ +apiVersion: crd.lagoon.sh/v1alpha1 +kind: DatabaseRequest +metadata: + labels: + app.kubernetes.io/name: credential-broken-seed-databaserequest + app.kubernetes.io/instance: credential-broken-seed-databaserequest-sample + app.kubernetes.io/part-of: dbaas-controller + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: dbaas-controller + name: credential-broken-seed-databaserequest-sample +spec: + seed: + name: credential-broken-seed-secret + namespace: default + name: credential-broken-seed-mysql-db + scope: development + type: mysql \ No newline at end of file diff --git a/test/e2e/testdata/crd_v1alpha1_non_existing_database_seed_databaserequest.yaml b/test/e2e/testdata/crd_v1alpha1_non_existing_database_seed_databaserequest.yaml new file mode 100644 index 0000000..5c551c6 --- /dev/null +++ b/test/e2e/testdata/crd_v1alpha1_non_existing_database_seed_databaserequest.yaml @@ -0,0 +1,17 @@ +apiVersion: crd.lagoon.sh/v1alpha1 +kind: DatabaseRequest +metadata: + labels: + app.kubernetes.io/name: non-existing-database-seed-databaserequest + app.kubernetes.io/instance: non-existing-database-seed-databaserequest-sample + app.kubernetes.io/part-of: dbaas-controller + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: dbaas-controller + name: non-existing-database-seed-databaserequest-sample +spec: + seed: + name: non-existing-database-seed-secret + namespace: default + name: non-existing-database-seed-mysql-db + scope: development + type: mysql \ No newline at end of file diff --git a/test/e2e/testdata/credential-broken-seed-secret.yaml b/test/e2e/testdata/credential-broken-seed-secret.yaml new file mode 100644 index 0000000..d718ffa --- /dev/null +++ b/test/e2e/testdata/credential-broken-seed-secret.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Secret +metadata: + name: credential-broken-seed-secret + namespace: default +type: Opaque +data: + #database: seed-database + database: c2VlZC1kYXRhYmFzZQ== + #password: invalid-seed-password + password: aW52YWxpZC1zZWVkLXBhc3N3b3Jk + #username: seed-username + username: c2VlZC11c2VybmFtZQ== + #hostname mysql-service.mysql + hostname: bXlzcWwtc2VydmljZS5teXNxbA== + #port 3306 + port: MzMwNg== + diff --git a/test/e2e/testdata/non-existing-database-seed-secret.yaml b/test/e2e/testdata/non-existing-database-seed-secret.yaml new file mode 100644 index 0000000..bf74eec --- /dev/null +++ b/test/e2e/testdata/non-existing-database-seed-secret.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Secret +metadata: + name: non-existing-database-seed-secret + namespace: default +type: Opaque +data: + #database: non-existing-seed-database + database: bm9uLWV4aXN0aW5pbmctc2VlZC1kYXRhYmFzZQ== + #password: seed-password + password: c2VlZC1wYXNzd29yZA== + #username: seed-username + username: c2VlZC11c2VybmFtZQ== + #hostname mysql-service.mysql + hostname: bXlzcWwtc2VydmljZS5teXNxbA== + #port 3306 + port: MzMwNg== + From 4eb18ab8c0f977b6d3ccbec86e8bcbaac6f17d65 Mon Sep 17 00:00:00 2001 From: Marco Cadetg Date: Wed, 5 Jun 2024 08:24:25 +0200 Subject: [PATCH 08/11] add error condition --- .../controller/databaserequest_controller.go | 17 +++++++++++++++++ internal/database/database.go | 3 +-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/internal/controller/databaserequest_controller.go b/internal/controller/databaserequest_controller.go index 0fc2c9c..1e4fd71 100644 --- a/internal/controller/databaserequest_controller.go +++ b/internal/controller/databaserequest_controller.go @@ -219,6 +219,8 @@ func (r *DatabaseRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } + // clear the error condition if set + meta.RemoveStatusCondition(&databaseRequest.Status.Conditions, "Error") if serviceChanged || secretChanged { if meta.SetStatusCondition(&databaseRequest.Status.Conditions, metav1.Condition{ Type: "Ready", @@ -268,6 +270,14 @@ func (r *DatabaseRequestReconciler) handleError( Message: err.Error(), }) + // add additional condition to reflect the error state more clearly + meta.SetStatusCondition(&databaseRequest.Status.Conditions, metav1.Condition{ + Type: "Error", + Status: metav1.ConditionTrue, + Reason: "ReconcileFailed", + Message: fmt.Sprintf("An error occurred during reconciliation: %v", err), + }) + // update the status if err := r.Status().Update(ctx, databaseRequest); err != nil { promDatabaseRequestReconcileErrorCounter.With( @@ -469,6 +479,13 @@ func (r *DatabaseRequestReconciler) deleteDatabase( return r.handleError(ctx, databaseRequest, "remove-finalizer", err) } } + // record the event + r.Recorder.Event( + databaseRequest, + v1.EventTypeNormal, + "DeletedDatabase", + fmt.Sprintf("Deleted database %s/%s", databaseRequest.Namespace, databaseRequest.Name), + ) // cleanup metrics promDatabaseRequestReconcileStatus.DeletePartialMatch(prometheus.Labels{ "name": databaseRequest.Name, diff --git a/internal/database/database.go b/internal/database/database.go index e49d846..331fccd 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -118,7 +118,6 @@ func (ri *RelationalDatabaseImpl) Ping(ctx context.Context, dsn string, dbType s if err := db.PingContext(ctx); err != nil { if dbType == mysql { - log.FromContext(ctx).Error(err, "Failed to ping MMMMMySQL database") var driverErr *md.MySQLError if errors.As(err, &driverErr) { switch driverErr.Number { @@ -127,7 +126,7 @@ func (ri *RelationalDatabaseImpl) Ping(ctx context.Context, dsn string, dbType s case 1049: return fmt.Errorf("failed to ping %s database due to database does not exist: %w", dbType, err) default: - return fmt.Errorf("failed to ping driveErr %s database: %w", dbType, err) + return fmt.Errorf("failed to ping %s database: %w", dbType, err) } } } else if dbType == postgres { From aaabf181b5f701bdab5574a3971ff879d47cb27c Mon Sep 17 00:00:00 2001 From: Marco Cadetg Date: Thu, 6 Jun 2024 09:38:54 +0200 Subject: [PATCH 09/11] ignore invalid seed --- .../controller/databaserequest_controller.go | 45 +++++++++++++------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/internal/controller/databaserequest_controller.go b/internal/controller/databaserequest_controller.go index 1e4fd71..e293b60 100644 --- a/internal/controller/databaserequest_controller.go +++ b/internal/controller/databaserequest_controller.go @@ -135,7 +135,7 @@ func (r *DatabaseRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ if controllerutil.AddFinalizer(databaseRequest, databaseRequestFinalizer) { if err := r.Update(ctx, databaseRequest); err != nil { - return r.handleError(ctx, databaseRequest, "add-finalizer", err) + return r.handleError(ctx, databaseRequest, "add-finalizer", err, false) } } @@ -154,7 +154,10 @@ func (r *DatabaseRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ if databaseRequest.Spec.Seed != nil { seedInfo, err := r.handleSeed(ctx, databaseRequest) if err != nil { - return r.handleError(ctx, databaseRequest, "handle-seed", err) + if errIsInvalidCredentials(err) || errIsDatabaseDoesNotExist(err) { + return r.handleError(ctx, databaseRequest, "invalid-seed", err, true) + } + return r.handleError(ctx, databaseRequest, "handle-seed", err, false) } // now let's update the database request with the connection reference databaseRequest.Spec.DatabaseConnectionReference = seedInfo.databaseProviderRef @@ -165,11 +168,11 @@ func (r *DatabaseRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ } else { if databaseRequest.Spec.DatabaseConnectionReference == nil { if err := r.createDatabase(ctx, databaseRequest); err != nil { - return r.handleError(ctx, databaseRequest, "create-database", err) + return r.handleError(ctx, databaseRequest, "create-database", err, false) } if databaseRequest.Spec.DatabaseConnectionReference == nil { return r.handleError( - ctx, databaseRequest, "missing-connection-reference", errors.New("missing database connection reference")) + ctx, databaseRequest, "missing-connection-reference", errors.New("missing database connection reference"), false) } } @@ -191,7 +194,7 @@ func (r *DatabaseRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ dbInfo, err = r.relDBInfo(ctx, databaseRequest) if err != nil { return r.handleError( - ctx, databaseRequest, fmt.Sprintf("get-%s-database-info", databaseRequest.Spec.Type), err) + ctx, databaseRequest, fmt.Sprintf("get-%s-database-info", databaseRequest.Spec.Type), err, false) } } else if databaseRequest.Spec.Type == mongodbType { logger.Info("Get mongodb database info") @@ -202,12 +205,12 @@ func (r *DatabaseRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ serviceChanged, err := r.handleService(ctx, dbInfo, databaseRequest) if err != nil { - return r.handleError(ctx, databaseRequest, "handle-service", err) + return r.handleError(ctx, databaseRequest, "handle-service", err, false) } secretChanged, err := r.handleSecret(ctx, dbInfo, databaseRequest) if err != nil { - return r.handleError(ctx, databaseRequest, "handle-secret", err) + return r.handleError(ctx, databaseRequest, "handle-secret", err, false) } promDatabaseRequestReconcileStatus.With(promLabels(databaseRequest, "")).Set(1) @@ -229,7 +232,7 @@ func (r *DatabaseRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ Message: "The database request has been changed", }) { if err := r.Status().Update(ctx, databaseRequest); err != nil { - return r.handleError(ctx, databaseRequest, "update-status", err) + return r.handleError(ctx, databaseRequest, "update-status", err, false) } } r.Recorder.Event(databaseRequest, "Normal", "DatabaseRequestUpdated", "The database request has been updated") @@ -242,7 +245,7 @@ func (r *DatabaseRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ Message: "The database request has been created", }) { if err := r.Status().Update(ctx, databaseRequest); err != nil { - return r.handleError(ctx, databaseRequest, "update-status", err) + return r.handleError(ctx, databaseRequest, "update-status", err, false) } } r.Recorder.Event(databaseRequest, "Normal", "DatabaseRequestUnchanged", "The database request has been created") @@ -256,6 +259,7 @@ func (r *DatabaseRequestReconciler) handleError( databaseRequest *crdv1alpha1.DatabaseRequest, promErr string, err error, + ignoreError bool, ) (ctrl.Result, error) { promDatabaseRequestReconcileErrorCounter.With( promLabels(databaseRequest, promErr)).Inc() @@ -285,7 +289,20 @@ func (r *DatabaseRequestReconciler) handleError( log.FromContext(ctx).Error(err, "Failed to update status") } + if ignoreError { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + +} + +func errIsInvalidCredentials(err error) bool { + return strings.Contains(err.Error(), "invalid credentials") +} + +func errIsDatabaseDoesNotExist(err error) bool { + return strings.Contains(err.Error(), "database does not exist") } func (r *DatabaseRequestReconciler) handleSeed( @@ -442,7 +459,7 @@ func (r *DatabaseRequestReconciler) deleteDatabase( // Implementing additional users would require to extend the logic here logger.Info("Dropping relational database") if err := r.relDBDeletion(ctx, databaseRequest); err != nil { - return r.handleError(ctx, databaseRequest, fmt.Sprintf("%s-drop", databaseRequest.Spec.Type), err) + return r.handleError(ctx, databaseRequest, fmt.Sprintf("%s-drop", databaseRequest.Spec.Type), err, false) } } else if databaseRequest.Spec.Type == mongodbType { // handle mongodb deletion @@ -450,7 +467,7 @@ func (r *DatabaseRequestReconciler) deleteDatabase( } else { // this should never happen, but just in case logger.Error(ErrInvalidDatabaseType, "Unsupported database type", "type", databaseRequest.Spec.Type) - return r.handleError(ctx, databaseRequest, "invalid-database-type", ErrInvalidDatabaseType) + return r.handleError(ctx, databaseRequest, "invalid-database-type", ErrInvalidDatabaseType, false) } } serviceName := databaseRequest.Spec.Name @@ -461,7 +478,7 @@ func (r *DatabaseRequestReconciler) deleteDatabase( }, }); err != nil { if !apierrors.IsNotFound(err) { - return r.handleError(ctx, databaseRequest, "delete-service", err) + return r.handleError(ctx, databaseRequest, "delete-service", err, false) } } if err := r.Delete(ctx, &v1.Secret{ @@ -471,12 +488,12 @@ func (r *DatabaseRequestReconciler) deleteDatabase( }, }); err != nil { if !apierrors.IsNotFound(err) { - return r.handleError(ctx, databaseRequest, "delete-secret", err) + return r.handleError(ctx, databaseRequest, "delete-secret", err, false) } } if controllerutil.RemoveFinalizer(databaseRequest, databaseRequestFinalizer) { if err := r.Update(ctx, databaseRequest); err != nil { - return r.handleError(ctx, databaseRequest, "remove-finalizer", err) + return r.handleError(ctx, databaseRequest, "remove-finalizer", err, false) } } // record the event From e7233f3d0e623132ad9ff6268c5df90d856eb054 Mon Sep 17 00:00:00 2001 From: Marco Cadetg Date: Thu, 6 Jun 2024 10:24:52 +0200 Subject: [PATCH 10/11] use syntax if err := ... --- internal/database/database.go | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/internal/database/database.go b/internal/database/database.go index 331fccd..5c9f7b0 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -307,34 +307,38 @@ func (ri *RelationalDatabaseImpl) CreateDatabase( return info, fmt.Errorf("create database failed to get %s database info: %w", dbType, err) } // Create the database - _, err = db.Exec(fmt.Sprintf("CREATE DATABASE \"%s\"", info.Dbname)) - if pqErr, ok := err.(*pq.Error); !ok || ok && pqErr.Code != "42P04" { - // either the error is not a pq.Error or it is a pq.Error but not a duplicate_database error - // 42P04 is the error code for duplicate_database - return info, fmt.Errorf( - "create %s database error in creating the database `%s`: %w", dbType, info.Dbname, err) + if _, err := db.Exec(fmt.Sprintf("CREATE DATABASE \"%s\"", info.Dbname)); err != nil { + if pqErr, ok := err.(*pq.Error); !ok || ok && pqErr.Code != "42P04" { + // either the error is not a pq.Error or it is a pq.Error but not a duplicate_database error + // 42P04 is the error code for duplicate_database + return info, fmt.Errorf( + "create %s database error in creating the database `%s`: %w", dbType, info.Dbname, err) + } } // Check if user exists and create or update the user var userExists int - err = db.QueryRow(fmt.Sprintf("SELECT 1 FROM pg_roles WHERE rolname='%s'", info.Username)).Scan(&userExists) - if err != nil && err != sql.ErrNoRows { + if err := db.QueryRow( + fmt.Sprintf("SELECT 1 FROM pg_roles WHERE rolname='%s'", info.Username), + ).Scan(&userExists); err != nil && err != sql.ErrNoRows { return info, fmt.Errorf( "create %s database error in check if user exists in database `%s`: %w", dbType, info.Dbname, err) } if userExists == 0 { // Create the user with encrypted password - _, err = db.Exec(fmt.Sprintf("CREATE USER \"%s\" WITH ENCRYPTED PASSWORD '%s'", info.Username, info.Password)) - if err != nil { + if _, err := db.Exec( + fmt.Sprintf("CREATE USER \"%s\" WITH ENCRYPTED PASSWORD '%s'", info.Username, info.Password), + ); err != nil { return info, fmt.Errorf( "create %s database error in create user in database `%s`: %w", dbType, info.Dbname, err) } } // Grant privileges - _, err = db.Exec(fmt.Sprintf("GRANT ALL PRIVILEGES ON DATABASE \"%s\" TO \"%s\"", info.Dbname, info.Username)) - if err != nil { + if _, err := db.Exec( + fmt.Sprintf("GRANT ALL PRIVILEGES ON DATABASE \"%s\" TO \"%s\"", info.Dbname, info.Username), + ); err != nil { return info, fmt.Errorf( "create %s database error in grant privileges in database `%s`: %w", dbType, info.Dbname, err) } From 72021085c7fe672e9fc0aefad11fa2d5492d4bc0 Mon Sep 17 00:00:00 2001 From: Marco Cadetg Date: Thu, 6 Jun 2024 10:25:19 +0200 Subject: [PATCH 11/11] add sleep to fix random test failures --- test/e2e/e2e_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index c93ec00..402e392 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -211,6 +211,11 @@ var _ = Describe("controller", Ordered, func() { cmd = exec.Command("kubectl", "apply", "-f", "test/e2e/testdata/mysql-client-pod.yaml") _, err = utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + // We need to wait here a bit because we changed the code to not retry on specific failures in the seed. + // As we can see this is problematic... before it would just have automatically retried and worked after the mysql + // pod was fully up. Now we need to sleep some arbitrary time to make sure the seed database is up + time.Sleep(10 * time.Second) } By("creating a DatabaseRequest resource")