devdb: change readinessProbe to startupProbe (#220)

ariga · Oct 23, 2024 · 0df58f9 · 0df58f9
1 parent 5c3e5f3
commit 0df58f9
Show file tree

Hide file tree

Showing 14 changed files with 155 additions and 156 deletions.
diff --git a/Makefile b/Makefile
@@ -81,7 +81,7 @@ test: manifests generate fmt vet envtest ## Run tests.
 .PHONY: test-e2e
 SKAFFOLD_PROFILE ?= kustomize
 test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
-	SKAFFOLD_PROFILE=${SKAFFOLD_PROFILE} go test ./test/e2e/ -v -run="^TestOperator/${TEST_RUN}"
+	SKAFFOLD_PROFILE=${SKAFFOLD_PROFILE} go test -timeout 20m -v ./test/e2e/ -run="^TestOperator/${TEST_RUN}"
 
 .PHONY: kind-image
 kind-image:

diff --git a/internal/controller/devdb.go b/internal/controller/devdb.go
@@ -192,10 +192,9 @@ func deploymentDevDB(key types.NamespacedName, targetURL url.URL) (*appsv1.Deplo
 	)
 	c := corev1.Container{
 		Name: drv.String(),
-		ReadinessProbe: &corev1.Probe{
-			InitialDelaySeconds: 10,
-			PeriodSeconds:       5,
-			TimeoutSeconds:      5,
+		StartupProbe: &corev1.Probe{
+			FailureThreshold: 30,
+			PeriodSeconds:    10,
 		},
 		SecurityContext: &corev1.SecurityContext{
 			RunAsNonRoot:             ptr.To(true),
@@ -215,7 +214,7 @@ func deploymentDevDB(key types.NamespacedName, targetURL url.URL) (*appsv1.Deplo
 		c.Ports = []corev1.ContainerPort{
 			{Name: drv.String(), ContainerPort: 5432},
 		}
-		c.ReadinessProbe.Exec = &corev1.ExecAction{
+		c.StartupProbe.Exec = &corev1.ExecAction{
 			Command: []string{"pg_isready"},
 		}
 		c.Env = []corev1.EnvVar{
@@ -236,7 +235,7 @@ func deploymentDevDB(key types.NamespacedName, targetURL url.URL) (*appsv1.Deplo
 		c.Ports = []corev1.ContainerPort{
 			{Name: drv.String(), ContainerPort: 1433},
 		}
-		c.ReadinessProbe.Exec = &corev1.ExecAction{
+		c.StartupProbe.Exec = &corev1.ExecAction{
 			Command: []string{
 				"/opt/mssql-tools18/bin/sqlcmd",
 				"-C", "-Q", "SELECT 1",
@@ -266,7 +265,7 @@ func deploymentDevDB(key types.NamespacedName, targetURL url.URL) (*appsv1.Deplo
 		c.Ports = []corev1.ContainerPort{
 			{Name: drv.String(), ContainerPort: 3306},
 		}
-		c.ReadinessProbe.Exec = &corev1.ExecAction{
+		c.StartupProbe.Exec = &corev1.ExecAction{
 			Command: []string{
 				"mysql",
 				"-h", "127.0.0.1",
@@ -292,7 +291,7 @@ func deploymentDevDB(key types.NamespacedName, targetURL url.URL) (*appsv1.Deplo
 		c.Ports = []corev1.ContainerPort{
 			{Name: drv.String(), ContainerPort: 3306},
 		}
-		c.ReadinessProbe.Exec = &corev1.ExecAction{
+		c.StartupProbe.Exec = &corev1.ExecAction{
 			Command: []string{
 				"mariadb",
 				"-h", "127.0.0.1",
@@ -318,7 +317,7 @@ func deploymentDevDB(key types.NamespacedName, targetURL url.URL) (*appsv1.Deplo
 		c.Ports = []corev1.ContainerPort{
 			{Name: drv.String(), ContainerPort: 9000},
 		}
-		c.ReadinessProbe.Exec = &corev1.ExecAction{
+		c.StartupProbe.Exec = &corev1.ExecAction{
 			Command: []string{
 				"clickhouse-client", "-q", "SELECT 1",
 			},

diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
@@ -131,6 +131,24 @@ func TestOperator(t *testing.T) {
 					ts.Fatalf("unexpected success")
 				}
 			},
+			// kubectl-wait-ready runs kubectl wait for the given resource to be ready
+			"kubectl-wait-ready": func(ts *testscript.TestScript, neg bool, args []string) {
+				if len(args) == 0 {
+					ts.Fatalf("usage: kubectl-wait-ready <resource> <name>")
+				}
+				err := ts.Exec("kubectl", append([]string{"-n", ts.Getenv("NAMESPACE"),
+					"wait", "--for=condition=ready",
+					// We need a timeout of 10m because we run the test in parallel
+					// and the controller-manager is not able to handle all the requests
+					// at the same time
+					"--timeout=10m",
+				}, args...)...)
+				if !neg {
+					ts.Check(err)
+				} else if err == nil {
+					ts.Fatalf("unexpected success")
+				}
+			},
 			// envfile read the file and using its content as environment variables
 			"envfile": func(ts *testscript.TestScript, neg bool, args []string) {
 				if neg {

diff --git a/test/e2e/testscript/migration-mysql.txtar b/test/e2e/testscript/migration-mysql.txtar
@@ -1,15 +1,15 @@
 env DB_URL=mysql://root:pass@mysql.${NAMESPACE}:3306/myapp
 kubectl apply -f database.yaml
-kubectl create secret generic mysql-credentials --from-literal=url=${DB_URL}
+kubectl create secret generic db-creds --from-literal=url=${DB_URL}
 # Wait for the DB ready before creating the schema
-kubectl wait --for=condition=ready --timeout=60s -l app=mysql pods
+kubectl-wait-ready -l app=mysql pods
 
 # Create configmap to store the migrations directory
 kubectl create configmap migration-dir --from-file=migrations-v1
 
 # Create the resource
 kubectl apply -f migration.yaml
-kubectl wait --for=condition=ready --timeout=120s AtlasMigration/sample
+kubectl-wait-ready AtlasMigration/mysql
 
 # Inspect the schema to ensure it's correct
 atlas schema inspect -u ${DB_URL} --exclude=atlas_schema_revisions
@@ -20,8 +20,8 @@ kubectl create configmap migration-dir --from-file=migrations-v2 --dry-run=clien
 stdin stdout
 kubectl apply -f -
 # Ensure the controller is aware of the change
-kubectl wait --for=condition=ready=false --timeout=60s AtlasMigration/sample
-kubectl wait --for=condition=ready --timeout=120s AtlasMigration/sample
+kubectl wait --for=condition=ready=false --timeout=120s AtlasMigration/mysql
+kubectl-wait-ready AtlasMigration/mysql
 
 # Inspect the schema to ensure it's correct
 atlas schema inspect -u ${DB_URL} --exclude=atlas_schema_revisions
@@ -32,10 +32,10 @@ kubectl create configmap migration-dir --from-file=migrations-v1 --dry-run=clien
 stdin stdout
 kubectl apply -f -
 # Expect migration is failured
-kubectl wait --timeout=120s --for=jsonpath='{.status.conditions[*].message}'='"Migrate down is not allowed"' AtlasMigration/sample
+kubectl wait --timeout=120s --for=jsonpath='{.status.conditions[*].message}'='"Migrate down is not allowed"' AtlasMigration/mysql
 # Patch the migration to allow down migration
-kubectl patch AtlasMigration/sample --type merge --patch-file ./migration-patch-down.yaml
-kubectl wait --timeout=120s --for=condition=ready AtlasMigration/sample
+kubectl patch AtlasMigration/mysql --type merge --patch-file ./migration-patch-down.yaml
+kubectl wait --timeout=120s --for=condition=ready AtlasMigration/mysql
 
 # Inspect the schema to ensure it's correct after down migration
 atlas schema inspect -u ${DB_URL} --exclude=atlas_schema_revisions
@@ -206,15 +206,15 @@ ALTER TABLE posts ADD COLUMN created_at datetime NOT NULL DEFAULT CURRENT_TIMEST
 apiVersion: db.atlasgo.io/v1alpha1
 kind: AtlasMigration
 metadata:
-  name: sample
+  name: mysql
 spec:
   dir:
     configMapRef:
       name: "migration-dir"
   urlFrom:
     secretKeyRef:
+      name: db-creds
       key: url
-      name: mysql-credentials
 -- migration-patch-down.yaml --
 apiVersion: db.atlasgo.io/v1alpha1
 kind: AtlasMigration
@@ -262,9 +262,8 @@ spec:
           ports:
             - containerPort: 3306
               name: mysql
-          readinessProbe:
-            initialDelaySeconds: 5
-            periodSeconds: 2
-            timeoutSeconds: 1
+          startupProbe:
             exec:
               command: [ "mysql", "-ppass", "-h", "127.0.0.1", "-e", "SELECT 1" ]
+            failureThreshold: 30
+            periodSeconds: 10
diff --git a/test/e2e/testscript/schema-clickhouse.txtar b/test/e2e/testscript/schema-clickhouse.txtar
@@ -1,8 +1,8 @@
 env DB_URL=clickhouse://root:pass@clickhouse.${NAMESPACE}:9000/myapp
 kubectl apply -f database.yaml
-kubectl create secret generic clickhouse-credentials --from-literal=url=${DB_URL}
+kubectl create secret generic db-creds --from-literal=url=${DB_URL}
 # Wait for the DB ready before creating the schema
-kubectl wait --for=condition=ready --timeout=60s -l app=clickhouse pods
+kubectl-wait-ready -l app=clickhouse pods
 
 # Create the secret to store ATLAS_TOKEN
 kubectl create secret generic atlas-token --from-literal=ATLAS_TOKEN=${ATLAS_TOKEN}
@@ -14,7 +14,7 @@ kubectl apply -f -
 
 # Create the schema
 kubectl apply -f schema.yaml
-kubectl wait --for=condition=ready --timeout=120s AtlasSchema/clickhouse
+kubectl-wait-ready AtlasSchema/clickhouse
 
 # Inspect the schema to ensure it's correct
 atlas schema inspect -u ${DB_URL}
@@ -65,8 +65,8 @@ metadata:
 spec:
   urlFrom:
     secretKeyRef:
+      name: db-creds
       key: url
-      name: clickhouse-credentials
   policy:
     lint:
       destructive:
@@ -126,9 +126,8 @@ spec:
           ports:
             - containerPort: 9000
               name: clickhouse
-          readinessProbe:
-            initialDelaySeconds: 5
-            periodSeconds: 2
-            timeoutSeconds: 1
+          startupProbe:
             exec:
               command: [ "clickhouse-client", "-q", "SELECT 1" ]
+            failureThreshold: 30
+            periodSeconds: 10
diff --git a/test/e2e/testscript/schema-manual-changes.txtar b/test/e2e/testscript/schema-manual-changes.txtar
@@ -1,8 +1,8 @@
 env DB_URL=postgres://root:pass@postgres.${NAMESPACE}:5432/postgres?sslmode=disable
 kubectl apply -f database.yaml
-kubectl create secret generic postgres-credentials --from-literal=url=${DB_URL}
+kubectl create secret generic db-creds --from-literal=url=${DB_URL}
 # Wait for the DB ready before creating the schema
-kubectl wait --for=condition=ready --timeout=60s -l app=postgres pods
+kubectl-wait-ready -l app=postgres pods
 
 # Create the secret to store ATLAS_TOKEN
 kubectl create secret generic atlas-token --from-literal=ATLAS_TOKEN=${ATLAS_TOKEN}
@@ -16,7 +16,7 @@ plans-rm stdout
 
 # Create the schema
 kubectl apply -f schema.yaml
-kubectl wait --for=condition=ready --timeout=120s AtlasSchema/postgres
+kubectl-wait-ready AtlasSchema/postgres
 
 # Inspect the schema to ensure it's correct
 atlas schema inspect -u ${DB_URL}
@@ -27,15 +27,15 @@ atlas schema apply --auto-approve --url=${DB_URL} --dev-url=${DEV_URL} --to=file
 
 kubectl patch -f schema.yaml --type merge --patch-file patch-comment.yaml
 # Ensure the controller is aware of the change
-kubectl wait --for=jsonpath='{.status.conditions[*].reason}'=ApprovalPending --timeout=60s AtlasSchemas/postgres
+kubectl wait --for=jsonpath='{.status.conditions[*].reason}'=ApprovalPending --timeout=120s AtlasSchemas/postgres
 
 # Get the plan URL from resource then approve it
 kubectl get AtlasSchemas/postgres -o go-template --template='{{ .status.planURL }}'
 envfile PLAN_URL=stdout
 atlas schema plan approve --url=${PLAN_URL}
 
 # The schema should be updated now
-kubectl wait --for=condition=ready --timeout=120s AtlasSchemas/postgres
+kubectl-wait-ready AtlasSchemas/postgres
 atlas schema inspect -u ${DB_URL}
 cmp stdout schema-v1.hcl
 
@@ -84,7 +84,7 @@ metadata:
 spec:
   urlFrom:
     secretKeyRef:
-      name: postgres-credentials
+      name: db-creds
       key: url
   cloud:
     repo: atlas-operator
@@ -149,12 +149,11 @@ spec:
           ports:
             - containerPort: 5432
               name: postgres
-          readinessProbe:
-            initialDelaySeconds: 5
-            periodSeconds: 2
-            timeoutSeconds: 1
+          startupProbe:
             exec:
               command: [ "pg_isready" ]
+            failureThreshold: 30
+            periodSeconds: 10
         - name: postgres-dev
           image: postgres:15.4
           securityContext:
@@ -172,9 +171,8 @@ spec:
           ports:
             - containerPort: 5433
               name: postgres-dev
-          readinessProbe:
-            initialDelaySeconds: 5
-            periodSeconds: 2
-            timeoutSeconds: 1
+          startupProbe:
             exec:
               command: [ "pg_isready" ]
+            failureThreshold: 30
+            periodSeconds: 10
diff --git a/test/e2e/testscript/schema-mariadb.txtar b/test/e2e/testscript/schema-mariadb.txtar
@@ -1,8 +1,8 @@
 env DB_URL=mariadb://root:pass@mariadb.${NAMESPACE}:3306/myapp
 kubectl apply -f database.yaml
-kubectl create secret generic mariadb-credentials --from-literal=url=${DB_URL}
+kubectl create secret generic db-creds --from-literal=url=${DB_URL}
 # Wait for the DB ready before creating the schema
-kubectl wait --for=condition=ready --timeout=60s -l app=mariadb pods
+kubectl-wait-ready -l app=mariadb pods
 
 # Sync the $WORK directory to the controller pod
 kubectl cp -n ${CONTROLLER_NS} ${WORK} ${CONTROLLER}:/tmp/${NAMESPACE}/
@@ -17,7 +17,7 @@ kubectl apply -f -
 
 # Create the schema
 kubectl apply -f schema.yaml
-kubectl wait --for=condition=ready --timeout=120s AtlasSchema/atlasschema-mariadb
+kubectl-wait-ready AtlasSchema/mariadb
 
 # Inspect the schema to ensure it's correct
 atlas schema inspect -u ${DB_URL}
@@ -29,8 +29,8 @@ stdin stdout
 kubectl apply -f -
 
 # Ensure the controller is aware of the change
-kubectl wait --for=condition=ready=false --timeout=60s AtlasSchema/atlasschema-mariadb
-kubectl wait --for=condition=ready --timeout=120s AtlasSchema/atlasschema-mariadb
+kubectl wait --for=condition=ready=false --timeout=120s AtlasSchema/mariadb
+kubectl-wait-ready AtlasSchema/mariadb
 
 # Inspect the schema to ensure it's correct
 atlas schema inspect -u ${DB_URL}
@@ -140,12 +140,12 @@ create table myapp.ignore_me (c int);
 apiVersion: db.atlasgo.io/v1alpha1
 kind: AtlasSchema
 metadata:
-  name: atlasschema-mariadb
+  name: mariadb
 spec:
   urlFrom:
     secretKeyRef:
+      name: db-creds
       key: url
-      name: mariadb-credentials
   policy:
     lint:
       destructive:
@@ -201,12 +201,11 @@ spec:
           ports:
             - containerPort: 3306
               name: mariadb
-          readinessProbe:
-            initialDelaySeconds: 5
-            periodSeconds: 2
-            timeoutSeconds: 1
+          startupProbe:
             exec:
               command: [ "mariadb", "-ppass", "-h", "127.0.0.1", "-e", "SELECT 1" ]
+            failureThreshold: 30
+            periodSeconds: 10
         - name: mariadb-dev
           image: mariadb:latest
           env:
@@ -219,9 +218,8 @@ spec:
           ports:
             - containerPort: 3307
               name: mariadb-dev
-          readinessProbe:
-            initialDelaySeconds: 5
-            periodSeconds: 2
-            timeoutSeconds: 1
+          startupProbe:
             exec:
               command: [ "mariadb", "-ppass", "-h", "127.0.0.1", "-e", "SELECT 1" ]
+            failureThreshold: 30
+            periodSeconds: 10