feat: add feature flag for separate weekly-random prune and check (#329)

cmd/template_backups_test.go

@@ -210,6 +210,31 @@ func TestBackupTemplateGeneration(t *testing.T) {
 			emptyDir:     true,
 			want:         "../internal/testdata/node/backup-templates/backup-7",
 		},
+		{
+			name: "test10 - generic backup with random check prune feature flags",
+			args: testdata.GetSeedData(
+				testdata.TestData{
+					ProjectName:     "example-project",
+					EnvironmentName: "main",
+					Branch:          "main",
+					EnvironmentType: "production",
+					LagoonYAML:      "../internal/testdata/node/lagoon.yml",
+					EnvVariables: []lagoon.EnvironmentVariable{
+						{
+							Name:  "LAGOON_FEATURE_FLAG_K8UP_WEEKLY_RANDOM_CHECK",
+							Value: "enabled",
+							Scope: "global",
+						},
+						{
+							Name:  "LAGOON_FEATURE_FLAG_K8UP_WEEKLY_RANDOM_PRUNE",
+							Value: "enabled",
+							Scope: "global",
+						},
+					},
+				}, true),
+			templatePath: "testdata/output",
+			want:         "../internal/testdata/node/backup-templates/backup-8",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/generator/backups.go

@@ -95,44 +95,46 @@ func generateBackupValues(
 
 	buildValues.Backup.BackupSchedule, err = helpers.ConvertCrontab(buildValues.Namespace, newBackupSchedule)
 	if err != nil {
-		return fmt.Errorf("Unable to convert crontab for default backup schedule: %v", err)
+		return fmt.Errorf("unable to convert crontab for default backup schedule: %v", err)
 	}
 
 	// start: get variables from the build pod that may have been added by the controller
 	flagCheckSchedule := helpers.GetEnv("K8UP_WEEKLY_RANDOM_FEATURE_FLAG", defaultCheckSchedule, debug)
-	if flagCheckSchedule == "enabled" {
+	lffCheckSchedule := CheckFeatureFlag("K8UP_WEEKLY_RANDOM_CHECK", mergedVariables, debug)
+	if flagCheckSchedule == "enabled" || lffCheckSchedule == "enabled" {
 		buildValues.Backup.CheckSchedule = "@weekly-random"
 	} else {
-		buildValues.Backup.CheckSchedule, err = helpers.ConvertCrontab(fmt.Sprintf("%s", buildValues.Namespace), defaultCheckSchedule)
+		buildValues.Backup.CheckSchedule, err = helpers.ConvertCrontab(buildValues.Namespace, defaultCheckSchedule)
 		if err != nil {
-			return fmt.Errorf("Unable to convert crontab for default check schedule: %v", err)
+			return fmt.Errorf("unable to convert crontab for default check schedule: %v", err)
 		}
 	}
 	flagPruneSchedule := helpers.GetEnv("K8UP_WEEKLY_RANDOM_FEATURE_FLAG", defaultPruneSchedule, debug)
-	if flagPruneSchedule == "enabled" {
+	lffPruneSchedule := CheckFeatureFlag("K8UP_WEEKLY_RANDOM_PRUNE", mergedVariables, debug)
+	if flagPruneSchedule == "enabled" || lffPruneSchedule == "enabled" {
 		buildValues.Backup.PruneSchedule = "@weekly-random"
 	} else {
-		buildValues.Backup.PruneSchedule, err = helpers.ConvertCrontab(fmt.Sprintf("%s", buildValues.Namespace), defaultPruneSchedule)
+		buildValues.Backup.PruneSchedule, err = helpers.ConvertCrontab(buildValues.Namespace, defaultPruneSchedule)
 		if err != nil {
-			return fmt.Errorf("Unable to convert crontab for default prune schedule: %v", err)
+			return fmt.Errorf("unable to convert crontab for default prune schedule: %v", err)
 		}
 	}
 
 	buildValues.Backup.PruneRetention.Hourly, err = helpers.EGetEnvInt("HOURLY_BACKUP_DEFAULT_RETENTION", hourlyDefaultBackupRetention, debug)
 	if err != nil {
-		return fmt.Errorf("Unable to convert hourly retention provided in the environment variable to integer")
+		return fmt.Errorf("unable to convert hourly retention provided in the environment variable to integer")
 	}
 	buildValues.Backup.PruneRetention.Daily, err = helpers.EGetEnvInt("DAILY_BACKUP_DEFAULT_RETENTION", dailyDefaultBackupRetention, debug)
 	if err != nil {
-		return fmt.Errorf("Unable to convert daily retention provided in the environment variable to integer")
+		return fmt.Errorf("unable to convert daily retention provided in the environment variable to integer")
 	}
 	buildValues.Backup.PruneRetention.Weekly, err = helpers.EGetEnvInt("WEEKLY_BACKUP_DEFAULT_RETENTION", weeklyDefaultBackupRetention, debug)
 	if err != nil {
-		return fmt.Errorf("Unable to convert weekly retention provided in the environment variable to integer")
+		return fmt.Errorf("unable to convert weekly retention provided in the environment variable to integer")
 	}
 	buildValues.Backup.PruneRetention.Monthly, err = helpers.EGetEnvInt("MONTHLY_BACKUP_DEFAULT_RETENTION", monthlyDefaultBackupRetention, debug)
 	if err != nil {
-		return fmt.Errorf("Unable to convert monthly retention provided in the environment variable to integer")
+		return fmt.Errorf("unable to convert monthly retention provided in the environment variable to integer")
 	}
 	// :end
 
@@ -151,7 +153,7 @@ func generateBackupValues(
 	if lYAML.BackupSchedule.Production != "" && buildValues.EnvironmentType == "production" {
 		buildValues.Backup.BackupSchedule, err = helpers.ConvertCrontab(buildValues.Namespace, lYAML.BackupSchedule.Production)
 		if err != nil {
-			return fmt.Errorf("Unable to convert crontab for default backup schedule from .lagoon.yml: %v", err)
+			return fmt.Errorf("unable to convert crontab for default backup schedule from .lagoon.yml: %v", err)
 		}
 	}
 

internal/generator/backups_test.go

@@ -691,6 +691,78 @@ func Test_generateBackupValues(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "test18 - LAGOON_FEATURE_FLAG_K8UP_WEEKLY_RANDOM_CHECK enabled",
+			args: args{
+				buildValues: &BuildValues{
+					BuildType:             "branch",
+					EnvironmentType:       "development",
+					Project:               "example-project",
+					Namespace:             "example-com-main",
+					DefaultBackupSchedule: "M H(22-2) * * *",
+				},
+				lYAML: &lagoon.YAML{},
+				mergedVariables: []lagoon.EnvironmentVariable{
+					{Name: "LAGOON_FEATURE_FLAG_K8UP_WEEKLY_RANDOM_CHECK", Value: "enabled", Scope: "global"},
+				},
+			},
+			vars: []helpers.EnvironmentVariable{},
+			want: &BuildValues{
+				BuildType:             "branch",
+				EnvironmentType:       "development",
+				Project:               "example-project",
+				Namespace:             "example-com-main",
+				DefaultBackupSchedule: "M H(22-2) * * *",
+				Backup: BackupConfiguration{
+					BackupSchedule: "31 1 * * *",
+					CheckSchedule:  "@weekly-random",
+					PruneSchedule:  "31 4 * * 0",
+					S3BucketName:   "baas-example-project",
+					PruneRetention: PruneRetention{
+						Hourly:  0,
+						Daily:   7,
+						Weekly:  6,
+						Monthly: 1,
+					},
+				},
+			},
+		},
+		{
+			name: "test19 - LAGOON_FEATURE_FLAG_K8UP_WEEKLY_RANDOM_PRUNE enabled",
+			args: args{
+				buildValues: &BuildValues{
+					BuildType:             "branch",
+					EnvironmentType:       "development",
+					Project:               "example-project",
+					Namespace:             "example-com-main",
+					DefaultBackupSchedule: "M H(22-2) * * *",
+				},
+				lYAML: &lagoon.YAML{},
+				mergedVariables: []lagoon.EnvironmentVariable{
+					{Name: "LAGOON_FEATURE_FLAG_K8UP_WEEKLY_RANDOM_PRUNE", Value: "enabled", Scope: "global"},
+				},
+			},
+			vars: []helpers.EnvironmentVariable{},
+			want: &BuildValues{
+				BuildType:             "branch",
+				EnvironmentType:       "development",
+				Project:               "example-project",
+				Namespace:             "example-com-main",
+				DefaultBackupSchedule: "M H(22-2) * * *",
+				Backup: BackupConfiguration{
+					BackupSchedule: "31 1 * * *",
+					CheckSchedule:  "31 6 * * 1",
+					PruneSchedule:  "@weekly-random",
+					S3BucketName:   "baas-example-project",
+					PruneRetention: PruneRetention{
+						Hourly:  0,
+						Daily:   7,
+						Weekly:  6,
+						Monthly: 1,
+					},
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/generator/generator.go

@@ -153,16 +153,16 @@ func NewGenerator(
 
 	// break out of the generator if these requirements are missing
 	if projectName == "" || environmentName == "" || environmentType == "" || buildType == "" {
-		return nil, fmt.Errorf("Missing arguments: project-name, environment-name, environment-type, or build-type not defined")
+		return nil, fmt.Errorf("missing arguments: project-name, environment-name, environment-type, or build-type not defined")
 	}
 	switch buildType {
 	case "branch", "promote":
 		if branch == "" {
-			return nil, fmt.Errorf("Missing arguments: branch not defined")
+			return nil, fmt.Errorf("missing arguments: branch not defined")
 		}
 	case "pullrequest":
 		if prNumber == "" || prHeadBranch == "" || prBaseBranch == "" {
-			return nil, fmt.Errorf("Missing arguments: pullrequest-number, pullrequest-head-branch, or pullrequest-base-branch not defined")
+			return nil, fmt.Errorf("missing arguments: pullrequest-number, pullrequest-head-branch, or pullrequest-base-branch not defined")
 		}
 	}
 
@@ -343,18 +343,18 @@ func LoadAndUnmarshalLagoonYml(lagoonYml string, lagoonYmlOverride string, lagoo
 		//Decode it
 		envLagoonYamlString, err := base64.StdEncoding.DecodeString(envLagoonYamlStringBase64)
 		if err != nil {
-			return fmt.Errorf("Unable to decode %v - is it base64 encoded?", lagoonYmlOverrideEnvVarName)
+			return fmt.Errorf("unable to decode %v - is it base64 encoded?", lagoonYmlOverrideEnvVarName)
 		}
 		envLagoonYaml := &lagoon.YAML{}
 		lEnvLagoonPolysite := make(map[string]interface{})
 
 		err = yaml.Unmarshal(envLagoonYamlString, envLagoonYaml)
 		if err != nil {
-			return fmt.Errorf("Unable to unmarshal env var %v: %v", lagoonYmlOverrideEnvVarName, err)
+			return fmt.Errorf("unable to unmarshal env var %v: %v", lagoonYmlOverrideEnvVarName, err)
 		}
 		err = yaml.Unmarshal(envLagoonYamlString, lEnvLagoonPolysite)
 		if err != nil {
-			return fmt.Errorf("Unable to unmarshal env var %v: %v", lagoonYmlOverrideEnvVarName, err)
+			return fmt.Errorf("unable to unmarshal env var %v: %v", lagoonYmlOverrideEnvVarName, err)
 		}
 
 		if _, ok := lEnvLagoonPolysite[projectName]; ok {
@@ -404,23 +404,23 @@ func CheckFeatureFlag(key string, envVariables []lagoon.EnvironmentVariable, deb
 	// check for force value
 	if value, ok := os.LookupEnv(fmt.Sprintf("LAGOON_FEATURE_FLAG_FORCE_%s", key)); ok {
 		if debug {
-			fmt.Println(fmt.Sprintf("Using forced flag value from build variable %s", fmt.Sprintf("LAGOON_FEATURE_FLAG_FORCE_%s", key)))
+			fmt.Printf("Using forced flag value from build variable %s\n", fmt.Sprintf("LAGOON_FEATURE_FLAG_FORCE_%s", key))
 		}
 		return value
 	}
 	// check lagoon environment variables
 	for _, lVar := range envVariables {
 		if strings.Contains(lVar.Name, fmt.Sprintf("LAGOON_FEATURE_FLAG_%s", key)) {
 			if debug {
-				fmt.Println(fmt.Sprintf("Using flag value from Lagoon environment variable %s", fmt.Sprintf("LAGOON_FEATURE_FLAG_%s", key)))
+				fmt.Printf("Using flag value from Lagoon environment variable %s\n", fmt.Sprintf("LAGOON_FEATURE_FLAG_%s", key))
 			}
 			return lVar.Value
 		}
 	}
 	// return default
 	if value, ok := os.LookupEnv(fmt.Sprintf("LAGOON_FEATURE_FLAG_DEFAULT_%s", key)); ok {
 		if debug {
-			fmt.Println(fmt.Sprintf("Using default flag value from build variable %s", fmt.Sprintf("LAGOON_FEATURE_FLAG_DEFAULT_%s", key)))
+			fmt.Printf("Using default flag value from build variable %s\n", fmt.Sprintf("LAGOON_FEATURE_FLAG_DEFAULT_%s", key))
 		}
 		return value
 	}

internal/generator/ingress.go

@@ -251,7 +251,7 @@ func generateActiveStandbyRoutes(
 ) (lagoon.RoutesV2, error) {
 	activeStanbyRoutes := &lagoon.RoutesV2{}
 	if lagoonYAML.ProductionRoutes != nil {
-		if buildValues.IsActiveEnvironment == true {
+		if buildValues.IsActiveEnvironment {
 			if lagoonYAML.ProductionRoutes.Active != nil {
 				if lagoonYAML.ProductionRoutes.Active.Routes != nil {
 					for _, routeMap := range lagoonYAML.ProductionRoutes.Active.Routes {
@@ -263,7 +263,7 @@ func generateActiveStandbyRoutes(
 				}
 			}
 		}
-		if buildValues.IsStandbyEnvironment == true {
+		if buildValues.IsStandbyEnvironment {
 			if lagoonYAML.ProductionRoutes.Standby != nil {
 				if lagoonYAML.ProductionRoutes.Standby.Routes != nil {
 					for _, routeMap := range lagoonYAML.ProductionRoutes.Standby.Routes {

internal/generator/services.go

@@ -160,36 +160,36 @@ func composeToServiceValues(
 		lagoonType = lagoon.CheckServiceLagoonLabel(composeServiceValues.Labels, "lagoon.type")
 	}
 	if lagoonType == "" {
-		return ServiceValues{}, fmt.Errorf("No lagoon.type has been set for service %s. If a Lagoon service is not required, please set the lagoon.type to 'none' for this service in docker-compose.yaml. See the Lagoon documentation for supported service types.", composeService)
+		return ServiceValues{}, fmt.Errorf("no lagoon.type has been set for service %s. If a Lagoon service is not required, please set the lagoon.type to 'none' for this service in docker-compose.yaml. See the Lagoon documentation for supported service types", composeService)
 	} else {
 		// if the lagoontype is populated, even none is valid as there may be a servicetype override in an environment variable
 		autogenEnabled := true
 		autogenTLSAcmeEnabled := true
 		// check if autogenerated routes are disabled
 		if lYAML.Routes.Autogenerate.Enabled != nil {
-			if *lYAML.Routes.Autogenerate.Enabled == false {
+			if !*lYAML.Routes.Autogenerate.Enabled {
 				autogenEnabled = false
 			}
 		}
 		// check if pullrequests autogenerated routes are disabled
 		if buildValues.BuildType == "pullrequest" && lYAML.Routes.Autogenerate.AllowPullRequests != nil {
-			if *lYAML.Routes.Autogenerate.AllowPullRequests == false {
+			if !*lYAML.Routes.Autogenerate.AllowPullRequests {
 				autogenEnabled = false
 			} else {
 				autogenEnabled = true
 			}
 		}
 		// check if this environment has autogenerated routes disabled
 		if lYAML.Environments[buildValues.Branch].AutogenerateRoutes != nil {
-			if *lYAML.Environments[buildValues.Branch].AutogenerateRoutes == false {
+			if !*lYAML.Environments[buildValues.Branch].AutogenerateRoutes {
 				autogenEnabled = false
 			} else {
 				autogenEnabled = true
 			}
 		}
 		// check if autogenerated routes tls-acme disabled
 		if lYAML.Routes.Autogenerate.TLSAcme != nil {
-			if *lYAML.Routes.Autogenerate.TLSAcme == false {
+			if !*lYAML.Routes.Autogenerate.TLSAcme {
 				autogenTLSAcmeEnabled = false
 			}
 		}
@@ -297,7 +297,7 @@ func composeToServiceValues(
 				// 	return ServiceValues{}, fmt.Errorf("Unable to check the DBaaS endpoint %s: %v", buildValues.DBaaSOperatorEndpoint, err)
 				// }
 				if debug {
-					fmt.Println(fmt.Sprintf("Unable to check the DBaaS endpoint %s, falling back to %s-single: %v", buildValues.DBaaSOperatorEndpoint, lagoonType, err))
+					fmt.Printf("Unable to check the DBaaS endpoint %s, falling back to %s-single: %v\n", buildValues.DBaaSOperatorEndpoint, lagoonType, err)
 				}
 				// normally we would fall back to doing a cluster capability check, this is phased out in the build tool, it isn't reliable
 				// and noone should be doing checks that way any more
@@ -326,10 +326,10 @@ func composeToServiceValues(
 					// 	return ServiceValues{}, err
 					// }
 					if debug {
-						fmt.Println(fmt.Sprintf(
-							"There was an error checking DBaaS endpoint %s, falling back to %s-single: %v",
+						fmt.Printf(
+							"There was an error checking DBaaS endpoint %s, falling back to %s-single: %v\n",
 							buildValues.DBaaSOperatorEndpoint, lagoonType, err,
-						))
+						)
 					}
 				}
 
@@ -379,7 +379,7 @@ func composeToServiceValues(
 			sPort, err := strconv.Atoi(servicePortOverride)
 			if err != nil {
 				return ServiceValues{}, fmt.Errorf(
-					"The provided service port %s for service %s is not a valid integer: %v",
+					"the provided service port %s for service %s is not a valid integer: %v",
 					servicePortOverride, composeService, err,
 				)
 			}
@@ -408,7 +408,7 @@ func getDBaasEnvironment(
 	}
 	exists, err := buildValues.DBaaSClient.CheckProvider(buildValues.DBaaSOperatorEndpoint, lagoonType, *dbaasEnvironment)
 	if err != nil {
-		return exists, fmt.Errorf("There was an error checking DBaaS endpoint %s: %v", buildValues.DBaaSOperatorEndpoint, err)
+		return exists, fmt.Errorf("there was an error checking DBaaS endpoint %s: %v", buildValues.DBaaSOperatorEndpoint, err)
 	}
 	return exists, nil
 }

internal/testdata/node/backup-templates/backup-8/k8up-lagoon-backup-schedule.yaml

@@ -0,0 +1,42 @@
+---
+apiVersion: backup.appuio.ch/v1alpha1
+kind: Schedule
+metadata:
+  annotations:
+    lagoon.sh/branch: main
+    lagoon.sh/version: v2.7.x
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/instance: k8up-lagoon-backup-schedule
+    app.kubernetes.io/managed-by: build-deploy-tool
+    app.kubernetes.io/name: k8up-schedule
+    lagoon.sh/buildType: branch
+    lagoon.sh/environment: main
+    lagoon.sh/environmentType: production
+    lagoon.sh/project: example-project
+    lagoon.sh/service: k8up-lagoon-backup-schedule
+    lagoon.sh/service-type: k8up-schedule
+    lagoon.sh/template: k8up-schedule-0.1.0
+  name: k8up-lagoon-backup-schedule
+spec:
+  backend:
+    repoPasswordSecretRef:
+      key: repo-pw
+      name: baas-repo-pw
+    s3:
+      bucket: baas-example-project
+  backup:
+    resources: {}
+    schedule: 48 22 * * *
+  check:
+    resources: {}
+    schedule: '@weekly-random'
+  prune:
+    resources: {}
+    retention:
+      keepDaily: 7
+      keepMonthly: 1
+      keepWeekly: 6
+    schedule: '@weekly-random'
+  resourceRequirementsTemplate: {}
+status: {}