From 02d1bbaf4a8379c51f31fcfe1d880b06ecc0053e Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Wed, 3 Jul 2024 19:13:23 +1000 Subject: [PATCH] feat: add feature flag for separate weekly-random prune and check (#329) --- cmd/template_backups_test.go | 25 +++++++ internal/generator/backups.go | 26 +++---- internal/generator/backups_test.go | 72 +++++++++++++++++++ internal/generator/generator.go | 18 ++--- internal/generator/ingress.go | 4 +- internal/generator/services.go | 22 +++--- .../backup-8/k8up-lagoon-backup-schedule.yaml | 42 +++++++++++ 7 files changed, 175 insertions(+), 34 deletions(-) create mode 100644 internal/testdata/node/backup-templates/backup-8/k8up-lagoon-backup-schedule.yaml diff --git a/cmd/template_backups_test.go b/cmd/template_backups_test.go index b738b17d..a45f95ac 100644 --- a/cmd/template_backups_test.go +++ b/cmd/template_backups_test.go @@ -210,6 +210,31 @@ func TestBackupTemplateGeneration(t *testing.T) { emptyDir: true, want: "../internal/testdata/node/backup-templates/backup-7", }, + { + name: "test10 - generic backup with random check prune feature flags", + args: testdata.GetSeedData( + testdata.TestData{ + ProjectName: "example-project", + EnvironmentName: "main", + Branch: "main", + EnvironmentType: "production", + LagoonYAML: "../internal/testdata/node/lagoon.yml", + EnvVariables: []lagoon.EnvironmentVariable{ + { + Name: "LAGOON_FEATURE_FLAG_K8UP_WEEKLY_RANDOM_CHECK", + Value: "enabled", + Scope: "global", + }, + { + Name: "LAGOON_FEATURE_FLAG_K8UP_WEEKLY_RANDOM_PRUNE", + Value: "enabled", + Scope: "global", + }, + }, + }, true), + templatePath: "testdata/output", + want: "../internal/testdata/node/backup-templates/backup-8", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/generator/backups.go b/internal/generator/backups.go index 077ba7df..591ae67b 100644 --- a/internal/generator/backups.go +++ b/internal/generator/backups.go @@ -95,44 +95,46 @@ func generateBackupValues( buildValues.Backup.BackupSchedule, err = helpers.ConvertCrontab(buildValues.Namespace, newBackupSchedule) if err != nil { - return fmt.Errorf("Unable to convert crontab for default backup schedule: %v", err) + return fmt.Errorf("unable to convert crontab for default backup schedule: %v", err) } // start: get variables from the build pod that may have been added by the controller flagCheckSchedule := helpers.GetEnv("K8UP_WEEKLY_RANDOM_FEATURE_FLAG", defaultCheckSchedule, debug) - if flagCheckSchedule == "enabled" { + lffCheckSchedule := CheckFeatureFlag("K8UP_WEEKLY_RANDOM_CHECK", mergedVariables, debug) + if flagCheckSchedule == "enabled" || lffCheckSchedule == "enabled" { buildValues.Backup.CheckSchedule = "@weekly-random" } else { - buildValues.Backup.CheckSchedule, err = helpers.ConvertCrontab(fmt.Sprintf("%s", buildValues.Namespace), defaultCheckSchedule) + buildValues.Backup.CheckSchedule, err = helpers.ConvertCrontab(buildValues.Namespace, defaultCheckSchedule) if err != nil { - return fmt.Errorf("Unable to convert crontab for default check schedule: %v", err) + return fmt.Errorf("unable to convert crontab for default check schedule: %v", err) } } flagPruneSchedule := helpers.GetEnv("K8UP_WEEKLY_RANDOM_FEATURE_FLAG", defaultPruneSchedule, debug) - if flagPruneSchedule == "enabled" { + lffPruneSchedule := CheckFeatureFlag("K8UP_WEEKLY_RANDOM_PRUNE", mergedVariables, debug) + if flagPruneSchedule == "enabled" || lffPruneSchedule == "enabled" { buildValues.Backup.PruneSchedule = "@weekly-random" } else { - buildValues.Backup.PruneSchedule, err = helpers.ConvertCrontab(fmt.Sprintf("%s", buildValues.Namespace), defaultPruneSchedule) + buildValues.Backup.PruneSchedule, err = helpers.ConvertCrontab(buildValues.Namespace, defaultPruneSchedule) if err != nil { - return fmt.Errorf("Unable to convert crontab for default prune schedule: %v", err) + return fmt.Errorf("unable to convert crontab for default prune schedule: %v", err) } } buildValues.Backup.PruneRetention.Hourly, err = helpers.EGetEnvInt("HOURLY_BACKUP_DEFAULT_RETENTION", hourlyDefaultBackupRetention, debug) if err != nil { - return fmt.Errorf("Unable to convert hourly retention provided in the environment variable to integer") + return fmt.Errorf("unable to convert hourly retention provided in the environment variable to integer") } buildValues.Backup.PruneRetention.Daily, err = helpers.EGetEnvInt("DAILY_BACKUP_DEFAULT_RETENTION", dailyDefaultBackupRetention, debug) if err != nil { - return fmt.Errorf("Unable to convert daily retention provided in the environment variable to integer") + return fmt.Errorf("unable to convert daily retention provided in the environment variable to integer") } buildValues.Backup.PruneRetention.Weekly, err = helpers.EGetEnvInt("WEEKLY_BACKUP_DEFAULT_RETENTION", weeklyDefaultBackupRetention, debug) if err != nil { - return fmt.Errorf("Unable to convert weekly retention provided in the environment variable to integer") + return fmt.Errorf("unable to convert weekly retention provided in the environment variable to integer") } buildValues.Backup.PruneRetention.Monthly, err = helpers.EGetEnvInt("MONTHLY_BACKUP_DEFAULT_RETENTION", monthlyDefaultBackupRetention, debug) if err != nil { - return fmt.Errorf("Unable to convert monthly retention provided in the environment variable to integer") + return fmt.Errorf("unable to convert monthly retention provided in the environment variable to integer") } // :end @@ -151,7 +153,7 @@ func generateBackupValues( if lYAML.BackupSchedule.Production != "" && buildValues.EnvironmentType == "production" { buildValues.Backup.BackupSchedule, err = helpers.ConvertCrontab(buildValues.Namespace, lYAML.BackupSchedule.Production) if err != nil { - return fmt.Errorf("Unable to convert crontab for default backup schedule from .lagoon.yml: %v", err) + return fmt.Errorf("unable to convert crontab for default backup schedule from .lagoon.yml: %v", err) } } diff --git a/internal/generator/backups_test.go b/internal/generator/backups_test.go index 956b993d..357faca3 100644 --- a/internal/generator/backups_test.go +++ b/internal/generator/backups_test.go @@ -691,6 +691,78 @@ func Test_generateBackupValues(t *testing.T) { }, }, }, + { + name: "test18 - LAGOON_FEATURE_FLAG_K8UP_WEEKLY_RANDOM_CHECK enabled", + args: args{ + buildValues: &BuildValues{ + BuildType: "branch", + EnvironmentType: "development", + Project: "example-project", + Namespace: "example-com-main", + DefaultBackupSchedule: "M H(22-2) * * *", + }, + lYAML: &lagoon.YAML{}, + mergedVariables: []lagoon.EnvironmentVariable{ + {Name: "LAGOON_FEATURE_FLAG_K8UP_WEEKLY_RANDOM_CHECK", Value: "enabled", Scope: "global"}, + }, + }, + vars: []helpers.EnvironmentVariable{}, + want: &BuildValues{ + BuildType: "branch", + EnvironmentType: "development", + Project: "example-project", + Namespace: "example-com-main", + DefaultBackupSchedule: "M H(22-2) * * *", + Backup: BackupConfiguration{ + BackupSchedule: "31 1 * * *", + CheckSchedule: "@weekly-random", + PruneSchedule: "31 4 * * 0", + S3BucketName: "baas-example-project", + PruneRetention: PruneRetention{ + Hourly: 0, + Daily: 7, + Weekly: 6, + Monthly: 1, + }, + }, + }, + }, + { + name: "test19 - LAGOON_FEATURE_FLAG_K8UP_WEEKLY_RANDOM_PRUNE enabled", + args: args{ + buildValues: &BuildValues{ + BuildType: "branch", + EnvironmentType: "development", + Project: "example-project", + Namespace: "example-com-main", + DefaultBackupSchedule: "M H(22-2) * * *", + }, + lYAML: &lagoon.YAML{}, + mergedVariables: []lagoon.EnvironmentVariable{ + {Name: "LAGOON_FEATURE_FLAG_K8UP_WEEKLY_RANDOM_PRUNE", Value: "enabled", Scope: "global"}, + }, + }, + vars: []helpers.EnvironmentVariable{}, + want: &BuildValues{ + BuildType: "branch", + EnvironmentType: "development", + Project: "example-project", + Namespace: "example-com-main", + DefaultBackupSchedule: "M H(22-2) * * *", + Backup: BackupConfiguration{ + BackupSchedule: "31 1 * * *", + CheckSchedule: "31 6 * * 1", + PruneSchedule: "@weekly-random", + S3BucketName: "baas-example-project", + PruneRetention: PruneRetention{ + Hourly: 0, + Daily: 7, + Weekly: 6, + Monthly: 1, + }, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/generator/generator.go b/internal/generator/generator.go index 709fd4f8..93530513 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -153,16 +153,16 @@ func NewGenerator( // break out of the generator if these requirements are missing if projectName == "" || environmentName == "" || environmentType == "" || buildType == "" { - return nil, fmt.Errorf("Missing arguments: project-name, environment-name, environment-type, or build-type not defined") + return nil, fmt.Errorf("missing arguments: project-name, environment-name, environment-type, or build-type not defined") } switch buildType { case "branch", "promote": if branch == "" { - return nil, fmt.Errorf("Missing arguments: branch not defined") + return nil, fmt.Errorf("missing arguments: branch not defined") } case "pullrequest": if prNumber == "" || prHeadBranch == "" || prBaseBranch == "" { - return nil, fmt.Errorf("Missing arguments: pullrequest-number, pullrequest-head-branch, or pullrequest-base-branch not defined") + return nil, fmt.Errorf("missing arguments: pullrequest-number, pullrequest-head-branch, or pullrequest-base-branch not defined") } } @@ -343,18 +343,18 @@ func LoadAndUnmarshalLagoonYml(lagoonYml string, lagoonYmlOverride string, lagoo //Decode it envLagoonYamlString, err := base64.StdEncoding.DecodeString(envLagoonYamlStringBase64) if err != nil { - return fmt.Errorf("Unable to decode %v - is it base64 encoded?", lagoonYmlOverrideEnvVarName) + return fmt.Errorf("unable to decode %v - is it base64 encoded?", lagoonYmlOverrideEnvVarName) } envLagoonYaml := &lagoon.YAML{} lEnvLagoonPolysite := make(map[string]interface{}) err = yaml.Unmarshal(envLagoonYamlString, envLagoonYaml) if err != nil { - return fmt.Errorf("Unable to unmarshal env var %v: %v", lagoonYmlOverrideEnvVarName, err) + return fmt.Errorf("unable to unmarshal env var %v: %v", lagoonYmlOverrideEnvVarName, err) } err = yaml.Unmarshal(envLagoonYamlString, lEnvLagoonPolysite) if err != nil { - return fmt.Errorf("Unable to unmarshal env var %v: %v", lagoonYmlOverrideEnvVarName, err) + return fmt.Errorf("unable to unmarshal env var %v: %v", lagoonYmlOverrideEnvVarName, err) } if _, ok := lEnvLagoonPolysite[projectName]; ok { @@ -404,7 +404,7 @@ func CheckFeatureFlag(key string, envVariables []lagoon.EnvironmentVariable, deb // check for force value if value, ok := os.LookupEnv(fmt.Sprintf("LAGOON_FEATURE_FLAG_FORCE_%s", key)); ok { if debug { - fmt.Println(fmt.Sprintf("Using forced flag value from build variable %s", fmt.Sprintf("LAGOON_FEATURE_FLAG_FORCE_%s", key))) + fmt.Printf("Using forced flag value from build variable %s\n", fmt.Sprintf("LAGOON_FEATURE_FLAG_FORCE_%s", key)) } return value } @@ -412,7 +412,7 @@ func CheckFeatureFlag(key string, envVariables []lagoon.EnvironmentVariable, deb for _, lVar := range envVariables { if strings.Contains(lVar.Name, fmt.Sprintf("LAGOON_FEATURE_FLAG_%s", key)) { if debug { - fmt.Println(fmt.Sprintf("Using flag value from Lagoon environment variable %s", fmt.Sprintf("LAGOON_FEATURE_FLAG_%s", key))) + fmt.Printf("Using flag value from Lagoon environment variable %s\n", fmt.Sprintf("LAGOON_FEATURE_FLAG_%s", key)) } return lVar.Value } @@ -420,7 +420,7 @@ func CheckFeatureFlag(key string, envVariables []lagoon.EnvironmentVariable, deb // return default if value, ok := os.LookupEnv(fmt.Sprintf("LAGOON_FEATURE_FLAG_DEFAULT_%s", key)); ok { if debug { - fmt.Println(fmt.Sprintf("Using default flag value from build variable %s", fmt.Sprintf("LAGOON_FEATURE_FLAG_DEFAULT_%s", key))) + fmt.Printf("Using default flag value from build variable %s\n", fmt.Sprintf("LAGOON_FEATURE_FLAG_DEFAULT_%s", key)) } return value } diff --git a/internal/generator/ingress.go b/internal/generator/ingress.go index 82247882..f62e17c5 100644 --- a/internal/generator/ingress.go +++ b/internal/generator/ingress.go @@ -251,7 +251,7 @@ func generateActiveStandbyRoutes( ) (lagoon.RoutesV2, error) { activeStanbyRoutes := &lagoon.RoutesV2{} if lagoonYAML.ProductionRoutes != nil { - if buildValues.IsActiveEnvironment == true { + if buildValues.IsActiveEnvironment { if lagoonYAML.ProductionRoutes.Active != nil { if lagoonYAML.ProductionRoutes.Active.Routes != nil { for _, routeMap := range lagoonYAML.ProductionRoutes.Active.Routes { @@ -263,7 +263,7 @@ func generateActiveStandbyRoutes( } } } - if buildValues.IsStandbyEnvironment == true { + if buildValues.IsStandbyEnvironment { if lagoonYAML.ProductionRoutes.Standby != nil { if lagoonYAML.ProductionRoutes.Standby.Routes != nil { for _, routeMap := range lagoonYAML.ProductionRoutes.Standby.Routes { diff --git a/internal/generator/services.go b/internal/generator/services.go index 1b539d80..89bd4a14 100644 --- a/internal/generator/services.go +++ b/internal/generator/services.go @@ -160,20 +160,20 @@ func composeToServiceValues( lagoonType = lagoon.CheckServiceLagoonLabel(composeServiceValues.Labels, "lagoon.type") } if lagoonType == "" { - return ServiceValues{}, fmt.Errorf("No lagoon.type has been set for service %s. If a Lagoon service is not required, please set the lagoon.type to 'none' for this service in docker-compose.yaml. See the Lagoon documentation for supported service types.", composeService) + return ServiceValues{}, fmt.Errorf("no lagoon.type has been set for service %s. If a Lagoon service is not required, please set the lagoon.type to 'none' for this service in docker-compose.yaml. See the Lagoon documentation for supported service types", composeService) } else { // if the lagoontype is populated, even none is valid as there may be a servicetype override in an environment variable autogenEnabled := true autogenTLSAcmeEnabled := true // check if autogenerated routes are disabled if lYAML.Routes.Autogenerate.Enabled != nil { - if *lYAML.Routes.Autogenerate.Enabled == false { + if !*lYAML.Routes.Autogenerate.Enabled { autogenEnabled = false } } // check if pullrequests autogenerated routes are disabled if buildValues.BuildType == "pullrequest" && lYAML.Routes.Autogenerate.AllowPullRequests != nil { - if *lYAML.Routes.Autogenerate.AllowPullRequests == false { + if !*lYAML.Routes.Autogenerate.AllowPullRequests { autogenEnabled = false } else { autogenEnabled = true @@ -181,7 +181,7 @@ func composeToServiceValues( } // check if this environment has autogenerated routes disabled if lYAML.Environments[buildValues.Branch].AutogenerateRoutes != nil { - if *lYAML.Environments[buildValues.Branch].AutogenerateRoutes == false { + if !*lYAML.Environments[buildValues.Branch].AutogenerateRoutes { autogenEnabled = false } else { autogenEnabled = true @@ -189,7 +189,7 @@ func composeToServiceValues( } // check if autogenerated routes tls-acme disabled if lYAML.Routes.Autogenerate.TLSAcme != nil { - if *lYAML.Routes.Autogenerate.TLSAcme == false { + if !*lYAML.Routes.Autogenerate.TLSAcme { autogenTLSAcmeEnabled = false } } @@ -297,7 +297,7 @@ func composeToServiceValues( // return ServiceValues{}, fmt.Errorf("Unable to check the DBaaS endpoint %s: %v", buildValues.DBaaSOperatorEndpoint, err) // } if debug { - fmt.Println(fmt.Sprintf("Unable to check the DBaaS endpoint %s, falling back to %s-single: %v", buildValues.DBaaSOperatorEndpoint, lagoonType, err)) + fmt.Printf("Unable to check the DBaaS endpoint %s, falling back to %s-single: %v\n", buildValues.DBaaSOperatorEndpoint, lagoonType, err) } // normally we would fall back to doing a cluster capability check, this is phased out in the build tool, it isn't reliable // and noone should be doing checks that way any more @@ -326,10 +326,10 @@ func composeToServiceValues( // return ServiceValues{}, err // } if debug { - fmt.Println(fmt.Sprintf( - "There was an error checking DBaaS endpoint %s, falling back to %s-single: %v", + fmt.Printf( + "There was an error checking DBaaS endpoint %s, falling back to %s-single: %v\n", buildValues.DBaaSOperatorEndpoint, lagoonType, err, - )) + ) } } @@ -379,7 +379,7 @@ func composeToServiceValues( sPort, err := strconv.Atoi(servicePortOverride) if err != nil { return ServiceValues{}, fmt.Errorf( - "The provided service port %s for service %s is not a valid integer: %v", + "the provided service port %s for service %s is not a valid integer: %v", servicePortOverride, composeService, err, ) } @@ -408,7 +408,7 @@ func getDBaasEnvironment( } exists, err := buildValues.DBaaSClient.CheckProvider(buildValues.DBaaSOperatorEndpoint, lagoonType, *dbaasEnvironment) if err != nil { - return exists, fmt.Errorf("There was an error checking DBaaS endpoint %s: %v", buildValues.DBaaSOperatorEndpoint, err) + return exists, fmt.Errorf("there was an error checking DBaaS endpoint %s: %v", buildValues.DBaaSOperatorEndpoint, err) } return exists, nil } diff --git a/internal/testdata/node/backup-templates/backup-8/k8up-lagoon-backup-schedule.yaml b/internal/testdata/node/backup-templates/backup-8/k8up-lagoon-backup-schedule.yaml new file mode 100644 index 00000000..65e347dd --- /dev/null +++ b/internal/testdata/node/backup-templates/backup-8/k8up-lagoon-backup-schedule.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: backup.appuio.ch/v1alpha1 +kind: Schedule +metadata: + annotations: + lagoon.sh/branch: main + lagoon.sh/version: v2.7.x + creationTimestamp: null + labels: + app.kubernetes.io/instance: k8up-lagoon-backup-schedule + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: k8up-schedule + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/service: k8up-lagoon-backup-schedule + lagoon.sh/service-type: k8up-schedule + lagoon.sh/template: k8up-schedule-0.1.0 + name: k8up-lagoon-backup-schedule +spec: + backend: + repoPasswordSecretRef: + key: repo-pw + name: baas-repo-pw + s3: + bucket: baas-example-project + backup: + resources: {} + schedule: 48 22 * * * + check: + resources: {} + schedule: '@weekly-random' + prune: + resources: {} + retention: + keepDaily: 7 + keepMonthly: 1 + keepWeekly: 6 + schedule: '@weekly-random' + resourceRequirementsTemplate: {} +status: {}