From 364b431389431dff14c0d0175be8782345e2e450 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Mon, 13 Jan 2025 07:53:15 +1100 Subject: [PATCH 1/6] feat: task qos --- Makefile | 8 +- cmd/main.go | 52 ++++- config/default/manager_auth_proxy_patch.yaml | 5 + .../controllers/v1beta2/build_controller.go | 1 + internal/controllers/v1beta2/build_helpers.go | 2 +- .../controllers/v1beta2/task_controller.go | 49 ++++- internal/controllers/v1beta2/task_helpers.go | 178 ++++++++++++++++ .../controllers/v1beta2/task_qoshandler.go | 192 ++++++++++++++++++ .../v1beta2/task_standardhandler.go | 29 +++ test-resources/Dockerfile.task | 4 + 10 files changed, 506 insertions(+), 14 deletions(-) create mode 100644 internal/controllers/v1beta2/task_qoshandler.go create mode 100644 internal/controllers/v1beta2/task_standardhandler.go create mode 100644 test-resources/Dockerfile.task diff --git a/Makefile b/Makefile index fb1e6855..989bd3f4 100644 --- a/Makefile +++ b/Makefile @@ -368,7 +368,7 @@ kind/clean: # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up inside github action. -test-e2e: +test-e2e: build-task-image export HARBOR_VERSION=$(HARBOR_VERSION) && \ export OVERRIDE_BUILD_DEPLOY_DIND_IMAGE=$(OVERRIDE_BUILD_DEPLOY_DIND_IMAGE) && \ go test ./test/e2e/ -v -ginkgo.v @@ -401,4 +401,8 @@ echo "Downloading $${package}" ;\ GOBIN=$(LOCALBIN) go install $${package} ;\ mv "$$(echo "$(1)" | sed "s/-$(3)$$//")" $(1) ;\ } -endef \ No newline at end of file +endef + +.PHONY: build-task-image +build-task-image: + docker build . -f test-resources/Dockerfile.task -t example.com/test-task-image:v0.0.1 diff --git a/cmd/main.go b/cmd/main.go index e2ad485e..d5769dbf 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -27,6 +27,7 @@ import ( "github.com/cheshir/go-mq/v2" str2duration "github.com/xhit/go-str2duration/v2" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" @@ -169,6 +170,14 @@ func main() { var qosMaxBuilds int var qosDefaultValue int + var lffTaskQoSEnabled bool + var qosMaxTasks int + var qosMaxNamespaceTasks int + var qosTasksDefaultValue int + + var taskImagePullPolicy string + var buildImagePullPolicy string + var lffRouterURL bool var enableDeprecatedAPIs bool @@ -363,11 +372,24 @@ func main() { flag.IntVar(&timeoutForLongRunningBuildPods, "timeout-longrunning-build-pod-cleanup", 6, "How many hours a build pod should run before forcefully closed.") flag.IntVar(&timeoutForLongRunningTaskPods, "timeout-longrunning-task-pod-cleanup", 6, "How many hours a task pod should run before forcefully closed.") - // QoS configuration + // Build QoS configuration flag.BoolVar(&lffQoSEnabled, "enable-qos", false, "Flag to enable this controller with QoS for builds.") - flag.IntVar(&qosMaxBuilds, "qos-max-builds", 20, "The number of builds that can run at any one time.") + flag.IntVar(&qosMaxBuilds, "qos-max-builds", 20, "The total number of builds that can run at any one time.") flag.IntVar(&qosDefaultValue, "qos-default", 5, "The default qos value to apply if one is not provided.") + // Task QoS configuration + flag.BoolVar(&lffTaskQoSEnabled, "enable-task-qos", false, "Flag to enable this controller with QoS for tasks.") + flag.IntVar(&qosMaxTasks, "qos-max-tasks", 200, "The total number of tasks that can run at any one time.") + flag.IntVar(&qosMaxNamespaceTasks, "qos-max-namespace-tasks", 20, "The total number of tasks that can run at any one time.") + flag.IntVar(&qosTasksDefaultValue, "task-qos-default", 5, "The default qos value to apply to tasks if one is not provided.") + + // flags to change the image pull policy used for tasks and builds + // defaults to Always, can change to another option as required. tests use IfNotPresent + flag.StringVar(&taskImagePullPolicy, "task-image-pull-policy", "Always", + "The image pull policy to use for tasks.") + flag.StringVar(&buildImagePullPolicy, "build-image-pull-policy", "Always", + "The image pull policy to use for builds.") + // If installing this controller from scratch, deprecated APIs should not be configured flag.BoolVar(&enableDeprecatedAPIs, "enable-deprecated-apis", false, "Flag to have this controller enable support for deprecated APIs.") @@ -698,6 +720,26 @@ func main() { DefaultValue: qosDefaultValue, } + taskQoSConfigv1beta2 := lagoonv1beta2ctrl.TaskQoS{ + MaxTasks: qosMaxTasks, + MaxNamespaceTasks: qosMaxNamespaceTasks, + } + + tipp := corev1.PullAlways + switch taskImagePullPolicy { + case "IfNotPresent": + tipp = corev1.PullIfNotPresent + case "Never": + tipp = corev1.PullNever + } + bipp := corev1.PullAlways + switch buildImagePullPolicy { + case "IfNotPresent": + bipp = corev1.PullIfNotPresent + case "Never": + bipp = corev1.PullNever + } + resourceCleanup := pruner.New(mgr.GetClient(), buildsToKeep, buildPodsToKeep, @@ -834,6 +876,7 @@ func main() { NoProxy: noProxy, }, UnauthenticatedRegistry: unauthenticatedRegistry, + ImagePullPolicy: bipp, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "LagoonBuild") os.Exit(1) @@ -860,6 +903,8 @@ func main() { Client: mgr.GetClient(), Log: ctrl.Log.WithName("v1beta2").WithName("LagoonTask"), Scheme: mgr.GetScheme(), + EnableMQ: enableMQ, + Messaging: messaging, ControllerNamespace: controllerNamespace, NamespacePrefix: namespacePrefix, RandomNamespacePrefix: randomPrefix, @@ -877,6 +922,9 @@ func main() { HTTPSProxy: httpsProxy, NoProxy: noProxy, }, + LFFTaskQoSEnabled: lffTaskQoSEnabled, + TaskQoS: taskQoSConfigv1beta2, + ImagePullPolicy: tipp, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "LagoonTask") os.Exit(1) diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 567b1d51..565d6c21 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -24,6 +24,11 @@ spec: - "--enable-deprecated-apis" - "--lagoon-feature-flag-support-k8upv2" - "--skip-tls-verify" + - "--enable-task-qos" + - "--qos-max-tasks=3" + - "--qos-max-namespace-tasks=3" + - "--task-image-pull-policy=IfNotPresent" + - "--build-image-pull-policy=IfNotPresent" ports: - containerPort: 8443 name: https diff --git a/internal/controllers/v1beta2/build_controller.go b/internal/controllers/v1beta2/build_controller.go index 566f763c..f6128c4d 100644 --- a/internal/controllers/v1beta2/build_controller.go +++ b/internal/controllers/v1beta2/build_controller.go @@ -78,6 +78,7 @@ type LagoonBuildReconciler struct { LagoonAPIConfiguration helpers.LagoonAPIConfiguration ProxyConfig ProxyConfig UnauthenticatedRegistry string + ImagePullPolicy corev1.PullPolicy } // BackupConfig holds all the backup configuration settings diff --git a/internal/controllers/v1beta2/build_helpers.go b/internal/controllers/v1beta2/build_helpers.go index ce8b6929..4bd9b732 100644 --- a/internal/controllers/v1beta2/build_helpers.go +++ b/internal/controllers/v1beta2/build_helpers.go @@ -831,7 +831,7 @@ func (r *LagoonBuildReconciler) processBuild(ctx context.Context, opLog logr.Log { Name: "lagoon-build", Image: buildImage, - ImagePullPolicy: "Always", + ImagePullPolicy: r.ImagePullPolicy, Env: podEnvs, VolumeMounts: volumeMounts, }, diff --git a/internal/controllers/v1beta2/task_controller.go b/internal/controllers/v1beta2/task_controller.go index 6eea9829..4cef8b0c 100644 --- a/internal/controllers/v1beta2/task_controller.go +++ b/internal/controllers/v1beta2/task_controller.go @@ -34,6 +34,7 @@ import ( lagooncrd "github.com/uselagoon/remote-controller/api/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/helpers" + "github.com/uselagoon/remote-controller/internal/messenger" "github.com/uselagoon/remote-controller/internal/metrics" ) @@ -42,6 +43,8 @@ type LagoonTaskReconciler struct { client.Client Log logr.Logger Scheme *runtime.Scheme + EnableMQ bool + Messaging *messenger.Messenger ControllerNamespace string NamespacePrefix string RandomNamespacePrefix bool @@ -49,6 +52,9 @@ type LagoonTaskReconciler struct { EnableDebug bool LagoonTargetName string ProxyConfig ProxyConfig + LFFTaskQoSEnabled bool + TaskQoS TaskQoS + ImagePullPolicy corev1.PullPolicy } var ( @@ -70,15 +76,40 @@ func (r *LagoonTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) // examine DeletionTimestamp to determine if object is under deletion if lagoonTask.ObjectMeta.DeletionTimestamp.IsZero() { - // check if the task that has been recieved is a standard or advanced task - if lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"] == lagooncrd.TaskStatusPending.String() && - lagoonTask.ObjectMeta.Labels["lagoon.sh/taskType"] == lagooncrd.TaskTypeStandard.String() { - return ctrl.Result{}, r.createStandardTask(ctx, &lagoonTask, opLog) - } - if lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"] == lagooncrd.TaskStatusPending.String() && - lagoonTask.ObjectMeta.Labels["lagoon.sh/taskType"] == lagooncrd.TaskTypeAdvanced.String() { - return ctrl.Result{}, r.createAdvancedTask(ctx, &lagoonTask, opLog) + if r.LFFTaskQoSEnabled { + // handle QoS tasks here + // if we do have a `lagoon.sh/taskStatus` set as running, then process it + runningNSTasks := &lagooncrd.LagoonTaskList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(req.Namespace), + client.MatchingLabels(map[string]string{ + "lagoon.sh/taskStatus": lagooncrd.TaskStatusRunning.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + // // list any tasks that are running + if err := r.List(ctx, runningNSTasks, listOption); err != nil { + return ctrl.Result{}, fmt.Errorf("unable to list tasks in the namespace, there may be none or something went wrong: %v", err) + } + for _, runningTask := range runningNSTasks.Items { + // if the running task is the one from this request then process it + if lagoonTask.ObjectMeta.Name == runningTask.ObjectMeta.Name { + // actually process the task here + if _, ok := lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStarted"]; !ok { + if lagoonTask.ObjectMeta.Labels["lagoon.sh/taskType"] == lagooncrd.TaskTypeStandard.String() { + return ctrl.Result{}, r.createStandardTask(ctx, &lagoonTask, opLog) + } + if lagoonTask.ObjectMeta.Labels["lagoon.sh/taskType"] == lagooncrd.TaskTypeAdvanced.String() { + return ctrl.Result{}, r.createAdvancedTask(ctx, &lagoonTask, opLog) + } + } + } // end check if running task is current LagoonTask + } // end loop for running tasks + // // once running tasks are processed, run the qos handler + return r.qosTaskProcessor(ctx, opLog, lagoonTask) } + // if qos is not enabled, just process it as a standard task + return r.standardTaskProcessor(ctx, opLog, lagoonTask) } else { // The object is being deleted if helpers.ContainsString(lagoonTask.ObjectMeta.Finalizers, taskFinalizer) { @@ -567,7 +598,7 @@ func (r *LagoonTaskReconciler) createAdvancedTask(ctx context.Context, lagoonTas { Name: "lagoon-task", Image: lagoonTask.Spec.AdvancedTask.RunnerImage, - ImagePullPolicy: "Always", + ImagePullPolicy: r.ImagePullPolicy, EnvFrom: []corev1.EnvFromSource{ { ConfigMapRef: &corev1.ConfigMapEnvSource{ diff --git a/internal/controllers/v1beta2/task_helpers.go b/internal/controllers/v1beta2/task_helpers.go index a77a3c13..05864679 100644 --- a/internal/controllers/v1beta2/task_helpers.go +++ b/internal/controllers/v1beta2/task_helpers.go @@ -2,9 +2,18 @@ package v1beta2 import ( "context" + "encoding/json" "fmt" + "sort" + "time" + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "github.com/uselagoon/machinery/api/schema" + lagooncrd "github.com/uselagoon/remote-controller/api/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/helpers" + "github.com/uselagoon/remote-controller/internal/metrics" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -57,3 +66,172 @@ func (r *LagoonMonitorReconciler) deleteActiveStandbyRole(ctx context.Context, d } return nil } + +// updateQueuedTask will update a task if it is queued +func (r *LagoonTaskReconciler) updateQueuedTask( + lagoonTask lagooncrd.LagoonTask, + queuePosition, queueLength int, + opLog logr.Logger, +) error { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Updating task %s to queued: %s", lagoonTask.ObjectMeta.Name, fmt.Sprintf("This task is currently queued in position %v/%v", queuePosition, queueLength))) + } + // if we get this handler, then it is likely that the task was in a pending or running state with no actual running pod + // so just set the logs to be cancellation message + allContainerLogs := []byte(fmt.Sprintf(`======================================== +%s +======================================== +`, fmt.Sprintf("This task is currently queued in position %v/%v", queuePosition, queueLength))) + // send any messages to lagoon message queues + opLog.Info(fmt.Sprintf("task %v", len(allContainerLogs))) + // update the deployment with the status, lagoon v2.12.0 supports queued status, otherwise use pending + r.taskStatusLogsToLagoonLogs(opLog, &lagoonTask, lagooncrd.TaskStatusQueued) //, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) + r.updateLagoonTask(opLog, &lagoonTask, lagooncrd.TaskStatusQueued) //, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) + r.taskLogsToLagoonLogs(opLog, &lagoonTask, allContainerLogs, lagooncrd.TaskStatusQueued) + return nil +} + +func sortTasks(pendingTasks *lagooncrd.LagoonTaskList) { + sort.Slice(pendingTasks.Items, func(i, j int) bool { + return pendingTasks.Items[i].ObjectMeta.CreationTimestamp.Before(&pendingTasks.Items[j].ObjectMeta.CreationTimestamp) + }) +} + +// taskStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging +func (r *LagoonTaskReconciler) taskStatusLogsToLagoonLogs( + opLog logr.Logger, + lagoonTask *lagooncrd.LagoonTask, + taskCondition lagooncrd.TaskStatusType, +) (bool, schema.LagoonLog) { + if r.EnableMQ { + msg := schema.LagoonLog{ + Severity: "info", + Project: lagoonTask.Spec.Project.Name, + Event: "task:job-kubernetes:" + taskCondition.ToLower(), //@TODO: this probably needs to be changed to a new task event for the controller + Meta: &schema.LagoonLogMeta{ + Task: &lagoonTask.Spec.Task, + ProjectName: lagoonTask.Spec.Project.Name, + Environment: lagoonTask.Spec.Environment.Name, + EnvironmentID: lagoonTask.Spec.Environment.ID, + ProjectID: lagoonTask.Spec.Project.ID, + JobName: lagoonTask.ObjectMeta.Name, + JobStatus: taskCondition.ToLower(), + RemoteID: string(lagoonTask.ObjectMeta.UID), + Key: lagoonTask.Spec.Key, + Cluster: r.LagoonTargetName, + }, + Message: fmt.Sprintf("*[%s]* %s Task `%s` %s", + lagoonTask.Spec.Project.Name, + lagoonTask.Spec.Environment.Name, + lagoonTask.ObjectMeta.Name, + taskCondition.ToLower(), + ), + } + // if we aren't being provided the lagoon config, we can skip adding the routes etc + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + // @TODO: if we can't publish the message because we are deleting the resource, then should we even + // bother to patch the resource?? + // leave it for now cause the resource will just be deleted anyway + if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + return true, msg + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + } + return false, schema.LagoonLog{} +} + +// updateLagoonTask sends the status of the task and deployment to the controllerhandler message queue in lagoon, +// this is for the handler in lagoon to process. +func (r *LagoonTaskReconciler) updateLagoonTask(opLog logr.Logger, + lagoonTask *lagooncrd.LagoonTask, + taskCondition lagooncrd.TaskStatusType, +) (bool, schema.LagoonMessage) { + if r.EnableMQ && lagoonTask != nil { + if taskCondition.ToLower() == "failed" || taskCondition.ToLower() == "complete" || taskCondition.ToLower() == "cancelled" { + time.AfterFunc(31*time.Second, func() { + metrics.TaskRunningStatus.Delete(prometheus.Labels{ + "task_namespace": lagoonTask.ObjectMeta.Namespace, + "task_name": lagoonTask.ObjectMeta.Name, + }) + }) + time.Sleep(2 * time.Second) // smol sleep to reduce race of final messages with previous messages + } + msg := schema.LagoonMessage{ + Type: "task", + Namespace: lagoonTask.ObjectMeta.Namespace, + Meta: &schema.LagoonLogMeta{ + Task: &lagoonTask.Spec.Task, + Environment: lagoonTask.Spec.Environment.Name, + Project: lagoonTask.Spec.Project.Name, + EnvironmentID: lagoonTask.Spec.Environment.ID, + ProjectID: lagoonTask.Spec.Project.ID, + JobName: lagoonTask.ObjectMeta.Name, + JobStatus: taskCondition.ToLower(), + RemoteID: string(lagoonTask.ObjectMeta.UID), + Key: lagoonTask.Spec.Key, + Cluster: r.LagoonTargetName, + }, + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + if err := r.Messaging.Publish("lagoon-tasks:controller", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + return true, msg + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + } + return false, schema.LagoonMessage{} +} + +// taskLogsToLagoonLogs sends the task logs to the lagoon-logs message queue +// it contains the actual pod log output that is sent to elasticsearch, it is what eventually is displayed in the UI +func (r *LagoonTaskReconciler) taskLogsToLagoonLogs( + opLog logr.Logger, + lagoonTask *lagooncrd.LagoonTask, + logs []byte, + taskCondition lagooncrd.TaskStatusType, +) (bool, schema.LagoonLog) { + if r.EnableMQ && lagoonTask != nil { + msg := schema.LagoonLog{ + Severity: "info", + Project: lagoonTask.Spec.Project.Name, + Event: "task-logs:job-kubernetes:" + lagoonTask.ObjectMeta.Name, + Meta: &schema.LagoonLogMeta{ + Task: &lagoonTask.Spec.Task, + Environment: lagoonTask.Spec.Environment.Name, + JobName: lagoonTask.ObjectMeta.Name, + JobStatus: taskCondition.ToLower(), + RemoteID: string(lagoonTask.ObjectMeta.UID), + Key: lagoonTask.Spec.Key, + Cluster: r.LagoonTargetName, + }, + } + // add the actual task log message + msg.Message = string(logs) + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + return true, msg + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + } + return false, schema.LagoonLog{} +} diff --git a/internal/controllers/v1beta2/task_qoshandler.go b/internal/controllers/v1beta2/task_qoshandler.go new file mode 100644 index 00000000..708f82b8 --- /dev/null +++ b/internal/controllers/v1beta2/task_qoshandler.go @@ -0,0 +1,192 @@ +package v1beta2 + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/go-logr/logr" + lagooncrd "github.com/uselagoon/remote-controller/api/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// TaskQoS is use for the quality of service configuration for lagoon tasks. +// if MaxTasks is lower than MaxNamespaceTasks, then the limit imposed is the lower value of MaxTasks +// MaxNamespaceTasks will only be compared against if MaxTasks is greater than MaxNamespaceTasks +type TaskQoS struct { + MaxTasks int + MaxNamespaceTasks int +} + +func (r *LagoonTaskReconciler) qosTaskProcessor(ctx context.Context, + opLog logr.Logger, + lagoonTask lagooncrd.LagoonTask) (ctrl.Result, error) { + if r.EnableDebug { + opLog.Info("Checking which task next") + } + // handle the QoS task process here + // if the task is already running, then there is no need to check which task can be started next + if lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"] == lagooncrd.TaskStatusRunning.String() { + // this is done so that all running state updates don't try to force the queue processor to run unnecessarily + // downside is that this can lead to queue/state changes being less frequent for queued tasks in the api + // any new tasks, or complete/failed/cancelled tasks will still force the whichtasknext processor to run though + return ctrl.Result{}, nil + } + // handle the QoS task process here + return ctrl.Result{}, r.whichTaskNext(ctx, opLog) +} + +func (r *LagoonTaskReconciler) whichTaskNext(ctx context.Context, opLog logr.Logger) error { + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabels(map[string]string{ + "lagoon.sh/taskStatus": lagooncrd.TaskStatusRunning.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + runningTasks := &lagooncrd.LagoonTaskList{} + if err := r.List(ctx, runningTasks, listOption); err != nil { + return fmt.Errorf("unable to list tasks in the cluster, there may be none or something went wrong: %v", err) + } + tasksToStart := r.TaskQoS.MaxTasks - len(runningTasks.Items) + if len(runningTasks.Items) >= r.TaskQoS.MaxTasks { + // if the maximum number of tasks is hit, then drop out and try again next time + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Currently %v running tasks, no room for new tasks to be started", len(runningTasks.Items))) + } + go r.processQueue(ctx, opLog, tasksToStart, true) + return nil + } + if tasksToStart > 0 { + opLog.Info(fmt.Sprintf("Currently %v running tasks, room for %v tasks to be started", len(runningTasks.Items), tasksToStart)) + // if there are any free slots to start a task, do that here + go r.processQueue(ctx, opLog, tasksToStart, false) + } + return nil +} + +var runningTaskQueueProcess bool + +// this is a processor for any tasks that are currently `queued` status. all normal task activity will still be performed +// this just allows the controller to update any tasks that are in the queue periodically +// if this ran on every single event, it would flood the queue with messages, so it is restricted using `runningTaskQueueProcess` global +// to only run the process at any one time til it is complete +// tasksToStart is the number of tasks that can be started at the time the process is called +// limitHit is used to determine if the task limit has been hit, this is used to prevent new tasks from being started inside this process +func (r *LagoonTaskReconciler) processQueue(ctx context.Context, opLog logr.Logger, tasksToStart int, limitHit bool) error { + // this should only ever be able to run one instance of at a time within a single controller + // this is because this process is quite heavy when it goes to submit the queue messages to the api + // the downside of this is that there can be delays with the messages it sends to the actual + // status of the tasks, but task complete/fail/cancel will always win out on the lagoon-core side + // so this isn't that much of an issue if there are some delays in the messages + opLog = opLog.WithName("QueueProcessor") + if !runningTaskQueueProcess { + runningTaskQueueProcess = true + if r.EnableDebug { + opLog.Info("Processing queue") + } + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabels(map[string]string{ + "lagoon.sh/taskStatus": lagooncrd.TaskStatusPending.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + pendingTasks := &lagooncrd.LagoonTaskList{} + if err := r.List(ctx, pendingTasks, listOption); err != nil { + runningTaskQueueProcess = false + return fmt.Errorf("unable to list tasks in the cluster, there may be none or something went wrong: %v", err) + } + if len(pendingTasks.Items) > 0 { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("There are %v pending tasks", len(pendingTasks.Items))) + } + // if we have any pending tasks, then grab the latest one and make it running + // if there are any other pending tasks, cancel them so only the latest one runs + sortTasks(pendingTasks) + for idx, pTask := range pendingTasks.Items { + // need to +1 to index because 0 + // if the `limitHit` is not set it means that task qos has reached the maximum that this remote has allowed to start + if idx+1 <= tasksToStart && !limitHit { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking if task %s can be started", pTask.ObjectMeta.Name)) + } + // if we do have a `lagoon.sh/taskStatus` set, then process as normal + runningNSTasks := &lagooncrd.LagoonTaskList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(pTask.ObjectMeta.Namespace), + client.MatchingLabels(map[string]string{ + "lagoon.sh/taskStatus": lagooncrd.TaskStatusRunning.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + // list any tasks that are running + if err := r.List(ctx, runningNSTasks, listOption); err != nil { + runningTaskQueueProcess = false + return fmt.Errorf("unable to list tasks in the namespace, there may be none or something went wrong: %v", err) + } + + // if there is a limit to the number of tasks per namespace, enforce that here + if len(runningNSTasks.Items) < r.TaskQoS.MaxNamespaceTasks { + pendingTasks := &lagooncrd.LagoonTaskList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(pTask.ObjectMeta.Namespace), + client.MatchingLabels(map[string]string{"lagoon.sh/taskStatus": lagooncrd.TaskStatusPending.String()}), + }) + if err := r.List(ctx, pendingTasks, listOption); err != nil { + return fmt.Errorf("unable to list tasks in the namespace, there may be none or something went wrong: %v", err) + } + /* + if namespaces or sorting allows for additional edge cases + then in the future this section should be updated to accomodate these additional rule sets + right now the sorting sorts by creation time, and then only the first pending item is started + all other tasks remain pending + */ + sortTasks(pendingTasks) + // opLog.Info(fmt.Sprintf("There are %v pending tasks", len(pendingTasks.Items))) + // if we have any pending tasks, then grab the latest one and make it running + // this is where the task controller will take over and start the pod + for idx, pTask := range pendingTasks.Items { + pendingTask := pTask.DeepCopy() + // if the task is the first in the sorted list of pending tasks in this namespace, then start it + if idx == 0 { + pendingTask.Labels["lagoon.sh/taskStatus"] = lagooncrd.TaskStatusRunning.String() + } + if err := r.Update(ctx, pendingTask); err != nil { + return err + } + } + // don't handle the queued process for this task, continue to next in the list + continue + } + // The object is not being deleted, so if it does not have our finalizer, + // then lets add the finalizer and update the object. This is equivalent + // registering our finalizer. + if !helpers.ContainsString(pTask.ObjectMeta.Finalizers, taskFinalizer) { + pTask.ObjectMeta.Finalizers = append(pTask.ObjectMeta.Finalizers, taskFinalizer) + // use patches to avoid update errors + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": pTask.ObjectMeta.Finalizers, + }, + }) + if err := r.Patch(ctx, &pTask, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + runningTaskQueueProcess = false + return err + } + } + } + // update the task to be queued, and add a log message with the task log with the current position in the queue + // this position will update as tasks are created/processed, so the position of a task could change depending on + // higher or lower priority tasks being created + if err := r.updateQueuedTask(pTask, (idx + 1), len(pendingTasks.Items), opLog); err != nil { + runningTaskQueueProcess = false + return nil + } + } + } + runningTaskQueueProcess = false + } + return nil +} diff --git a/internal/controllers/v1beta2/task_standardhandler.go b/internal/controllers/v1beta2/task_standardhandler.go new file mode 100644 index 00000000..3f6a5348 --- /dev/null +++ b/internal/controllers/v1beta2/task_standardhandler.go @@ -0,0 +1,29 @@ +package v1beta2 + +import ( + "context" + + "github.com/go-logr/logr" + lagooncrd "github.com/uselagoon/remote-controller/api/lagoon/v1beta2" + ctrl "sigs.k8s.io/controller-runtime" +) + +func (r *LagoonTaskReconciler) standardTaskProcessor(ctx context.Context, + opLog logr.Logger, + lagoonTask lagooncrd.LagoonTask) (ctrl.Result, error) { + // check if we get a lagoontask that hasn't got any taskstatus + // this means it was created by the message queue handler + // so we should do the steps required for a lagoon task and then copy the task + // into the created namespace + if _, ok := lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStarted"]; !ok { + if lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"] == lagooncrd.TaskStatusPending.String() && + lagoonTask.ObjectMeta.Labels["lagoon.sh/taskType"] == lagooncrd.TaskTypeStandard.String() { + return ctrl.Result{}, r.createStandardTask(ctx, &lagoonTask, opLog) + } + if lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"] == lagooncrd.TaskStatusPending.String() && + lagoonTask.ObjectMeta.Labels["lagoon.sh/taskType"] == lagooncrd.TaskTypeAdvanced.String() { + return ctrl.Result{}, r.createAdvancedTask(ctx, &lagoonTask, opLog) + } + } + return ctrl.Result{}, nil +} diff --git a/test-resources/Dockerfile.task b/test-resources/Dockerfile.task new file mode 100644 index 00000000..b517b103 --- /dev/null +++ b/test-resources/Dockerfile.task @@ -0,0 +1,4 @@ +# this is an image used for the qos task test to ensure that the task runs for a long enough time +FROM alpine + +CMD ["sleep", "30"] \ No newline at end of file From 4fbf700e88d544e15dff6ca351b3ccf612c22a20 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Mon, 13 Jan 2025 18:49:50 +1100 Subject: [PATCH 2/6] test: limited coverage for task and build qos --- Makefile | 2 +- .../v1beta2/podmonitor_taskhandlers.go | 3 +- test/e2e/e2e_buildqos.go | 114 +++++++++++++ test/e2e/e2e_taskqos.go | 114 +++++++++++++ test/e2e/e2e_test.go | 159 ++++++++++++------ .../buildqos/lagoon-build-xpyz5m1.json | 47 ++++++ .../buildqos/lagoon-build-xpyz5m2.json | 47 ++++++ .../buildqos/lagoon-build-xpyz5m3.json | 47 ++++++ .../buildqos/lagoon-build-xpyz5m4.json | 47 ++++++ test/e2e/testdata/lagoon-task-7m5zypx.json | 34 ++++ .../e2e/testdata/remove-environment-dev1.json | 10 ++ .../e2e/testdata/remove-environment-dev2.json | 10 ++ .../e2e/testdata/remove-environment-dev3.json | 10 ++ ...ment.json => remove-environment-main.json} | 0 .../testdata/taskqos/lagoon-task-1xpyz5m.json | 35 ++++ .../testdata/taskqos/lagoon-task-2xpyz5m.json | 35 ++++ .../testdata/taskqos/lagoon-task-3xpyz5m.json | 35 ++++ .../testdata/taskqos/lagoon-task-4xpyz5m.json | 35 ++++ test/utils/utils.go | 68 ++++++++ 19 files changed, 803 insertions(+), 49 deletions(-) create mode 100644 test/e2e/e2e_buildqos.go create mode 100644 test/e2e/e2e_taskqos.go create mode 100644 test/e2e/testdata/buildqos/lagoon-build-xpyz5m1.json create mode 100644 test/e2e/testdata/buildqos/lagoon-build-xpyz5m2.json create mode 100644 test/e2e/testdata/buildqos/lagoon-build-xpyz5m3.json create mode 100644 test/e2e/testdata/buildqos/lagoon-build-xpyz5m4.json create mode 100644 test/e2e/testdata/lagoon-task-7m5zypx.json create mode 100644 test/e2e/testdata/remove-environment-dev1.json create mode 100644 test/e2e/testdata/remove-environment-dev2.json create mode 100644 test/e2e/testdata/remove-environment-dev3.json rename test/e2e/testdata/{remove-environment.json => remove-environment-main.json} (100%) create mode 100644 test/e2e/testdata/taskqos/lagoon-task-1xpyz5m.json create mode 100644 test/e2e/testdata/taskqos/lagoon-task-2xpyz5m.json create mode 100644 test/e2e/testdata/taskqos/lagoon-task-3xpyz5m.json create mode 100644 test/e2e/testdata/taskqos/lagoon-task-4xpyz5m.json diff --git a/Makefile b/Makefile index 989bd3f4..3c749f03 100644 --- a/Makefile +++ b/Makefile @@ -371,7 +371,7 @@ kind/clean: test-e2e: build-task-image export HARBOR_VERSION=$(HARBOR_VERSION) && \ export OVERRIDE_BUILD_DEPLOY_DIND_IMAGE=$(OVERRIDE_BUILD_DEPLOY_DIND_IMAGE) && \ - go test ./test/e2e/ -v -ginkgo.v + go test ./test/e2e/ -v -ginkgo.v -timeout 20m .PHONY: github/test-e2e github/test-e2e: local-dev/tools install-lagoon-remote test-e2e diff --git a/internal/controllers/v1beta2/podmonitor_taskhandlers.go b/internal/controllers/v1beta2/podmonitor_taskhandlers.go index a33852c4..6ecf221f 100644 --- a/internal/controllers/v1beta2/podmonitor_taskhandlers.go +++ b/internal/controllers/v1beta2/podmonitor_taskhandlers.go @@ -332,7 +332,8 @@ Task %s mergeMap := map[string]interface{}{ "metadata": map[string]interface{}{ "labels": map[string]interface{}{ - "lagoon.sh/taskStatus": taskCondition.String(), + "lagoon.sh/taskStatus": taskCondition.String(), + "lagoon.sh/taskStarted": "true", }, }, } diff --git a/test/e2e/e2e_buildqos.go b/test/e2e/e2e_buildqos.go new file mode 100644 index 00000000..c8f6f00c --- /dev/null +++ b/test/e2e/e2e_buildqos.go @@ -0,0 +1,114 @@ +package e2e + +import ( + "fmt" + "os/exec" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "github.com/uselagoon/remote-controller/test/utils" +) + +func testBuildQoS(timeout string, duration, interval time.Duration) { + var builds = []struct { + Name string + Namespace string + }{ + { + Name: "xpyz5m1", + Namespace: "nginx-example-main", + }, + { + Name: "xpyz5m2", + Namespace: "nginx-example-dev1", + }, + { + Name: "xpyz5m3", + Namespace: "nginx-example-dev2", + }, + { + Name: "xpyz5m4", + Namespace: "nginx-example-dev3", + }, + } + + ginkgo.By("validating that lagoonbuilds are queing") + for _, build := range builds { + ginkgo.By("creating a LagoonBuild resource via rabbitmq") + cmd := exec.Command( + "curl", + "-s", + "-u", + "guest:guest", + "-H", + "'Accept: application/json'", + "-H", + "'Content-Type:application/json'", + "-X", + "POST", + "-d", + fmt.Sprintf("@test/e2e/testdata/buildqos/lagoon-build-%s.json", build.Name), + "http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish", + ) + _, err := utils.Run(cmd) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + time.Sleep(1 * time.Second) + } + time.Sleep(5 * time.Second) + for _, build := range builds { + ginkgo.By("validating that the LagoonBuild build pod is created") + cmd := exec.Command( + utils.Kubectl(), + "-n", build.Namespace, + "wait", + "--for=condition=Ready", + "pod", + fmt.Sprintf("lagoon-build-%s", build.Name), + fmt.Sprintf("--timeout=%s", timeout), + ) + _, err := utils.Run(cmd) + if build.Name == "xpyz5m4" { + // should fail because it gets queued at 3 builds max + gomega.ExpectWithOffset(1, err).To(gomega.HaveOccurred()) + // then wait for the build pod to start + verifyBuildRuns := func() error { + cmd = exec.Command(utils.Kubectl(), "get", + "pods", fmt.Sprintf("lagoon-build-%s", build.Name), + "-o", "jsonpath={.status.phase}", + "-n", build.Namespace, + ) + status, err := utils.Run(cmd) + if err != nil { + return err + } + if string(status) != "Running" { + return fmt.Errorf("build pod in %s status", status) + } + return nil + } + // if this fails, then qos didn't start the pod for some reason in the duration available + gomega.EventuallyWithOffset(1, verifyBuildRuns, duration, interval).Should(gomega.Succeed()) + + ginkgo.By("validating that the lagoon-build pod completes as expected") + verifyBuildPodCompletes := func() error { + // Validate pod status + cmd = exec.Command(utils.Kubectl(), "get", + "pods", fmt.Sprintf("lagoon-build-%s", build.Name), "-o", "jsonpath={.status.phase}", + "-n", build.Namespace, + ) + status, err := utils.Run(cmd) + gomega.ExpectWithOffset(2, err).NotTo(gomega.HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("controller pod in %s status", status) + } + return nil + } + gomega.EventuallyWithOffset(1, verifyBuildPodCompletes, duration, interval).Should(gomega.Succeed()) + } else { + // should pass because qos builds + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + } + } + +} diff --git a/test/e2e/e2e_taskqos.go b/test/e2e/e2e_taskqos.go new file mode 100644 index 00000000..5e4e5546 --- /dev/null +++ b/test/e2e/e2e_taskqos.go @@ -0,0 +1,114 @@ +package e2e + +import ( + "fmt" + "os/exec" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "github.com/uselagoon/remote-controller/test/utils" +) + +func testTaskQoS(timeout string, duration, interval time.Duration) { + var tasks = []struct { + Name string + Namespace string + }{ + { + Name: "1xpyz5m", + Namespace: "nginx-example-main", + }, + { + Name: "2xpyz5m", + Namespace: "nginx-example-dev1", + }, + { + Name: "3xpyz5m", + Namespace: "nginx-example-dev2", + }, + { + Name: "4xpyz5m", + Namespace: "nginx-example-dev3", + }, + } + ginkgo.By("validating that lagoontasks are queing") + // these tasks use the task test image with a sleep to ensure there is enough + // time for QoS to be tested + for _, task := range tasks { + ginkgo.By("creating a LagoonTask resource via rabbitmq") + cmd := exec.Command( + "curl", + "-s", + "-u", + "guest:guest", + "-H", + "'Accept: application/json'", + "-H", + "'Content-Type:application/json'", + "-X", + "POST", + "-d", + fmt.Sprintf("@test/e2e/testdata/taskqos/lagoon-task-%s.json", task.Name), + "http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish", + ) + _, err := utils.Run(cmd) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + time.Sleep(1 * time.Second) + } + time.Sleep(5 * time.Second) + for _, task := range tasks { + ginkgo.By("validating that the LagoonTask task pod is created") + cmd := exec.Command( + utils.Kubectl(), + "-n", task.Namespace, + "wait", + "--for=condition=Ready", + "pod", + fmt.Sprintf("lagoon-task-%s", task.Name), + fmt.Sprintf("--timeout=%s", timeout), + ) + _, err := utils.Run(cmd) + if task.Name == "4xpyz5m" { + // should fail because it gets queued at 3 tasks max + gomega.ExpectWithOffset(1, err).To(gomega.HaveOccurred()) + // then wait for the task pod to start + verifyTaskRuns := func() error { + cmd = exec.Command(utils.Kubectl(), "get", + "pods", fmt.Sprintf("lagoon-task-%s", task.Name), + "-o", "jsonpath={.status.phase}", + "-n", task.Namespace, + ) + status, err := utils.Run(cmd) + if err != nil { + return err + } + if string(status) != "Running" { + return fmt.Errorf("task pod in %s status", status) + } + return nil + } + // if this fails, then qos didn't start the pod for some reason in the duration available + gomega.EventuallyWithOffset(1, verifyTaskRuns, duration, interval).Should(gomega.Succeed()) + + ginkgo.By("validating that the lagoon-task pod completes as expected") + verifyTaskPodCompletes := func() error { + // Validate pod status + cmd = exec.Command(utils.Kubectl(), "get", + "pods", fmt.Sprintf("lagoon-task-%s", task.Name), "-o", "jsonpath={.status.phase}", + "-n", task.Namespace, + ) + status, err := utils.Run(cmd) + gomega.ExpectWithOffset(2, err).NotTo(gomega.HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("controller pod in %s status", status) + } + return nil + } + gomega.EventuallyWithOffset(1, verifyTaskPodCompletes, duration, interval).Should(gomega.Succeed()) + } else { + // should pass because qos tasks + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + } + } +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index a9503e7e..6a9a631b 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -35,6 +35,9 @@ const ( ) var ( + // projectimage stores the name of the image used in the example + projectimage = "example.com/remote-controller:v0.0.1" + testtaskimage = "example.com/test-task-image:v0.0.1" harborversion string builddeployimage string @@ -73,19 +76,45 @@ var _ = Describe("controller", Ordered, func() { // when running a re-test, it is best to make sure the old namespace doesn't exist By("removing existing test resources") // remove the old namespace - cmd = exec.Command(utils.Kubectl(), "delete", "ns", "nginx-example-main") - _, _ = utils.Run(cmd) + utils.CleanupNamespace("nginx-example-main") + utils.CleanupNamespace("nginx-example-dev1") + utils.CleanupNamespace("nginx-example-dev2") + utils.CleanupNamespace("nginx-example-dev3") + // clean up the k8up crds utils.UninstallK8upCRDs() }) // comment to prevent cleaning up controller namespace and local services AfterAll(func() { + By("dump controller logs") + cmd := exec.Command(utils.Kubectl(), "get", + "pods", "-l", "control-plane=controller-manager", + "-o", "go-template={{ range .items }}"+ + "{{ if not .metadata.deletionTimestamp }}"+ + "{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}{{ end }}", + "-n", namespace, + ) + podOutput, err := utils.Run(cmd) + if err == nil { + podNames := utils.GetNonEmptyLines(string(podOutput)) + controllerPodName := podNames[0] + cmd = exec.Command(utils.Kubectl(), "logs", + controllerPodName, "-c", "manager", + "-n", namespace, + ) + podlogs, err := utils.Run(cmd) + if err == nil { + fmt.Fprintf(GinkgoWriter, "info: %s\n", podlogs) + } + } + By("stop metrics consumer") utils.StopMetricsConsumer() By("removing manager namespace") - cmd := exec.Command(utils.Kubectl(), "delete", "ns", namespace) + cmd = exec.Command(utils.Kubectl(), "delete", "ns", namespace) _, _ = utils.Run(cmd) By("stop local services") @@ -98,9 +127,6 @@ var _ = Describe("controller", Ordered, func() { var controllerPodName string var err error - // projectimage stores the name of the image used in the example - var projectimage = "example.com/remote-controller:v0.0.1" - By("building the manager(Operator) image") cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) _, err = utils.Run(cmd) @@ -110,6 +136,10 @@ var _ = Describe("controller", Ordered, func() { err = utils.LoadImageToKindClusterWithName(projectimage) ExpectWithOffset(1, err).NotTo(HaveOccurred()) + By("loading the the test-task-image image on Kind") + err = utils.LoadImageToKindClusterWithName(testtaskimage) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + By("installing CRDs") cmd = exec.Command("make", "install") _, err = utils.Run(cmd) @@ -158,7 +188,7 @@ var _ = Describe("controller", Ordered, func() { By("start metrics consumer") Expect(utils.StartMetricsConsumer()).To(Succeed()) - time.Sleep(30 * time.Second) + time.Sleep(10 * time.Second) By("validating that lagoonbuilds are working") for _, name := range []string{"7m5zypx", "8m5zypx", "9m5zypx"} { @@ -271,7 +301,7 @@ var _ = Describe("controller", Ordered, func() { EventuallyWithOffset(1, verifyOnlyOneBuildPod, duration, interval).Should(Succeed()) By("validating that LagoonTasks are working") - for _, name := range []string{"1m5zypx"} { + for _, name := range []string{"1m5zypx", "7m5zypx"} { if name == "1m5zypx" { By("creating dynamic secret resource") cmd = exec.Command( @@ -282,16 +312,37 @@ var _ = Describe("controller", Ordered, func() { ) _, err = utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("creating a LagoonTask resource") + cmd = exec.Command( + utils.Kubectl(), + "apply", + "-f", + fmt.Sprintf("test/e2e/testdata/lagoon-task-%s.yaml", name), + ) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } else { + By("creating a LagoonTask resource via rabbitmq") + cmd := exec.Command( + "curl", + "-s", + "-u", + "guest:guest", + "-H", + "'Accept: application/json'", + "-H", + "'Content-Type:application/json'", + "-X", + "POST", + "-d", + fmt.Sprintf("@test/e2e/testdata/lagoon-task-%s.json", name), + "http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish", + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) } - By("creating a LagoonTask resource") - cmd = exec.Command( - utils.Kubectl(), - "apply", - "-f", - fmt.Sprintf("test/e2e/testdata/lagoon-task-%s.yaml", name), - ) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + time.Sleep(10 * time.Second) By("validating that the lagoon-task pod completes as expected") verifyTaskPodCompletes := func() error { @@ -331,7 +382,7 @@ var _ = Describe("controller", Ordered, func() { err := utils.InstallK8upCRD(name) ExpectWithOffset(1, err).NotTo(HaveOccurred()) - time.Sleep(5 * time.Second) + time.Sleep(10 * time.Second) By(fmt.Sprintf("creating a %s restore task via rabbitmq", name)) cmd = exec.Command( @@ -400,40 +451,54 @@ var _ = Describe("controller", Ordered, func() { } EventuallyWithOffset(1, verifyRobotCredentialsRotate, duration, interval).Should(Succeed()) - By("delete environment via rabbitmq") - cmd = exec.Command( - "curl", - "-s", - "-u", - "guest:guest", - "-H", - "'Accept: application/json'", - "-H", - "'Content-Type:application/json'", - "-X", - "POST", - "-d", - "@test/e2e/testdata/remove-environment.json", - "http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish", - ) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + // this tests the build qos functionality by starting 4 builds with a qos of max 3 (defined in the controller config provided by kustomize) + testBuildQoS(timeout, duration, interval) - By("validating that the namespace deletes") - verifyNamespaceRemoved := func() error { - cmd = exec.Command(utils.Kubectl(), "get", - "namespace", "nginx-example-main", "-o", "jsonpath={.status.phase}", + time.Sleep(5 * time.Second) + + // this tests the task qos functionality by starting 4 tasks with a qos of max 3 (defined in the controller config provided by kustomize) + testTaskQoS(timeout, duration, interval) + + time.Sleep(5 * time.Second) + + By("delete environments via rabbitmq") + for _, env := range []string{"main", "dev1", "dev2", "dev3"} { + cmd = exec.Command( + "curl", + "-s", + "-u", + "guest:guest", + "-H", + "'Accept: application/json'", + "-H", + "'Content-Type:application/json'", + "-X", + "POST", + "-d", + fmt.Sprintf("@test/e2e/testdata/remove-environment-%s.json", env), + "http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish", ) - status, err := utils.Run(cmd) - if err == nil { - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - if string(status) == "Active" || string(status) == "Terminating" { - return fmt.Errorf("namespace in %s status\n", status) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } + By("validating that the namespaces are deleted") + for _, env := range []string{"main", "dev1", "dev2", "dev3"} { + verifyNamespaceRemoved := func() error { + cmd = exec.Command(utils.Kubectl(), "get", + "namespace", fmt.Sprintf("nginx-example-%s", env), + "-o", "jsonpath={.status.phase}", + ) + status, err := utils.Run(cmd) + if err == nil { + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + if string(status) == "Active" || string(status) == "Terminating" { + return fmt.Errorf("namespace in %s status\n", status) + } } + return nil } - return nil + EventuallyWithOffset(1, verifyNamespaceRemoved, duration, interval).Should(Succeed()) } - EventuallyWithOffset(1, verifyNamespaceRemoved, duration, interval).Should(Succeed()) By("validating that unauthenticated metrics requests fail") runCmd := `curl -s -k https://remote-controller-controller-manager-metrics-service.remote-controller-system.svc.cluster.local:8443/metrics | grep -v "#" | grep "lagoon_"` diff --git a/test/e2e/testdata/buildqos/lagoon-build-xpyz5m1.json b/test/e2e/testdata/buildqos/lagoon-build-xpyz5m1.json new file mode 100644 index 00000000..b727eb9b --- /dev/null +++ b/test/e2e/testdata/buildqos/lagoon-build-xpyz5m1.json @@ -0,0 +1,47 @@ +{ + "properties":{ + "delivery_mode":2 + }, + "routing_key":"ci-local-controller-kubernetes:builddeploy", + "payload":"{ + \"metadata\": { + \"name\": \"lagoon-build-xpyz5m1\" + }, + \"spec\": { + \"build\": { + \"ci\": \"true\", + \"type\": \"branch\" + }, + \"gitReference\": \"origin\/main\", + \"project\": { + \"name\": \"nginx-example\", + \"environment\": \"main\", + \"uiLink\": \"https:\/\/dashboard.amazeeio.cloud\/projects\/project\/project-environment\/deployments\/lagoon-build-xpyz5m1\", + \"routerPattern\": \"main-nginx-example\", + \"environmentType\": \"production\", + \"productionEnvironment\": \"main\", + \"standbyEnvironment\": \"\", + \"gitUrl\": \"https:\/\/github.com\/shreddedbacon\/lagoon-nginx-example.git\", + \"deployTarget\": \"kind\", + \"organization\": { + \"id\": 123, + \"name\": \"test-org\" + }, + \"projectSecret\": \"4d6e7dd0f013a75d62a0680139fa82d350c2a1285f43f867535bad1143f228b1\", + \"key\": \"LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlDWFFJQkFBS0JnUUNjc1g2RG5KNXpNb0RqQ2R6a1JFOEg2TEh2TDQzaUhsekJLTWo4T1VNV05ZZG5YekdqCkR5Mkp1anQ3ZDNlMTVLeC8zOFo5UzJLdHNnVFVtWi9lUlRQSTdabE1idHRJK250UmtyblZLblBWNzhEeEFKNW8KTGZtQndmdWE2MnlVYnl0cnpYQ2pwVVJrQUlBMEZiR2VqS2Rvd3cxcnZGMzJoZFUzQ3ZIcG5rKzE2d0lEQVFBQgpBb0dCQUkrV0dyL1NDbVMzdCtIVkRPVGtMNk9vdVR6Y1QrRVFQNkVGbGIrRFhaV0JjZFhwSnB3c2NXZFBEK2poCkhnTEJUTTFWS3hkdnVEcEE4aW83cUlMTzJWYm1MeGpNWGk4TUdwY212dXJFNVJydTZTMXJzRDl2R0c5TGxoR3UKK0pUSmViMVdaZFduWFZ2am5LbExrWEV1eUthbXR2Z253Um5xNld5V05OazJ6SktoQWtFQThFenpxYnowcFVuTApLc241K2k0NUdoRGVpRTQvajRtamo1b1FHVzJxbUZWT2pHaHR1UGpaM2lwTis0RGlTRkFyMkl0b2VlK085d1pyCkRINHBkdU5YOFFKQkFLYnVOQ3dXK29sYXA4R2pUSk1TQjV1MW8wMVRHWFdFOGhVZG1leFBBdjl0cTBBT0gzUUQKUTIrM0RsaVY0ektoTlMra2xaSkVjNndzS0YyQmJIby81NXNDUVFETlBJd24vdERja3loSkJYVFJyc1RxZEZuOApCUWpZYVhBZTZEQ3o1eXg3S3ZFSmp1K1h1a01xTXV1ajBUSnpITFkySHVzK3FkSnJQVG9VMDNSS3JHV2hBa0JFCnB3aXI3Vk5pYy9jMFN2MnVLcWNZWWM1a2ViMnB1R0I3VUs1Q0lvaWdGakZzNmFJRDYyZXJwVVJ3S0V6RlFNbUgKNjQ5Y0ZXemhMVlA0aU1iZFREVHJBa0FFMTZXU1A3WXBWOHV1eFVGMGV0L3lFR3dURVpVU2R1OEppSTBHN0tqagpqcVR6RjQ3YkJZc0pIYTRYcWpVb2E3TXgwcS9FSUtRWkJ2NGFvQm42bGFOQwotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQ==\", + \"monitoring\": { + \"contact\": \"1234\", + \"statuspageID\": \"1234\" + }, + \"variables\": { + \"project\": \"W3sibmFtZSI6IkxBR09PTl9TWVNURU1fUk9VVEVSX1BBVFRFUk4iLCJ2YWx1ZSI6IiR7ZW52aXJvbm1lbnR9LiR7cHJvamVjdH0uZXhhbXBsZS5jb20iLCJzY29wZSI6ImludGVybmFsX3N5c3RlbSJ9XQ==\", + \"environment\": \"W10=\" + } + }, + \"branch\": { + \"name\": \"main\" + } + } + }", + "payload_encoding":"string" +} \ No newline at end of file diff --git a/test/e2e/testdata/buildqos/lagoon-build-xpyz5m2.json b/test/e2e/testdata/buildqos/lagoon-build-xpyz5m2.json new file mode 100644 index 00000000..9404ffd1 --- /dev/null +++ b/test/e2e/testdata/buildqos/lagoon-build-xpyz5m2.json @@ -0,0 +1,47 @@ +{ + "properties":{ + "delivery_mode":2 + }, + "routing_key":"ci-local-controller-kubernetes:builddeploy", + "payload":"{ + \"metadata\": { + \"name\": \"lagoon-build-xpyz5m2\" + }, + \"spec\": { + \"build\": { + \"ci\": \"true\", + \"type\": \"branch\" + }, + \"gitReference\": \"origin\/test\", + \"project\": { + \"name\": \"nginx-example\", + \"environment\": \"dev1\", + \"uiLink\": \"https:\/\/dashboard.amazeeio.cloud\/projects\/project\/project-environment\/deployments\/lagoon-build-xpyz5m2\", + \"routerPattern\": \"dev1-nginx-example\", + \"environmentType\": \"development\", + \"productionEnvironment\": \"main\", + \"standbyEnvironment\": \"\", + \"gitUrl\": \"https:\/\/github.com\/shreddedbacon\/lagoon-nginx-example.git\", + \"deployTarget\": \"kind\", + \"organization\": { + \"id\": 123, + \"name\": \"test-org\" + }, + \"projectSecret\": \"4d6e7dd0f013a75d62a0680139fa82d350c2a1285f43f867535bad1143f228b1\", + \"key\": \"LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlDWFFJQkFBS0JnUUNjc1g2RG5KNXpNb0RqQ2R6a1JFOEg2TEh2TDQzaUhsekJLTWo4T1VNV05ZZG5YekdqCkR5Mkp1anQ3ZDNlMTVLeC8zOFo5UzJLdHNnVFVtWi9lUlRQSTdabE1idHRJK250UmtyblZLblBWNzhEeEFKNW8KTGZtQndmdWE2MnlVYnl0cnpYQ2pwVVJrQUlBMEZiR2VqS2Rvd3cxcnZGMzJoZFUzQ3ZIcG5rKzE2d0lEQVFBQgpBb0dCQUkrV0dyL1NDbVMzdCtIVkRPVGtMNk9vdVR6Y1QrRVFQNkVGbGIrRFhaV0JjZFhwSnB3c2NXZFBEK2poCkhnTEJUTTFWS3hkdnVEcEE4aW83cUlMTzJWYm1MeGpNWGk4TUdwY212dXJFNVJydTZTMXJzRDl2R0c5TGxoR3UKK0pUSmViMVdaZFduWFZ2am5LbExrWEV1eUthbXR2Z253Um5xNld5V05OazJ6SktoQWtFQThFenpxYnowcFVuTApLc241K2k0NUdoRGVpRTQvajRtamo1b1FHVzJxbUZWT2pHaHR1UGpaM2lwTis0RGlTRkFyMkl0b2VlK085d1pyCkRINHBkdU5YOFFKQkFLYnVOQ3dXK29sYXA4R2pUSk1TQjV1MW8wMVRHWFdFOGhVZG1leFBBdjl0cTBBT0gzUUQKUTIrM0RsaVY0ektoTlMra2xaSkVjNndzS0YyQmJIby81NXNDUVFETlBJd24vdERja3loSkJYVFJyc1RxZEZuOApCUWpZYVhBZTZEQ3o1eXg3S3ZFSmp1K1h1a01xTXV1ajBUSnpITFkySHVzK3FkSnJQVG9VMDNSS3JHV2hBa0JFCnB3aXI3Vk5pYy9jMFN2MnVLcWNZWWM1a2ViMnB1R0I3VUs1Q0lvaWdGakZzNmFJRDYyZXJwVVJ3S0V6RlFNbUgKNjQ5Y0ZXemhMVlA0aU1iZFREVHJBa0FFMTZXU1A3WXBWOHV1eFVGMGV0L3lFR3dURVpVU2R1OEppSTBHN0tqagpqcVR6RjQ3YkJZc0pIYTRYcWpVb2E3TXgwcS9FSUtRWkJ2NGFvQm42bGFOQwotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQ==\", + \"monitoring\": { + \"contact\": \"1234\", + \"statuspageID\": \"1234\" + }, + \"variables\": { + \"project\": \"W3sibmFtZSI6IkxBR09PTl9TWVNURU1fUk9VVEVSX1BBVFRFUk4iLCJ2YWx1ZSI6IiR7ZW52aXJvbm1lbnR9LiR7cHJvamVjdH0uZXhhbXBsZS5jb20iLCJzY29wZSI6ImludGVybmFsX3N5c3RlbSJ9XQ==\", + \"environment\": \"W10=\" + } + }, + \"branch\": { + \"name\": \"dev1\" + } + } + }", + "payload_encoding":"string" +} \ No newline at end of file diff --git a/test/e2e/testdata/buildqos/lagoon-build-xpyz5m3.json b/test/e2e/testdata/buildqos/lagoon-build-xpyz5m3.json new file mode 100644 index 00000000..38179a3e --- /dev/null +++ b/test/e2e/testdata/buildqos/lagoon-build-xpyz5m3.json @@ -0,0 +1,47 @@ +{ + "properties":{ + "delivery_mode":2 + }, + "routing_key":"ci-local-controller-kubernetes:builddeploy", + "payload":"{ + \"metadata\": { + \"name\": \"lagoon-build-xpyz5m3\" + }, + \"spec\": { + \"build\": { + \"ci\": \"true\", + \"type\": \"branch\" + }, + \"gitReference\": \"origin\/test\", + \"project\": { + \"name\": \"nginx-example\", + \"environment\": \"dev2\", + \"uiLink\": \"https:\/\/dashboard.amazeeio.cloud\/projects\/project\/project-environment\/deployments\/lagoon-build-xpyz5m3\", + \"routerPattern\": \"dev2-nginx-example\", + \"environmentType\": \"development\", + \"productionEnvironment\": \"main\", + \"standbyEnvironment\": \"\", + \"gitUrl\": \"https:\/\/github.com\/shreddedbacon\/lagoon-nginx-example.git\", + \"deployTarget\": \"kind\", + \"organization\": { + \"id\": 123, + \"name\": \"test-org\" + }, + \"projectSecret\": \"4d6e7dd0f013a75d62a0680139fa82d350c2a1285f43f867535bad1143f228b1\", + \"key\": \"LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlDWFFJQkFBS0JnUUNjc1g2RG5KNXpNb0RqQ2R6a1JFOEg2TEh2TDQzaUhsekJLTWo4T1VNV05ZZG5YekdqCkR5Mkp1anQ3ZDNlMTVLeC8zOFo5UzJLdHNnVFVtWi9lUlRQSTdabE1idHRJK250UmtyblZLblBWNzhEeEFKNW8KTGZtQndmdWE2MnlVYnl0cnpYQ2pwVVJrQUlBMEZiR2VqS2Rvd3cxcnZGMzJoZFUzQ3ZIcG5rKzE2d0lEQVFBQgpBb0dCQUkrV0dyL1NDbVMzdCtIVkRPVGtMNk9vdVR6Y1QrRVFQNkVGbGIrRFhaV0JjZFhwSnB3c2NXZFBEK2poCkhnTEJUTTFWS3hkdnVEcEE4aW83cUlMTzJWYm1MeGpNWGk4TUdwY212dXJFNVJydTZTMXJzRDl2R0c5TGxoR3UKK0pUSmViMVdaZFduWFZ2am5LbExrWEV1eUthbXR2Z253Um5xNld5V05OazJ6SktoQWtFQThFenpxYnowcFVuTApLc241K2k0NUdoRGVpRTQvajRtamo1b1FHVzJxbUZWT2pHaHR1UGpaM2lwTis0RGlTRkFyMkl0b2VlK085d1pyCkRINHBkdU5YOFFKQkFLYnVOQ3dXK29sYXA4R2pUSk1TQjV1MW8wMVRHWFdFOGhVZG1leFBBdjl0cTBBT0gzUUQKUTIrM0RsaVY0ektoTlMra2xaSkVjNndzS0YyQmJIby81NXNDUVFETlBJd24vdERja3loSkJYVFJyc1RxZEZuOApCUWpZYVhBZTZEQ3o1eXg3S3ZFSmp1K1h1a01xTXV1ajBUSnpITFkySHVzK3FkSnJQVG9VMDNSS3JHV2hBa0JFCnB3aXI3Vk5pYy9jMFN2MnVLcWNZWWM1a2ViMnB1R0I3VUs1Q0lvaWdGakZzNmFJRDYyZXJwVVJ3S0V6RlFNbUgKNjQ5Y0ZXemhMVlA0aU1iZFREVHJBa0FFMTZXU1A3WXBWOHV1eFVGMGV0L3lFR3dURVpVU2R1OEppSTBHN0tqagpqcVR6RjQ3YkJZc0pIYTRYcWpVb2E3TXgwcS9FSUtRWkJ2NGFvQm42bGFOQwotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQ==\", + \"monitoring\": { + \"contact\": \"1234\", + \"statuspageID\": \"1234\" + }, + \"variables\": { + \"project\": \"W3sibmFtZSI6IkxBR09PTl9TWVNURU1fUk9VVEVSX1BBVFRFUk4iLCJ2YWx1ZSI6IiR7ZW52aXJvbm1lbnR9LiR7cHJvamVjdH0uZXhhbXBsZS5jb20iLCJzY29wZSI6ImludGVybmFsX3N5c3RlbSJ9XQ==\", + \"environment\": \"W10=\" + } + }, + \"branch\": { + \"name\": \"dev2\" + } + } + }", + "payload_encoding":"string" +} \ No newline at end of file diff --git a/test/e2e/testdata/buildqos/lagoon-build-xpyz5m4.json b/test/e2e/testdata/buildqos/lagoon-build-xpyz5m4.json new file mode 100644 index 00000000..44dd4b88 --- /dev/null +++ b/test/e2e/testdata/buildqos/lagoon-build-xpyz5m4.json @@ -0,0 +1,47 @@ +{ + "properties":{ + "delivery_mode":2 + }, + "routing_key":"ci-local-controller-kubernetes:builddeploy", + "payload":"{ + \"metadata\": { + \"name\": \"lagoon-build-xpyz5m4\" + }, + \"spec\": { + \"build\": { + \"ci\": \"true\", + \"type\": \"branch\" + }, + \"gitReference\": \"origin\/test\", + \"project\": { + \"name\": \"nginx-example\", + \"environment\": \"dev3\", + \"uiLink\": \"https:\/\/dashboard.amazeeio.cloud\/projects\/project\/project-environment\/deployments\/lagoon-build-xpyz5m4\", + \"routerPattern\": \"dev3-nginx-example\", + \"environmentType\": \"development\", + \"productionEnvironment\": \"main\", + \"standbyEnvironment\": \"\", + \"gitUrl\": \"https:\/\/github.com\/shreddedbacon\/lagoon-nginx-example.git\", + \"deployTarget\": \"kind\", + \"organization\": { + \"id\": 123, + \"name\": \"test-org\" + }, + \"projectSecret\": \"4d6e7dd0f013a75d62a0680139fa82d350c2a1285f43f867535bad1143f228b1\", + \"key\": \"LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlDWFFJQkFBS0JnUUNjc1g2RG5KNXpNb0RqQ2R6a1JFOEg2TEh2TDQzaUhsekJLTWo4T1VNV05ZZG5YekdqCkR5Mkp1anQ3ZDNlMTVLeC8zOFo5UzJLdHNnVFVtWi9lUlRQSTdabE1idHRJK250UmtyblZLblBWNzhEeEFKNW8KTGZtQndmdWE2MnlVYnl0cnpYQ2pwVVJrQUlBMEZiR2VqS2Rvd3cxcnZGMzJoZFUzQ3ZIcG5rKzE2d0lEQVFBQgpBb0dCQUkrV0dyL1NDbVMzdCtIVkRPVGtMNk9vdVR6Y1QrRVFQNkVGbGIrRFhaV0JjZFhwSnB3c2NXZFBEK2poCkhnTEJUTTFWS3hkdnVEcEE4aW83cUlMTzJWYm1MeGpNWGk4TUdwY212dXJFNVJydTZTMXJzRDl2R0c5TGxoR3UKK0pUSmViMVdaZFduWFZ2am5LbExrWEV1eUthbXR2Z253Um5xNld5V05OazJ6SktoQWtFQThFenpxYnowcFVuTApLc241K2k0NUdoRGVpRTQvajRtamo1b1FHVzJxbUZWT2pHaHR1UGpaM2lwTis0RGlTRkFyMkl0b2VlK085d1pyCkRINHBkdU5YOFFKQkFLYnVOQ3dXK29sYXA4R2pUSk1TQjV1MW8wMVRHWFdFOGhVZG1leFBBdjl0cTBBT0gzUUQKUTIrM0RsaVY0ektoTlMra2xaSkVjNndzS0YyQmJIby81NXNDUVFETlBJd24vdERja3loSkJYVFJyc1RxZEZuOApCUWpZYVhBZTZEQ3o1eXg3S3ZFSmp1K1h1a01xTXV1ajBUSnpITFkySHVzK3FkSnJQVG9VMDNSS3JHV2hBa0JFCnB3aXI3Vk5pYy9jMFN2MnVLcWNZWWM1a2ViMnB1R0I3VUs1Q0lvaWdGakZzNmFJRDYyZXJwVVJ3S0V6RlFNbUgKNjQ5Y0ZXemhMVlA0aU1iZFREVHJBa0FFMTZXU1A3WXBWOHV1eFVGMGV0L3lFR3dURVpVU2R1OEppSTBHN0tqagpqcVR6RjQ3YkJZc0pIYTRYcWpVb2E3TXgwcS9FSUtRWkJ2NGFvQm42bGFOQwotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQ==\", + \"monitoring\": { + \"contact\": \"1234\", + \"statuspageID\": \"1234\" + }, + \"variables\": { + \"project\": \"W3sibmFtZSI6IkxBR09PTl9TWVNURU1fUk9VVEVSX1BBVFRFUk4iLCJ2YWx1ZSI6IiR7ZW52aXJvbm1lbnR9LiR7cHJvamVjdH0uZXhhbXBsZS5jb20iLCJzY29wZSI6ImludGVybmFsX3N5c3RlbSJ9XQ==\", + \"environment\": \"W10=\" + } + }, + \"branch\": { + \"name\": \"dev3\" + } + } + }", + "payload_encoding":"string" +} \ No newline at end of file diff --git a/test/e2e/testdata/lagoon-task-7m5zypx.json b/test/e2e/testdata/lagoon-task-7m5zypx.json new file mode 100644 index 00000000..80baa0fd --- /dev/null +++ b/test/e2e/testdata/lagoon-task-7m5zypx.json @@ -0,0 +1,34 @@ +{ + "properties":{ + "delivery_mode":2 + }, + "routing_key":"ci-local-controller-kubernetes:jobs", + "payload":"{ + \"key\": \"deploytarget:task:advanced\", + \"project\": { + \"id\": 1, + \"name\": \"nginx-example\", + \"variables\": { + \"project\": \"W3sibmFtZSI6IkxBR09PTl9TWVNURU1fUk9VVEVSX1BBVFRFUk4iLCJ2YWx1ZSI6IiR7ZW52aXJvbm1lbnR9LiR7cHJvamVjdH0uZXhhbXBsZS5jb20iLCJzY29wZSI6ImludGVybmFsX3N5c3RlbSJ9XQ==\", + \"environment\": \"W10=\" + } + }, + \"environment\": { + \"id\": 1, + \"environmentType\": \"production\", + \"name\": \"main\", + \"project\": \"nginx-example\" + }, + \"task\": { + \"id\": \"1\", + \"name\": \"lagoon-task-7m5zypx\", + \"taskName\": \"lagoon-task-7m5zypx\", + \"service\": \"nginx\", + \"command\": \"env\" + }, + \"misc\": { + \"name\": \"lagoon-task-7m5zypx\" + } + }", + "payload_encoding":"string" +} \ No newline at end of file diff --git a/test/e2e/testdata/remove-environment-dev1.json b/test/e2e/testdata/remove-environment-dev1.json new file mode 100644 index 00000000..54a19c9c --- /dev/null +++ b/test/e2e/testdata/remove-environment-dev1.json @@ -0,0 +1,10 @@ +{"properties":{"delivery_mode":2},"routing_key":"ci-local-controller-kubernetes:remove", + "payload":"{ + \"projectName\": \"nginx-example\", + \"type\":\"branch\", + \"forceDeleteProductionEnvironment\":true, + \"branch\":\"dev1\", + \"openshiftProjectName\":\"nginx-example-dev1\" + }", +"payload_encoding":"string" +} \ No newline at end of file diff --git a/test/e2e/testdata/remove-environment-dev2.json b/test/e2e/testdata/remove-environment-dev2.json new file mode 100644 index 00000000..71eb7a08 --- /dev/null +++ b/test/e2e/testdata/remove-environment-dev2.json @@ -0,0 +1,10 @@ +{"properties":{"delivery_mode":2},"routing_key":"ci-local-controller-kubernetes:remove", + "payload":"{ + \"projectName\": \"nginx-example\", + \"type\":\"branch\", + \"forceDeleteProductionEnvironment\":true, + \"branch\":\"dev2\", + \"openshiftProjectName\":\"nginx-example-dev2\" + }", +"payload_encoding":"string" +} \ No newline at end of file diff --git a/test/e2e/testdata/remove-environment-dev3.json b/test/e2e/testdata/remove-environment-dev3.json new file mode 100644 index 00000000..480077f4 --- /dev/null +++ b/test/e2e/testdata/remove-environment-dev3.json @@ -0,0 +1,10 @@ +{"properties":{"delivery_mode":2},"routing_key":"ci-local-controller-kubernetes:remove", + "payload":"{ + \"projectName\": \"nginx-example\", + \"type\":\"branch\", + \"forceDeleteProductionEnvironment\":true, + \"branch\":\"dev3\", + \"openshiftProjectName\":\"nginx-example-dev3\" + }", +"payload_encoding":"string" +} \ No newline at end of file diff --git a/test/e2e/testdata/remove-environment.json b/test/e2e/testdata/remove-environment-main.json similarity index 100% rename from test/e2e/testdata/remove-environment.json rename to test/e2e/testdata/remove-environment-main.json diff --git a/test/e2e/testdata/taskqos/lagoon-task-1xpyz5m.json b/test/e2e/testdata/taskqos/lagoon-task-1xpyz5m.json new file mode 100644 index 00000000..f6de879f --- /dev/null +++ b/test/e2e/testdata/taskqos/lagoon-task-1xpyz5m.json @@ -0,0 +1,35 @@ +{ + "properties":{ + "delivery_mode":2 + }, + "routing_key":"ci-local-controller-kubernetes:misc", + "payload":"{ + \"advancedTask\": { + \"runnerImage\": \"example.com/test-task-image:v0.0.1\" + }, + \"key\": \"deploytarget:task:advanced\", + \"project\": { + \"id\": 1, + \"name\": \"nginx-example\", + \"variables\": { + \"project\": \"W3sibmFtZSI6IkxBR09PTl9TWVNURU1fUk9VVEVSX1BBVFRFUk4iLCJ2YWx1ZSI6IiR7ZW52aXJvbm1lbnR9LiR7cHJvamVjdH0uZXhhbXBsZS5jb20iLCJzY29wZSI6ImludGVybmFsX3N5c3RlbSJ9XQ==\", + \"environment\": \"W10=\" + } + }, + \"environment\": { + \"id\": 1, + \"environmentType\": \"production\", + \"name\": \"main\", + \"project\": \"nginx-example\" + }, + \"task\": { + \"id\": \"1\", + \"name\": \"lagoon-task-1xpyz5m\", + \"taskName\": \"lagoon-task-1xpyz5m\" + }, + \"misc\": { + \"name\": \"lagoon-task-1xpyz5m\" + } + }", + "payload_encoding":"string" +} \ No newline at end of file diff --git a/test/e2e/testdata/taskqos/lagoon-task-2xpyz5m.json b/test/e2e/testdata/taskqos/lagoon-task-2xpyz5m.json new file mode 100644 index 00000000..14344366 --- /dev/null +++ b/test/e2e/testdata/taskqos/lagoon-task-2xpyz5m.json @@ -0,0 +1,35 @@ +{ + "properties":{ + "delivery_mode":2 + }, + "routing_key":"ci-local-controller-kubernetes:misc", + "payload":"{ + \"advancedTask\": { + \"runnerImage\": \"example.com/test-task-image:v0.0.1\" + }, + \"key\": \"deploytarget:task:advanced\", + \"project\": { + \"id\": 1, + \"name\": \"nginx-example\", + \"variables\": { + \"project\": \"W3sibmFtZSI6IkxBR09PTl9TWVNURU1fUk9VVEVSX1BBVFRFUk4iLCJ2YWx1ZSI6IiR7ZW52aXJvbm1lbnR9LiR7cHJvamVjdH0uZXhhbXBsZS5jb20iLCJzY29wZSI6ImludGVybmFsX3N5c3RlbSJ9XQ==\", + \"environment\": \"W10=\" + } + }, + \"environment\": { + \"id\": 1, + \"environmentType\": \"production\", + \"name\": \"dev1\", + \"project\": \"nginx-example\" + }, + \"task\": { + \"id\": \"1\", + \"name\": \"lagoon-task-2xpyz5m\", + \"taskName\": \"lagoon-task-2xpyz5m\" + }, + \"misc\": { + \"name\": \"lagoon-task-2xpyz5m\" + } + }", + "payload_encoding":"string" +} \ No newline at end of file diff --git a/test/e2e/testdata/taskqos/lagoon-task-3xpyz5m.json b/test/e2e/testdata/taskqos/lagoon-task-3xpyz5m.json new file mode 100644 index 00000000..756275ce --- /dev/null +++ b/test/e2e/testdata/taskqos/lagoon-task-3xpyz5m.json @@ -0,0 +1,35 @@ +{ + "properties":{ + "delivery_mode":2 + }, + "routing_key":"ci-local-controller-kubernetes:misc", + "payload":"{ + \"advancedTask\": { + \"runnerImage\": \"example.com/test-task-image:v0.0.1\" + }, + \"key\": \"deploytarget:task:advanced\", + \"project\": { + \"id\": 1, + \"name\": \"nginx-example\", + \"variables\": { + \"project\": \"W3sibmFtZSI6IkxBR09PTl9TWVNURU1fUk9VVEVSX1BBVFRFUk4iLCJ2YWx1ZSI6IiR7ZW52aXJvbm1lbnR9LiR7cHJvamVjdH0uZXhhbXBsZS5jb20iLCJzY29wZSI6ImludGVybmFsX3N5c3RlbSJ9XQ==\", + \"environment\": \"W10=\" + } + }, + \"environment\": { + \"id\": 1, + \"environmentType\": \"production\", + \"name\": \"dev2\", + \"project\": \"nginx-example\" + }, + \"task\": { + \"id\": \"1\", + \"name\": \"lagoon-task-3xpyz5m\", + \"taskName\": \"lagoon-task-3xpyz5m\" + }, + \"misc\": { + \"name\": \"lagoon-task-3xpyz5m\" + } + }", + "payload_encoding":"string" +} \ No newline at end of file diff --git a/test/e2e/testdata/taskqos/lagoon-task-4xpyz5m.json b/test/e2e/testdata/taskqos/lagoon-task-4xpyz5m.json new file mode 100644 index 00000000..22976dd0 --- /dev/null +++ b/test/e2e/testdata/taskqos/lagoon-task-4xpyz5m.json @@ -0,0 +1,35 @@ +{ + "properties":{ + "delivery_mode":2 + }, + "routing_key":"ci-local-controller-kubernetes:misc", + "payload":"{ + \"advancedTask\": { + \"runnerImage\": \"example.com/test-task-image:v0.0.1\" + }, + \"key\": \"deploytarget:task:advanced\", + \"project\": { + \"id\": 1, + \"name\": \"nginx-example\", + \"variables\": { + \"project\": \"W3sibmFtZSI6IkxBR09PTl9TWVNURU1fUk9VVEVSX1BBVFRFUk4iLCJ2YWx1ZSI6IiR7ZW52aXJvbm1lbnR9LiR7cHJvamVjdH0uZXhhbXBsZS5jb20iLCJzY29wZSI6ImludGVybmFsX3N5c3RlbSJ9XQ==\", + \"environment\": \"W10=\" + } + }, + \"environment\": { + \"id\": 1, + \"environmentType\": \"production\", + \"name\": \"dev3\", + \"project\": \"nginx-example\" + }, + \"task\": { + \"id\": \"1\", + \"name\": \"lagoon-task-4xpyz5m\", + \"taskName\": \"lagoon-task-4xpyz5m\" + }, + \"misc\": { + \"name\": \"lagoon-task-4xpyz5m\" + } + }", + "payload_encoding":"string" +} \ No newline at end of file diff --git a/test/utils/utils.go b/test/utils/utils.go index 66dd6a2c..d118c61b 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -34,6 +34,10 @@ func warnError(err error) { fmt.Fprintf(ginkgo.GinkgoWriter, "warning: %v\n", err) } +func infoError(err error) { + fmt.Fprintf(ginkgo.GinkgoWriter, "info: %v\n", err) +} + var kubectlPath, kindPath string func init() { @@ -69,6 +73,70 @@ func StopLocalServices() { } } +// CleanupNamespace cleans up a namespace and all potentially stuck resources +func CleanupNamespace(namespace string) { + cmd := exec.Command("kubectl", "delete", "ns", namespace, "--timeout=30s") + if _, err := Run(cmd); err != nil { + infoError(err) + } + // check for builds + cmd = exec.Command("kubectl", "-n", namespace, "get", "lagoonbuilds", + "-o", "go-template={{ range .items }}"+ + "{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}", + ) + output, err := Run(cmd) + if err != nil { + infoError(err) + } + builds := GetNonEmptyLines(string(output)) + if len(builds) > 0 { + for _, build := range builds { + fmt.Fprintf(ginkgo.GinkgoWriter, "info: %v\n", "patching stuck builds for removal") + cmd = exec.Command("kubectl", "-n", namespace, "patch", "lagoonbuild", + build, "--type", "merge", + "-p", "{\"metadata\":{\"finalizers\":null}}", + ) + _, err := Run(cmd) + if err != nil { + infoError(err) + } + } + cmd = exec.Command("kubectl", "delete", "ns", namespace) + if _, err := Run(cmd); err != nil { + infoError(err) + } + } + // check for tasks + cmd = exec.Command("kubectl", "-n", namespace, "get", "lagoontasks", + "-o", "go-template={{ range .items }}"+ + "{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}", + ) + tasksoutput, err := Run(cmd) + if err != nil { + infoError(err) + } + tasks := GetNonEmptyLines(string(tasksoutput)) + if len(tasks) > 0 { + for _, task := range tasks { + fmt.Fprintf(ginkgo.GinkgoWriter, "info: %v\n", "patching stuck tasks for removal") + cmd = exec.Command("kubectl", "-n", namespace, "patch", "lagoontask", + task, "--type", "merge", + "-p", "{\"metadata\":{\"finalizers\":null}}", + ) + _, err := Run(cmd) + if err != nil { + infoError(err) + } + } + cmd = exec.Command("kubectl", "delete", "ns", namespace) + if _, err := Run(cmd); err != nil { + infoError(err) + } + } +} + // InstallBulkStorage installs the bulk storage class. func InstallBulkStorage() error { cmd := exec.Command(kubectlPath, "apply", "-f", "test/e2e/testdata/bulk-storageclass.yaml") From 97eef5f680d2f07467f313102955686040d51d16 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Tue, 14 Jan 2025 13:53:25 +1100 Subject: [PATCH 3/6] fix: messaging and references --- .../v1beta2/podmonitor_taskhandlers.go | 35 ++++++++++++++++--- internal/messenger/consumer.go | 8 ++--- internal/messenger/tasks_restore.go | 1 - 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/internal/controllers/v1beta2/podmonitor_taskhandlers.go b/internal/controllers/v1beta2/podmonitor_taskhandlers.go index 6ecf221f..64c73282 100644 --- a/internal/controllers/v1beta2/podmonitor_taskhandlers.go +++ b/internal/controllers/v1beta2/podmonitor_taskhandlers.go @@ -126,7 +126,7 @@ func (r *LagoonMonitorReconciler) taskLogsToLagoonLogs(opLog logr.Logger, Environment: lagoonTask.Spec.Environment.Name, JobName: lagoonTask.ObjectMeta.Name, JobStatus: condition, - RemoteID: string(jobPod.ObjectMeta.UID), + RemoteID: string(lagoonTask.ObjectMeta.UID), Key: lagoonTask.Spec.Key, Cluster: r.LagoonTargetName, }, @@ -152,6 +152,15 @@ Logs on pod %s, assigned to cluster %s // really matter if we don't smootly transition in what we send back to lagoon return err } + if r.EnableDebug { + opLog.Info( + fmt.Sprintf( + "Published event %s for %s to lagoon-logs exchange", + fmt.Sprintf("task-logs:job-kubernetes:%s", jobPod.ObjectMeta.Name), + jobPod.ObjectMeta.Name, + ), + ) + } // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } @@ -186,7 +195,7 @@ func (r *LagoonMonitorReconciler) updateLagoonTask(opLog logr.Logger, ProjectID: lagoonTask.Spec.Project.ID, JobName: lagoonTask.ObjectMeta.Name, JobStatus: condition, - RemoteID: string(jobPod.ObjectMeta.UID), + RemoteID: string(lagoonTask.ObjectMeta.UID), Key: lagoonTask.Spec.Key, Cluster: r.LagoonTargetName, }, @@ -217,6 +226,14 @@ func (r *LagoonMonitorReconciler) updateLagoonTask(opLog logr.Logger, // really matter if we don't smootly transition in what we send back to lagoon return err } + if r.EnableDebug { + opLog.Info( + fmt.Sprintf( + "Published task update message for %s to lagoon-tasks:controller queue", + jobPod.ObjectMeta.Name, + ), + ) + } // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } @@ -226,7 +243,6 @@ func (r *LagoonMonitorReconciler) updateLagoonTask(opLog logr.Logger, // taskStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging func (r *LagoonMonitorReconciler) taskStatusLogsToLagoonLogs(opLog logr.Logger, lagoonTask *lagooncrd.LagoonTask, - jobPod *corev1.Pod, condition string, ) error { if r.EnableMQ && lagoonTask != nil { @@ -242,7 +258,7 @@ func (r *LagoonMonitorReconciler) taskStatusLogsToLagoonLogs(opLog logr.Logger, ProjectID: lagoonTask.Spec.Project.ID, JobName: lagoonTask.ObjectMeta.Name, JobStatus: condition, - RemoteID: string(jobPod.ObjectMeta.UID), + RemoteID: string(lagoonTask.ObjectMeta.UID), Key: lagoonTask.Spec.Key, Cluster: r.LagoonTargetName, }, @@ -263,6 +279,15 @@ func (r *LagoonMonitorReconciler) taskStatusLogsToLagoonLogs(opLog logr.Logger, // really matter if we don't smootly transition in what we send back to lagoon return err } + if r.EnableDebug { + opLog.Info( + fmt.Sprintf( + "Published event %s for %s to lagoon-logs exchange", + fmt.Sprintf("task:job-kubernetes:%s", condition), + lagoonTask.ObjectMeta.Name, + ), + ) + } // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } @@ -347,7 +372,7 @@ Task %s // send any messages to lagoon message queues // update the deployment with the status - if err = r.taskStatusLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower()); err != nil { + if err = r.taskStatusLogsToLagoonLogs(opLog, &lagoonTask, taskCondition.ToLower()); err != nil { opLog.Error(err, "unable to publish task status logs") } if err = r.updateLagoonTask(opLog, &lagoonTask, &jobPod, taskCondition.ToLower()); err != nil { diff --git a/internal/messenger/consumer.go b/internal/messenger/consumer.go index 4817f1d5..0bd0a23d 100644 --- a/internal/messenger/consumer.go +++ b/internal/messenger/consumer.go @@ -103,7 +103,7 @@ func (m *Messenger) Consumer(targetName string) { //error { message.Ack(false) // ack to remove from queue }) if err != nil { - log.Fatalf(fmt.Sprintf("Failed to set handler to consumer `%s`: %v", "builddeploy-queue", err)) + log.Fatalf("Failed to set handler to consumer `%s`: %v", "builddeploy-queue", err) } // Handle any tasks that go to the `remove` queue @@ -227,7 +227,7 @@ func (m *Messenger) Consumer(targetName string) { //error { message.Ack(false) // ack to remove from queue }) if err != nil { - log.Fatalf(fmt.Sprintf("Failed to set handler to consumer `%s`: %v", "remove-queue", err)) + log.Fatalf("Failed to set handler to consumer `%s`: %v", "remove-queue", err) } // Handle any tasks that go to the `jobs` queue @@ -285,7 +285,7 @@ func (m *Messenger) Consumer(targetName string) { //error { message.Ack(false) // ack to remove from queue }) if err != nil { - log.Fatalf(fmt.Sprintf("Failed to set handler to consumer `%s`: %v", "jobs-queue", err)) + log.Fatalf("Failed to set handler to consumer `%s`: %v", "jobs-queue", err) } // Handle any tasks that go to the `misc` queue @@ -448,7 +448,7 @@ func (m *Messenger) Consumer(targetName string) { //error { message.Ack(false) // ack to remove from queue }) if err != nil { - log.Fatalf(fmt.Sprintf("Failed to set handler to consumer `%s`: %v", "misc-queue", err)) + log.Fatalf("Failed to set handler to consumer `%s`: %v", "misc-queue", err) } <-forever } diff --git a/internal/messenger/tasks_restore.go b/internal/messenger/tasks_restore.go index ad481984..dc5d593b 100644 --- a/internal/messenger/tasks_restore.go +++ b/internal/messenger/tasks_restore.go @@ -30,7 +30,6 @@ func (m *Messenger) ResticRestore(namespace string, jobSpec *lagoonv1beta2.Lagoo // just log the error then return return nil } - // check if k8up crds exist in the cluster k8upv1alpha1Exists := false k8upv1Exists := false From c63fe6ee23a0f70f8049bfd2a6373fa55473b26c Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Tue, 14 Jan 2025 13:54:13 +1100 Subject: [PATCH 4/6] test: adjustments for broker and other minor tweaks --- test/e2e/e2e_test.go | 8 +++++++ .../testdata/taskqos/lagoon-task-2xpyz5m.json | 2 +- .../testdata/taskqos/lagoon-task-3xpyz5m.json | 2 +- .../testdata/taskqos/lagoon-task-4xpyz5m.json | 2 +- test/utils/utils.go | 24 +++++++++++++------ 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 6a9a631b..f2f67d68 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -108,6 +108,14 @@ var _ = Describe("controller", Ordered, func() { if err == nil { fmt.Fprintf(GinkgoWriter, "info: %s\n", podlogs) } + // cmd = exec.Command(utils.Kubectl(), "logs", + // controllerPodName, "-c", "manager", + // "-n", namespace, "--previous", + // ) + // podlogs, err = utils.Run(cmd) + // if err == nil { + // fmt.Fprintf(GinkgoWriter, "info: previous %s\n", podlogs) + // } } By("stop metrics consumer") diff --git a/test/e2e/testdata/taskqos/lagoon-task-2xpyz5m.json b/test/e2e/testdata/taskqos/lagoon-task-2xpyz5m.json index 14344366..41f1c75a 100644 --- a/test/e2e/testdata/taskqos/lagoon-task-2xpyz5m.json +++ b/test/e2e/testdata/taskqos/lagoon-task-2xpyz5m.json @@ -18,7 +18,7 @@ }, \"environment\": { \"id\": 1, - \"environmentType\": \"production\", + \"environmentType\": \"development\", \"name\": \"dev1\", \"project\": \"nginx-example\" }, diff --git a/test/e2e/testdata/taskqos/lagoon-task-3xpyz5m.json b/test/e2e/testdata/taskqos/lagoon-task-3xpyz5m.json index 756275ce..8d9b490a 100644 --- a/test/e2e/testdata/taskqos/lagoon-task-3xpyz5m.json +++ b/test/e2e/testdata/taskqos/lagoon-task-3xpyz5m.json @@ -18,7 +18,7 @@ }, \"environment\": { \"id\": 1, - \"environmentType\": \"production\", + \"environmentType\": \"development\", \"name\": \"dev2\", \"project\": \"nginx-example\" }, diff --git a/test/e2e/testdata/taskqos/lagoon-task-4xpyz5m.json b/test/e2e/testdata/taskqos/lagoon-task-4xpyz5m.json index 22976dd0..e3f5d901 100644 --- a/test/e2e/testdata/taskqos/lagoon-task-4xpyz5m.json +++ b/test/e2e/testdata/taskqos/lagoon-task-4xpyz5m.json @@ -18,7 +18,7 @@ }, \"environment\": { \"id\": 1, - \"environmentType\": \"production\", + \"environmentType\": \"development\", \"name\": \"dev3\", \"project\": \"nginx-example\" }, diff --git a/test/utils/utils.go b/test/utils/utils.go index d118c61b..6bb041c6 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -21,6 +21,7 @@ import ( "os" "os/exec" "strings" + "time" "github.com/onsi/ginkgo/v2" ) @@ -62,6 +63,12 @@ func Kubectl() string { func StartLocalServices() error { cmd := exec.Command("docker", "compose", "up", "-d") _, err := Run(cmd) + if err != nil { + return err + } + time.Sleep(10 * time.Second) + cmd = exec.Command("docker", "compose", "exec", "local-broker", "rabbitmqadmin", "declare", "exchange", "--vhost=/", "name=lagoon-logs", "type=direct") + _, err = Run(cmd) return err } @@ -75,12 +82,15 @@ func StopLocalServices() { // CleanupNamespace cleans up a namespace and all potentially stuck resources func CleanupNamespace(namespace string) { - cmd := exec.Command("kubectl", "delete", "ns", namespace, "--timeout=30s") + cmd := exec.Command(kubectlPath, "delete", "ns", namespace, "--timeout=30s") if _, err := Run(cmd); err != nil { + if strings.Contains(err.Error(), "not found") { + return + } infoError(err) } // check for builds - cmd = exec.Command("kubectl", "-n", namespace, "get", "lagoonbuilds", + cmd = exec.Command(kubectlPath, "-n", namespace, "get", "lagoonbuilds", "-o", "go-template={{ range .items }}"+ "{{ .metadata.name }}"+ "{{ \"\\n\" }}{{ end }}", @@ -93,7 +103,7 @@ func CleanupNamespace(namespace string) { if len(builds) > 0 { for _, build := range builds { fmt.Fprintf(ginkgo.GinkgoWriter, "info: %v\n", "patching stuck builds for removal") - cmd = exec.Command("kubectl", "-n", namespace, "patch", "lagoonbuild", + cmd = exec.Command(kubectlPath, "-n", namespace, "patch", "lagoonbuild", build, "--type", "merge", "-p", "{\"metadata\":{\"finalizers\":null}}", ) @@ -102,13 +112,13 @@ func CleanupNamespace(namespace string) { infoError(err) } } - cmd = exec.Command("kubectl", "delete", "ns", namespace) + cmd = exec.Command(kubectlPath, "delete", "ns", namespace) if _, err := Run(cmd); err != nil { infoError(err) } } // check for tasks - cmd = exec.Command("kubectl", "-n", namespace, "get", "lagoontasks", + cmd = exec.Command(kubectlPath, "-n", namespace, "get", "lagoontasks", "-o", "go-template={{ range .items }}"+ "{{ .metadata.name }}"+ "{{ \"\\n\" }}{{ end }}", @@ -121,7 +131,7 @@ func CleanupNamespace(namespace string) { if len(tasks) > 0 { for _, task := range tasks { fmt.Fprintf(ginkgo.GinkgoWriter, "info: %v\n", "patching stuck tasks for removal") - cmd = exec.Command("kubectl", "-n", namespace, "patch", "lagoontask", + cmd = exec.Command(kubectlPath, "-n", namespace, "patch", "lagoontask", task, "--type", "merge", "-p", "{\"metadata\":{\"finalizers\":null}}", ) @@ -130,7 +140,7 @@ func CleanupNamespace(namespace string) { infoError(err) } } - cmd = exec.Command("kubectl", "delete", "ns", namespace) + cmd = exec.Command(kubectlPath, "delete", "ns", namespace) if _, err := Run(cmd); err != nil { infoError(err) } From d56d0d39ee1db13128963e5b0efc8a899341cd6c Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Wed, 5 Feb 2025 10:50:32 +1100 Subject: [PATCH 5/6] chore: add warning to pullpolicy flags --- cmd/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index d5769dbf..3db9a9d3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -385,10 +385,11 @@ func main() { // flags to change the image pull policy used for tasks and builds // defaults to Always, can change to another option as required. tests use IfNotPresent + // these flags are used for stability in testing, in actual production use you should never change these. flag.StringVar(&taskImagePullPolicy, "task-image-pull-policy", "Always", - "The image pull policy to use for tasks.") + "The image pull policy to use for tasks. Changing this can have a negative impact and result in tasks failing.") flag.StringVar(&buildImagePullPolicy, "build-image-pull-policy", "Always", - "The image pull policy to use for builds.") + "The image pull policy to use for builds. Changing this can have a negative impact and result in builds failing.") // If installing this controller from scratch, deprecated APIs should not be configured flag.BoolVar(&enableDeprecatedAPIs, "enable-deprecated-apis", false, "Flag to have this controller enable support for deprecated APIs.") From c9d91956d9a6a2505738ee48205b89394690004c Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Wed, 5 Feb 2025 12:09:50 +1100 Subject: [PATCH 6/6] chore: remove unused flag --- cmd/main.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 3db9a9d3..33a1772c 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -173,7 +173,6 @@ func main() { var lffTaskQoSEnabled bool var qosMaxTasks int var qosMaxNamespaceTasks int - var qosTasksDefaultValue int var taskImagePullPolicy string var buildImagePullPolicy string @@ -381,7 +380,6 @@ func main() { flag.BoolVar(&lffTaskQoSEnabled, "enable-task-qos", false, "Flag to enable this controller with QoS for tasks.") flag.IntVar(&qosMaxTasks, "qos-max-tasks", 200, "The total number of tasks that can run at any one time.") flag.IntVar(&qosMaxNamespaceTasks, "qos-max-namespace-tasks", 20, "The total number of tasks that can run at any one time.") - flag.IntVar(&qosTasksDefaultValue, "task-qos-default", 5, "The default qos value to apply to tasks if one is not provided.") // flags to change the image pull policy used for tasks and builds // defaults to Always, can change to another option as required. tests use IfNotPresent