From 9a376b85b29ebe18c638a45579555c861b5fb963 Mon Sep 17 00:00:00 2001 From: Scott Leggett Date: Fri, 12 Jul 2024 16:16:17 +0800 Subject: [PATCH] feat: support the new idle labels and annotations The idle labels and annotations can now have the lagoon.sh suffix as amazee.io. This change adds support for both suffixes, with idling.lagoon.sh taking priority over idling.amazee.io. --- go.mod | 2 + go.sum | 4 ++ internal/k8s/client.go | 2 +- internal/k8s/exec.go | 81 +++++++++++++++++------- internal/k8s/exec_test.go | 130 +++++++++++++++++++++++++++++++++++++- 5 files changed, 194 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 387cd99b..7cda3199 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require ( github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect @@ -77,6 +78,7 @@ require ( golang.org/x/term v0.24.0 // indirect golang.org/x/text v0.18.0 // indirect google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index b6457b10..33fd6229 100644 --- a/go.sum +++ b/go.sum @@ -128,6 +128,8 @@ github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -226,6 +228,8 @@ google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWn gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/k8s/client.go b/internal/k8s/client.go index bc656b3e..b68b828a 100644 --- a/internal/k8s/client.go +++ b/internal/k8s/client.go @@ -22,7 +22,7 @@ var timeoutSeconds = int64(timeout / time.Second) // Client is a k8s client. type Client struct { config *rest.Config - clientset *kubernetes.Clientset + clientset kubernetes.Interface logStreamIDs sync.Map } diff --git a/internal/k8s/exec.go b/internal/k8s/exec.go index 3cd296ad..6750605f 100644 --- a/internal/k8s/exec.go +++ b/internal/k8s/exec.go @@ -18,8 +18,21 @@ import ( "k8s.io/client-go/tools/remotecommand" ) -const ( - idleAnnotation = "idling.amazee.io/unidle-replicas" +var ( + // idleReplicaAnnotations are used to determine how many replicas to set when + // scaling up a deployment from idle. The annotations are in priority order + // from high to low. The first annotation found on a deployment will be used. + idleReplicaAnnotations = []string{ + "idling.lagoon.sh/unidle-replicas", + "idling.amazee.io/unidle-replicas", + } + // idleWatchLabels are used to select deployments to scale when unidling a + // namespace. The labels are in priority order from high to low. The first + // label found on any deployment will be used. + idleWatchLabels = []string{ + "idling.lagoon.sh/watch=true", + "idling.amazee.io/watch=true", + } ) // podContainer returns the first pod and first container inside that pod for @@ -68,33 +81,57 @@ func (c *Client) hasRunningPod(ctx context.Context, } } -// unidleReplicas checks the unidle-replicas annotation for the number of -// replicas to restore. If the label cannot be read or parsed, 1 is returned. -// The return value is clamped to the interval [1,16]. +// unidleReplicas checks the idleReplicaAnnotations for the number of replicas +// to restore. If the labels cannot be found or parsed, 1 is returned. The +// return value is clamped to the interval [1,16]. func unidleReplicas(deploy appsv1.Deployment) int { - rs, ok := deploy.Annotations[idleAnnotation] - if !ok { - return 1 - } - r, err := strconv.Atoi(rs) - if err != nil || r < 1 { - return 1 + for _, ra := range idleReplicaAnnotations { + rs, ok := deploy.Annotations[ra] + if !ok { + continue + } + r, err := strconv.Atoi(rs) + if err != nil || r < 1 { + return 1 + } + if r > 16 { + return 16 + } + return r } - if r > 16 { - return 16 + return 1 +} + +// idledDeploys returns the DeploymentList of idled deployments in the given +// namespace. +func (c *Client) idledDeploys(ctx context.Context, namespace string) ( + *appsv1.DeploymentList, error, +) { + var deploys *appsv1.DeploymentList + for _, selector := range idleWatchLabels { + deploys, err := c.clientset.AppsV1().Deployments(namespace).List(ctx, + metav1.ListOptions{ + LabelSelector: selector, + }) + if err != nil { + return nil, fmt.Errorf("couldn't select deploys by label: %v", err) + } + if deploys != nil && len(deploys.Items) > 0 { + return deploys, nil + } } - return r + return deploys, nil } -// unidleNamespace scales all deployments with the -// "idling.amazee.io/watch=true" label up to the number of replicas in the -// "idling.amazee.io/unidle-replicas" label. +// unidleNamespace scales all deployments with the idleWatchLabels up to the +// number of replicas in the idleReplicaAnnotations. func (c *Client) unidleNamespace(ctx context.Context, namespace string) error { - deploys, err := c.clientset.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: "idling.amazee.io/watch=true", - }) + deploys, err := c.idledDeploys(ctx, namespace) if err != nil { - return fmt.Errorf("couldn't select deploys by label: %v", err) + return fmt.Errorf("couldn't get idled deploys: %v", err) + } + if deploys == nil { + return nil // no deploys to unidle } for _, deploy := range deploys.Items { // check if idled diff --git a/internal/k8s/exec_test.go b/internal/k8s/exec_test.go index 3dba5d04..e92daeab 100644 --- a/internal/k8s/exec_test.go +++ b/internal/k8s/exec_test.go @@ -1,14 +1,16 @@ package k8s import ( + "context" "testing" "github.com/alecthomas/assert/v2" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" ) -func TestUnidleReplicas(t *testing.T) { +func TestUnidleReplicasParsing(t *testing.T) { var testCases = map[string]struct { input string expect int @@ -28,10 +30,134 @@ func TestUnidleReplicas(t *testing.T) { t.Run(name, func(tt *testing.T) { deploy := appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{idleAnnotation: tc.input}, + Annotations: map[string]string{idleReplicaAnnotations[0]: tc.input}, }, } assert.Equal(tt, tc.expect, unidleReplicas(deploy), name) }) } } + +func TestUnidleReplicasLabels(t *testing.T) { + for _, ra := range idleReplicaAnnotations { + t.Run(ra, func(tt *testing.T) { + deploy := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ra: "9"}, + }, + } + assert.Equal(tt, 9, unidleReplicas(deploy), ra) + }) + } +} + +func deployNames(deploys *appsv1.DeploymentList) []string { + var names []string + if deploys == nil { + return names // no deploys to unidle + } + for _, deploy := range deploys.Items { + names = append(names, deploy.Name) + } + return names +} + +func TestIdledDeployLabels(t *testing.T) { + testNS := "testns" + var testCases = map[string]struct { + deploys *appsv1.DeploymentList + expect []string + }{ + "prefer lagoon.sh": { + deploys: &appsv1.DeploymentList{ + Items: []appsv1.Deployment{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "one", + Namespace: testNS, + Labels: map[string]string{ + "idling.lagoon.sh/watch": "true", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "two", + Namespace: testNS, + Labels: map[string]string{ + "idling.amazee.io/watch": "true", + }, + }, + }, + }, + }, + expect: []string{"one"}, + }, + "fall back to amazee.io": { + deploys: &appsv1.DeploymentList{ + Items: []appsv1.Deployment{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "one", + Namespace: testNS, + Labels: map[string]string{ + "idling.amazee.io/watch": "true", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "two", + Namespace: testNS, + Labels: map[string]string{ + "idling.amazee.io/watch": "true", + }, + }, + }, + }, + }, + expect: []string{"one", "two"}, + }, + "ignore mislabelled deploys": { + deploys: &appsv1.DeploymentList{ + Items: []appsv1.Deployment{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "one", + Namespace: testNS, + Labels: map[string]string{ + "idling.foo/watch": "true", + }, + }, + }, + }, + }, + }, + "ignore other namespaces": { + deploys: &appsv1.DeploymentList{ + Items: []appsv1.Deployment{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "one", + Namespace: "wrongns", + Labels: map[string]string{ + "idling.lagoon.sh/watch": "true", + }, + }, + }, + }, + }, + }, + } + for name, tc := range testCases { + t.Run(name, func(tt *testing.T) { + // create fake Kubernetes client with test deploys + c := &Client{ + clientset: fake.NewSimpleClientset(tc.deploys), + } + deploys, err := c.idledDeploys(context.Background(), testNS) + assert.NoError(tt, err, name) + assert.Equal(tt, tc.expect, deployNames(deploys), name) + }) + } +}