From 6159e4fe6d378ab06bffee97184f12bc0e56b1e6 Mon Sep 17 00:00:00 2001 From: Zlatko Bratkovic Date: Mon, 6 Nov 2023 13:37:11 +0100 Subject: [PATCH] MAJOR: crd: add job for custom resource definition handling CRDs are not properly handled with external tools, Helm and similar options cannot handle upgrading of custom resource definitions --- .gitignore | 1 + ...nd.yaml => backends.core.haproxy.org.yaml} | 0 ...ts.yaml => defaults.core.haproxy.org.yaml} | 0 crs/definition/embed.go | 37 +++++++ ...bal.yaml => globals.core.haproxy.org.yaml} | 0 ...nd.yaml => backends.core.haproxy.org.yaml} | 0 ...ts.yaml => defaults.core.haproxy.org.yaml} | 0 ...bal.yaml => globals.core.haproxy.org.yaml} | 0 deploy/tests/config/crd/job-crd.yaml | 16 +++ deploy/tests/config/crd/rbac.yaml | 31 ++++++ deploy/tests/create.sh | 12 +- go.mod | 5 +- go.sum | 20 +++- main.go | 16 +++ pkg/job/crd-check.go | 103 +++++++++++++++++ pkg/k8s/crs-monitor.go | 104 ++++++++++++++++++ pkg/k8s/main.go | 24 ++-- pkg/utils/flags.go | 1 + 18 files changed, 348 insertions(+), 22 deletions(-) rename crs/definition/{backend.yaml => backends.core.haproxy.org.yaml} (100%) rename crs/definition/{defaults.yaml => defaults.core.haproxy.org.yaml} (100%) create mode 100644 crs/definition/embed.go rename crs/definition/{global.yaml => globals.core.haproxy.org.yaml} (100%) rename crs/definition/upgrade/{backend.yaml => backends.core.haproxy.org.yaml} (100%) rename crs/definition/upgrade/{defaults.yaml => defaults.core.haproxy.org.yaml} (100%) rename crs/definition/upgrade/{global.yaml => globals.core.haproxy.org.yaml} (100%) create mode 100644 deploy/tests/config/crd/job-crd.yaml create mode 100644 deploy/tests/config/crd/rbac.yaml create mode 100644 pkg/job/crd-check.go create mode 100644 pkg/k8s/crs-monitor.go diff --git a/.gitignore b/.gitignore index 78863928..017b3207 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ kubernetes-ingress dist/ .code-generator/ bin/golangci-lint +.local/* diff --git a/crs/definition/backend.yaml b/crs/definition/backends.core.haproxy.org.yaml similarity index 100% rename from crs/definition/backend.yaml rename to crs/definition/backends.core.haproxy.org.yaml diff --git a/crs/definition/defaults.yaml b/crs/definition/defaults.core.haproxy.org.yaml similarity index 100% rename from crs/definition/defaults.yaml rename to crs/definition/defaults.core.haproxy.org.yaml diff --git a/crs/definition/embed.go b/crs/definition/embed.go new file mode 100644 index 00000000..96d94186 --- /dev/null +++ b/crs/definition/embed.go @@ -0,0 +1,37 @@ +package definition + +import _ "embed" + +//go:embed defaults.core.haproxy.org.yaml +var DefaultsV1alpha2 []byte + +//go:embed globals.core.haproxy.org.yaml +var GlobalsV1alpha2 []byte + +//go:embed backends.core.haproxy.org.yaml +var BackendsV1alpha2 []byte + +//go:embed upgrade/defaults.core.haproxy.org.yaml +var DefaultsV1alpha1V1alpha2 []byte + +//go:embed upgrade/globals.core.haproxy.org.yaml +var GlobalsV1alpha1V1alpha2 []byte + +//go:embed upgrade/backends.core.haproxy.org.yaml +var BackendsV1alpha1V1alpha2 []byte + +func GetCRDs() map[string][]byte { + return map[string][]byte{ + "defaults.core.haproxy.org": DefaultsV1alpha2, + "globals.core.haproxy.org": GlobalsV1alpha2, + "backends.core.haproxy.org": BackendsV1alpha2, + } +} + +func GetCRDsUpgrade() map[string][]byte { + return map[string][]byte{ + "defaults.core.haproxy.org": DefaultsV1alpha1V1alpha2, + "globals.core.haproxy.org": GlobalsV1alpha1V1alpha2, + "backends.core.haproxy.org": BackendsV1alpha1V1alpha2, + } +} diff --git a/crs/definition/global.yaml b/crs/definition/globals.core.haproxy.org.yaml similarity index 100% rename from crs/definition/global.yaml rename to crs/definition/globals.core.haproxy.org.yaml diff --git a/crs/definition/upgrade/backend.yaml b/crs/definition/upgrade/backends.core.haproxy.org.yaml similarity index 100% rename from crs/definition/upgrade/backend.yaml rename to crs/definition/upgrade/backends.core.haproxy.org.yaml diff --git a/crs/definition/upgrade/defaults.yaml b/crs/definition/upgrade/defaults.core.haproxy.org.yaml similarity index 100% rename from crs/definition/upgrade/defaults.yaml rename to crs/definition/upgrade/defaults.core.haproxy.org.yaml diff --git a/crs/definition/upgrade/global.yaml b/crs/definition/upgrade/globals.core.haproxy.org.yaml similarity index 100% rename from crs/definition/upgrade/global.yaml rename to crs/definition/upgrade/globals.core.haproxy.org.yaml diff --git a/deploy/tests/config/crd/job-crd.yaml b/deploy/tests/config/crd/job-crd.yaml new file mode 100644 index 00000000..b495f5e2 --- /dev/null +++ b/deploy/tests/config/crd/job-crd.yaml @@ -0,0 +1,16 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: haproxy-ingress-crd # each deploymnent should have a unique name, in example we always recreate the custer + namespace: haproxy-controller +spec: + template: + spec: + serviceAccountName: haproxy-kubernetes-ingress-crd + containers: + - name: haproxy-ingress-crd + image: haproxytech/kubernetes-ingress:latest + imagePullPolicy: Never + command: ["./haproxy-ingress-controller","--job-check-crd"] + restartPolicy: Never + backoffLimit: 0 diff --git a/deploy/tests/config/crd/rbac.yaml b/deploy/tests/config/crd/rbac.yaml new file mode 100644 index 00000000..30ca1cae --- /dev/null +++ b/deploy/tests/config/crd/rbac.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: haproxy-kubernetes-ingress-crd + namespace: haproxy-controller +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: haproxy-kubernetes-ingress-crd +rules: + - apiGroups: + - "apiextensions.k8s.io" + resources: + - customresourcedefinitions + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: haproxy-kubernetes-ingress-crd + namespace: haproxy-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: haproxy-kubernetes-ingress-crd +subjects: + - kind: ServiceAccount + name: haproxy-kubernetes-ingress-crd + namespace: haproxy-controller diff --git a/deploy/tests/create.sh b/deploy/tests/create.sh index 59eb6a2f..8988c9f8 100755 --- a/deploy/tests/create.sh +++ b/deploy/tests/create.sh @@ -11,7 +11,6 @@ if [ -n "${CI_ENV}" ]; then echo "building image for ingress controller" if [ -n "${GITLAB_CI}" ]; then echo "haproxytech/kubernetes-ingress image already available from previous stage" - #docker build --build-arg TARGETPLATFORM="linux/amd64" -t haproxytech/kubernetes-ingress -f build/Dockerfile . else docker build --build-arg TARGETPLATFORM="linux/amd64" -t haproxytech/kubernetes-ingress -f build/Dockerfile . fi @@ -44,13 +43,16 @@ fi echo "loading image http-echo in kind" kind load docker-image haproxytech/http-echo:latest --name=$clustername +printf %80s |tr " " "="; echo "" +echo "Create HAProxy namespace ..." +kubectl apply -f $DIR/config/0.namespace.yaml +printf %80s |tr " " "="; echo "" echo "Install custom resource definitions ..." -kubectl apply -f $DIR/../../crs/definition/backend.yaml -kubectl apply -f $DIR/../../crs/definition/defaults.yaml -kubectl apply -f $DIR/../../crs/definition/global.yaml +kubectl apply -f $DIR/../../deploy/tests/config/crd/rbac.yaml +kubectl apply -f $DIR/../../deploy/tests/config/crd/job-crd.yaml +kubectl wait --for=condition=complete --timeout=60s job/haproxy-ingress-crd -n haproxy-controller echo "deploying Ingress Controller ..." -kubectl apply -f $DIR/config/0.namespace.yaml kubectl apply -f $DIR/config/1.rbac.yaml kubectl apply -f $DIR/config/2.configmap.yaml kubectl apply -f $DIR/config/3.ingress-controller.yaml diff --git a/go.mod b/go.mod index 6ed7810e..d1da19fe 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,10 @@ require ( github.com/valyala/fasthttp v1.50.0 go.uber.org/automaxprocs v1.5.3 k8s.io/api v0.25.2 + k8s.io/apiextensions-apiserver v0.25.2 k8s.io/apimachinery v0.25.2 k8s.io/client-go v0.25.2 + sigs.k8s.io/yaml v1.4.0 ) require ( @@ -58,6 +60,8 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oklog/ulid v1.3.1 // indirect + github.com/onsi/ginkgo/v2 v2.11.0 // indirect + github.com/onsi/gomega v1.27.10 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect @@ -83,5 +87,4 @@ require ( k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index af637ced..6518ab5f 100644 --- a/go.sum +++ b/go.sum @@ -127,6 +127,8 @@ github.com/go-openapi/validate v0.22.1 h1:G+c2ub6q47kfX1sOBLwIQwzBVt8qmOAARyo/9F github.com/go-openapi/validate v0.22.1/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= @@ -186,6 +188,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v1.0.1 h1:Lh/jXZmvZxb0BBeSY5VKEfidcbcbenKjZFzM/q0fSeU= github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -266,10 +270,10 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= -github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= -github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= -github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= @@ -457,6 +461,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -509,6 +515,8 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.25.2 h1:v6G8RyFcwf0HR5jQGIAYlvtRNrxMJQG1xJzaSeVnIS8= k8s.io/api v0.25.2/go.mod h1:qP1Rn4sCVFwx/xIhe+we2cwBLTXNcheRyYXwajonhy0= +k8s.io/apiextensions-apiserver v0.25.2 h1:8uOQX17RE7XL02ngtnh3TgifY7EhekpK+/piwzQNnBo= +k8s.io/apiextensions-apiserver v0.25.2/go.mod h1:iRwwRDlWPfaHhuBfQ0WMa5skdQfrE18QXJaJvIDLvE8= k8s.io/apimachinery v0.25.2 h1:WbxfAjCx+AeN8Ilp9joWnyJ6xu9OMeS/fsfjK/5zaQs= k8s.io/apimachinery v0.25.2/go.mod h1:hqqA1X0bsgsxI6dXsJ4HnNTBOmJNxyPp8dw3u2fSHwA= k8s.io/client-go v0.25.2 h1:SUPp9p5CwM0yXGQrwYurw9LWz+YtMwhWd0GqOsSiefo= @@ -524,5 +532,5 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMm sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/main.go b/main.go index 75d8e5aa..aa5ca809 100644 --- a/main.go +++ b/main.go @@ -34,6 +34,7 @@ import ( "github.com/haproxytech/kubernetes-ingress/pkg/annotations" "github.com/haproxytech/kubernetes-ingress/pkg/controller" + "github.com/haproxytech/kubernetes-ingress/pkg/job" "github.com/haproxytech/kubernetes-ingress/pkg/k8s" "github.com/haproxytech/kubernetes-ingress/pkg/store" "github.com/haproxytech/kubernetes-ingress/pkg/utils" @@ -70,6 +71,21 @@ func main() { parser.WriteHelp(os.Stdout) return } + if osArgs.JobCheckCRD { + logger.Print(IngressControllerInfo) + logger.Print(job.IngressControllerCRDUpdater) + logger.Infof("HAProxy Ingress Controller CRD Updater %s %s%s", GitTag, GitCommit, GitDirty) + logger.Infof("Build from: %s", GitRepo) + + err := job.CRDRefresh(logger, osArgs) + if err != nil { + logger.Error(err) + os.Exit(1) //nolint:gocritic + } + // exit, this is just a job + os.Exit(0) + } + logger.ShowFilename(false) exit := logInfo(logger, osArgs) if exit { diff --git a/pkg/job/crd-check.go b/pkg/job/crd-check.go new file mode 100644 index 00000000..c9157209 --- /dev/null +++ b/pkg/job/crd-check.go @@ -0,0 +1,103 @@ +// Copyright 2023 HAProxy Technologies LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package job + +import ( + "context" + + "github.com/haproxytech/kubernetes-ingress/crs/definition" + "github.com/haproxytech/kubernetes-ingress/pkg/k8s" + "github.com/haproxytech/kubernetes-ingress/pkg/utils" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + apiError "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" +) + +func CRDRefresh(log utils.Logger, osArgs utils.OSArgs) error { + log.Info("checking CRDS") + config, err := k8s.GetRestConfig(osArgs) + if err != nil { + return err + } + + // Create a new clientset for the apiextensions API group + clientset := apiextensionsclientset.NewForConfigOrDie(config) + + // Check if the CRD exists + crds := definition.GetCRDs() + crdsUpgrade := definition.GetCRDsUpgrade() + for crdName, crdDef := range crds { + // CustomResourceDefinition object + var crd apiextensionsv1.CustomResourceDefinition + err = yaml.Unmarshal(crdDef, &crd) + if err != nil { + return err + } + log.Info("") + log.Infof("checking CRD %s", crdName) + + existingVersion, err := clientset.ApiextensionsV1().CustomResourceDefinitions().Get(context.Background(), crdName, metav1.GetOptions{}) + if err != nil { + if !apiError.IsNotFound(err) { + return err + } + log.Infof("CRD %s does not exist", crdName) + // Create the CRD + _, err = clientset.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), &crd, metav1.CreateOptions{}) + if err != nil { + return err + } + log.Infof("CRD %s created", crdName) + continue + } + log.Infof("CRD %s exists", crdName) + versions := existingVersion.Spec.Versions + if len(versions) == 2 { + log.Infof("CRD %s exists as v1alpha1 and v1alpha2, nothing to do", crdName) + continue + } + // check if we have alpha 2 or we need to upgrade for alpha2 + crd.ObjectMeta.ResourceVersion = existingVersion.ObjectMeta.ResourceVersion + if versions[0].Name == "v1alpha2" { + log.Infof("CRD %s exists as v1alpha2, nothing to do", crdName) + continue + } + err = yaml.Unmarshal(crdsUpgrade[crdName], &crd) + if err != nil { + return err + } + // Upgrade the CRDl + _, err = clientset.ApiextensionsV1().CustomResourceDefinitions().Update(context.Background(), &crd, metav1.UpdateOptions{}) + if err != nil { + return err + } + } + + log.Info("") + log.Info("CRD update done") + return nil +} + +// IngressControllerCRDUpdater console pretty print +const IngressControllerCRDUpdater = ` + ____ ____ ____ _ _ _ _ + / ___| _ \| _ \ | | | |_ __ __| | __ _| |_ ___ _ __ +| | | |_) | | | | | | | | '_ \ / _` + "`" + ` |/ _` + "`" + ` | __/ _ \ '__| +| |___| _ <| |_| | | |_| | |_) | (_| | (_| | || __/ | + \____|_| \_\____/ \___/| .__/ \__,_|\__,_|\__\___|_| + |_| + +` diff --git a/pkg/k8s/crs-monitor.go b/pkg/k8s/crs-monitor.go new file mode 100644 index 00000000..fca7e72f --- /dev/null +++ b/pkg/k8s/crs-monitor.go @@ -0,0 +1,104 @@ +// Copyright 2019 HAProxy Technologies LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k8s + +import ( + "strings" + "time" + + "k8s.io/client-go/tools/cache" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + // apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + apiextensionsinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions" +) + +func (k k8s) runCRDefinitionsInformer(eventChan chan string, stop chan struct{}) { + // Create a new informer factory with the clientset. + + factory := apiextensionsinformers.NewSharedInformerFactoryWithOptions(k.apiExtensionsClient, k.cacheResyncPeriod) + informer := factory.Apiextensions().V1().CustomResourceDefinitions().Informer() + informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + crd := obj.(*apiextensionsv1.CustomResourceDefinition) + if crd.Spec.Group != "core.haproxy.org" { + return + } + if !(crd.Spec.Names.Kind == "Global" || crd.Spec.Names.Kind == "Defaults" || crd.Spec.Names.Kind == "Backend") { + return + } + for _, version := range crd.Spec.Versions { + if version.Name == "v1alpha2" { + time.Sleep(time.Second * 5) // a little delay is needed to let CRD API be created + eventChan <- crd.Spec.Names.Kind + return + } + } + }, + }) + + go informer.Run(stop) + + if !cache.WaitForCacheSync(stop, informer.HasSynced) { + logger.Error("Caches are not populated due to an underlying error, cannot monitor CRS creation") + } +} + +func (k k8s) RunCRSCreationMonitoring(eventChan chan SyncDataEvent, stop chan struct{}) { + count := 0 + for key := range k.crs { + if strings.Contains(key, "core.haproxy.org/v1alpha2") { + count++ + } + } + if count > 2 { + // all crds are already in list + return + } + + eventCRS := make(chan string) + k.runCRDefinitionsInformer(eventCRS, stop) + go func(chan string) { + for { + select { + case crdName := <-eventCRS: + if _, ok := k.crs["core.haproxy.org/v1alpha2 - "+crdName]; ok { + // we have already created watchers for this CRD + continue + } + informersSyncedEvent := &[]cache.InformerSynced{} + for _, namespace := range k.whiteListedNS { + crs := map[string]CR{} + switch crdName { + case "Backend": + crs[crdName] = NewBackendCR() + case "Defaults": + crs[crdName] = NewDefaultsCR() + case "Global": + crs[crdName] = NewGlobalCR() + } + logger.Info("Custom resource definition created, adding CR watcher for " + crs[crdName].GetKind()) + k.runCRInformers(eventChan, stop, namespace, informersSyncedEvent, crs) + } + + if !cache.WaitForCacheSync(stop, *informersSyncedEvent...) { + logger.Error("Caches are not populated due to an underlying error, cannot monitor new CRDs") + } + case <-stop: + return + } + } + }(eventCRS) +} diff --git a/pkg/k8s/main.go b/pkg/k8s/main.go index efa14d25..39c4c208 100644 --- a/pkg/k8s/main.go +++ b/pkg/k8s/main.go @@ -21,16 +21,16 @@ import ( "strconv" "time" + crclientset "github.com/haproxytech/kubernetes-ingress/crs/generated/clientset/versioned" + crinformers "github.com/haproxytech/kubernetes-ingress/crs/generated/informers/externalversions" + "github.com/haproxytech/kubernetes-ingress/pkg/ingress" + "github.com/haproxytech/kubernetes-ingress/pkg/utils" + crdclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" k8sinformers "k8s.io/client-go/informers" k8sclientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" - - crclientset "github.com/haproxytech/kubernetes-ingress/crs/generated/clientset/versioned" - crinformers "github.com/haproxytech/kubernetes-ingress/crs/generated/informers/externalversions" - "github.com/haproxytech/kubernetes-ingress/pkg/ingress" - "github.com/haproxytech/kubernetes-ingress/pkg/utils" ) var logger = utils.GetLogger() @@ -62,6 +62,7 @@ type k8s struct { builtInClient *k8sclientset.Clientset crClient *crclientset.Clientset crs map[string]CR + apiExtensionsClient *crdclientset.Clientset whiteListedNS []string publishSvc *utils.NamespaceValue syncPeriod time.Duration @@ -72,7 +73,7 @@ type k8s struct { } func New(osArgs utils.OSArgs, whitelist map[string]struct{}, publishSvc *utils.NamespaceValue) K8s { //nolint:ireturn - restconfig, err := getRestConfig(osArgs) + restconfig, err := GetRestConfig(osArgs) logger.Panic(err) builtInClient := k8sclientset.NewForConfigOrDie(restconfig) if k8sVersion, errVer := builtInClient.Discovery().ServerVersion(); errVer != nil { @@ -85,6 +86,7 @@ func New(osArgs utils.OSArgs, whitelist map[string]struct{}, publishSvc *utils.N k := k8s{ builtInClient: builtInClient, crClient: crclientset.NewForConfigOrDie(restconfig), + apiExtensionsClient: crdclientset.NewForConfigOrDie(restconfig), crs: map[string]CR{}, whiteListedNS: getWhitelistedNS(whitelist, osArgs.ConfigMap.Namespace), publishSvc: publishSvc, @@ -122,8 +124,10 @@ func (k k8s) MonitorChanges(eventChan chan SyncDataEvent, stop chan struct{}) { k.runPodInformer(eventChan, stop, informersSynced) for _, namespace := range k.whiteListedNS { k.runInformers(eventChan, stop, namespace, informersSynced) - k.runCRInformers(eventChan, stop, namespace, informersSynced) + k.runCRInformers(eventChan, stop, namespace, informersSynced, k.crs) } + // check if we need to also watch CRS creation (in case not all alpha2 definitions are already installed) + k.RunCRSCreationMonitoring(eventChan, stop) if !cache.WaitForCacheSync(stop, *informersSynced...) { logger.Panic("Caches are not populated due to an underlying error, cannot run the Ingress Controller") @@ -153,9 +157,9 @@ func (k k8s) registerCoreCR(cr CR, groupVersion string) { } } -func (k k8s) runCRInformers(eventChan chan SyncDataEvent, stop chan struct{}, namespace string, informersSynced *[]cache.InformerSynced) { +func (k k8s) runCRInformers(eventChan chan SyncDataEvent, stop chan struct{}, namespace string, informersSynced *[]cache.InformerSynced, crs map[string]CR) { informerFactory := crinformers.NewSharedInformerFactoryWithOptions(k.crClient, k.cacheResyncPeriod, crinformers.WithNamespace(namespace)) - for _, cr := range k.crs { + for _, cr := range crs { informer := cr.GetInformer(eventChan, informerFactory) go informer.Run(stop) *informersSynced = append(*informersSynced, informer.HasSynced) @@ -233,7 +237,7 @@ func (k k8s) endpointsMirroring() bool { return true } -func getRestConfig(osArgs utils.OSArgs) (restConfig *rest.Config, err error) { +func GetRestConfig(osArgs utils.OSArgs) (restConfig *rest.Config, err error) { if osArgs.External { kubeconfig := filepath.Join(utils.HomeDir(), ".kube", "config") if osArgs.KubeConfig != "" { diff --git a/pkg/utils/flags.go b/pkg/utils/flags.go index 6140a604..63c9fdf1 100644 --- a/pkg/utils/flags.go +++ b/pkg/utils/flags.go @@ -106,4 +106,5 @@ type OSArgs struct { //nolint:maligned PprofEnabled bool `long:"pprof" short:"p" description:"enable pprof"` PrometheusEnabled bool `long:"prometheus" description:"enable prometheus of IC data"` ChannelSize int64 `long:"channel-size" description:"sets the size of controller buffers used to receive and send k8s events.NOTE: increase the value to accommodate large number of resources "` + JobCheckCRD bool `long:"job-check-crd" description:"does not execute IC, but adds/updates CRDs"` }