From 7320a087f66373d8323e6ca4d1cc3e11fde0a9a4 Mon Sep 17 00:00:00 2001 From: Helene Durand Date: Thu, 5 Sep 2024 10:40:15 +0200 Subject: [PATCH] MEDIUM: add ingress.class annotation to TCP CR --- documentation/custom-resource-tcp.md | 30 ++++++++ .../tcp-cr-full-example/tcp-cr-full.yaml | 2 + pkg/controller/handler.go | 2 +- pkg/handler/tcp-cr.go | 69 ++++++++++++++++++- pkg/k8s/cr-tcp.go | 9 +-- pkg/store/types-tcp-cr.go | 12 ++-- 6 files changed, 114 insertions(+), 10 deletions(-) diff --git a/documentation/custom-resource-tcp.md b/documentation/custom-resource-tcp.md index 9e6dbaa6..22e54597 100644 --- a/documentation/custom-resource-tcp.md +++ b/documentation/custom-resource-tcp.md @@ -35,6 +35,8 @@ kind: TCP metadata: name: tcp-1 namespace: test + annotations: + ingress.class: haproxy spec: - name: tcp-http-echo-8443 frontend: @@ -103,6 +105,30 @@ Note that in the TCP CR : Except the frontend keyword `default_backend`, all other lines are not automatically generated but are in a flexible way handled by the `frontend` section in the TCP CR. +## ingress.class + +Starting `3.1`, the TCP Custom Resource managed by the Ingress Controller can be filtered using the `ingress.class` annotation. +It behaves the same way as `Ingress`: + +| ingress.class controller flag | TCP CR ingress.class annotation | Behavior | +|------------------|---------------------|-----------------| +| '' (not set) | * (any value) | TCP CR managed by IC | +| \ | Same value as controller | TCP CR managed by IC | +| \ | Value different from controller | TCP CR not managed by IC, frontend and backend deleted if existing | +| \ | '' (empty, not set)| if controller `empty-ingress-class` flag is set, TCP CR managed by IC, otherwise ignored (and frontend and backend are deleted)| + + +### Migration 3.0 to 3.1: action required regarding ingress.class annotation + +If some TCP CRs were deployed with Ingress Controller version <= v3.0, and the Ingress Controller has a `ingress.class` flag for the controller, the TCP CRs need to have the same value for the `ingress.class` annotation in the TCP CR. + +If the annotation is not set, the corresponding backends and frontends in the haproxy configuration would be deleted: +- except if the controller `empty-ingress-class` flag is set (same behavior as for `Ingress`). + +The setting of the `ingress.class` to the TCP CRs should be done **prior to the upgrade to** `v3.1`. It will not be used in v3.0 but needs to be there starting v3.1. + + + ## Pod and Service definitions with the following Kubernetes Service and Pod manifests: @@ -258,6 +284,8 @@ kind: TCP metadata: name: tcp-2 namespace: test + annotations: + ingress.class: haproxy spec: - name: tcp-http-echo-test2-8443 frontend: @@ -297,6 +325,8 @@ kind: TCP metadata: name: tcp-1 namespace: test + annotations: + ingress.class: haproxy spec: - name: tcp-http-echo-443 frontend: diff --git a/documentation/tcp-cr-full-example/tcp-cr-full.yaml b/documentation/tcp-cr-full-example/tcp-cr-full.yaml index 4ce14679..f6a7e452 100644 --- a/documentation/tcp-cr-full-example/tcp-cr-full.yaml +++ b/documentation/tcp-cr-full-example/tcp-cr-full.yaml @@ -2,6 +2,8 @@ apiVersion: ingress.v1.haproxy.org/v1 kind: TCP metadata: name: tcp-1 + annotations: + ingress.class: haproxy spec: - name: tcp-test frontend: diff --git a/pkg/controller/handler.go b/pkg/controller/handler.go index 928b8440..002aa2d2 100644 --- a/pkg/controller/handler.go +++ b/pkg/controller/handler.go @@ -52,7 +52,7 @@ func (c *HAProxyController) initHandlers() { &handler.PatternFiles{}, annotations.ConfigSnippetHandler{}, c.updateStatusManager, - handler.TCPCustomResource{}, + handler.NewTCPCustomResource(c.osArgs.IngressClass, c.osArgs.EmptyIngressClass), } defer func() { c.updateHandlers = append(c.updateHandlers, handler.Refresh{}) }() diff --git a/pkg/handler/tcp-cr.go b/pkg/handler/tcp-cr.go index 21b514d5..fafa5e34 100644 --- a/pkg/handler/tcp-cr.go +++ b/pkg/handler/tcp-cr.go @@ -17,6 +17,7 @@ package handler import ( "fmt" "strings" + "sync" "github.com/haproxytech/client-native/v5/models" v1 "github.com/haproxytech/kubernetes-ingress/crs/api/ingress/v1" @@ -42,7 +43,10 @@ import ( const tcpServicePrefix = "tcpcr" -type TCPCustomResource struct{} +type TCPCustomResource struct { + controllerIngressClass string + allowEmptyIngressClass bool +} type tcpcontext struct { k store.K8s @@ -50,11 +54,52 @@ type tcpcontext struct { h haproxy.HAProxy } +var syncIngressClassLog sync.Once + +func NewTCPCustomResource(controllerIngressClass string, allowEmptyIngressClass bool) TCPCustomResource { + return TCPCustomResource{ + controllerIngressClass: controllerIngressClass, + allowEmptyIngressClass: allowEmptyIngressClass, + } +} + +func logTCPMigration30To31Warning() { + logger := utils.GetLogger() + // For 3.0, (WARNING) + // Starting from 3.1, if ingress.class is set for controller, you will need to set the ingress.class annotation in the TCP CRD + // - Setting the ingress.class annotation in the TCP CRD in 3.0 is highly recommended before migration to 3.1 + // - empty-ingress-class controller option will also impact TCP CRD starting 3.1 + logger.Warning("Using TCP CRD without ingress.class annotation will work only in 3.0") + logger.Warning("If you are using TCP CRDS without ingress.class annotation and ingress.class is set for the controller,an action is required before migrating to 3.1") + logger.Warning("Please read https://github.com/haproxytech/kubernetes-ingress/blob/master/documentation/custom-resource-tcp.md for more information") +} + func (handler TCPCustomResource) Update(k store.K8s, h haproxy.HAProxy, a annotations.Annotations) (err error) { var errs utils.Errors for _, ns := range k.Namespaces { for _, tcpCR := range ns.CRs.TCPsPerCR { + //---------------------------------- + // ingress.class migration + // To log in 3.0 + // Not in 3.1 + syncIngressClassLog.Do(func() { + logTCPMigration30To31Warning() + }) + + // >= v3.1 for ingress.class + // Not in 3.0 + // supported := handler.isSupportedIngressClass(tcpCR) + // if !supported { + // for _, atcp := range tcpCR.Items { + // owner := atcp.Owner() + // k.FrontendRC.RemoveOwner(owner) + // } + // continue + // } + // end ingress.class migration + //---------------------------- + // Cleanup will done after Haproxy config transaction succeeds if tcpCR.Status == store.DELETED { continue @@ -302,3 +347,25 @@ func (handler TCPCustomResource) reconcileAdditionalBackends(ctx tcpcontext, ser } return errors.Result() } + +// func (handler TCPCustomResource) isSupportedIngressClass(tcps *store.TCPs) bool { +// var supported bool +// tcpIgClassAnn := tcps.IngressClass + +// switch handler.controllerIngressClass { +// case "", tcpIgClassAnn: +// supported = true +// default: // mismatch osArgs.Ingress and TCP IngressClass annotation +// if tcpIgClassAnn == "" { +// supported = handler.allowEmptyIngressClass +// if !supported { +// utils.GetLogger().Warningf("[SKIP] TCP %s/%s ingress.class annotation='%s' does not match with controller ingress.class flag '%s' and controller flag 'empty-ingress-class' is false", +// tcps.Namespace, tcps.Name, tcpIgClassAnn, handler.controllerIngressClass) +// } +// } else { +// utils.GetLogger().Warningf("[SKIP] TCP %s/%s ingress.class annotation='%s' does not match with controller ingress.class flag '%s'", +// tcps.Namespace, tcps.Name, tcpIgClassAnn, handler.controllerIngressClass) +// } +// } +// return supported +// } diff --git a/pkg/k8s/cr-tcp.go b/pkg/k8s/cr-tcp.go index 1ef9544e..f929ad98 100644 --- a/pkg/k8s/cr-tcp.go +++ b/pkg/k8s/cr-tcp.go @@ -71,10 +71,11 @@ func convertToStoreTCP(k8sData interface{}, status store.Status) *store.TCPs { return nil } storeTCP := store.TCPs{ - Status: status, - Namespace: data.GetNamespace(), - Name: data.GetName(), - Items: make([]*store.TCPResource, 0), + Status: status, + Namespace: data.GetNamespace(), + IngressClass: data.Annotations["ingress.class"], + Name: data.GetName(), + Items: make([]*store.TCPResource, 0), } for _, tcp := range data.Spec { storeTCP.Items = append(storeTCP.Items, &store.TCPResource{ diff --git a/pkg/store/types-tcp-cr.go b/pkg/store/types-tcp-cr.go index 868d137f..db16c188 100644 --- a/pkg/store/types-tcp-cr.go +++ b/pkg/store/types-tcp-cr.go @@ -45,10 +45,11 @@ type TCPResource struct { type TCPResourceList []*TCPResource type TCPs struct { - Status Status `json:"status,omitempty"` - Namespace string `json:"namespace,omitempty"` - Name string `json:"name,omitempty"` - Items TCPResourceList `json:"items"` + Status Status `json:"status,omitempty"` + IngressClass string `json:"ingress_class,omitempty"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + Items TCPResourceList `json:"items"` } func (a *TCPs) Equal(b *TCPs, opt ...models.Options) bool { @@ -64,6 +65,9 @@ func (a *TCPs) Equal(b *TCPs, opt ...models.Options) bool { if a.Namespace != b.Namespace { return false } + if a.IngressClass != b.IngressClass { + return false + } // Always ordered before being added into the store, so no need to order here a.Items.Order() b.Items.Order()