From 41db688653bfed532dbe0b088a9d68b3b0ad3729 Mon Sep 17 00:00:00 2001 From: Josh Robson Chase Date: Thu, 5 Oct 2023 12:29:59 -0400 Subject: [PATCH 01/10] some fixes for the tunnel controller --- api/v1alpha1/tcpedge_types.go | 12 ------------ api/v1alpha1/tunnel_types.go | 12 ++++++++++++ helm/ingress-controller/Chart.lock | 6 +++--- internal/controllers/tunnel_controller.go | 9 +++++++-- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/api/v1alpha1/tcpedge_types.go b/api/v1alpha1/tcpedge_types.go index 9bb96868..24458792 100644 --- a/api/v1alpha1/tcpedge_types.go +++ b/api/v1alpha1/tcpedge_types.go @@ -31,13 +31,6 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -type TunnelGroupBackend struct { - ngrokAPICommon `json:",inline"` - - // Labels to watch for tunnels on this backend - Labels map[string]string `json:"labels,omitempty"` -} - // TCPEdgeSpec defines the desired state of TCPEdge type TCPEdgeSpec struct { ngrokAPICommon `json:",inline"` @@ -51,11 +44,6 @@ type TCPEdgeSpec struct { IPRestriction *EndpointIPPolicy `json:"ipRestriction,omitempty"` } -type TunnelGroupBackendStatus struct { - // ID is the unique identifier for this backend - ID string `json:"id,omitempty"` -} - // TCPEdgeStatus defines the observed state of TCPEdge type TCPEdgeStatus struct { // ID is the unique identifier for this edge diff --git a/api/v1alpha1/tunnel_types.go b/api/v1alpha1/tunnel_types.go index 0a257b42..55473142 100644 --- a/api/v1alpha1/tunnel_types.go +++ b/api/v1alpha1/tunnel_types.go @@ -82,6 +82,18 @@ type TunnelList struct { Items []Tunnel `json:"items"` } +type TunnelGroupBackend struct { + ngrokAPICommon `json:",inline"` + + // Labels to watch for tunnels on this backend + Labels map[string]string `json:"labels,omitempty"` +} + +type TunnelGroupBackendStatus struct { + // ID is the unique identifier for this backend + ID string `json:"id,omitempty"` +} + func init() { SchemeBuilder.Register(&Tunnel{}, &TunnelList{}) } diff --git a/helm/ingress-controller/Chart.lock b/helm/ingress-controller/Chart.lock index 15297d9b..29361a33 100644 --- a/helm/ingress-controller/Chart.lock +++ b/helm/ingress-controller/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: common repository: https://charts.bitnami.com/bitnami - version: 2.11.0 -digest: sha256:86246e3c038aa9f3b1f544f3427bdc1c4980a87565f9e5297c5d2110d73c29f8 -generated: "2023-09-12T14:39:25.654472-04:00" + version: 2.13.2 +digest: sha256:2672c3a43386aa82424bca0a5b774ea94e167c7c90604cd66520afde23238e37 +generated: "2023-10-05T10:48:29.016056701-04:00" diff --git a/internal/controllers/tunnel_controller.go b/internal/controllers/tunnel_controller.go index e9550179..d38c962f 100644 --- a/internal/controllers/tunnel_controller.go +++ b/internal/controllers/tunnel_controller.go @@ -69,6 +69,7 @@ func (r *TunnelReconciler) SetupWithManager(mgr ctrl.Manager) error { kubeType: "v1alpha1.Tunnel", update: r.update, delete: r.delete, + statusID: r.statusID, } cont, err := controller.NewUnmanaged("tunnel-controller", mgr, controller.Options{ @@ -108,11 +109,15 @@ func (r *TunnelReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } func (r *TunnelReconciler) update(ctx context.Context, tunnel *ingressv1alpha1.Tunnel) error { - tunnelName := fmt.Sprintf("%s/%s", tunnel.Namespace, tunnel.Name) + tunnelName := r.statusID(tunnel) return r.TunnelDriver.CreateTunnel(ctx, tunnelName, tunnel.Spec.Labels, tunnel.Spec.BackendConfig, tunnel.Spec.ForwardsTo) } func (r *TunnelReconciler) delete(ctx context.Context, tunnel *ingressv1alpha1.Tunnel) error { - tunnelName := fmt.Sprintf("%s/%s", tunnel.Namespace, tunnel.Name) + tunnelName := r.statusID(tunnel) return r.TunnelDriver.DeleteTunnel(ctx, tunnelName) } + +func (r *TunnelReconciler) statusID(tunnel *ingressv1alpha1.Tunnel) string { + return fmt.Sprintf("%s/%s", tunnel.Namespace, tunnel.Name) +} From 06c848286e8495fea91aa825341e5ef9fb2a519a Mon Sep 17 00:00:00 2001 From: Josh Robson Chase Date: Thu, 5 Oct 2023 12:30:52 -0400 Subject: [PATCH 02/10] basic tls edge support --- api/v1alpha1/tlsedge_types.go | 96 ++++++ api/v1alpha1/zz_generated.deepcopy.go | 107 ++++++ .../crds/ingress.k8s.ngrok.com_tlsedges.yaml | 126 +++++++ .../templates/rbac/role.yaml | 26 ++ .../templates/rbac/tlsedge_editor_role.yaml | 31 ++ .../templates/rbac/tlsedge_viewer_role.yaml | 27 ++ internal/controllers/tlsedge_controller.go | 319 ++++++++++++++++++ internal/ngrokapi/clientset.go | 8 + main.go | 10 + 9 files changed, 750 insertions(+) create mode 100644 api/v1alpha1/tlsedge_types.go create mode 100644 helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tlsedges.yaml create mode 100644 helm/ingress-controller/templates/rbac/tlsedge_editor_role.yaml create mode 100644 helm/ingress-controller/templates/rbac/tlsedge_viewer_role.yaml create mode 100644 internal/controllers/tlsedge_controller.go diff --git a/api/v1alpha1/tlsedge_types.go b/api/v1alpha1/tlsedge_types.go new file mode 100644 index 00000000..211f97b4 --- /dev/null +++ b/api/v1alpha1/tlsedge_types.go @@ -0,0 +1,96 @@ +/* +MIT License + +Copyright (c) 2022 ngrok, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// TLSEdgeSpec defines the desired state of TLSEdge +type TLSEdgeSpec struct { + ngrokAPICommon `json:",inline"` + + // Backend is the definition for the tunnel group backend + // that serves traffic for this edge + // +kubebuilder:validation:Required + Backend TunnelGroupBackend `json:"backend,omitempty"` + + // Hostports is a list of hostports served by this edge + // +kubebuilder:validation:Required + Hostports []string `json:"hostports,omitempty"` + + // IPRestriction is an IPRestriction to apply to this route + IPRestriction *EndpointIPPolicy `json:"ipRestriction,omitempty"` + + // TODO: mutual tls, tls termination +} + +// TLSEdgeStatus defines the observed state of TLSEdge +type TLSEdgeStatus struct { + // ID is the unique identifier for this edge + ID string `json:"id,omitempty"` + + // URI is the URI of the edge + URI string `json:"uri,omitempty"` + + // Hostports served by this edge + Hostports []string `json:"hostports,omitempty"` + + // Backend stores the status of the tunnel group backend, + // mainly the ID of the backend + Backend TunnelGroupBackendStatus `json:"backend,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:printcolumn:name="ID",type=string,JSONPath=`.status.id`,description="Domain ID" +//+kubebuilder:printcolumn:name="Hostports",type=string,JSONPath=`.status.hostports`,description="Hostports" +//+kubebuilder:printcolumn:name="Backend ID",type=string,JSONPath=`.status.backend.id`,description="Tunnel Group Backend ID" +//+kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`,description="Age" + +// TLSEdge is the Schema for the tlsedges API +type TLSEdge struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec TLSEdgeSpec `json:"spec,omitempty"` + Status TLSEdgeStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// TLSEdgeList contains a list of TLSEdge +type TLSEdgeList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []TLSEdge `json:"items"` +} + +func init() { + SchemeBuilder.Register(&TLSEdge{}, &TLSEdgeList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index c3ba508e..df6fa334 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1147,6 +1147,113 @@ func (in *TCPEdgeStatus) DeepCopy() *TCPEdgeStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSEdge) DeepCopyInto(out *TLSEdge) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSEdge. +func (in *TLSEdge) DeepCopy() *TLSEdge { + if in == nil { + return nil + } + out := new(TLSEdge) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TLSEdge) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSEdgeList) DeepCopyInto(out *TLSEdgeList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]TLSEdge, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSEdgeList. +func (in *TLSEdgeList) DeepCopy() *TLSEdgeList { + if in == nil { + return nil + } + out := new(TLSEdgeList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TLSEdgeList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSEdgeSpec) DeepCopyInto(out *TLSEdgeSpec) { + *out = *in + out.ngrokAPICommon = in.ngrokAPICommon + in.Backend.DeepCopyInto(&out.Backend) + if in.Hostports != nil { + in, out := &in.Hostports, &out.Hostports + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.IPRestriction != nil { + in, out := &in.IPRestriction, &out.IPRestriction + *out = new(EndpointIPPolicy) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSEdgeSpec. +func (in *TLSEdgeSpec) DeepCopy() *TLSEdgeSpec { + if in == nil { + return nil + } + out := new(TLSEdgeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSEdgeStatus) DeepCopyInto(out *TLSEdgeStatus) { + *out = *in + if in.Hostports != nil { + in, out := &in.Hostports, &out.Hostports + *out = make([]string, len(*in)) + copy(*out, *in) + } + out.Backend = in.Backend +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSEdgeStatus. +func (in *TLSEdgeStatus) DeepCopy() *TLSEdgeStatus { + if in == nil { + return nil + } + out := new(TLSEdgeStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Tunnel) DeepCopyInto(out *Tunnel) { *out = *in diff --git a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tlsedges.yaml b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tlsedges.yaml new file mode 100644 index 00000000..1615c7bd --- /dev/null +++ b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tlsedges.yaml @@ -0,0 +1,126 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: tlsedges.ingress.k8s.ngrok.com +spec: + group: ingress.k8s.ngrok.com + names: + kind: TLSEdge + listKind: TLSEdgeList + plural: tlsedges + singular: tlsedge + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Domain ID + jsonPath: .status.id + name: ID + type: string + - description: Hostports + jsonPath: .status.hostports + name: Hostports + type: string + - description: Tunnel Group Backend ID + jsonPath: .status.backend.id + name: Backend ID + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: TLSEdge is the Schema for the tlsedges API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TLSEdgeSpec defines the desired state of TLSEdge + properties: + backend: + description: Backend is the definition for the tunnel group backend + that serves traffic for this edge + properties: + description: + default: Created by kubernetes-ingress-controller + description: Description is a human-readable description of the + object in the ngrok API/Dashboard + type: string + labels: + additionalProperties: + type: string + description: Labels to watch for tunnels on this backend + type: object + metadata: + default: '{"owned-by":"kubernetes-ingress-controller"}' + description: Metadata is a string of arbitrary data associated + with the object in the ngrok API/Dashboard + type: string + type: object + description: + default: Created by kubernetes-ingress-controller + description: Description is a human-readable description of the object + in the ngrok API/Dashboard + type: string + hostports: + description: Hostports is a list of hostports served by this edge + items: + type: string + type: array + ipRestriction: + description: IPRestriction is an IPRestriction to apply to this route + properties: + policies: + items: + type: string + type: array + type: object + metadata: + default: '{"owned-by":"kubernetes-ingress-controller"}' + description: Metadata is a string of arbitrary data associated with + the object in the ngrok API/Dashboard + type: string + type: object + status: + description: TLSEdgeStatus defines the observed state of TLSEdge + properties: + backend: + description: Backend stores the status of the tunnel group backend, + mainly the ID of the backend + properties: + id: + description: ID is the unique identifier for this backend + type: string + type: object + hostports: + description: Hostports served by this edge + items: + type: string + type: array + id: + description: ID is the unique identifier for this edge + type: string + uri: + description: URI is the URI of the edge + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/helm/ingress-controller/templates/rbac/role.yaml b/helm/ingress-controller/templates/rbac/role.yaml index f514256f..3a624ca6 100644 --- a/helm/ingress-controller/templates/rbac/role.yaml +++ b/helm/ingress-controller/templates/rbac/role.yaml @@ -151,6 +151,32 @@ rules: - get - patch - update +- apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges/finalizers + verbs: + - update +- apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges/status + verbs: + - get + - patch + - update - apiGroups: - ingress.k8s.ngrok.com resources: diff --git a/helm/ingress-controller/templates/rbac/tlsedge_editor_role.yaml b/helm/ingress-controller/templates/rbac/tlsedge_editor_role.yaml new file mode 100644 index 00000000..9d052319 --- /dev/null +++ b/helm/ingress-controller/templates/rbac/tlsedge_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit tlsedges. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: tlsedge-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: ngrok-ingress-controller + app.kubernetes.io/part-of: ngrok-ingress-controller + app.kubernetes.io/managed-by: kustomize + name: tlsedge-editor-role +rules: +- apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges/status + verbs: + - get diff --git a/helm/ingress-controller/templates/rbac/tlsedge_viewer_role.yaml b/helm/ingress-controller/templates/rbac/tlsedge_viewer_role.yaml new file mode 100644 index 00000000..a9eb99d9 --- /dev/null +++ b/helm/ingress-controller/templates/rbac/tlsedge_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view tlsedges. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: tlsedge-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: ngrok-ingress-controller + app.kubernetes.io/part-of: ngrok-ingress-controller + app.kubernetes.io/managed-by: kustomize + name: tlsedge-viewer-role +rules: +- apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges + verbs: + - get + - list + - watch +- apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges/status + verbs: + - get diff --git a/internal/controllers/tlsedge_controller.go b/internal/controllers/tlsedge_controller.go new file mode 100644 index 00000000..49feda7b --- /dev/null +++ b/internal/controllers/tlsedge_controller.go @@ -0,0 +1,319 @@ +/* +MIT License + +Copyright (c) 2022 ngrok, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package controllers + +import ( + "context" + + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/go-logr/logr" + ingressv1alpha1 "github.com/ngrok/kubernetes-ingress-controller/api/v1alpha1" + "github.com/ngrok/kubernetes-ingress-controller/internal/ngrokapi" + "github.com/ngrok/ngrok-api-go/v5" +) + +// TLSEdgeReconciler reconciles a TLSEdge object +type TLSEdgeReconciler struct { + client.Client + + Log logr.Logger + Scheme *runtime.Scheme + Recorder record.EventRecorder + + ipPolicyResolver + + NgrokClientset ngrokapi.Clientset + + controller *baseController[*ingressv1alpha1.TLSEdge] +} + +// SetupWithManager sets up the controller with the Manager. +func (r *TLSEdgeReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.ipPolicyResolver = ipPolicyResolver{client: mgr.GetClient()} + + r.controller = &baseController[*ingressv1alpha1.TLSEdge]{ + Kube: r.Client, + Log: r.Log, + Recorder: r.Recorder, + + kubeType: "v1alpha1.TLSEdge", + statusID: func(cr *ingressv1alpha1.TLSEdge) string { return cr.Status.ID }, + create: r.create, + update: r.update, + delete: r.delete, + } + + return ctrl.NewControllerManagedBy(mgr). + For(&ingressv1alpha1.TLSEdge{}). + Watches( + &source.Kind{Type: &ingressv1alpha1.IPPolicy{}}, + handler.EnqueueRequestsFromMapFunc(r.listTLSEdgesForIPPolicy), + ). + Complete(r) +} + +//+kubebuilder:rbac:groups=ingress.k8s.ngrok.com,resources=tlsedges,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=ingress.k8s.ngrok.com,resources=tlsedges/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=ingress.k8s.ngrok.com,resources=tlsedges/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.1/pkg/reconcile +func (r *TLSEdgeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + return r.controller.reconcile(ctx, req, new(ingressv1alpha1.TLSEdge)) +} + +func (r *TLSEdgeReconciler) create(ctx context.Context, edge *ingressv1alpha1.TLSEdge) error { + if err := r.reconcileTunnelGroupBackend(ctx, edge); err != nil { + return err + } + + // Try to find the edge by the backend labels + resp, err := r.findEdgeByBackendLabels(ctx, edge.Spec.Backend.Labels) + if err != nil { + return err + } + + if resp != nil { + return r.updateEdgeStatus(ctx, edge, resp) + } + + // No edge has been created for this edge, create one + r.Log.Info("Creating new TLSEdge", "namespace", edge.Namespace, "name", edge.Name) + resp, err = r.NgrokClientset.TLSEdges().Create(ctx, &ngrok.TLSEdgeCreate{ + Hostports: edge.Spec.Hostports, + Description: edge.Spec.Description, + Metadata: edge.Spec.Metadata, + Backend: &ngrok.EndpointBackendMutate{ + BackendID: edge.Status.Backend.ID, + }, + }) + if err != nil { + return err + } + r.Log.Info("Created new TLSEdge", "edge.ID", resp.ID, "name", edge.Name, "namespace", edge.Namespace) + + if err := r.updateEdgeStatus(ctx, edge, resp); err != nil { + return err + } + + return r.updateIPRestrictionRouteModule(ctx, edge, resp) +} + +func (r *TLSEdgeReconciler) update(ctx context.Context, edge *ingressv1alpha1.TLSEdge) error { + if err := r.reconcileTunnelGroupBackend(ctx, edge); err != nil { + return err + } + + resp, err := r.NgrokClientset.TLSEdges().Get(ctx, edge.Status.ID) + if err != nil { + // If we can't find the edge in the ngrok API, it's been deleted, so clear the ID + // and requeue the edge. When it gets reconciled again, it will be recreated. + if ngrok.IsNotFound(err) { + r.Log.Info("TLSEdge not found, clearing ID and requeuing", "edge.ID", edge.Status.ID) + edge.Status.ID = "" + //nolint:errcheck + r.Status().Update(ctx, edge) + } + return err + } + + // If the backend or hostports do not match, update the edge with the desired backend and hostports + if resp.Backend.Backend.ID != edge.Status.Backend.ID || + !slices.Equal(resp.Hostports, edge.Status.Hostports) { + resp, err = r.NgrokClientset.TLSEdges().Update(ctx, &ngrok.TLSEdgeUpdate{ + ID: resp.ID, + Description: pointer.String(edge.Spec.Description), + Metadata: pointer.String(edge.Spec.Metadata), + Hostports: edge.Spec.Hostports, + Backend: &ngrok.EndpointBackendMutate{ + BackendID: edge.Status.Backend.ID, + }, + }) + if err != nil { + return err + } + } + + return r.updateEdgeStatus(ctx, edge, resp) +} + +func (r *TLSEdgeReconciler) delete(ctx context.Context, edge *ingressv1alpha1.TLSEdge) error { + err := r.NgrokClientset.TLSEdges().Delete(ctx, edge.Status.ID) + if err == nil || ngrok.IsNotFound(err) { + edge.Status.ID = "" + } + return err +} + +func (r *TLSEdgeReconciler) reconcileTunnelGroupBackend(ctx context.Context, edge *ingressv1alpha1.TLSEdge) error { + specBackend := edge.Spec.Backend + // First make sure the tunnel group backend matches + if edge.Status.Backend.ID != "" { + // A backend has already been created for this edge, make sure the labels match + backend, err := r.NgrokClientset.TunnelGroupBackends().Get(ctx, edge.Status.Backend.ID) + if err != nil { + if ngrok.IsNotFound(err) { + r.Log.Info("TunnelGroupBackend not found, clearing ID and requeuing", "TunnelGroupBackend.ID", edge.Status.Backend.ID) + edge.Status.Backend.ID = "" + //nolint:errcheck + r.Status().Update(ctx, edge) + } + return err + } + + // If the labels don't match, update the backend with the desired labels + if !maps.Equal(backend.Labels, specBackend.Labels) { + _, err = r.NgrokClientset.TunnelGroupBackends().Update(ctx, &ngrok.TunnelGroupBackendUpdate{ + ID: backend.ID, + Metadata: pointer.String(specBackend.Metadata), + Description: pointer.String(specBackend.Description), + Labels: specBackend.Labels, + }) + if err != nil { + return err + } + } + return nil + } + + // No backend has been created for this edge, create one + backend, err := r.NgrokClientset.TunnelGroupBackends().Create(ctx, &ngrok.TunnelGroupBackendCreate{ + Metadata: edge.Spec.Backend.Metadata, + Description: edge.Spec.Backend.Description, + Labels: edge.Spec.Backend.Labels, + }) + if err != nil { + return err + } + edge.Status.Backend.ID = backend.ID + + return r.Status().Update(ctx, edge) +} + +func (r *TLSEdgeReconciler) findEdgeByBackendLabels(ctx context.Context, backendLabels map[string]string) (*ngrok.TLSEdge, error) { + r.Log.Info("Searching for existing TLSEdge with backend labels", "labels", backendLabels) + iter := r.NgrokClientset.TLSEdges().List(&ngrok.Paging{}) + for iter.Next(ctx) { + edge := iter.Item() + if edge.Backend == nil { + continue + } + + backend, err := r.NgrokClientset.TunnelGroupBackends().Get(ctx, edge.Backend.Backend.ID) + if err != nil { + // If we get an error looking up the backend, return the error and + // hopefully the next reconcile will fix it. + return nil, err + } + if backend == nil { + continue + } + + if maps.Equal(backend.Labels, backendLabels) { + r.Log.Info("Found existing TLSEdge with matching backend labels", "labels", backendLabels, "edge.ID", edge.ID) + return edge, nil + } + } + return nil, iter.Err() +} + +func (r *TLSEdgeReconciler) updateEdgeStatus(ctx context.Context, edge *ingressv1alpha1.TLSEdge, remoteEdge *ngrok.TLSEdge) error { + edge.Status.ID = remoteEdge.ID + edge.Status.URI = remoteEdge.URI + edge.Status.Hostports = remoteEdge.Hostports + edge.Status.Backend.ID = remoteEdge.Backend.Backend.ID + + return r.Status().Update(ctx, edge) +} + +func (r *TLSEdgeReconciler) updateIPRestrictionRouteModule(ctx context.Context, edge *ingressv1alpha1.TLSEdge, remoteEdge *ngrok.TLSEdge) error { + if edge.Spec.IPRestriction == nil || len(edge.Spec.IPRestriction.IPPolicies) == 0 { + return r.NgrokClientset.EdgeModules().TLS().IPRestriction().Delete(ctx, edge.Status.ID) + } + policyIds, err := r.ipPolicyResolver.resolveIPPolicyNamesorIds(ctx, edge.Namespace, edge.Spec.IPRestriction.IPPolicies) + if err != nil { + return err + } + r.Log.Info("Resolved IP Policy NamesOrIDs to IDs", "policyIds", policyIds) + + _, err = r.NgrokClientset.EdgeModules().TLS().IPRestriction().Replace(ctx, &ngrok.EdgeIPRestrictionReplace{ + ID: edge.Status.ID, + Module: ngrok.EndpointIPPolicyMutate{ + IPPolicyIDs: policyIds, + }, + }) + return err +} + +func (r *TLSEdgeReconciler) listTLSEdgesForIPPolicy(obj client.Object) []reconcile.Request { + r.Log.Info("Listing TLSEdges for ip policy to determine if they need to be reconciled") + policy, ok := obj.(*ingressv1alpha1.IPPolicy) + if !ok { + r.Log.Error(nil, "failed to convert object to IPPolicy", "object", obj) + return []reconcile.Request{} + } + + edges := &ingressv1alpha1.TLSEdgeList{} + if err := r.Client.List(context.Background(), edges); err != nil { + r.Log.Error(err, "failed to list TLSEdges for ippolicy", "name", policy.Name, "namespace", policy.Namespace) + return []reconcile.Request{} + } + + recs := []reconcile.Request{} + + for _, edge := range edges.Items { + if edge.Spec.IPRestriction == nil { + continue + } + for _, edgePolicyID := range edge.Spec.IPRestriction.IPPolicies { + if edgePolicyID == policy.Name || edgePolicyID == policy.Status.ID { + recs = append(recs, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: edge.GetName(), + Namespace: edge.GetNamespace(), + }, + }) + break + } + } + } + + r.Log.Info("IPPolicy change triggered TLSEdge reconciliation", "count", len(recs), "policy", policy.Name, "namespace", policy.Namespace) + return recs +} diff --git a/internal/ngrokapi/clientset.go b/internal/ngrokapi/clientset.go index 66033732..40e5ad1c 100644 --- a/internal/ngrokapi/clientset.go +++ b/internal/ngrokapi/clientset.go @@ -6,6 +6,7 @@ import ( https_edges "github.com/ngrok/ngrok-api-go/v5/edges/https" https_edge_routes "github.com/ngrok/ngrok-api-go/v5/edges/https_routes" tcp_edges "github.com/ngrok/ngrok-api-go/v5/edges/tcp" + tls_edges "github.com/ngrok/ngrok-api-go/v5/edges/tls" "github.com/ngrok/ngrok-api-go/v5/ip_policies" "github.com/ngrok/ngrok-api-go/v5/ip_policy_rules" "github.com/ngrok/ngrok-api-go/v5/reserved_addrs" @@ -21,6 +22,7 @@ type Clientset interface { IPPolicyRules() *ip_policy_rules.Client TCPAddresses() *reserved_addrs.Client TCPEdges() *tcp_edges.Client + TLSEdges() *tls_edges.Client TunnelGroupBackends() *tunnel_group_backends.Client } @@ -33,6 +35,7 @@ type DefaultClientset struct { ipPolicyRulesClient *ip_policy_rules.Client tcpAddrsClient *reserved_addrs.Client tcpEdgesClient *tcp_edges.Client + tlsEdgesClient *tls_edges.Client tunnelGroupBackendsClient *tunnel_group_backends.Client } @@ -47,6 +50,7 @@ func NewClientSet(config *ngrok.ClientConfig) *DefaultClientset { ipPolicyRulesClient: ip_policy_rules.NewClient(config), tcpAddrsClient: reserved_addrs.NewClient(config), tcpEdgesClient: tcp_edges.NewClient(config), + tlsEdgesClient: tls_edges.NewClient(config), tunnelGroupBackendsClient: tunnel_group_backends.NewClient(config), } } @@ -79,6 +83,10 @@ func (c *DefaultClientset) TCPAddresses() *reserved_addrs.Client { return c.tcpAddrsClient } +func (c *DefaultClientset) TLSEdges() *tls_edges.Client { + return c.tlsEdgesClient +} + func (c *DefaultClientset) TCPEdges() *tcp_edges.Client { return c.tcpEdgesClient } diff --git a/main.go b/main.go index b4f17aba..eb4c7cc2 100644 --- a/main.go +++ b/main.go @@ -222,6 +222,16 @@ func runController(ctx context.Context, opts managerOpts) error { setupLog.Error(err, "unable to create controller", "controller", "TCPEdge") os.Exit(1) } + if err = (&controllers.TLSEdgeReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("tls-edge"), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("tls-edge-controller"), + NgrokClientset: ngrokClientset, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "TLSEdge") + os.Exit(1) + } if err = (&controllers.HTTPSEdgeReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("https-edge"), From 5e1e53974c6e8e2c4899565cc3b5110bbe1f4a41 Mon Sep 17 00:00:00 2001 From: Josh Robson Chase Date: Thu, 5 Oct 2023 13:10:29 -0400 Subject: [PATCH 03/10] retry domain-still-attached errors --- internal/controllers/domain_controller.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/controllers/domain_controller.go b/internal/controllers/domain_controller.go index 9e6f1a07..46498cd7 100644 --- a/internal/controllers/domain_controller.go +++ b/internal/controllers/domain_controller.go @@ -33,6 +33,7 @@ import ( "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/go-logr/logr" ingressv1alpha1 "github.com/ngrok/kubernetes-ingress-controller/api/v1alpha1" @@ -68,6 +69,15 @@ func (r *DomainReconciler) SetupWithManager(mgr ctrl.Manager) error { create: r.create, update: r.update, delete: r.delete, + errResult: func(op baseControllerOp, cr *ingressv1alpha1.Domain, err error) (reconcile.Result, error) { + // Domain still attached to an edge, probably a race condition. + // Schedule for retry, and hopefully the edge will be gone + // eventually. + if ngrok.IsErrorCode(err, 446) { + return ctrl.Result{}, err + } + return reconcileResultFromError(err) + }, } return ctrl.NewControllerManagedBy(mgr). From 2d819d2acccbca141597c6fa6a0efed3d9ba16eb Mon Sep 17 00:00:00 2001 From: Josh Robson Chase Date: Thu, 5 Oct 2023 15:21:32 -0400 Subject: [PATCH 04/10] support mutual TLS and TLS termination --- api/v1alpha1/ngrok_common.go | 16 ++++++ api/v1alpha1/tlsedge_types.go | 4 +- api/v1alpha1/zz_generated.deepcopy.go | 50 +++++++++++++++++ .../crds/ingress.k8s.ngrok.com_tlsedges.yaml | 22 ++++++++ internal/controllers/tlsedge_controller.go | 55 +++++++++++++++++++ 5 files changed, 146 insertions(+), 1 deletion(-) diff --git a/api/v1alpha1/ngrok_common.go b/api/v1alpha1/ngrok_common.go index cba02644..7034412b 100644 --- a/api/v1alpha1/ngrok_common.go +++ b/api/v1alpha1/ngrok_common.go @@ -56,6 +56,22 @@ type EndpointHeaders struct { Response *EndpointResponseHeaders `json:"response,omitempty"` } +type EndpointMutualTLS struct { + // List of CA IDs that will be used to validate incoming connections to the + // edge. + CertificateAuthorities []string `json:"certificateAuthorities,omitempty"` +} + +type EndpointTLSTermination struct { + // TerminateAt determines where the TLS connection should be terminated. + // "edge" if the ngrok edge should terminate TLS traffic, "upstream" if TLS + // traffic should be passed through to the upstream ngrok agent / + // application server for termination. + TerminateAt string `json:"terminateAt,omitempty"` + // MinVersion is the minimum TLS version to allow for connections to the edge + MinVersion *string `json:"minVersion,omitempty"` +} + type EndpointTLSTerminationAtEdge struct { // MinVersion is the minimum TLS version to allow for connections to the edge MinVersion string `json:"minVersion,omitempty"` diff --git a/api/v1alpha1/tlsedge_types.go b/api/v1alpha1/tlsedge_types.go index 211f97b4..2b5b5adb 100644 --- a/api/v1alpha1/tlsedge_types.go +++ b/api/v1alpha1/tlsedge_types.go @@ -47,7 +47,9 @@ type TLSEdgeSpec struct { // IPRestriction is an IPRestriction to apply to this route IPRestriction *EndpointIPPolicy `json:"ipRestriction,omitempty"` - // TODO: mutual tls, tls termination + TLSTermination *EndpointTLSTermination `json:"tlsTermination,omitempty"` + + MutualTLS *EndpointMutualTLS `json:"mutualTls,omitempty"` } // TLSEdgeStatus defines the observed state of TLSEdge diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index df6fa334..6f6f9396 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -221,6 +221,26 @@ func (in *EndpointIPPolicy) DeepCopy() *EndpointIPPolicy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EndpointMutualTLS) DeepCopyInto(out *EndpointMutualTLS) { + *out = *in + if in.CertificateAuthorities != nil { + in, out := &in.CertificateAuthorities, &out.CertificateAuthorities + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EndpointMutualTLS. +func (in *EndpointMutualTLS) DeepCopy() *EndpointMutualTLS { + if in == nil { + return nil + } + out := new(EndpointMutualTLS) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EndpointOAuth) DeepCopyInto(out *EndpointOAuth) { *out = *in @@ -518,6 +538,26 @@ func (in *EndpointSAML) DeepCopy() *EndpointSAML { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EndpointTLSTermination) DeepCopyInto(out *EndpointTLSTermination) { + *out = *in + if in.MinVersion != nil { + in, out := &in.MinVersion, &out.MinVersion + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EndpointTLSTermination. +func (in *EndpointTLSTermination) DeepCopy() *EndpointTLSTermination { + if in == nil { + return nil + } + out := new(EndpointTLSTermination) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EndpointTLSTerminationAtEdge) DeepCopyInto(out *EndpointTLSTerminationAtEdge) { *out = *in @@ -1221,6 +1261,16 @@ func (in *TLSEdgeSpec) DeepCopyInto(out *TLSEdgeSpec) { *out = new(EndpointIPPolicy) (*in).DeepCopyInto(*out) } + if in.TLSTermination != nil { + in, out := &in.TLSTermination, &out.TLSTermination + *out = new(EndpointTLSTermination) + (*in).DeepCopyInto(*out) + } + if in.MutualTLS != nil { + in, out := &in.MutualTLS, &out.MutualTLS + *out = new(EndpointMutualTLS) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSEdgeSpec. diff --git a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tlsedges.yaml b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tlsedges.yaml index 1615c7bd..dde03d15 100644 --- a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tlsedges.yaml +++ b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tlsedges.yaml @@ -95,6 +95,28 @@ spec: description: Metadata is a string of arbitrary data associated with the object in the ngrok API/Dashboard type: string + mutualTls: + properties: + certificateAuthorities: + description: List of CA IDs that will be used to validate incoming + connections to the edge. + items: + type: string + type: array + type: object + tlsTermination: + properties: + minVersion: + description: MinVersion is the minimum TLS version to allow for + connections to the edge + type: string + terminateAt: + description: TerminateAt determines where the TLS connection should + be terminated. "edge" if the ngrok edge should terminate TLS + traffic, "upstream" if TLS traffic should be passed through + to the upstream ngrok agent / application server for termination. + type: string + type: object type: object status: description: TLSEdgeStatus defines the observed state of TLSEdge diff --git a/internal/controllers/tlsedge_controller.go b/internal/controllers/tlsedge_controller.go index 49feda7b..a2962705 100644 --- a/internal/controllers/tlsedge_controller.go +++ b/internal/controllers/tlsedge_controller.go @@ -170,6 +170,14 @@ func (r *TLSEdgeReconciler) update(ctx context.Context, edge *ingressv1alpha1.TL } } + if err := r.setTLSTermination(ctx, resp, edge.Spec.TLSTermination); err != nil { + return err + } + + if err := r.setMutualTLS(ctx, resp, edge.Spec.MutualTLS); err != nil { + return err + } + return r.updateEdgeStatus(ctx, edge, resp) } @@ -226,6 +234,53 @@ func (r *TLSEdgeReconciler) reconcileTunnelGroupBackend(ctx context.Context, edg return r.Status().Update(ctx, edge) } +func (r *TLSEdgeReconciler) setMutualTLS(ctx context.Context, edge *ngrok.TLSEdge, mutualTls *ingressv1alpha1.EndpointMutualTLS) error { + log := ctrl.LoggerFrom(ctx) + + client := r.NgrokClientset.EdgeModules().TLS().MutualTLS() + if mutualTls == nil { + if edge.MutualTls == nil { + log.V(1).Info("Edge Mutual TLS matches spec") + return nil + } + + log.Info("Deleting Edge Mutual TLS") + return client.Delete(ctx, edge.ID) + } + + _, err := client.Replace(ctx, &ngrok.EdgeMutualTLSReplace{ + ID: edge.ID, + Module: ngrok.EndpointMutualTLSMutate{ + CertificateAuthorityIDs: mutualTls.CertificateAuthorities, + }, + }) + return err +} + +func (r *TLSEdgeReconciler) setTLSTermination(ctx context.Context, edge *ngrok.TLSEdge, tlsTermination *ingressv1alpha1.EndpointTLSTermination) error { + log := ctrl.LoggerFrom(ctx) + + client := r.NgrokClientset.EdgeModules().TLS().TLSTermination() + if tlsTermination == nil { + if edge.TlsTermination == nil { + log.V(1).Info("Edge TLS termination matches spec") + return nil + } + + log.Info("Deleting Edge TLS termination") + return client.Delete(ctx, edge.ID) + } + + _, err := client.Replace(ctx, &ngrok.EdgeTLSTerminationReplace{ + ID: edge.ID, + Module: ngrok.EndpointTLSTermination{ + TerminateAt: tlsTermination.TerminateAt, + MinVersion: tlsTermination.MinVersion, + }, + }) + return err +} + func (r *TLSEdgeReconciler) findEdgeByBackendLabels(ctx context.Context, backendLabels map[string]string) (*ngrok.TLSEdge, error) { r.Log.Info("Searching for existing TLSEdge with backend labels", "labels", backendLabels) iter := r.NgrokClientset.TLSEdges().List(&ngrok.Paging{}) From 4cd46f3cef2217b337057d7e84cff382ab077c96 Mon Sep 17 00:00:00 2001 From: Josh Robson Chase Date: Thu, 5 Oct 2023 15:28:19 -0400 Subject: [PATCH 05/10] update helm snapshot --- .../controller-deployment_test.yaml.snap | 56 ++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/helm/ingress-controller/tests/__snapshot__/controller-deployment_test.yaml.snap b/helm/ingress-controller/tests/__snapshot__/controller-deployment_test.yaml.snap index 4ea3b4e3..ba2dcc46 100644 --- a/helm/ingress-controller/tests/__snapshot__/controller-deployment_test.yaml.snap +++ b/helm/ingress-controller/tests/__snapshot__/controller-deployment_test.yaml.snap @@ -4,7 +4,7 @@ Should match all-options snapshot: kind: Deployment metadata: annotations: - checksum/controller-role: 55beaaec6ab70343a40b69e51ce45a3c7a1e4c6c48053390666d585b2f0f3458 + checksum/controller-role: 935bd1fb3894f82f10c4e873691dfb57e2f67cf250ef103844c6dfda7622ded2 checksum/rbac: d31fdcb337a6f1ee71323040c2cbc4d5580d73ae5f7623cd19be57db97f748c1 labels: app.kubernetes.io/component: controller @@ -357,6 +357,32 @@ Should match all-options snapshot: - get - patch - update + - apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges/finalizers + verbs: + - update + - apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges/status + verbs: + - get + - patch + - update - apiGroups: - ingress.k8s.ngrok.com resources: @@ -415,7 +441,7 @@ Should match default snapshot: kind: Deployment metadata: annotations: - checksum/controller-role: 55beaaec6ab70343a40b69e51ce45a3c7a1e4c6c48053390666d585b2f0f3458 + checksum/controller-role: 935bd1fb3894f82f10c4e873691dfb57e2f67cf250ef103844c6dfda7622ded2 checksum/rbac: d31fdcb337a6f1ee71323040c2cbc4d5580d73ae5f7623cd19be57db97f748c1 labels: app.kubernetes.io/component: controller @@ -755,6 +781,32 @@ Should match default snapshot: - get - patch - update + - apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges/finalizers + verbs: + - update + - apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges/status + verbs: + - get + - patch + - update - apiGroups: - ingress.k8s.ngrok.com resources: From 58decd72842726dfd930dcc412f024b1327fd1cc Mon Sep 17 00:00:00 2001 From: Josh Robson Chase Date: Fri, 6 Oct 2023 09:48:04 -0400 Subject: [PATCH 06/10] remove 'route' from tls and tcp edge docs and function names --- api/v1alpha1/tcpedge_types.go | 2 +- api/v1alpha1/tlsedge_types.go | 2 +- internal/controllers/tcpedge_controller.go | 4 ++-- internal/controllers/tlsedge_controller.go | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/tcpedge_types.go b/api/v1alpha1/tcpedge_types.go index 24458792..9b6385eb 100644 --- a/api/v1alpha1/tcpedge_types.go +++ b/api/v1alpha1/tcpedge_types.go @@ -40,7 +40,7 @@ type TCPEdgeSpec struct { // +kubebuilder:validation:Required Backend TunnelGroupBackend `json:"backend,omitempty"` - // IPRestriction is an IPRestriction to apply to this route + // IPRestriction is an IPRestriction to apply to this edge IPRestriction *EndpointIPPolicy `json:"ipRestriction,omitempty"` } diff --git a/api/v1alpha1/tlsedge_types.go b/api/v1alpha1/tlsedge_types.go index 2b5b5adb..051a1777 100644 --- a/api/v1alpha1/tlsedge_types.go +++ b/api/v1alpha1/tlsedge_types.go @@ -44,7 +44,7 @@ type TLSEdgeSpec struct { // +kubebuilder:validation:Required Hostports []string `json:"hostports,omitempty"` - // IPRestriction is an IPRestriction to apply to this route + // IPRestriction is an IPRestriction to apply to this edge IPRestriction *EndpointIPPolicy `json:"ipRestriction,omitempty"` TLSTermination *EndpointTLSTermination `json:"tlsTermination,omitempty"` diff --git a/internal/controllers/tcpedge_controller.go b/internal/controllers/tcpedge_controller.go index f99ca1c4..aafd96d1 100644 --- a/internal/controllers/tcpedge_controller.go +++ b/internal/controllers/tcpedge_controller.go @@ -136,7 +136,7 @@ func (r *TCPEdgeReconciler) create(ctx context.Context, edge *ingressv1alpha1.TC return err } - return r.updateIPRestrictionRouteModule(ctx, edge, resp) + return r.updateIPRestrictionModule(ctx, edge, resp) } func (r *TCPEdgeReconciler) update(ctx context.Context, edge *ingressv1alpha1.TCPEdge) error { @@ -317,7 +317,7 @@ func (r *TCPEdgeReconciler) descriptionForEdge(edge *ingressv1alpha1.TCPEdge) st return fmt.Sprintf("Reserved for %s/%s", edge.Namespace, edge.Name) } -func (r *TCPEdgeReconciler) updateIPRestrictionRouteModule(ctx context.Context, edge *ingressv1alpha1.TCPEdge, remoteEdge *ngrok.TCPEdge) error { +func (r *TCPEdgeReconciler) updateIPRestrictionModule(ctx context.Context, edge *ingressv1alpha1.TCPEdge, remoteEdge *ngrok.TCPEdge) error { if edge.Spec.IPRestriction == nil || len(edge.Spec.IPRestriction.IPPolicies) == 0 { return r.NgrokClientset.EdgeModules().TCP().IPRestriction().Delete(ctx, edge.Status.ID) } diff --git a/internal/controllers/tlsedge_controller.go b/internal/controllers/tlsedge_controller.go index a2962705..5602ba32 100644 --- a/internal/controllers/tlsedge_controller.go +++ b/internal/controllers/tlsedge_controller.go @@ -132,7 +132,7 @@ func (r *TLSEdgeReconciler) create(ctx context.Context, edge *ingressv1alpha1.TL return err } - return r.updateIPRestrictionRouteModule(ctx, edge, resp) + return r.updateIPRestrictionModule(ctx, edge, resp) } func (r *TLSEdgeReconciler) update(ctx context.Context, edge *ingressv1alpha1.TLSEdge) error { @@ -317,7 +317,7 @@ func (r *TLSEdgeReconciler) updateEdgeStatus(ctx context.Context, edge *ingressv return r.Status().Update(ctx, edge) } -func (r *TLSEdgeReconciler) updateIPRestrictionRouteModule(ctx context.Context, edge *ingressv1alpha1.TLSEdge, remoteEdge *ngrok.TLSEdge) error { +func (r *TLSEdgeReconciler) updateIPRestrictionModule(ctx context.Context, edge *ingressv1alpha1.TLSEdge, remoteEdge *ngrok.TLSEdge) error { if edge.Spec.IPRestriction == nil || len(edge.Spec.IPRestriction.IPPolicies) == 0 { return r.NgrokClientset.EdgeModules().TLS().IPRestriction().Delete(ctx, edge.Status.ID) } From a58a189ec8b91672ed74c223d07f1ac043c6f54b Mon Sep 17 00:00:00 2001 From: Josh Robson Chase Date: Fri, 6 Oct 2023 09:58:49 -0400 Subject: [PATCH 07/10] update consistency and error handling --- internal/controllers/tcpedge_controller.go | 26 ++++++++++++++-------- internal/controllers/tlsedge_controller.go | 26 ++++++++++++++-------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/internal/controllers/tcpedge_controller.go b/internal/controllers/tcpedge_controller.go index aafd96d1..1db42f45 100644 --- a/internal/controllers/tcpedge_controller.go +++ b/internal/controllers/tcpedge_controller.go @@ -115,7 +115,7 @@ func (r *TCPEdgeReconciler) create(ctx context.Context, edge *ingressv1alpha1.TC } if resp != nil { - return r.updateEdgeStatus(ctx, edge, resp) + return r.updateEdge(ctx, edge, resp) } // No edge has been created for this edge, create one @@ -132,11 +132,7 @@ func (r *TCPEdgeReconciler) create(ctx context.Context, edge *ingressv1alpha1.TC } r.Log.Info("Created new TCPEdge", "edge.ID", resp.ID, "name", edge.Name, "namespace", edge.Namespace) - if err := r.updateEdgeStatus(ctx, edge, resp); err != nil { - return err - } - - return r.updateIPRestrictionModule(ctx, edge, resp) + return r.updateEdge(ctx, edge, resp) } func (r *TCPEdgeReconciler) update(ctx context.Context, edge *ingressv1alpha1.TCPEdge) error { @@ -155,8 +151,7 @@ func (r *TCPEdgeReconciler) update(ctx context.Context, edge *ingressv1alpha1.TC if ngrok.IsNotFound(err) { r.Log.Info("TCPEdge not found, clearing ID and requeuing", "edge.ID", edge.Status.ID) edge.Status.ID = "" - //nolint:errcheck - r.Status().Update(ctx, edge) + err = r.Status().Update(ctx, edge) } return err } @@ -178,7 +173,7 @@ func (r *TCPEdgeReconciler) update(ctx context.Context, edge *ingressv1alpha1.TC } } - return r.updateEdgeStatus(ctx, edge, resp) + return r.updateEdge(ctx, edge, resp) } func (r *TCPEdgeReconciler) delete(ctx context.Context, edge *ingressv1alpha1.TCPEdge) error { @@ -261,6 +256,19 @@ func (r *TCPEdgeReconciler) findEdgeByBackendLabels(ctx context.Context, backend return nil, iter.Err() } +// Update the edge status and modules, called from both create and update. +func (r *TCPEdgeReconciler) updateEdge(ctx context.Context, edge *ingressv1alpha1.TCPEdge, remoteEdge *ngrok.TCPEdge) error { + if err := r.updateEdgeStatus(ctx, edge, remoteEdge); err != nil { + return err + } + + if err := r.updateIPRestrictionModule(ctx, edge, remoteEdge); err != nil { + return err + } + + return nil +} + func (r *TCPEdgeReconciler) updateEdgeStatus(ctx context.Context, edge *ingressv1alpha1.TCPEdge, remoteEdge *ngrok.TCPEdge) error { edge.Status.ID = remoteEdge.ID edge.Status.URI = remoteEdge.URI diff --git a/internal/controllers/tlsedge_controller.go b/internal/controllers/tlsedge_controller.go index 5602ba32..974e86f5 100644 --- a/internal/controllers/tlsedge_controller.go +++ b/internal/controllers/tlsedge_controller.go @@ -110,7 +110,7 @@ func (r *TLSEdgeReconciler) create(ctx context.Context, edge *ingressv1alpha1.TL } if resp != nil { - return r.updateEdgeStatus(ctx, edge, resp) + return r.updateEdge(ctx, edge, resp) } // No edge has been created for this edge, create one @@ -128,11 +128,7 @@ func (r *TLSEdgeReconciler) create(ctx context.Context, edge *ingressv1alpha1.TL } r.Log.Info("Created new TLSEdge", "edge.ID", resp.ID, "name", edge.Name, "namespace", edge.Namespace) - if err := r.updateEdgeStatus(ctx, edge, resp); err != nil { - return err - } - - return r.updateIPRestrictionModule(ctx, edge, resp) + return r.updateEdge(ctx, edge, resp) } func (r *TLSEdgeReconciler) update(ctx context.Context, edge *ingressv1alpha1.TLSEdge) error { @@ -147,8 +143,7 @@ func (r *TLSEdgeReconciler) update(ctx context.Context, edge *ingressv1alpha1.TL if ngrok.IsNotFound(err) { r.Log.Info("TLSEdge not found, clearing ID and requeuing", "edge.ID", edge.Status.ID) edge.Status.ID = "" - //nolint:errcheck - r.Status().Update(ctx, edge) + err = r.Status().Update(ctx, edge) } return err } @@ -170,6 +165,15 @@ func (r *TLSEdgeReconciler) update(ctx context.Context, edge *ingressv1alpha1.TL } } + return r.updateEdge(ctx, edge, resp) +} + +// Update the edge status and modules, called from both create and update. +func (r *TLSEdgeReconciler) updateEdge(ctx context.Context, edge *ingressv1alpha1.TLSEdge, resp *ngrok.TLSEdge) error { + if err := r.updateEdgeStatus(ctx, edge, resp); err != nil { + return err + } + if err := r.setTLSTermination(ctx, resp, edge.Spec.TLSTermination); err != nil { return err } @@ -178,7 +182,11 @@ func (r *TLSEdgeReconciler) update(ctx context.Context, edge *ingressv1alpha1.TL return err } - return r.updateEdgeStatus(ctx, edge, resp) + if err := r.updateIPRestrictionModule(ctx, edge, resp); err != nil { + return err + } + + return nil } func (r *TLSEdgeReconciler) delete(ctx context.Context, edge *ingressv1alpha1.TLSEdge) error { From f0338f2c0c77f74382b4ff99a19147e0d27da721 Mon Sep 17 00:00:00 2001 From: Josh Robson Chase Date: Fri, 6 Oct 2023 10:03:06 -0400 Subject: [PATCH 08/10] specialize errResult for TLSEdge --- internal/controllers/tlsedge_controller.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/internal/controllers/tlsedge_controller.go b/internal/controllers/tlsedge_controller.go index 974e86f5..83da976c 100644 --- a/internal/controllers/tlsedge_controller.go +++ b/internal/controllers/tlsedge_controller.go @@ -26,6 +26,7 @@ package controllers import ( "context" + "errors" "golang.org/x/exp/maps" "golang.org/x/exp/slices" @@ -41,6 +42,7 @@ import ( "github.com/go-logr/logr" ingressv1alpha1 "github.com/ngrok/kubernetes-ingress-controller/api/v1alpha1" + ierr "github.com/ngrok/kubernetes-ingress-controller/internal/errors" "github.com/ngrok/kubernetes-ingress-controller/internal/ngrokapi" "github.com/ngrok/ngrok-api-go/v5" ) @@ -74,6 +76,15 @@ func (r *TLSEdgeReconciler) SetupWithManager(mgr ctrl.Manager) error { create: r.create, update: r.update, delete: r.delete, + errResult: func(op baseControllerOp, cr *ingressv1alpha1.TLSEdge, err error) (ctrl.Result, error) { + if errors.As(err, &ierr.ErrInvalidConfiguration{}) { + return ctrl.Result{}, nil + } + if ngrok.IsErrorCode(err, 7117) { // https://ngrok.com/docs/errors/err_ngrok_7117, domain not found + return ctrl.Result{}, err + } + return reconcileResultFromError(err) + }, } return ctrl.NewControllerManagedBy(mgr). From 6093da9e9e723d4b5105b78e375f1d56aecbe219 Mon Sep 17 00:00:00 2001 From: Josh Robson Chase Date: Fri, 6 Oct 2023 10:06:25 -0400 Subject: [PATCH 09/10] run generators --- .../templates/crds/ingress.k8s.ngrok.com_tcpedges.yaml | 2 +- .../templates/crds/ingress.k8s.ngrok.com_tlsedges.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tcpedges.yaml b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tcpedges.yaml index ffb4af82..3ae1b679 100644 --- a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tcpedges.yaml +++ b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tcpedges.yaml @@ -78,7 +78,7 @@ spec: in the ngrok API/Dashboard type: string ipRestriction: - description: IPRestriction is an IPRestriction to apply to this route + description: IPRestriction is an IPRestriction to apply to this edge properties: policies: items: diff --git a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tlsedges.yaml b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tlsedges.yaml index dde03d15..7f804456 100644 --- a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tlsedges.yaml +++ b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tlsedges.yaml @@ -83,7 +83,7 @@ spec: type: string type: array ipRestriction: - description: IPRestriction is an IPRestriction to apply to this route + description: IPRestriction is an IPRestriction to apply to this edge properties: policies: items: From fcf379d913f7c179cca4d5dc57de880415f03cfb Mon Sep 17 00:00:00 2001 From: Josh Robson Chase Date: Mon, 9 Oct 2023 13:48:10 -0400 Subject: [PATCH 10/10] put back the error handling on NotFound errors --- internal/controllers/tcpedge_controller.go | 3 ++- internal/controllers/tlsedge_controller.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/controllers/tcpedge_controller.go b/internal/controllers/tcpedge_controller.go index 1db42f45..14947205 100644 --- a/internal/controllers/tcpedge_controller.go +++ b/internal/controllers/tcpedge_controller.go @@ -151,7 +151,8 @@ func (r *TCPEdgeReconciler) update(ctx context.Context, edge *ingressv1alpha1.TC if ngrok.IsNotFound(err) { r.Log.Info("TCPEdge not found, clearing ID and requeuing", "edge.ID", edge.Status.ID) edge.Status.ID = "" - err = r.Status().Update(ctx, edge) + //nolint:errcheck + r.Status().Update(ctx, edge) } return err } diff --git a/internal/controllers/tlsedge_controller.go b/internal/controllers/tlsedge_controller.go index 83da976c..e33e5930 100644 --- a/internal/controllers/tlsedge_controller.go +++ b/internal/controllers/tlsedge_controller.go @@ -154,7 +154,8 @@ func (r *TLSEdgeReconciler) update(ctx context.Context, edge *ingressv1alpha1.TL if ngrok.IsNotFound(err) { r.Log.Info("TLSEdge not found, clearing ID and requeuing", "edge.ID", edge.Status.ID) edge.Status.ID = "" - err = r.Status().Update(ctx, edge) + //nolint:errcheck + r.Status().Update(ctx, edge) } return err }