diff --git a/cmd/cl-controlplane/app/server.go b/cmd/cl-controlplane/app/server.go index 59711b2c9..5c219168f 100644 --- a/cmd/cl-controlplane/app/server.go +++ b/cmd/cl-controlplane/app/server.go @@ -23,7 +23,9 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + discv1 "k8s.io/api/discovery/v1" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" @@ -152,6 +154,10 @@ func (o *Options) Run() error { return fmt.Errorf("unable to add discovery v1 objects to scheme: %w", err) } + if err := appsv1.AddToScheme(scheme); err != nil { + return fmt.Errorf("unable to add core appsv1 objects to scheme: %w", err) + } + // set logger for controller-runtime components ctrl.SetLogger(logrusr.New(logrus.WithField("component", "k8s.controller-runtime"))) diff --git a/config/crds/clusterlink.net_imports.yaml b/config/crds/clusterlink.net_imports.yaml index 31fc41e31..577752070 100644 --- a/config/crds/clusterlink.net_imports.yaml +++ b/config/crds/clusterlink.net_imports.yaml @@ -40,6 +40,10 @@ spec: spec: description: Spec represents the attributes of the imported service. properties: + alias: + description: Alias is an optional external DNS name for this imported + service + type: string lbScheme: default: round-robin description: LBScheme is the load-balancing scheme to use (e.g., random, diff --git a/config/operator/rbac/role.yaml b/config/operator/rbac/role.yaml index 3b42b3e67..f448aa647 100644 --- a/config/operator/rbac/role.yaml +++ b/config/operator/rbac/role.yaml @@ -4,6 +4,15 @@ kind: ClusterRole metadata: name: cl-operator-manager-role rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - update + - watch - apiGroups: - "" resources: diff --git a/pkg/apis/clusterlink.net/v1alpha1/import.go b/pkg/apis/clusterlink.net/v1alpha1/import.go index c4d628fcb..0824f9353 100644 --- a/pkg/apis/clusterlink.net/v1alpha1/import.go +++ b/pkg/apis/clusterlink.net/v1alpha1/import.go @@ -65,6 +65,8 @@ type ImportSpec struct { // +kubebuilder:default="round-robin" // LBScheme is the load-balancing scheme to use (e.g., random, static, round-robin) LBScheme LBScheme `json:"lbScheme"` + // Alias is an optional external DNS name for this imported service + Alias string `json:"alias,omitempty"` } const ( diff --git a/pkg/bootstrap/platform/k8s.go b/pkg/bootstrap/platform/k8s.go index efc84ec0e..86cc8c8d9 100644 --- a/pkg/bootstrap/platform/k8s.go +++ b/pkg/bootstrap/platform/k8s.go @@ -198,6 +198,9 @@ kind: ClusterRole metadata: name: cl-controlplane rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "get", "list", "update", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["create", "update"] diff --git a/pkg/controlplane/control/dns.go b/pkg/controlplane/control/dns.go new file mode 100644 index 000000000..82b275e0a --- /dev/null +++ b/pkg/controlplane/control/dns.go @@ -0,0 +1,178 @@ +// Copyright (c) The ClusterLink Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package control + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/sirupsen/logrus" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Restart coredns deployment. +func restartCoreDNS(ctx context.Context, mClient client.Client, logger *logrus.Entry) error { + logger.Infof("restarting coredns deployment") + patch := []byte( + fmt.Sprintf( + `{"spec": {"template": {"metadata": {"annotations":{"kubectl.kubernetes.io/restartedAt": %q}}}}}`, + time.Now().String(), + ), + ) + + return mClient.Patch(ctx, &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "coredns", + }, + }, client.RawPatch(types.StrategicMergePatchType, patch)) +} + +// Add coredns rewrite for a given external dns service. +func addCoreDNSRewrite(ctx context.Context, mClient client.Client, logger *logrus.Entry, name *types.NamespacedName, + alias string, +) error { + corednsName := types.NamespacedName{ + Name: "coredns", + Namespace: "kube-system", + } + var cm v1.ConfigMap + + if err := mClient.Get(ctx, corednsName, &cm); err != nil { + if k8serrors.IsNotFound(err) { + logger.Warnf("coredns configmap not found.") + return nil + } + return err + } + if _, ok := cm.Data["Corefile"]; !ok { + return errors.New("coredns configmap['Corefile'] not found") + } + + data := cm.Data["Corefile"] + // remove trailing end-of-line + data = strings.TrimSuffix(data, "\n") + // break into lines + lines := strings.Split(data, "\n") + serviceFqdn := fmt.Sprintf("%s.%s.svc.cluster.local", name.Name, name.Namespace) + + coreFileUpdated := false + rewriteLine := "" + for i, line := range lines { + if strings.Contains(line, serviceFqdn) { + // matched line already exists + break + } + // ready marker is reached - matched line not found, append it here + if strings.Contains(line, " ready") { + if strings.HasPrefix(alias, "*.") { // wildcard DNS + alias = strings.TrimPrefix(alias, "*") + alias = strings.ReplaceAll(alias, ".", "\\.") + alias = "(.*)" + alias + + rewriteLine = fmt.Sprintf(" rewrite name regex %s %s answer auto", alias, serviceFqdn) + } else { + rewriteLine = fmt.Sprintf(" rewrite name %s %s", alias, serviceFqdn) + } + // add matched line + lines = append(lines[:i+1], lines[i:]...) + lines[i] = rewriteLine + coreFileUpdated = true + break + } + } + + if coreFileUpdated { + // update configmap and restart the pods + var newLines string + for _, line := range lines { + // return back EOL + newLines += (line + "\n") + } + cm.Data["Corefile"] = newLines + if err := mClient.Update(ctx, &cm); err != nil { + return err + } + + if err := restartCoreDNS(ctx, mClient, logger); err != nil { + return err + } + } + + return nil +} + +// Remove coredns rewrite for a given external dns service. +func removeCoreDNSRewrite(ctx context.Context, mClient client.Client, logger *logrus.Entry, name *types.NamespacedName) error { + corednsName := types.NamespacedName{ + Name: "coredns", + Namespace: "kube-system", + } + var cm v1.ConfigMap + + if err := mClient.Get(ctx, corednsName, &cm); err != nil { + if k8serrors.IsNotFound(err) { + logger.Warnf("coredns configmap not found.") + return nil + } + return err + } + if _, ok := cm.Data["Corefile"]; !ok { + return errors.New("coredns configmap['Corefile'] not found") + } + + data := cm.Data["Corefile"] + // remove trailing end-of-line + dataEol := strings.TrimSuffix(data, "\n") + // break into lines + lines := strings.Split(dataEol, "\n") + serviceFqdn := fmt.Sprintf("%s.%s.svc.cluster.local", name.Name, name.Namespace) + + coreFileUpdated := false + for i, line := range lines { + if strings.Contains(line, serviceFqdn) { + // remove matched line + lines = append(lines[:i], lines[i+1:]...) + coreFileUpdated = true + break + } + } + + if coreFileUpdated { + // update configmap and restart the pods + var newLines string + for _, line := range lines { + // return back EOL + newLines += (line + "\n") + } + cm.Data["Corefile"] = newLines + if err := mClient.Update(ctx, &cm); err != nil { + return err + } + + if err := restartCoreDNS(ctx, mClient, logger); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/controlplane/control/manager.go b/pkg/controlplane/control/manager.go index 4453f7677..0f35f8f70 100644 --- a/pkg/controlplane/control/manager.go +++ b/pkg/controlplane/control/manager.go @@ -253,23 +253,31 @@ func (m *Manager) addImport(ctx context.Context, imp *v1alpha1.Import) (err erro return err } - if imp.Namespace == m.namespace { - return nil - } + if imp.Namespace != m.namespace { + userService := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: imp.Name, + Namespace: imp.Namespace, + Labels: make(map[string]string), + }, + Spec: v1.ServiceSpec{ + ExternalName: fmt.Sprintf("%s.%s.svc.cluster.local", serviceName, m.namespace), + Type: v1.ServiceTypeExternalName, + }, + } - userService := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: imp.Name, - Namespace: imp.Namespace, - Labels: make(map[string]string), - }, - Spec: v1.ServiceSpec{ - ExternalName: fmt.Sprintf("%s.%s.svc.cluster.local", serviceName, m.namespace), - Type: v1.ServiceTypeExternalName, - }, + if err := m.addImportService(ctx, imp, userService); err != nil { + return err + } } - return m.addImportService(ctx, imp, userService) + if imp.Spec.Alias != "" { + if err := addCoreDNSRewrite(ctx, m.client, m.logger, &importName, imp.Spec.Alias); err != nil { + m.logger.Errorf("failed to configure CoreDNS: %v.", err) + return err + } + } + return nil } // deleteImport removes the listening socket of a previously imported service. @@ -277,7 +285,7 @@ func (m *Manager) deleteImport(ctx context.Context, name types.NamespacedName) e m.logger.Infof("Deleting import '%s/%s'.", name.Namespace, name.Name) // delete user service - errs := make([]error, 3) + errs := make([]error, 4) errs[0] = m.deleteImportService(ctx, name, name) if name.Namespace != m.namespace { @@ -294,6 +302,8 @@ func (m *Manager) deleteImport(ctx context.Context, name types.NamespacedName) e m.ports.Release(name) + errs[3] = removeCoreDNSRewrite(ctx, m.client, m.logger, &name) + return errors.Join(errs...) } @@ -889,7 +899,6 @@ func generateJWKSecret() ([]byte, error) { Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rsaKey), }) - if err != nil { return nil, fmt.Errorf("cannot encode JWK key: %w", err) } diff --git a/pkg/operator/controller/instance_controller.go b/pkg/operator/controller/instance_controller.go index f3b128268..f1121af36 100644 --- a/pkg/operator/controller/instance_controller.go +++ b/pkg/operator/controller/instance_controller.go @@ -72,6 +72,7 @@ type InstanceReconciler struct { // +kubebuilder:rbac:groups="discovery.k8s.io",resources=endpointslices,verbs=list;get;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=nodes,verbs=list;get;watch // +kubebuilder:rbac:groups="",resources=pods,verbs=list;get;watch +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;update;watch // +kubebuilder:rbac:groups=clusterlink.net,resources=exports;peers;accesspolicies;privilegedaccesspolicies,verbs=list;get;watch // +kubebuilder:rbac:groups=clusterlink.net,resources=imports,verbs=get;list;watch;update // +kubebuilder:rbac:groups=clusterlink.net,resources=peers/status;exports/status;imports/status,verbs=update @@ -445,6 +446,13 @@ func (r *InstanceReconciler) createAccessControl(ctx context.Context, name, name "get", "list", "watch", "create", "delete", "update", }, }, + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{ + "get", "list", "update", "watch", + }, + }, { APIGroups: []string{"coordination.k8s.io"}, Resources: []string{"leases"}, @@ -464,6 +472,11 @@ func (r *InstanceReconciler) createAccessControl(ctx context.Context, name, name Resources: []string{"pods"}, Verbs: []string{"get", "list", "watch"}, }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get", "list", "watch", "update", "patch"}, + }, { APIGroups: []string{"clusterlink.net"}, Resources: []string{"peers", "exports", "accesspolicies", "privilegedaccesspolicies"},