Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add CoreDNS rewrite support for external services. #656

Closed
wants to merge 21 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions cmd/cl-controlplane/app/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@ import (
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"

discv1 "k8s.io/api/discovery/v1"
"k8s.io/client-go/rest"
ctrl "sigs.k8s.io/controller-runtime"
Expand Down Expand Up @@ -152,6 +154,10 @@ func (o *Options) Run() error {
return fmt.Errorf("unable to add discovery v1 objects to scheme: %w", err)
}

if err := appsv1.AddToScheme(scheme); err != nil {
return fmt.Errorf("unable to add core appsv1 objects to scheme: %w", err)
}

// set logger for controller-runtime components
ctrl.SetLogger(logrusr.New(logrus.WithField("component", "k8s.controller-runtime")))

Expand Down
4 changes: 4 additions & 0 deletions config/crds/clusterlink.net_imports.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ spec:
spec:
description: Spec represents the attributes of the imported service.
properties:
alias:
description: Alias is an optional external DNS name for this imported
service
type: string
lbScheme:
default: round-robin
description: LBScheme is the load-balancing scheme to use (e.g., random,
Expand Down
9 changes: 9 additions & 0 deletions config/operator/rbac/role.yaml
aviweit marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,15 @@ kind: ClusterRole
metadata:
name: cl-operator-manager-role
rules:
- apiGroups:
aviweit marked this conversation as resolved.
Show resolved Hide resolved
- ""
resources:
- configmaps
verbs:
- get
- list
- update
- watch
- apiGroups:
- ""
resources:
Expand Down
2 changes: 2 additions & 0 deletions pkg/apis/clusterlink.net/v1alpha1/import.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ type ImportSpec struct {
// +kubebuilder:default="round-robin"
// LBScheme is the load-balancing scheme to use (e.g., random, static, round-robin)
LBScheme LBScheme `json:"lbScheme"`
// Alias is an optional external DNS name for this imported service
Alias string `json:"alias,omitempty"`
}

const (
Expand Down
3 changes: 3 additions & 0 deletions pkg/bootstrap/platform/k8s.go
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,9 @@ kind: ClusterRole
metadata:
name: cl-controlplane
rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["create", "get", "list", "update", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update"]
Expand Down
178 changes: 178 additions & 0 deletions pkg/controlplane/control/dns.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
// Copyright (c) The ClusterLink Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package control

import (
"context"
"errors"
"fmt"
"strings"
"time"

"github.com/sirupsen/logrus"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)

// Restart coredns deployment.
func restartCoreDNS(ctx context.Context, mClient client.Client, logger *logrus.Entry) error {
logger.Infof("restarting coredns deployment")
patch := []byte(
fmt.Sprintf(
`{"spec": {"template": {"metadata": {"annotations":{"kubectl.kubernetes.io/restartedAt": %q}}}}}`,
time.Now().String(),
),
)

return mClient.Patch(ctx, &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Namespace: "kube-system",
Name: "coredns",
},
}, client.RawPatch(types.StrategicMergePatchType, patch))
}

// Add coredns rewrite for a given external dns service.
func addCoreDNSRewrite(ctx context.Context, mClient client.Client, logger *logrus.Entry, name *types.NamespacedName,
alias string,
) error {
corednsName := types.NamespacedName{
Name: "coredns",
Namespace: "kube-system",
}
var cm v1.ConfigMap

if err := mClient.Get(ctx, corednsName, &cm); err != nil {
if k8serrors.IsNotFound(err) {
logger.Warnf("coredns configmap not found.")
return nil
}
return err
}
if _, ok := cm.Data["Corefile"]; !ok {
return errors.New("coredns configmap['Corefile'] not found")
}

data := cm.Data["Corefile"]
// remove trailing end-of-line
data = strings.TrimSuffix(data, "\n")
// break into lines
lines := strings.Split(data, "\n")
serviceFqdn := fmt.Sprintf("%s.%s.svc.cluster.local", name.Name, name.Namespace)

coreFileUpdated := false
rewriteLine := ""
for i, line := range lines {
if strings.Contains(line, serviceFqdn) {
// matched line already exists
break
}
// ready marker is reached - matched line not found, append it here
if strings.Contains(line, " ready") {
if strings.HasPrefix(alias, "*.") { // wildcard DNS
alias = strings.TrimPrefix(alias, "*")
alias = strings.ReplaceAll(alias, ".", "\\.")
alias = "(.*)" + alias

rewriteLine = fmt.Sprintf(" rewrite name regex %s %s answer auto", alias, serviceFqdn)
} else {
rewriteLine = fmt.Sprintf(" rewrite name %s %s", alias, serviceFqdn)
}
// add matched line
lines = append(lines[:i+1], lines[i:]...)
lines[i] = rewriteLine
coreFileUpdated = true
break
}
}

if coreFileUpdated {
// update configmap and restart the pods
var newLines string
for _, line := range lines {
// return back EOL
newLines += (line + "\n")
}
cm.Data["Corefile"] = newLines
if err := mClient.Update(ctx, &cm); err != nil {
return err
}

if err := restartCoreDNS(ctx, mClient, logger); err != nil {
return err
}
}

return nil
}

// Remove coredns rewrite for a given external dns service.
func removeCoreDNSRewrite(ctx context.Context, mClient client.Client, logger *logrus.Entry, name *types.NamespacedName) error {
corednsName := types.NamespacedName{
Name: "coredns",
Namespace: "kube-system",
}
var cm v1.ConfigMap

if err := mClient.Get(ctx, corednsName, &cm); err != nil {
if k8serrors.IsNotFound(err) {
logger.Warnf("coredns configmap not found.")
return nil
}
return err
}
if _, ok := cm.Data["Corefile"]; !ok {
return errors.New("coredns configmap['Corefile'] not found")
}

data := cm.Data["Corefile"]
// remove trailing end-of-line
dataEol := strings.TrimSuffix(data, "\n")
// break into lines
lines := strings.Split(dataEol, "\n")
serviceFqdn := fmt.Sprintf("%s.%s.svc.cluster.local", name.Name, name.Namespace)

coreFileUpdated := false
for i, line := range lines {
if strings.Contains(line, serviceFqdn) {
// remove matched line
lines = append(lines[:i], lines[i+1:]...)
coreFileUpdated = true
break
}
}

if coreFileUpdated {
// update configmap and restart the pods
var newLines string
for _, line := range lines {
// return back EOL
newLines += (line + "\n")
}
cm.Data["Corefile"] = newLines
if err := mClient.Update(ctx, &cm); err != nil {
return err
}

if err := restartCoreDNS(ctx, mClient, logger); err != nil {
return err
}
}

return nil
}
41 changes: 25 additions & 16 deletions pkg/controlplane/control/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -253,31 +253,39 @@ func (m *Manager) addImport(ctx context.Context, imp *v1alpha1.Import) (err erro
return err
}

if imp.Namespace == m.namespace {
return nil
}
if imp.Namespace != m.namespace {
userService := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: imp.Name,
Namespace: imp.Namespace,
Labels: make(map[string]string),
},
Spec: v1.ServiceSpec{
ExternalName: fmt.Sprintf("%s.%s.svc.cluster.local", serviceName, m.namespace),
Type: v1.ServiceTypeExternalName,
},
}

userService := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: imp.Name,
Namespace: imp.Namespace,
Labels: make(map[string]string),
},
Spec: v1.ServiceSpec{
ExternalName: fmt.Sprintf("%s.%s.svc.cluster.local", serviceName, m.namespace),
Type: v1.ServiceTypeExternalName,
},
if err := m.addImportService(ctx, imp, userService); err != nil {
return err
}
}

return m.addImportService(ctx, imp, userService)
if imp.Spec.Alias != "" {
if err := addCoreDNSRewrite(ctx, m.client, m.logger, &importName, imp.Spec.Alias); err != nil {
m.logger.Errorf("failed to configure CoreDNS: %v.", err)
return err
}
}
return nil
}

// deleteImport removes the listening socket of a previously imported service.
func (m *Manager) deleteImport(ctx context.Context, name types.NamespacedName) error {
m.logger.Infof("Deleting import '%s/%s'.", name.Namespace, name.Name)

// delete user service
errs := make([]error, 3)
errs := make([]error, 4)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: I realize it is not part of the new code, but I wonder if using append won't be a cleaner solution instead of assigning specific array entries. Unless, of course, there is a reliance on the specific order even when there are nil errors in the array.
Alternately, this might be better served by errors.Join().
@kfirtoledo @orozery - not familiar with this code well enough to say. Please weigh in.

errs[0] = m.deleteImportService(ctx, name, name)

if name.Namespace != m.namespace {
Expand All @@ -294,6 +302,8 @@ func (m *Manager) deleteImport(ctx context.Context, name types.NamespacedName) e

m.ports.Release(name)

errs[3] = removeCoreDNSRewrite(ctx, m.client, m.logger, &name)

return errors.Join(errs...)
}

Expand Down Expand Up @@ -889,7 +899,6 @@ func generateJWKSecret() ([]byte, error) {
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(rsaKey),
})

if err != nil {
return nil, fmt.Errorf("cannot encode JWK key: %w", err)
}
Expand Down
13 changes: 13 additions & 0 deletions pkg/operator/controller/instance_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ type InstanceReconciler struct {
// +kubebuilder:rbac:groups="discovery.k8s.io",resources=endpointslices,verbs=list;get;watch;create;update;patch;delete
// +kubebuilder:rbac:groups="",resources=nodes,verbs=list;get;watch
// +kubebuilder:rbac:groups="",resources=pods,verbs=list;get;watch
// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;update;watch
// +kubebuilder:rbac:groups=clusterlink.net,resources=exports;peers;accesspolicies;privilegedaccesspolicies,verbs=list;get;watch
aviweit marked this conversation as resolved.
Show resolved Hide resolved
// +kubebuilder:rbac:groups=clusterlink.net,resources=imports,verbs=get;list;watch;update
// +kubebuilder:rbac:groups=clusterlink.net,resources=peers/status;exports/status;imports/status,verbs=update
Expand Down Expand Up @@ -445,6 +446,13 @@ func (r *InstanceReconciler) createAccessControl(ctx context.Context, name, name
"get", "list", "watch", "create", "delete", "update",
},
},
{
APIGroups: []string{""},
Resources: []string{"configmaps"},
Verbs: []string{
"get", "list", "update", "watch",
},
},
aviweit marked this conversation as resolved.
Show resolved Hide resolved
{
APIGroups: []string{"coordination.k8s.io"},
Resources: []string{"leases"},
Expand All @@ -464,6 +472,11 @@ func (r *InstanceReconciler) createAccessControl(ctx context.Context, name, name
Resources: []string{"pods"},
Verbs: []string{"get", "list", "watch"},
},
{
APIGroups: []string{"apps"},
Resources: []string{"deployments"},
Verbs: []string{"get", "list", "watch", "update", "patch"},
},
{
APIGroups: []string{"clusterlink.net"},
Resources: []string{"peers", "exports", "accesspolicies", "privilegedaccesspolicies"},
Expand Down
Loading