From b697bef0a29683276d69b683972a23489789da4f Mon Sep 17 00:00:00 2001 From: Kfir Toledo Date: Wed, 26 Jun 2024 22:00:31 +0300 Subject: [PATCH] bootstrap: Add Loadbalancer service and namespace to bootstrap file. (#661) Add creating ingress service and namespace when using k8s bootstrap. Signed-off-by: Kfir Toledo --- cmd/cl-dataplane/app/server.go | 2 + cmd/clusterlink/cmd/deploy/deploy_peer.go | 10 +- pkg/bootstrap/platform/k8s.go | 103 +++++++++++++++--- pkg/controlplane/control/manager.go | 4 +- .../controller/instance_controller.go | 11 +- .../controller/instance_controller_test.go | 5 +- tests/e2e/k8s/test_import.go | 3 +- tests/e2e/k8s/util/fabric.go | 19 ++-- .../content/en/docs/main/tasks/operator.md | 26 ++++- 9 files changed, 137 insertions(+), 46 deletions(-) diff --git a/cmd/cl-dataplane/app/server.go b/cmd/cl-dataplane/app/server.go index ffac48945..641e25362 100644 --- a/cmd/cl-dataplane/app/server.go +++ b/cmd/cl-dataplane/app/server.go @@ -38,6 +38,8 @@ const ( // Name is the app label of dataplane pods. Name = "cl-dataplane" + // IngressSvcName is the ingress service name for the dataplane pods. + IngressSvcName = "clusterlink" ) // Options contains everything necessary to create and run a dataplane. diff --git a/cmd/clusterlink/cmd/deploy/deploy_peer.go b/cmd/clusterlink/cmd/deploy/deploy_peer.go index aff94c0c7..33b0ea799 100644 --- a/cmd/clusterlink/cmd/deploy/deploy_peer.go +++ b/cmd/clusterlink/cmd/deploy/deploy_peer.go @@ -197,6 +197,11 @@ func (o *PeerOptions) Run() error { Tag: o.Tag, } + // Create clusterlink instance YAML for the operator. + if o.IngressPort != apis.DefaultExternalPort { // Set the port config only if it has changed. + platformCfg.IngressPort = o.IngressPort + } + if o.StartInstance == NoStart { // Create a YAML file for deployment without using the operator. k8sConfig, err := platform.K8SConfig(platformCfg) @@ -260,11 +265,6 @@ func (o *PeerOptions) Run() error { return err } - // Create clusterlink instance YAML for the operator. - if o.IngressPort != apis.DefaultExternalPort { // Set the port config only if it has changed. - platformCfg.IngressPort = o.IngressPort - } - instance, err := platform.K8SClusterLinkInstanceConfig(platformCfg, "cl-instance") if err != nil { return err diff --git a/pkg/bootstrap/platform/k8s.go b/pkg/bootstrap/platform/k8s.go index efc84ec0e..ed07cd239 100644 --- a/pkg/bootstrap/platform/k8s.go +++ b/pkg/bootstrap/platform/k8s.go @@ -24,9 +24,16 @@ import ( apis "github.com/clusterlink-net/clusterlink/pkg/apis/clusterlink.net/v1alpha1" cpapi "github.com/clusterlink-net/clusterlink/pkg/controlplane/api" dpapi "github.com/clusterlink-net/clusterlink/pkg/dataplane/api" + corev1 "k8s.io/api/core/v1" ) const ( + nsTemplate = `--- + apiVersion: v1 + kind: Namespace + metadata: + name: {{.namespace}} +` certsTemplate = `--- apiVersion: v1 kind: Secret @@ -64,7 +71,6 @@ data: {{.peerKeyFile}}: {{.peerKey}} {{.fabricCertFile}}: {{.fabricCert}} ` - k8sTemplate = `--- apiVersion: apps/v1 kind: Deployment @@ -181,18 +187,6 @@ spec: - name: controlplane port: {{.controlplanePort}} --- -apiVersion: v1 -kind: Service -metadata: - name: cl-dataplane - namespace: {{.namespace}} -spec: - selector: - app: {{ .dataplaneAppName }} - ports: - - name: dataplane - port: {{.dataplanePort}} ---- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -237,7 +231,8 @@ roleRef: subjects: - kind: ServiceAccount name: default - namespace: {{.namespace}}` + namespace: {{.namespace}} +` ClusterLinkInstanceTemplate = `apiVersion: clusterlink.net/v1alpha1 kind: Instance metadata: @@ -263,6 +258,24 @@ spec: namespace: {{.namespace}} tag: {{.tag}} ` + ingressTemplate = `--- +apiVersion: v1 +kind: Service +metadata: + name: {{.dataplaneService}} + namespace: {{.namespace}} +spec: + type: {{.ingressType}} + ports: + - name: dataplane + port: {{.ingressPort }} + targetPort: {{.dataplanePort}} +{{ if .ingressNodePort }} + nodePort: {{.ingressNodePort }} +{{ end }} + selector: + app: {{.dataplaneAppName}} + ` ) // K8SConfig returns a kubernetes deployment file. @@ -299,19 +312,36 @@ func K8SConfig(config *Config) ([]byte, error) { "dataplanePort": dpapi.ListenPort, } - var k8sConfig bytes.Buffer - t := template.Must(template.New("").Parse(k8sTemplate)) + var k8sConfig, nsConfig bytes.Buffer + // ClusterLink namespace + t := template.Must(template.New("").Parse(nsTemplate)) + if err := t.Execute(&nsConfig, args); err != nil { + return nil, fmt.Errorf("cannot create K8s namespace from template: %w", err) + } + + // ClusterLink components + t = template.Must(template.New("").Parse(k8sTemplate)) if err := t.Execute(&k8sConfig, args); err != nil { return nil, fmt.Errorf("cannot create k8s configuration from template: %w", err) } + // ClusterLink certificates certConfig, err := K8SCertificateConfig(config) if err != nil { return nil, err } - k8sBytes := certConfig + // ClusterLink ingress service + ingressConfig, err := k8SIngressConfig(config) + if err != nil { + return nil, err + } + + k8sBytes := nsConfig.Bytes() + k8sBytes = append(k8sBytes, certConfig...) k8sBytes = append(k8sBytes, k8sConfig.Bytes()...) + k8sBytes = append(k8sBytes, ingressConfig...) + return k8sBytes, nil } @@ -372,6 +402,7 @@ func K8SClusterLinkInstanceConfig(config *Config, name string) ([]byte, error) { } args["ingressPort"] = config.IngressPort } + var clConfig bytes.Buffer t := template.Must(template.New("").Parse(ClusterLinkInstanceTemplate)) if err := t.Execute(&clConfig, args); err != nil { @@ -404,3 +435,41 @@ func K8SEmptyCertificateConfig(config *Config) ([]byte, error) { return certConfig.Bytes(), nil } + +// k8SIngressConfig returns a kubernetes ingress service. +func k8SIngressConfig(config *Config) ([]byte, error) { + var ingressConfig bytes.Buffer + + ingressType := string(corev1.ServiceTypeClusterIP) + if config.IngressType == string(apis.IngressTypeNodePort) || config.IngressType == string(apis.IngressTypeLoadBalancer) { + ingressType = config.IngressType + } + + args := map[string]interface{}{ + "namespace": config.Namespace, + "ingressPort": apis.DefaultExternalPort, + "ingressType": ingressType, + + "dataplaneService": dpapp.IngressSvcName, + "dataplaneAppName": dpapp.Name, + "dataplanePort": dpapi.ListenPort, + } + + if config.IngressPort != 0 { + if config.IngressType == string(apis.IngressTypeNodePort) { + args["ingressNodePort"] = config.IngressPort + if (config.IngressPort < 30000) || (config.IngressPort > 32767) { + return nil, fmt.Errorf("nodeport number %v is not in the valid range (30000:32767)", config.IngressPort) + } + } else { + args["ingressPort"] = config.IngressPort + } + } + + t := template.Must(template.New("").Parse(ingressTemplate)) + if err := t.Execute(&ingressConfig, args); err != nil { + return nil, fmt.Errorf("cannot create K8s namespace from template: %w", err) + } + + return ingressConfig.Bytes(), nil +} diff --git a/pkg/controlplane/control/manager.go b/pkg/controlplane/control/manager.go index 015e7b0b6..3d59e8d35 100644 --- a/pkg/controlplane/control/manager.go +++ b/pkg/controlplane/control/manager.go @@ -449,7 +449,7 @@ func (m *Manager) checkJWKSecret(ctx context.Context, name types.NamespacedName) // addEndpointSlice adds a dataplane / import endpoint slices. func (m *Manager) addEndpointSlice(ctx context.Context, endpointSlice *discv1.EndpointSlice) error { - if endpointSlice.Labels[discv1.LabelServiceName] == dpapp.Name && endpointSlice.Namespace == m.namespace { + if endpointSlice.Labels[discv1.LabelServiceName] == dpapp.IngressSvcName && endpointSlice.Namespace == m.namespace { m.logger.Infof("Adding a dataplane endpoint slice: %s", endpointSlice.Name) mergeImportList := v1alpha1.ImportList{} @@ -808,7 +808,7 @@ func (m *Manager) addImportEndpointSlices(ctx context.Context, imp *v1alpha1.Imp err := m.client.List( ctx, &dataplaneEndpointSliceList, - client.MatchingLabels{discv1.LabelServiceName: dpapp.Name}, + client.MatchingLabels{discv1.LabelServiceName: dpapp.IngressSvcName}, client.InNamespace(m.namespace)) if err != nil { return err diff --git a/pkg/operator/controller/instance_controller.go b/pkg/operator/controller/instance_controller.go index f3b128268..bdb44acbf 100644 --- a/pkg/operator/controller/instance_controller.go +++ b/pkg/operator/controller/instance_controller.go @@ -44,7 +44,6 @@ const ( ControlPlaneName = "cl-controlplane" DataPlaneName = "cl-dataplane" GoDataPlaneName = "cl-go-dataplane" - IngressName = "clusterlink" OperatorNamespace = "clusterlink-operator" InstanceNamespace = "clusterlink-system" FinalizerName = "instance.clusterlink.net/finalizer" @@ -204,10 +203,6 @@ func (r *InstanceReconciler) applyClusterLink(ctx context.Context, instance *clu } // Create datapalne components - if err := r.createService(ctx, DataPlaneName, instance.Spec.Namespace, dpapi.ListenPort); err != nil { - return err - } - if err := r.applyDataplane(ctx, instance); err != nil { return err } @@ -516,7 +511,7 @@ func (r *InstanceReconciler) createExternalService(ctx context.Context, instance // Create a Service object service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: IngressName, + Name: dpapp.IngressSvcName, Namespace: instance.Spec.Namespace, Annotations: instance.Spec.Ingress.Annotations, }, @@ -632,7 +627,7 @@ func (r *InstanceReconciler) deleteClusterLink(ctx context.Context, namespace st } // Delete external ingress service - ingerssObj := metav1.ObjectMeta{Name: IngressName, Namespace: namespace} + ingerssObj := metav1.ObjectMeta{Name: dpapp.IngressSvcName, Namespace: namespace} return r.deleteResource(ctx, &corev1.Service{ObjectMeta: ingerssObj}) } @@ -720,7 +715,7 @@ func (r *InstanceReconciler) checkDataplaneStatus(ctx context.Context, instance // checkIngressStatus check the status of the ingress components. func (r *InstanceReconciler) checkIngressStatus(ctx context.Context, instance *clusterlink.Instance) (bool, error) { - ingress := types.NamespacedName{Name: IngressName, Namespace: instance.Spec.Namespace} + ingress := types.NamespacedName{Name: dpapp.IngressSvcName, Namespace: instance.Spec.Namespace} serviceStatus, err := r.checkExternalServiceStatus(ctx, ingress, &instance.Status.Ingress) if err != nil { return false, err diff --git a/pkg/operator/controller/instance_controller_test.go b/pkg/operator/controller/instance_controller_test.go index 523219e5e..a00e21ed0 100644 --- a/pkg/operator/controller/instance_controller_test.go +++ b/pkg/operator/controller/instance_controller_test.go @@ -35,6 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" + dpapp "github.com/clusterlink-net/clusterlink/cmd/cl-dataplane/app" clusterlink "github.com/clusterlink-net/clusterlink/pkg/apis/clusterlink.net/v1alpha1" "github.com/clusterlink-net/clusterlink/pkg/operator/controller" ) @@ -150,8 +151,8 @@ func TestClusterLinkController(t *testing.T) { } roleResource := []client.Object{&rbacv1.ClusterRole{}, &rbacv1.ClusterRoleBinding{}} dpID := types.NamespacedName{Name: controller.DataPlaneName, Namespace: controller.InstanceNamespace} - dpResource := []client.Object{&appsv1.Deployment{}, &corev1.Service{}} - ingressID := types.NamespacedName{Name: controller.IngressName, Namespace: controller.InstanceNamespace} + dpResource := []client.Object{&appsv1.Deployment{}} + ingressID := types.NamespacedName{Name: dpapp.IngressSvcName, Namespace: controller.InstanceNamespace} t.Run("Create ClusterLink deployment", func(t *testing.T) { // Create ClusterLink namespaces diff --git a/tests/e2e/k8s/test_import.go b/tests/e2e/k8s/test_import.go index 5bad2b92c..4d6f0f8eb 100644 --- a/tests/e2e/k8s/test_import.go +++ b/tests/e2e/k8s/test_import.go @@ -23,6 +23,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + dpapp "github.com/clusterlink-net/clusterlink/cmd/cl-dataplane/app" "github.com/clusterlink-net/clusterlink/pkg/apis/clusterlink.net/v1alpha1" "github.com/clusterlink-net/clusterlink/pkg/controlplane/control" "github.com/clusterlink-net/clusterlink/tests/e2e/k8s/services" @@ -407,7 +408,7 @@ func (s *TestSuite) TestImportMerge() { // delete dataplane endpoint slice by deleting the dataplane service var dataplaneService v1.Service require.Nil(s.T(), cl[0].Cluster().Resources().Get( - context.Background(), "cl-dataplane", cl[0].Namespace(), &dataplaneService)) + context.Background(), dpapp.IngressSvcName, cl[0].Namespace(), &dataplaneService)) require.Nil(s.T(), cl[0].Cluster().Resources().Delete( context.Background(), &dataplaneService)) diff --git a/tests/e2e/k8s/util/fabric.go b/tests/e2e/k8s/util/fabric.go index 863f5e490..e53491faa 100644 --- a/tests/e2e/k8s/util/fabric.go +++ b/tests/e2e/k8s/util/fabric.go @@ -20,10 +20,12 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/e2e-framework/klient/wait" "sigs.k8s.io/e2e-framework/klient/wait/conditions" + dpapp "github.com/clusterlink-net/clusterlink/cmd/cl-dataplane/app" "github.com/clusterlink-net/clusterlink/cmd/clusterlink/config" "github.com/clusterlink-net/clusterlink/pkg/apis/clusterlink.net/v1alpha1" "github.com/clusterlink-net/clusterlink/pkg/bootstrap" @@ -142,13 +144,6 @@ func (f *Fabric) SwitchToNewNamespace(name string, appendName bool) error { f.baseNamespace = name } - // create new namespace - for _, p := range f.peers { - if err := p.cluster.CreateNamespace(name); err != nil { - return fmt.Errorf("cannot create namespace %s: %w", name, err) - } - } - if f.namespace != "" { // delete old namespace for _, p := range f.peers { @@ -165,7 +160,7 @@ func (f *Fabric) SwitchToNewNamespace(name string, appendName bool) error { } } - if err := p.cluster.DeleteNamespace(f.namespace); err != nil { + if err := p.cluster.DeleteNamespace(f.namespace); err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("cannot delete namespace %s: %w", f.namespace, err) } } @@ -181,6 +176,11 @@ var deployFunc func(target *peer, cfg *PeerConfig) error func (f *Fabric) deployUsingOperator(target *peer, cfg *PeerConfig) error { instanceName := "cl-instance" + f.namespace + // Create namespace to run ClusterLink + if err := target.cluster.CreateNamespace(f.namespace); err != nil { + return fmt.Errorf("cannot create namespace %s: %w", f.namespace, err) + } + // Create ClusterLink instance instance, err := f.generateClusterlinkInstance(instanceName, target, cfg) if err != nil { @@ -259,9 +259,8 @@ func (f *Fabric) deployClusterLink(target *peer, cfg *PeerConfig) (*ClusterLink, return nil, fmt.Errorf("namespace not set") } - svcNodePort := "cl-dataplane" + svcNodePort := dpapp.IngressSvcName if cfg.DeployWithOperator { - svcNodePort = controller.IngressName deployFunc = f.deployUsingOperator } else { deployFunc = f.deployUsingK8sYAML diff --git a/website/content/en/docs/main/tasks/operator.md b/website/content/en/docs/main/tasks/operator.md index f70239198..d958a640d 100644 --- a/website/content/en/docs/main/tasks/operator.md +++ b/website/content/en/docs/main/tasks/operator.md @@ -61,7 +61,7 @@ The deployment process can be split into two steps: 1. Deploy only ClusterLink operator: ```sh - clusterlink deploy peer ---name --fabric --start operator + clusterlink deploy peer --name --fabric --start operator ``` The `start` flag will deploy only the ClusterLink operator and the certificate's secrets as described in the [common use case][] above. @@ -121,6 +121,30 @@ The `deploy peer` {{< anchor commandline-flags >}} command has the following fla - **path**: Represents the path where the peer and fabric certificates are stored, by default is the working current working directory. +## Manual Deployment without the operator + +To deploy the ClusterLink without using the Operator, follow the instructions below: + +1. Create a `k8s.yaml` file to deploy ClusterLink without the operator: + + ```sh + clusterlink deploy peer --name --fabric --start none + ``` + + The `k8s.yaml` file contains the deployment of all ClusterLink components and can be configured for various purposes, such as adding sidecar pods or managing the ClusterLink certificates. + +1. Deploy ClusterLink CRDs: + + ```sh + curl -L https://github.com/clusterlink-net/clusterlink/archive/refs/heads/main.tar.gz | tar -xzO clusterlink-main/config/crds | kubectl apply -f - + ``` + +1. Apply the `k8s.yaml` file to the cluster: + + ```sh + kubectl apply .///k8s.yaml + ``` + ## Manual Deployment without CLI To deploy the ClusterLink without using the CLI, follow the instructions below: