bootstrap: Add Loadbalancer service and namespace to bootstrap file. (#…

…661)

Add creating ingress service and namespace when using k8s bootstrap.

Signed-off-by: Kfir Toledo <[email protected]>

cmd/cl-dataplane/app/server.go

@@ -38,6 +38,8 @@ const (
 
 	// Name is the app label of dataplane pods.
 	Name = "cl-dataplane"
+	// IngressSvcName is the ingress service name for the dataplane pods.
+	IngressSvcName = "clusterlink"
 )
 
 // Options contains everything necessary to create and run a dataplane.

cmd/clusterlink/cmd/deploy/deploy_peer.go

@@ -197,6 +197,11 @@ func (o *PeerOptions) Run() error {
 		Tag:                     o.Tag,
 	}
 
+	// Create clusterlink instance YAML for the operator.
+	if o.IngressPort != apis.DefaultExternalPort { // Set the port config only if it has changed.
+		platformCfg.IngressPort = o.IngressPort
+	}
+
 	if o.StartInstance == NoStart {
 		// Create a YAML file for deployment without using the operator.
 		k8sConfig, err := platform.K8SConfig(platformCfg)
@@ -260,11 +265,6 @@ func (o *PeerOptions) Run() error {
 		return err
 	}
 
-	// Create clusterlink instance YAML for the operator.
-	if o.IngressPort != apis.DefaultExternalPort { // Set the port config only if it has changed.
-		platformCfg.IngressPort = o.IngressPort
-	}
-
 	instance, err := platform.K8SClusterLinkInstanceConfig(platformCfg, "cl-instance")
 	if err != nil {
 		return err

pkg/bootstrap/platform/k8s.go

@@ -24,9 +24,16 @@ import (
 	apis "github.com/clusterlink-net/clusterlink/pkg/apis/clusterlink.net/v1alpha1"
 	cpapi "github.com/clusterlink-net/clusterlink/pkg/controlplane/api"
 	dpapi "github.com/clusterlink-net/clusterlink/pkg/dataplane/api"
+	corev1 "k8s.io/api/core/v1"
 )
 
 const (
+	nsTemplate = `---
+  apiVersion: v1
+  kind: Namespace
+  metadata:
+    name: {{.namespace}}
+`
 	certsTemplate = `---
 apiVersion: v1
 kind: Secret
@@ -64,7 +71,6 @@ data:
   {{.peerKeyFile}}: {{.peerKey}}
   {{.fabricCertFile}}: {{.fabricCert}}
 `
-
 	k8sTemplate = `---
 apiVersion: apps/v1
 kind: Deployment
@@ -181,18 +187,6 @@ spec:
     - name: controlplane
       port: {{.controlplanePort}}
 ---
-apiVersion: v1
-kind: Service
-metadata:
-  name: cl-dataplane
-  namespace: {{.namespace}}
-spec:
-  selector:
-    app: {{ .dataplaneAppName }}
-  ports:
-    - name: dataplane
-      port: {{.dataplanePort}}
----
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -237,7 +231,8 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: default
-  namespace: {{.namespace}}`
+  namespace: {{.namespace}}
+`
 	ClusterLinkInstanceTemplate = `apiVersion: clusterlink.net/v1alpha1
 kind: Instance
 metadata:
@@ -263,6 +258,24 @@ spec:
   namespace: {{.namespace}}
   tag: {{.tag}}
 `
+	ingressTemplate = `---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{.dataplaneService}}
+  namespace: {{.namespace}}
+spec:
+  type: {{.ingressType}}
+  ports:
+    - name: dataplane
+      port: {{.ingressPort }}
+      targetPort: {{.dataplanePort}}
+{{ if .ingressNodePort }}
+      nodePort: {{.ingressNodePort }}
+{{ end }}
+  selector:
+    app:  {{.dataplaneAppName}}
+  `
 )
 
 // K8SConfig returns a kubernetes deployment file.
@@ -299,19 +312,36 @@ func K8SConfig(config *Config) ([]byte, error) {
 		"dataplanePort":    dpapi.ListenPort,
 	}
 
-	var k8sConfig bytes.Buffer
-	t := template.Must(template.New("").Parse(k8sTemplate))
+	var k8sConfig, nsConfig bytes.Buffer
+	// ClusterLink namespace
+	t := template.Must(template.New("").Parse(nsTemplate))
+	if err := t.Execute(&nsConfig, args); err != nil {
+		return nil, fmt.Errorf("cannot create K8s namespace from template: %w", err)
+	}
+
+	// ClusterLink components
+	t = template.Must(template.New("").Parse(k8sTemplate))
 	if err := t.Execute(&k8sConfig, args); err != nil {
 		return nil, fmt.Errorf("cannot create k8s configuration from template: %w", err)
 	}
 
+	// ClusterLink certificates
 	certConfig, err := K8SCertificateConfig(config)
 	if err != nil {
 		return nil, err
 	}
 
-	k8sBytes := certConfig
+	// ClusterLink ingress service
+	ingressConfig, err := k8SIngressConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	k8sBytes := nsConfig.Bytes()
+	k8sBytes = append(k8sBytes, certConfig...)
 	k8sBytes = append(k8sBytes, k8sConfig.Bytes()...)
+	k8sBytes = append(k8sBytes, ingressConfig...)
+
 	return k8sBytes, nil
 }
 
@@ -372,6 +402,7 @@ func K8SClusterLinkInstanceConfig(config *Config, name string) ([]byte, error) {
 		}
 		args["ingressPort"] = config.IngressPort
 	}
+
 	var clConfig bytes.Buffer
 	t := template.Must(template.New("").Parse(ClusterLinkInstanceTemplate))
 	if err := t.Execute(&clConfig, args); err != nil {
@@ -404,3 +435,41 @@ func K8SEmptyCertificateConfig(config *Config) ([]byte, error) {
 
 	return certConfig.Bytes(), nil
 }
+
+// k8SIngressConfig returns a kubernetes ingress service.
+func k8SIngressConfig(config *Config) ([]byte, error) {
+	var ingressConfig bytes.Buffer
+
+	ingressType := string(corev1.ServiceTypeClusterIP)
+	if config.IngressType == string(apis.IngressTypeNodePort) || config.IngressType == string(apis.IngressTypeLoadBalancer) {
+		ingressType = config.IngressType
+	}
+
+	args := map[string]interface{}{
+		"namespace":   config.Namespace,
+		"ingressPort": apis.DefaultExternalPort,
+		"ingressType": ingressType,
+
+		"dataplaneService": dpapp.IngressSvcName,
+		"dataplaneAppName": dpapp.Name,
+		"dataplanePort":    dpapi.ListenPort,
+	}
+
+	if config.IngressPort != 0 {
+		if config.IngressType == string(apis.IngressTypeNodePort) {
+			args["ingressNodePort"] = config.IngressPort
+			if (config.IngressPort < 30000) || (config.IngressPort > 32767) {
+				return nil, fmt.Errorf("nodeport number %v is not in the valid range (30000:32767)", config.IngressPort)
+			}
+		} else {
+			args["ingressPort"] = config.IngressPort
+		}
+	}
+
+	t := template.Must(template.New("").Parse(ingressTemplate))
+	if err := t.Execute(&ingressConfig, args); err != nil {
+		return nil, fmt.Errorf("cannot create K8s namespace from template: %w", err)
+	}
+
+	return ingressConfig.Bytes(), nil
+}

pkg/controlplane/control/manager.go

@@ -449,7 +449,7 @@ func (m *Manager) checkJWKSecret(ctx context.Context, name types.NamespacedName)
 
 // addEndpointSlice adds a dataplane / import endpoint slices.
 func (m *Manager) addEndpointSlice(ctx context.Context, endpointSlice *discv1.EndpointSlice) error {
-	if endpointSlice.Labels[discv1.LabelServiceName] == dpapp.Name && endpointSlice.Namespace == m.namespace {
+	if endpointSlice.Labels[discv1.LabelServiceName] == dpapp.IngressSvcName && endpointSlice.Namespace == m.namespace {
 		m.logger.Infof("Adding a dataplane endpoint slice: %s", endpointSlice.Name)
 
 		mergeImportList := v1alpha1.ImportList{}
@@ -808,7 +808,7 @@ func (m *Manager) addImportEndpointSlices(ctx context.Context, imp *v1alpha1.Imp
 	err := m.client.List(
 		ctx,
 		&dataplaneEndpointSliceList,
-		client.MatchingLabels{discv1.LabelServiceName: dpapp.Name},
+		client.MatchingLabels{discv1.LabelServiceName: dpapp.IngressSvcName},
 		client.InNamespace(m.namespace))
 	if err != nil {
 		return err

pkg/operator/controller/instance_controller.go

@@ -44,7 +44,6 @@ const (
 	ControlPlaneName  = "cl-controlplane"
 	DataPlaneName     = "cl-dataplane"
 	GoDataPlaneName   = "cl-go-dataplane"
-	IngressName       = "clusterlink"
 	OperatorNamespace = "clusterlink-operator"
 	InstanceNamespace = "clusterlink-system"
 	FinalizerName     = "instance.clusterlink.net/finalizer"
@@ -204,10 +203,6 @@ func (r *InstanceReconciler) applyClusterLink(ctx context.Context, instance *clu
 	}
 
 	// Create datapalne components
-	if err := r.createService(ctx, DataPlaneName, instance.Spec.Namespace, dpapi.ListenPort); err != nil {
-		return err
-	}
-
 	if err := r.applyDataplane(ctx, instance); err != nil {
 		return err
 	}
@@ -516,7 +511,7 @@ func (r *InstanceReconciler) createExternalService(ctx context.Context, instance
 	// Create a Service object
 	service := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:        IngressName,
+			Name:        dpapp.IngressSvcName,
 			Namespace:   instance.Spec.Namespace,
 			Annotations: instance.Spec.Ingress.Annotations,
 		},
@@ -632,7 +627,7 @@ func (r *InstanceReconciler) deleteClusterLink(ctx context.Context, namespace st
 	}
 
 	// Delete external ingress service
-	ingerssObj := metav1.ObjectMeta{Name: IngressName, Namespace: namespace}
+	ingerssObj := metav1.ObjectMeta{Name: dpapp.IngressSvcName, Namespace: namespace}
 	return r.deleteResource(ctx, &corev1.Service{ObjectMeta: ingerssObj})
 }
 
@@ -720,7 +715,7 @@ func (r *InstanceReconciler) checkDataplaneStatus(ctx context.Context, instance
 
 // checkIngressStatus check the status of the ingress components.
 func (r *InstanceReconciler) checkIngressStatus(ctx context.Context, instance *clusterlink.Instance) (bool, error) {
-	ingress := types.NamespacedName{Name: IngressName, Namespace: instance.Spec.Namespace}
+	ingress := types.NamespacedName{Name: dpapp.IngressSvcName, Namespace: instance.Spec.Namespace}
 	serviceStatus, err := r.checkExternalServiceStatus(ctx, ingress, &instance.Status.Ingress)
 	if err != nil {
 		return false, err

pkg/operator/controller/instance_controller_test.go

@@ -35,6 +35,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 
+	dpapp "github.com/clusterlink-net/clusterlink/cmd/cl-dataplane/app"
 	clusterlink "github.com/clusterlink-net/clusterlink/pkg/apis/clusterlink.net/v1alpha1"
 	"github.com/clusterlink-net/clusterlink/pkg/operator/controller"
 )
@@ -150,8 +151,8 @@ func TestClusterLinkController(t *testing.T) {
 	}
 	roleResource := []client.Object{&rbacv1.ClusterRole{}, &rbacv1.ClusterRoleBinding{}}
 	dpID := types.NamespacedName{Name: controller.DataPlaneName, Namespace: controller.InstanceNamespace}
-	dpResource := []client.Object{&appsv1.Deployment{}, &corev1.Service{}}
-	ingressID := types.NamespacedName{Name: controller.IngressName, Namespace: controller.InstanceNamespace}
+	dpResource := []client.Object{&appsv1.Deployment{}}
+	ingressID := types.NamespacedName{Name: dpapp.IngressSvcName, Namespace: controller.InstanceNamespace}
 
 	t.Run("Create ClusterLink deployment", func(t *testing.T) {
 		// Create ClusterLink namespaces

tests/e2e/k8s/test_import.go

@@ -23,6 +23,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 
+	dpapp "github.com/clusterlink-net/clusterlink/cmd/cl-dataplane/app"
 	"github.com/clusterlink-net/clusterlink/pkg/apis/clusterlink.net/v1alpha1"
 	"github.com/clusterlink-net/clusterlink/pkg/controlplane/control"
 	"github.com/clusterlink-net/clusterlink/tests/e2e/k8s/services"
@@ -407,7 +408,7 @@ func (s *TestSuite) TestImportMerge() {
 	// delete dataplane endpoint slice by deleting the dataplane service
 	var dataplaneService v1.Service
 	require.Nil(s.T(), cl[0].Cluster().Resources().Get(
-		context.Background(), "cl-dataplane", cl[0].Namespace(), &dataplaneService))
+		context.Background(), dpapp.IngressSvcName, cl[0].Namespace(), &dataplaneService))
 	require.Nil(s.T(), cl[0].Cluster().Resources().Delete(
 		context.Background(), &dataplaneService))
 

tests/e2e/k8s/util/fabric.go

@@ -20,10 +20,12 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/e2e-framework/klient/wait"
 	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
 
+	dpapp "github.com/clusterlink-net/clusterlink/cmd/cl-dataplane/app"
 	"github.com/clusterlink-net/clusterlink/cmd/clusterlink/config"
 	"github.com/clusterlink-net/clusterlink/pkg/apis/clusterlink.net/v1alpha1"
 	"github.com/clusterlink-net/clusterlink/pkg/bootstrap"
@@ -142,13 +144,6 @@ func (f *Fabric) SwitchToNewNamespace(name string, appendName bool) error {
 		f.baseNamespace = name
 	}
 
-	// create new namespace
-	for _, p := range f.peers {
-		if err := p.cluster.CreateNamespace(name); err != nil {
-			return fmt.Errorf("cannot create namespace %s: %w", name, err)
-		}
-	}
-
 	if f.namespace != "" {
 		// delete old namespace
 		for _, p := range f.peers {
@@ -165,7 +160,7 @@ func (f *Fabric) SwitchToNewNamespace(name string, appendName bool) error {
 				}
 			}
 
-			if err := p.cluster.DeleteNamespace(f.namespace); err != nil {
+			if err := p.cluster.DeleteNamespace(f.namespace); err != nil && !apierrors.IsNotFound(err) {
 				return fmt.Errorf("cannot delete namespace %s: %w", f.namespace, err)
 			}
 		}
@@ -181,6 +176,11 @@ var deployFunc func(target *peer, cfg *PeerConfig) error
 func (f *Fabric) deployUsingOperator(target *peer, cfg *PeerConfig) error {
 	instanceName := "cl-instance" + f.namespace
 
+	// Create namespace to run ClusterLink
+	if err := target.cluster.CreateNamespace(f.namespace); err != nil {
+		return fmt.Errorf("cannot create namespace %s: %w", f.namespace, err)
+	}
+
 	// Create ClusterLink instance
 	instance, err := f.generateClusterlinkInstance(instanceName, target, cfg)
 	if err != nil {
@@ -259,9 +259,8 @@ func (f *Fabric) deployClusterLink(target *peer, cfg *PeerConfig) (*ClusterLink,
 		return nil, fmt.Errorf("namespace not set")
 	}
 
-	svcNodePort := "cl-dataplane"
+	svcNodePort := dpapp.IngressSvcName
 	if cfg.DeployWithOperator {
-		svcNodePort = controller.IngressName
 		deployFunc = f.deployUsingOperator
 	} else {
 		deployFunc = f.deployUsingK8sYAML