Support multiple controlplane replicas

This commit adds an option for deploying clusterlink with multiple controlplane replicas.

Signed-off-by: Or Ozeri <[email protected]>

cmd/clusterlink/cmd/deploy/deploy_peer.go

@@ -73,7 +73,9 @@ type PeerOptions struct {
 	ContainerRegistry string
 	// Tag represents the tag of the project images.
 	Tag string
-	// Dataplanes is the number of dataplanes to create.
+	// ControlplaneReplicas is the number of controlplanes to create.
+	ControlplaneReplicas uint16
+	// DataplaneReplicas is the number of dataplanes to create.
 	DataplaneReplicas uint16
 	// DataplaneType is the type of dataplane to create (envoy or go-based)
 	DataplaneType string
@@ -130,6 +132,7 @@ func (o *PeerOptions) AddFlags(fs *pflag.FlagSet) {
 		"For example: --ingress-annotations <key1>=<value1> --ingress-annotations <key2>=<value2>.")
 	fs.StringVar(&o.DataplaneType, "dataplane", platform.DataplaneTypeEnvoy,
 		"Type of dataplane, Supported values: \"envoy\", \"go\"")
+	fs.Uint16Var(&o.ControlplaneReplicas, "controlplane-replicas", 1, "Number of controlplanes.")
 	fs.Uint16Var(&o.DataplaneReplicas, "dataplane-replicas", 1, "Number of dataplanes.")
 	fs.StringVar(&o.LogLevel, "log-level", "info",
 		"The log level. One of fatal, error, warn, info, debug.")
@@ -185,6 +188,7 @@ func (o *PeerOptions) Run() error {
 		FabricCertificate:       fabricCert,
 		PeerCertificate:         peerCert,
 		CACertificate:           caCert,
+		Controlplanes:           o.ControlplaneReplicas,
 		ControlplaneCertificate: controlplaneCert,
 		DataplaneCertificate:    dataplaneCert,
 		Dataplanes:              o.DataplaneReplicas,

pkg/bootstrap/platform/config.go

@@ -36,6 +36,9 @@ type Config struct {
 	// PeerCertificate is the peer certificate.
 	PeerCertificate *bootstrap.Certificate
 
+	// Controlplanes is the number of controlplane servers to run.
+	Controlplanes uint16
+
 	// Dataplanes is the number of dataplane servers to run.
 	Dataplanes uint16
 	// DataplaneType is the type of dataplane to create (envoy or go-based)

pkg/bootstrap/platform/k8s.go

@@ -80,7 +80,7 @@ metadata:
   labels:
     app: cl-controlplane
 spec:
-  replicas: 1
+  replicas: {{.controlplanes}}
   selector:
     matchLabels:
       app: cl-controlplane
@@ -285,11 +285,27 @@ func K8SConfig(config *Config) ([]byte, error) {
 		containerRegistry = config.ContainerRegistry + "/"
 	}
 
+	controlplanes := config.Controlplanes
+	if controlplanes == 0 {
+		controlplanes = 1
+	}
+
+	dataplanes := config.Dataplanes
+	if dataplanes == 0 {
+		dataplanes = 1
+	}
+
+	dataplaneType := config.DataplaneType
+	if dataplaneType == "" {
+		dataplaneType = DataplaneTypeEnvoy
+	}
+
 	args := map[string]interface{}{
 		"peer":              config.Peer,
 		"namespace":         config.Namespace,
-		"dataplanes":        config.Dataplanes,
-		"dataplaneType":     config.DataplaneType,
+		"controlplanes":     controlplanes,
+		"dataplanes":        dataplanes,
+		"dataplaneType":     dataplaneType,
 		"logLevel":          config.LogLevel,
 		"containerRegistry": containerRegistry,
 		"tag":               config.Tag,
@@ -386,6 +402,7 @@ func K8SClusterLinkInstanceConfig(config *Config, name string) ([]byte, error) {
 
 	args := map[string]interface{}{
 		"name":               name,
+		"controlplanes":      config.Controlplanes,
 		"dataplanes":         config.Dataplanes,
 		"dataplaneType":      config.DataplaneType,
 		"logLevel":           config.LogLevel,

tests/e2e/k8s/suite.go

@@ -197,7 +197,6 @@ func (s *TestSuite) RunOnAllDataplaneTypes(test func(cfg *util.PeerConfig)) {
 		s.RunSubTest(dataplaneType, func() {
 			test(&util.PeerConfig{
 				DataplaneType: dataplaneType,
-				Dataplanes:    1,
 			})
 		})
 	}

tests/e2e/k8s/test_operator.go

@@ -23,7 +23,6 @@ import (
 	"sigs.k8s.io/e2e-framework/klient/wait"
 
 	clusterlink "github.com/clusterlink-net/clusterlink/pkg/apis/clusterlink.net/v1alpha1"
-	"github.com/clusterlink-net/clusterlink/pkg/bootstrap/platform"
 	"github.com/clusterlink-net/clusterlink/pkg/operator/controller"
 	"github.com/clusterlink-net/clusterlink/tests/e2e/k8s/services"
 	"github.com/clusterlink-net/clusterlink/tests/e2e/k8s/services/httpecho"
@@ -34,8 +33,6 @@ import (
 func (s *TestSuite) TestOperator() {
 	// Deploy ClusterLink with operator
 	cfg := &util.PeerConfig{
-		DataplaneType:      platform.DataplaneTypeEnvoy,
-		Dataplanes:         1,
 		DeployWithOperator: true,
 	}
 	cl, err := s.fabric.DeployClusterlinks(2, cfg)

tests/e2e/k8s/test_redundancy.go

@@ -0,0 +1,52 @@
+// Copyright (c) The ClusterLink Authors.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package k8s
+
+import (
+	"github.com/clusterlink-net/clusterlink/tests/e2e/k8s/services/httpecho"
+	"github.com/clusterlink-net/clusterlink/tests/e2e/k8s/util"
+	"github.com/stretchr/testify/require"
+)
+
+func (s *TestSuite) TestRedundancy() {
+	s.RunOnAllDataplaneTypes(func(cfg *util.PeerConfig) {
+		cfg.Controlplanes = 3
+		cfg.Dataplanes = 3
+		cl, err := s.fabric.DeployClusterlinks(2, cfg)
+		require.Nil(s.T(), err)
+
+		require.Nil(s.T(), cl[0].CreateService(&httpEchoService))
+		require.Nil(s.T(), cl[0].CreateExport(&httpEchoService))
+		require.Nil(s.T(), cl[0].CreatePolicy(util.PolicyAllowAll))
+		require.Nil(s.T(), cl[1].CreatePeer(cl[0]))
+
+		importedService := &util.Service{
+			Name: httpEchoService.Name,
+			Port: 80,
+		}
+		require.Nil(s.T(), cl[1].CreateImport(importedService, cl[0], httpEchoService.Name))
+
+		require.Nil(s.T(), cl[1].CreatePolicy(util.PolicyAllowAll))
+
+		data, err := cl[1].AccessService(httpecho.GetEchoValue, importedService, true, nil)
+		require.Nil(s.T(), err)
+		require.Equal(s.T(), cl[0].Name(), data)
+
+		for i := 0; i < 100; i++ {
+			data, err := cl[1].AccessService(httpecho.GetEchoValue, importedService, false, nil)
+			require.Nil(s.T(), err)
+			require.Equal(s.T(), cl[0].Name(), data)
+		}
+	})
+}

tests/e2e/k8s/util/fabric.go

@@ -29,12 +29,13 @@ import (
 	"github.com/clusterlink-net/clusterlink/cmd/clusterlink/config"
 	"github.com/clusterlink-net/clusterlink/pkg/apis/clusterlink.net/v1alpha1"
 	"github.com/clusterlink-net/clusterlink/pkg/bootstrap"
-	"github.com/clusterlink-net/clusterlink/pkg/bootstrap/platform"
 	"github.com/clusterlink-net/clusterlink/pkg/operator/controller"
 )
 
 // PeerConfig is a peer configuration.
 type PeerConfig struct {
+	// Controlplanes is the number of controlplane instances.
+	Controlplanes uint16
 	// DataplaneType is the dataplane type (envoy / go).
 	DataplaneType string
 	// Dataplanes is the number of dataplane instances.
@@ -316,10 +317,7 @@ func (f *Fabric) DeployClusterlinks(peerCount uint8, cfg *PeerConfig) ([]*Cluste
 
 	if cfg == nil {
 		// default config
-		cfg = &PeerConfig{
-			DataplaneType: platform.DataplaneTypeEnvoy,
-			Dataplanes:    1,
-		}
+		cfg = &PeerConfig{}
 	}
 
 	clusterlinks := make([]*ClusterLink, peerCount)

tests/e2e/k8s/util/k8s_yaml.go

@@ -59,6 +59,7 @@ func (f *Fabric) generateK8SYAML(p *peer, cfg *PeerConfig) (string, error) {
 		FabricCertificate:       f.cert,
 		PeerCertificate:         p.peerCert,
 		CACertificate:           p.caCert,
+		Controlplanes:           cfg.Controlplanes,
 		ControlplaneCertificate: p.controlplaneCert,
 		DataplaneCertificate:    p.dataplaneCert,
 		Dataplanes:              cfg.Dataplanes,
@@ -186,6 +187,7 @@ func (f *Fabric) generateClusterlinkInstance(name string, p *peer, cfg *PeerConf
 
 	instance, err := platform.K8SClusterLinkInstanceConfig(&platform.Config{
 		Peer:              p.cluster.Name(),
+		Controlplanes:     cfg.Controlplanes,
 		Dataplanes:        cfg.Dataplanes,
 		DataplaneType:     cfg.DataplaneType,
 		LogLevel:          logLevel,