Skip to content

Commit

Permalink
Support multicluster control plane
Browse files Browse the repository at this point in the history 
Signed-off-by: Tamal Saha <[email protected]>
  • Loading branch information
@tamalsaha
tamalsaha committed Mar 8, 2024
1 parent 15c91e1 commit ca6dffa
Show file tree
Hide file tree
Showing 4 changed files with 146 additions and 90 deletions.
81 changes: 61 additions & 20 deletions cmd/addon-manager/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/klog/v2"
"k8s.io/klog/v2/textlogger"
"open-cluster-management.io/addon-framework/pkg/addonmanager"
Expand Down Expand Up @@ -70,6 +71,7 @@ func main() {
var signerSecretNamespace, signerSecretName string
var agentInstallAll bool
var enableKubeApiProxy bool
var mcKubeconfig string

logger := textlogger.NewLogger(textlogger.NewConfig())
klog.SetOutput(os.Stdout)
Expand All @@ -93,37 +95,70 @@ func main() {
"Configure the install strategy of agent on managed clusters. "+
"Enabling this will automatically install agent on all managed cluster.")
flag.BoolVar(&enableKubeApiProxy, "enable-kube-api-proxy", true, "Enable proxy to agent kube-apiserver")
flag.StringVar(&mcKubeconfig, "multicluster-kubeconfig", "",
"The path to multicluster-controlplane kubeconfig")

flag.Parse()

// pipe controller-runtime logs to klog
ctrl.SetLogger(logger)

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
hostManager, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Scheme: scheme,
Metrics: metricsserver.Options{BindAddress: metricsAddr},
HealthProbeBindAddress: probeAddr,
LeaderElection: enableLeaderElection,
LeaderElectionID: "cluster-proxy-addon-manager",
})
if err != nil {
setupLog.Error(err, "unable to start manager")
setupLog.Error(err, "unable to create host manager")
os.Exit(1)
}

nativeClient, err := kubernetes.NewForConfig(mgr.GetConfig())
hostNativeClient, err := kubernetes.NewForConfig(hostManager.GetConfig())
if err != nil {
setupLog.Error(err, "unable to set up kubernetes native client")
os.Exit(1)
}

addonClient, err := addonclient.NewForConfig(mgr.GetConfig())
mcMode := mcKubeconfig != ""
var mcManager ctrl.Manager
var mcNativeClient kubernetes.Interface
if mcMode {
mcConfig, err := clientcmd.BuildConfigFromFlags("", mcKubeconfig)
if err != nil {
setupLog.Error(err, "unable to build multicluster rest config")
os.Exit(1)
}
mcManager, err = ctrl.NewManager(mcConfig, ctrl.Options{
Scheme: scheme,
Metrics: metricsserver.Options{BindAddress: ""},
HealthProbeBindAddress: "",
LeaderElection: enableLeaderElection,
LeaderElectionID: "cluster-proxy-addon-manager",
})
if err != nil {
setupLog.Error(err, "unable to create mc manager")
os.Exit(1)
}

mcNativeClient, err = kubernetes.NewForConfig(mcManager.GetConfig())
if err != nil {
setupLog.Error(err, "unable to set up multicluster native client")
os.Exit(1)
}
} else {
mcManager = hostManager
mcNativeClient = hostNativeClient
}

addonClient, err := addonclient.NewForConfig(mcManager.GetConfig())
if err != nil {
setupLog.Error(err, "unable to set up ocm addon client")
os.Exit(1)
}

supportsV1CSR, supportsV1beta1CSR, err := addonutil.IsCSRSupported(nativeClient)
supportsV1CSR, supportsV1beta1CSR, err := addonutil.IsCSRSupported(mcNativeClient)
if err != nil {
setupLog.Error(err, "unable to detect available CSR API versions")
os.Exit(1)
Expand All @@ -138,43 +173,45 @@ func main() {
os.Exit(1)
}

nativeInformer := informers.NewSharedInformerFactoryWithOptions(nativeClient, 0)
hostNativeInformer := informers.NewSharedInformerFactoryWithOptions(hostNativeClient, 0)

// loading self-signer
selfSigner, err := selfsigned.NewSelfSignerFromSecretOrGenerate(
nativeClient, signerSecretNamespace, signerSecretName)
hostNativeClient, signerSecretNamespace, signerSecretName)
if err != nil {
setupLog.Error(err, "failed loading self-signer")
os.Exit(1)
}

if err := controllers.RegisterClusterManagementAddonReconciler(
mgr,
hostManager,
selfSigner,
nativeClient,
nativeInformer.Core().V1().Secrets(),
hostNativeClient,
hostNativeInformer.Core().V1().Secrets(),
mcManager,
mcMode,
supportsV1CSR,
); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "ClusterManagementAddonReconciler")
os.Exit(1)
}

if err := controllers.RegisterServiceResolverReconciler(mgr); err != nil {
if err := controllers.RegisterServiceResolverReconciler(mcManager); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "ServiceResolverReconciler")
os.Exit(1)
}

//+kubebuilder:scaffold:builder
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
if err := hostManager.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
os.Exit(1)
}
if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
if err := hostManager.AddReadyzCheck("readyz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up ready check")
os.Exit(1)
}

addonManager, err := addonmanager.New(mgr.GetConfig())
addonManager, err := addonmanager.New(mcManager.GetConfig())
if err != nil {
setupLog.Error(err, "unable to set up ready check")
os.Exit(1)
Expand All @@ -184,8 +221,9 @@ func main() {
selfSigner,
signerSecretNamespace,
supportsV1CSR,
mgr.GetClient(),
nativeClient,
mcManager.GetClient(),
mcNativeClient,
hostNativeClient,
agentInstallAll,
enableKubeApiProxy,
addonClient,
Expand All @@ -202,16 +240,19 @@ func main() {

ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler())
defer cancel()
go nativeInformer.Start(ctx.Done())
go hostNativeInformer.Start(ctx.Done())
go func() {
if err := addonManager.Start(ctx); err != nil {
setupLog.Error(err, "unable to start addon manager")
os.Exit(1)
}
}()
setupLog.Info("starting manager")
if err := mgr.Start(ctx); err != nil {
setupLog.Error(err, "problem running manager")
if mcMode {
go mcManager.Start(ctx)
}
setupLog.Info("starting host manager")
if err := hostManager.Start(ctx); err != nil {
setupLog.Error(err, "problem running host manager")
os.Exit(1)
}
}
23 changes: 12 additions & 11 deletions pkg/proxyagent/agent/agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,9 @@ func NewAgentAddon(
signer selfsigned.SelfSigner,
signerNamespace string,
v1CSRSupported bool,
runtimeClient client.Client,
nativeClient kubernetes.Interface,
mcRtc client.Client,
mcNativeClient kubernetes.Interface,
hostNativeClient kubernetes.Interface,
agentInstallAll bool,
enableKubeApiProxy bool,
addonClient addonclient.Interface) (agent.AgentAddon, error) {
Expand Down Expand Up @@ -93,7 +94,7 @@ func NewAgentAddon(
CSRApproveCheck: func(cluster *clusterv1.ManagedCluster, addon *addonv1alpha1.ManagedClusterAddOn, csr *csrv1.CertificateSigningRequest) bool {
return cluster.Spec.HubAcceptsClient
},
PermissionConfig: utils.NewRBACPermissionConfigBuilder(nativeClient).
PermissionConfig: utils.NewRBACPermissionConfigBuilder(mcNativeClient).
WithStaticRole(&rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster-proxy-addon-agent",
Expand Down Expand Up @@ -134,7 +135,7 @@ func NewAgentAddon(
utils.AddOnDeploymentConfigGVR,
).
WithGetValuesFuncs(
GetClusterProxyValueFunc(runtimeClient, nativeClient, signerNamespace, caCertData, v1CSRSupported, enableKubeApiProxy),
GetClusterProxyValueFunc(mcRtc, hostNativeClient, signerNamespace, caCertData, v1CSRSupported, enableKubeApiProxy),
addonfactory.GetAddOnDeploymentConfigValues(
utils.NewAddOnDeploymentConfigGetter(addonClient),
toAgentAddOnChartValues(caCertData),
Expand All @@ -149,8 +150,8 @@ func NewAgentAddon(
}

func GetClusterProxyValueFunc(
runtimeClient client.Client,
nativeClient kubernetes.Interface,
mcRtc client.Client,
hostNativeClient kubernetes.Interface,
signerNamespace string,
caCertData []byte,
v1CSRSupported bool,
Expand All @@ -173,7 +174,7 @@ func GetClusterProxyValueFunc(
}

proxyConfig := &proxyv1alpha1.ManagedProxyConfiguration{}
if err := runtimeClient.Get(context.TODO(), types.NamespacedName{
if err := mcRtc.Get(context.TODO(), types.NamespacedName{
Name: managedProxyConfigurations[0],
}, proxyConfig); err != nil {
return nil, err
Expand All @@ -188,7 +189,7 @@ func GetClusterProxyValueFunc(
// find the referenced proxy load-balancer prescribed in the proxy config if there's any
var proxyServerLoadBalancer *corev1.Service
if proxyConfig.Spec.ProxyServer.Entrypoint.Type == proxyv1alpha1.EntryPointTypeLoadBalancerService {
entrySvc, err := nativeClient.CoreV1().
entrySvc, err := hostNativeClient.CoreV1().
Services(proxyConfig.Spec.ProxyServer.Namespace).
Get(context.TODO(),
proxyConfig.Spec.ProxyServer.Entrypoint.LoadBalancerService.Name,
Expand Down Expand Up @@ -228,7 +229,7 @@ func GetClusterProxyValueFunc(
// "agent-client" to the managed clusters.
certDataBase64, keyDataBase64 := "", ""
if !v1CSRSupported {
agentClientSecret, err := nativeClient.CoreV1().
agentClientSecret, err := hostNativeClient.CoreV1().
Secrets(signerNamespace).
Get(context.TODO(), common.AgentClientSecretName, metav1.GetOptions{})
if err != nil {
Expand All @@ -249,7 +250,7 @@ func GetClusterProxyValueFunc(

// List all available managedClusterSets
managedClusterSetList := &clusterv1beta2.ManagedClusterSetList{}
err = runtimeClient.List(context.TODO(), managedClusterSetList)
err = mcRtc.List(context.TODO(), managedClusterSetList)
if err != nil {
return nil, err
}
Expand All @@ -261,7 +262,7 @@ func GetClusterProxyValueFunc(

// List all available serviceResolvers
serviceResolverList := &proxyv1alpha1.ManagedProxyServiceResolverList{}
err = runtimeClient.List(context.TODO(), serviceResolverList)
err = mcRtc.List(context.TODO(), serviceResolverList)
if err != nil {
return nil, err
}
Expand Down
Loading

0 comments on commit ca6dffa

Please sign in to comment.