Skip to content

Commit

Permalink
KUBESAW-172: Restart host-operator at the end of the register-member …
Browse files Browse the repository at this point in the history 
…command (#90)

* KUBESAW-172: Restart host-operator at the end of the register-member command

Signed-off-by: Feny Mehta <[email protected]>

* Register member test cases

Signed-off-by: Feny Mehta <[email protected]>

* clean up

Signed-off-by: Feny Mehta <[email protected]>

* e2e error fix

Signed-off-by: Feny Mehta <[email protected]>

* rename to restartFunc in reg-mem

Signed-off-by: Feny Mehta <[email protected]>

* some changes

Signed-off-by: Feny Mehta <[email protected]>

* clean up

Signed-off-by: Feny Mehta <[email protected]>

* cosmetic changes and cleanup

Signed-off-by: Feny Mehta <[email protected]>

* improving the readability and cosmetic changes

Signed-off-by: Feny Mehta <[email protected]>

---------

Signed-off-by: Feny Mehta <[email protected]>
Co-authored-by: Francisc Munteanu <[email protected]>
Co-authored-by: Matous Jobanek <[email protected]>
  • Loading branch information
@fbm3307 @mfrancisc @MatousJobanek
3 people authored Dec 23, 2024
1 parent f99bb4a commit ec701fa
Show file tree
Hide file tree
Showing 6 changed files with 175 additions and 67 deletions.
28 changes: 24 additions & 4 deletions pkg/cmd/adm/register_member.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/cli-runtime/pkg/genericclioptions"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
Expand Down Expand Up @@ -72,7 +73,7 @@ func NewRegisterMemberCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, args []string) error {
term := ioutils.NewTerminal(cmd.InOrStdin, cmd.OutOrStdout)
ctx := newExtendedCommandContext(term, client.DefaultNewClientFromRestConfig)
return registerMemberCluster(ctx, commandArgs)
return registerMemberCluster(ctx, commandArgs, restart)
},
}

Expand All @@ -94,7 +95,7 @@ func NewRegisterMemberCmd() *cobra.Command {
return cmd
}

func registerMemberCluster(ctx *extendedCommandContext, args registerMemberArgs) error {
func registerMemberCluster(ctx *extendedCommandContext, args registerMemberArgs, restart restartFunc) error {
validated, err := validateArgs(ctx, args)
if err != nil {
return err
Expand All @@ -114,7 +115,7 @@ func registerMemberCluster(ctx *extendedCommandContext, args registerMemberArgs)
return nil
}

return validated.perform(ctx)
return validated.perform(ctx, restart)
}

func (v *registerMemberValidated) getSourceAndTargetClusters(sourceClusterType configuration.ClusterType) (clusterData, clusterData) {
Expand Down Expand Up @@ -470,7 +471,7 @@ func (v *registerMemberValidated) confirmationPrompt() ioutils.ConfirmationMessa
return ioutils.WithMessagef(sb.String(), args...)
}

func (v *registerMemberValidated) perform(ctx *extendedCommandContext) error {
func (v *registerMemberValidated) perform(ctx *extendedCommandContext, restart restartFunc) error {
// add the host entry to the member cluster first. We assume that there is just 1 toolchain cluster entry in the member
// cluster (i.e. it just points back to the host), so there's no need to determine the number of entries with the same
// API endpoint.
Expand All @@ -483,6 +484,11 @@ func (v *registerMemberValidated) perform(ctx *extendedCommandContext) error {
return err
}

// restart Host Operator using the adm-restart command
if err := restart(ctx.CommandContext, configuration.HostName, v.getRegMemConfigFlagsAndClient); err != nil {
return err
}

exampleSPC := &toolchainv1alpha1.SpaceProvisionerConfig{
TypeMeta: metav1.TypeMeta{
Kind: "SpaceProvisionerConfig",
Expand All @@ -509,6 +515,20 @@ until the SpaceProvisionerConfig.spec.enabled is set to true.
`, v.hostClusterData.apiEndpoint))
}

func (v *registerMemberValidated) getRegMemConfigFlagsAndClient(_ *clicontext.CommandContext, _ string) (kubeConfigFlag *genericclioptions.ConfigFlags, rccl runtimeclient.Client, err error) {
kubeConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag()

kubeConfigFlags.ClusterName = nil // `cluster` flag is redefined for our own purpose
kubeConfigFlags.AuthInfoName = nil // unused here, so we can hide it
kubeConfigFlags.Context = nil // unused here, so we can hide it

kubeConfigFlags.Namespace = &v.hostClusterData.namespace
kubeConfigFlags.APIServer = &v.hostClusterData.apiEndpoint
kubeConfigFlags.KubeConfig = &v.hostClusterData.kubeConfig

return kubeConfigFlags, v.hostClusterData.client, nil
}

func findToolchainClusterForMember(allToolchainClusters []toolchainv1alpha1.ToolchainCluster, memberAPIEndpoint, memberOperatorNamespace string) *toolchainv1alpha1.ToolchainCluster {
for _, tc := range allToolchainClusters {
if tc.Status.APIEndpoint == memberAPIEndpoint && tc.Status.OperatorNamespace == memberOperatorNamespace {
Expand Down
107 changes: 91 additions & 16 deletions pkg/cmd/adm/register_member_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/codeready-toolchain/toolchain-common/pkg/test"
"github.com/ghodss/yaml"
"github.com/kubesaw/ksctl/pkg/configuration"
clicontext "github.com/kubesaw/ksctl/pkg/context"
. "github.com/kubesaw/ksctl/pkg/test"
"github.com/kubesaw/ksctl/pkg/utils"
"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -82,7 +83,9 @@ func TestRegisterMember(t *testing.T) {
}

// when
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false))
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return nil
})

// then
require.NoError(t, err)
Expand Down Expand Up @@ -115,7 +118,9 @@ func TestRegisterMember(t *testing.T) {
ctx := newExtendedCommandContext(term, newClient)

// when
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false))
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return nil
})

// then
require.Error(t, err)
Expand All @@ -135,7 +140,9 @@ func TestRegisterMember(t *testing.T) {
ctx := newExtendedCommandContext(term, newClient)

// when
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false))
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return nil
})

// then
require.Error(t, err)
Expand All @@ -155,7 +162,9 @@ func TestRegisterMember(t *testing.T) {
ctx := newExtendedCommandContext(term, newClient)

// when
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false))
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return nil
})

// then
require.NoError(t, err)
Expand All @@ -171,7 +180,9 @@ func TestRegisterMember(t *testing.T) {
ctx := newExtendedCommandContext(term, newClient)

// when
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, true))
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, true), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return nil
})

// then
require.NoError(t, err)
Expand Down Expand Up @@ -206,7 +217,9 @@ func TestRegisterMember(t *testing.T) {
mockCreateToolchainClusterWithReadyCondition(t, fakeClient)

// when
err := registerMemberCluster(ctx, newRegisterMemberArgsWithSuffix(hostKubeconfig, memberKubeconfig, false, "2"))
err := registerMemberCluster(ctx, newRegisterMemberArgsWithSuffix(hostKubeconfig, memberKubeconfig, false, "2"), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return nil
})

// then
require.NoError(t, err)
Expand All @@ -227,8 +240,12 @@ func TestRegisterMember(t *testing.T) {
ctx2 := newExtendedCommandContext(term2, newClient)

// when
err1 := registerMemberCluster(ctx1, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false))
err2 := registerMemberCluster(ctx2, newRegisterMemberArgsWithSuffix(hostKubeconfig, memberKubeconfig, false, "1"))
err1 := registerMemberCluster(ctx1, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return nil
})
err2 := registerMemberCluster(ctx2, newRegisterMemberArgsWithSuffix(hostKubeconfig, memberKubeconfig, false, "1"), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return nil
})

// then
require.NoError(t, err1)
Expand All @@ -250,8 +267,12 @@ func TestRegisterMember(t *testing.T) {
ctx2 := newExtendedCommandContext(term2, newClient)

// when
err1 := registerMemberCluster(ctx1, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false))
err2 := registerMemberCluster(ctx2, newRegisterMemberArgsWithSuffix(hostKubeconfig, memberKubeconfig, false, ""))
err1 := registerMemberCluster(ctx1, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return nil
})
err2 := registerMemberCluster(ctx2, newRegisterMemberArgsWithSuffix(hostKubeconfig, memberKubeconfig, false, ""), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return nil
})

// then
require.NoError(t, err1)
Expand Down Expand Up @@ -305,7 +326,9 @@ func TestRegisterMember(t *testing.T) {
require.NoError(t, fakeClient.Create(context.TODO(), preexistingToolchainCluster2.DeepCopy()))

// when
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false))
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return nil
})

// then
require.Error(t, err)
Expand Down Expand Up @@ -337,7 +360,9 @@ func TestRegisterMember(t *testing.T) {
require.NoError(t, fakeClient.Create(context.TODO(), preexistingToolchainCluster.DeepCopy()))

// when
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false))
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return nil
})

// then
require.Error(t, err)
Expand Down Expand Up @@ -369,7 +394,9 @@ func TestRegisterMember(t *testing.T) {
require.NoError(t, fakeClient.Create(context.TODO(), preexistingToolchainCluster.DeepCopy()))

// when
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false))
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return nil
})

// then
require.Error(t, err)
Expand Down Expand Up @@ -402,7 +429,9 @@ func TestRegisterMember(t *testing.T) {
require.NoError(t, fakeClient.Create(context.TODO(), preexistingToolchainCluster.DeepCopy()))

// when
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false))
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return nil
})

// then
require.Error(t, err)
Expand All @@ -419,7 +448,9 @@ func TestRegisterMember(t *testing.T) {
ctx := newExtendedCommandContext(term, newClient)

// when
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false))
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return nil
})

// then
require.Error(t, err)
Expand All @@ -438,7 +469,9 @@ func TestRegisterMember(t *testing.T) {
ctx := newExtendedCommandContext(term, newClient)

// when
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false))
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return nil
})

// then
require.Error(t, err)
Expand All @@ -449,6 +482,41 @@ func TestRegisterMember(t *testing.T) {
require.NoError(t, fakeClient.List(context.TODO(), tcs, runtimeclient.InNamespace(test.MemberOperatorNs)))
assert.Empty(t, tcs.Items)
})

t.Run("reports error when host-operator is not restarted", func(t *testing.T) {
// given
term := NewFakeTerminalWithResponse("Y")
newClient, fakeClient := newFakeClientsFromRestConfig(t, &toolchainClusterMemberSa, &toolchainClusterHostSa)
mockCreateToolchainClusterWithReadyCondition(t, fakeClient)
ctx := newExtendedCommandContext(term, newClient)

// when
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
return fmt.Errorf("restart did not happen")
})

// then
require.EqualError(t, err, "restart did not happen")
})

t.Run("Register-member calls restart ", func(t *testing.T) {
// given
term := NewFakeTerminalWithResponse("Y")
newClient, fakeClient := newFakeClientsFromRestConfig(t, &toolchainClusterMemberSa, &toolchainClusterHostSa)
mockCreateToolchainClusterWithReadyCondition(t, fakeClient)
ctx := newExtendedCommandContext(term, newClient)
called := 0
// when
err := registerMemberCluster(ctx, newRegisterMemberArgsWith(hostKubeconfig, memberKubeconfig, false), func(_ *clicontext.CommandContext, _ string, _ ConfigFlagsAndClientGetterFunc) error {
called++
return mockRestartReg(ctx.CommandContext, configuration.HostName, nil)
})

// then
require.NoError(t, err)
assert.Equal(t, 1, called)
})

}

func mockCreateToolchainClusterInNamespaceWithReadyCondition(t *testing.T, fakeClient *test.FakeClient, namespace string) {
Expand Down Expand Up @@ -585,3 +653,10 @@ func defaultRegisterMemberArgs() registerMemberArgs {

return args
}

func mockRestartReg(ctx *clicontext.CommandContext, clusterName string, _ ConfigFlagsAndClientGetterFunc) error {
if clusterName == "host" && ctx != nil {
return nil
}
return fmt.Errorf("cluster name is wrong")
}
51 changes: 31 additions & 20 deletions pkg/cmd/adm/restart.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,9 @@ import (
)

type (
RolloutRestartFunc func(ctx *clicontext.CommandContext, deployment appsv1.Deployment) error
RolloutStatusCheckerFunc func(ctx *clicontext.CommandContext, deployment appsv1.Deployment) error
RolloutRestartFunc func(ctx *clicontext.CommandContext, deployment appsv1.Deployment) error
RolloutStatusCheckerFunc func(ctx *clicontext.CommandContext, deployment appsv1.Deployment) error
ConfigFlagsAndClientGetterFunc func(ctx *clicontext.CommandContext, clusterName string) (kubeConfigFlag *genericclioptions.ConfigFlags, rccl runtimeclient.Client, err error)
)

// NewRestartCmd() is a function to restart the whole operator, it relies on the target cluster and fetches the cluster config
Expand All @@ -45,52 +46,62 @@ func NewRestartCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, args []string) error {
term := ioutils.NewTerminal(cmd.InOrStdin, cmd.OutOrStdout)
ctx := clicontext.NewCommandContext(term, client.DefaultNewClient)
return restart(ctx, args[0])
return restart(ctx, args[0], getConfigFlagsAndClient)
},
}
return command
}

func restart(ctx *clicontext.CommandContext, clusterName string) error {
kubeConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag()
func restart(ctx *clicontext.CommandContext, clusterName string, configFlagsClientGetter ConfigFlagsAndClientGetterFunc) error {
ioStreams := genericiooptions.IOStreams{
In: os.Stdin,
Out: os.Stdout,
ErrOut: os.Stderr,
}

kubeConfigFlags, cl, err := configFlagsClientGetter(ctx, clusterName)
if err != nil {
return err
}
factory := cmdutil.NewFactory(cmdutil.NewMatchVersionFlags(kubeConfigFlags))

if !ctx.AskForConfirmation(
ioutils.WithMessagef("restart all the deployments in the cluster '%s' and namespace '%s' \n", clusterName, *kubeConfigFlags.Namespace)) {
return nil
}

return restartDeployments(ctx, cl, *kubeConfigFlags.Namespace, func(ctx *clicontext.CommandContext, deployment appsv1.Deployment) error {
return checkRolloutStatus(ctx, factory, ioStreams, deployment)
}, func(ctx *clicontext.CommandContext, deployment appsv1.Deployment) error {
return restartNonOlmDeployments(ctx, deployment, factory, ioStreams)
})
}

func getConfigFlagsAndClient(ctx *clicontext.CommandContext, clusterName string) (kubeConfigFlag *genericclioptions.ConfigFlags, rccl runtimeclient.Client, err error) {
kubeConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag()

kubeConfigFlags.ClusterName = nil // `cluster` flag is redefined for our own purpose
kubeConfigFlags.AuthInfoName = nil // unused here, so we can hide it
kubeConfigFlags.Context = nil // unused here, so we can hide it

cfg, err := configuration.LoadClusterConfig(ctx, clusterName)
if err != nil {
return err
return nil, nil, err
}
kubeConfigFlags.Namespace = &cfg.OperatorNamespace
kubeConfigFlags.APIServer = &cfg.ServerAPI
kubeConfigFlags.BearerToken = &cfg.Token
kubeconfig, err := client.EnsureKsctlConfigFile()
if err != nil {
return err
return nil, nil, err
}
kubeConfigFlags.KubeConfig = &kubeconfig
factory := cmdutil.NewFactory(cmdutil.NewMatchVersionFlags(kubeConfigFlags))

if !ctx.AskForConfirmation(
ioutils.WithMessagef("restart all the deployments in the cluster '%s' and namespace '%s' \n", clusterName, cfg.OperatorNamespace)) {
return nil
}

cl, err := ctx.NewClient(cfg.Token, cfg.ServerAPI)
if err != nil {
return err
return nil, nil, err
}

return restartDeployments(ctx, cl, cfg.OperatorNamespace, func(ctx *clicontext.CommandContext, deployment appsv1.Deployment) error {
return checkRolloutStatus(ctx, factory, ioStreams, deployment)
}, func(ctx *clicontext.CommandContext, deployment appsv1.Deployment) error {
return restartNonOlmDeployments(ctx, deployment, factory, ioStreams)
})
return kubeConfigFlags, cl, nil
}

// This function has the whole logic of getting the list of olm and non-olm based deployment, then proceed on restarting/deleting accordingly
Expand Down
Loading

0 comments on commit ec701fa

Please sign in to comment.