From b932df0a92e948c3bc30342a5cb52084de454c2b Mon Sep 17 00:00:00 2001 From: wkd-woo Date: Mon, 29 Apr 2024 14:40:37 +0900 Subject: [PATCH] feat: exporter with tls Signed-off-by: wkd-woo --- controllers/rediscluster_controller.go | 33 ++++++++++++++++++++++ controllers/redisreplication_controller.go | 20 +++++++------ k8sutils/statefulset.go | 4 ++- 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/controllers/rediscluster_controller.go b/controllers/rediscluster_controller.go index 7cf371721..0a7c580f3 100644 --- a/controllers/rediscluster_controller.go +++ b/controllers/rediscluster_controller.go @@ -107,6 +107,39 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{RequeueAfter: time.Second * 60}, nil } + if followerReplicas < instance.Status.ReadyFollowerReplicas { + reqLogger.Info("Redis cluster is downscaling...", "Ready.ReadyFollowerReplicas", instance.Status.ReadyFollowerReplicas, "Expected.ReadFollowerReplicas", followerReplicas) + + // loop count times to remove the latest leader/follower pod + count := instance.Status.ReadyLeaderReplicas - leaderReplicas + for i := int32(0); i < count; i++ { + reqLogger.Info("Redis cluster is downscaling", "The times of loop", i) + + // Imp if the last index of leader sts is not leader make it then + // check whether the redis is leader or not ? + // if not true then make it leader pod + if !(k8sutils.VerifyLeaderPod(ctx, r.K8sClient, r.Log, instance)) { + // lastLeaderPod is slaving right now Make it the master Pod + // We have to bring a manual failover here to make it a leaderPod + // clusterFailover should also include the clusterReplicate since we have to map the followers to new leader + k8sutils.ClusterFailover(ctx, r.K8sClient, r.Log, instance) + } + // Step 1 Remove the Follower Node + k8sutils.RemoveRedisFollowerNodesFromCluster(ctx, r.K8sClient, r.Log, instance) + // Step 2 Reshard the Cluster + k8sutils.ReshardRedisCluster(r.K8sClient, r.Log, instance, true) + } + reqLogger.Info("Redis cluster is downscaled... Rebalancing the cluster") + // Step 3 Rebalance the cluster + k8sutils.RebalanceRedisCluster(r.K8sClient, r.Log, instance) + reqLogger.Info("Redis cluster is downscaled... Rebalancing the cluster is done") + err = k8sutils.UpdateRedisClusterStatus(instance, status.RedisClusterReady, status.ReadyClusterReason, leaderReplicas, leaderReplicas, r.Dk8sClient) + if err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: time.Second * 60}, nil + } + // Mark the cluster status as initializing if there are no leader or follower nodes if instance.Status.ReadyLeaderReplicas == 0 && instance.Status.ReadyFollowerReplicas == 0 { err = k8sutils.UpdateRedisClusterStatus(instance, status.RedisClusterInitializing, status.InitializingClusterLeaderReason, 0, 0, r.Dk8sClient) diff --git a/controllers/redisreplication_controller.go b/controllers/redisreplication_controller.go index 0b86f1080..72349b00c 100644 --- a/controllers/redisreplication_controller.go +++ b/controllers/redisreplication_controller.go @@ -86,10 +86,13 @@ func (r *RedisReplicationReconciler) Reconcile(ctx context.Context, req ctrl.Req if len(masterNodes) == int(leaderReplicas) && followerReplicas != 0 && len(slaveNodes) != 0 { realMaster = k8sutils.GetRedisReplicationRealMaster(ctx, r.K8sClient, r.Log, instance, masterNodes) if err = k8sutils.UpdateRoleLabelPod(ctx, r.K8sClient, r.Log, instance, "master", []string{realMaster}); err != nil { + return ctrl.Result{Requeue: true}, err + } + if err = k8sutils.UpdateRoleLabelPod(ctx, r.K8sClient, r.Log, instance, "slave", slaveNodes); err != nil { return ctrl.Result{RequeueAfter: time.Second * 1}, err } } - return ctrl.Result{RequeueAfter: time.Second * 10}, nil + return ctrl.Result{RequeueAfter: time.Second * 1}, nil } var realMaster string @@ -101,28 +104,27 @@ func (r *RedisReplicationReconciler) Reconcile(ctx context.Context, req ctrl.Req if len(slaveNodes) == 0 { realMaster = masterNodes[0] } + if err = k8sutils.UpdateRoleLabelPod(ctx, r.K8sClient, r.Log, instance, "master", []string{realMaster}); err != nil { + return ctrl.Result{Requeue: true}, err + } err = k8sutils.CreateMasterSlaveReplication(ctx, r.K8sClient, r.Log, instance, masterNodes, realMaster) if err != nil { - return ctrl.Result{RequeueAfter: time.Second * 60}, err - } - if err = k8sutils.UpdateRoleLabelPod(ctx, r.K8sClient, r.Log, instance, "master", []string{realMaster}); err != nil { - return ctrl.Result{RequeueAfter: time.Second}, err + return ctrl.Result{Requeue: true}, err } if err = k8sutils.UpdateRoleLabelPod(ctx, r.K8sClient, r.Log, instance, "slave", slaveNodes); err != nil { - return ctrl.Result{RequeueAfter: time.Second}, err + return ctrl.Result{Requeue: true}, err } } - realMaster = k8sutils.GetRedisReplicationRealMaster(ctx, r.K8sClient, r.Log, instance, masterNodes) slaveNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "slave") if err = r.UpdateRedisReplicationMaster(ctx, instance, realMaster); err != nil { return ctrl.Result{}, err } if err = k8sutils.UpdateRoleLabelPod(ctx, r.K8sClient, r.Log, instance, "master", []string{realMaster}); err != nil { - return ctrl.Result{RequeueAfter: time.Second}, err + return ctrl.Result{}, err } if err = k8sutils.UpdateRoleLabelPod(ctx, r.K8sClient, r.Log, instance, "slave", slaveNodes); err != nil { - return ctrl.Result{RequeueAfter: time.Second}, err + return ctrl.Result{}, err } reqLogger.Info("Will reconcile redis operator in again 10 seconds") diff --git a/k8sutils/statefulset.go b/k8sutils/statefulset.go index 788851be1..7359c3d36 100644 --- a/k8sutils/statefulset.go +++ b/k8sutils/statefulset.go @@ -491,6 +491,7 @@ func enableRedisMonitoring(params containerParameters) corev1.Container { func getExporterEnvironmentVariables(params containerParameters) []corev1.EnvVar { var envVars []corev1.EnvVar + redisHost := "redis://localhost:" if params.TLSConfig != nil { envVars = append(envVars, corev1.EnvVar{ Name: "REDIS_EXPORTER_TLS_CLIENT_KEY_FILE", @@ -508,6 +509,7 @@ func getExporterEnvironmentVariables(params containerParameters) []corev1.EnvVar Name: "REDIS_EXPORTER_SKIP_TLS_VERIFICATION", Value: "true", }) + redisHost = "rediss://localhost:" } if params.RedisExporterPort != nil { envVars = append(envVars, corev1.EnvVar{ @@ -518,7 +520,7 @@ func getExporterEnvironmentVariables(params containerParameters) []corev1.EnvVar if params.Port != nil { envVars = append(envVars, corev1.EnvVar{ Name: "REDIS_ADDR", - Value: fmt.Sprintf("redis://localhost:%d", *params.Port), + Value: redisHost + strconv.Itoa(*params.Port), }) } if params.EnabledPassword != nil && *params.EnabledPassword {