diff --git a/commands/cmd_add_node.go b/commands/cmd_add_node.go index 3068bb0..f194e25 100644 --- a/commands/cmd_add_node.go +++ b/commands/cmd_add_node.go @@ -114,6 +114,13 @@ func (c *CmdAddNode) setLocalFlags(cmd *cobra.Command) { "", "[Use only with support guidance] A comma-separated list of node names that exist in the cluster.", ) + cmd.Flags().StringVar( + &c.addNodeOptions.ComputeGroup, + "compute-group", + "", + util.GetEonFlagMsg("The new or existing compute group for the new nodes. "+ + "If specified, the new nodes will be compute-only nodes."), + ) cmd.Flags().IntVar( &c.addNodeOptions.TimeOut, "add-node-timeout", diff --git a/vclusterops/add_node.go b/vclusterops/add_node.go index 2ef79e8..95344ca 100644 --- a/vclusterops/add_node.go +++ b/vclusterops/add_node.go @@ -16,6 +16,7 @@ package vclusterops import ( + "errors" "fmt" "strings" @@ -46,6 +47,9 @@ type VAddNodeOptions struct { // Names of the existing nodes in the cluster. This option can be // used to remove partially added nodes from catalog. ExpectedNodeNames []string + // Name of the compute group for the new node(s). If provided, this indicates the new nodes + // will be compute nodes. + ComputeGroup string // timeout for polling nodes in seconds when we add Nodes TimeOut int @@ -185,7 +189,7 @@ func (vcc VClusterCommands) VAddNode(options *VAddNodeOptions) (VCoordinationDat // add_node is aborted if requirements are not met. // Here we check whether the nodes being added already exist - err = checkAddNodeRequirements(&vdb, options.NewHosts) + err = options.checkAddNodeRequirements(&vdb, options.NewHosts) if err != nil { return vdb, err } @@ -208,13 +212,18 @@ func (vcc VClusterCommands) VAddNode(options *VAddNodeOptions) (VCoordinationDat } // checkAddNodeRequirements returns an error if at least one of the nodes -// to add already exists in db. -func checkAddNodeRequirements(vdb *VCoordinationDatabase, hostsToAdd []string) error { +// to add already exists in db, or if attempting to add compute nodes to +// an enterprise db. +func (options *VAddNodeOptions) checkAddNodeRequirements(vdb *VCoordinationDatabase, hostsToAdd []string) error { // we don't want any of the new host to be part of the db. if nodes, _ := vdb.containNodes(hostsToAdd); len(nodes) != 0 { return fmt.Errorf("%s already exist in the database", strings.Join(nodes, ",")) } + if !vdb.IsEon && options.ComputeGroup != "" { + return errors.New("cannot add compute nodes to an Enterprise mode database") + } + return nil } @@ -225,8 +234,8 @@ func (options *VAddNodeOptions) completeVDBSetting(vdb *VCoordinationDatabase) e vdb.DepotPrefix = options.DepotPrefix hostNodeMap := makeVHostNodeMap() - // TODO: we set the depot and data path from /nodes rather than manually - // (VER-92725). This is useful for nmaDeleteDirectoriesOp. + // We could set the depot and data path from /nodes rather than manually. + // This would be useful for nmaDeleteDirectoriesOp. for h, vnode := range vdb.HostNodeMap { dataPath := vdb.GenDataPath(vnode.Name) vnode.StorageLocations = append(vnode.StorageLocations, dataPath) @@ -237,6 +246,12 @@ func (options *VAddNodeOptions) completeVDBSetting(vdb *VCoordinationDatabase) e } vdb.HostNodeMap = hostNodeMap + // Compute nodes currently do not have depot support, so skip setting up + // the depot for now. This doesn't affect directory preparation. + if options.ComputeGroup != "" { + vdb.UseDepot = false + } + return nil } @@ -260,6 +275,7 @@ func (vcc VClusterCommands) trimNodesInCatalog(vdb *VCoordinationDatabase, expectedNodeNames[nodeName] = struct{}{} } + subscribingHostsCount := 0 var aliveHosts []string var nodesToTrim []string nodeNamesInCatalog := make(map[string]any) @@ -268,13 +284,19 @@ func (vcc VClusterCommands) trimNodesInCatalog(vdb *VCoordinationDatabase, if _, ok := expectedNodeNames[vnode.Name]; ok { // catalog node is expected aliveHosts = append(aliveHosts, h) existingHostNodeMap[h] = vnode + // This could be counting a DOWN compute node as counting towards + // k-safety. When compute nodes can be identified when down or offline, + // this should do so instead of checking state. + if vnode.State != util.NodeComputeState { + subscribingHostsCount++ + } } else if vnode.Sandbox != "" { // add sandbox node to allExistingHostNodeMap as well existingHostNodeMap[h] = vnode } else { // main cluster catalog node is not expected, trim it // cannot trim UP nodes - if vnode.State == util.NodeUpState { - return existingHostNodeMap, fmt.Errorf("cannot trim the UP node %s (address %s)", - vnode.Name, h) + if vnode.State == util.NodeUpState || vnode.State == util.NodeComputeState { + return existingHostNodeMap, fmt.Errorf("cannot trim the %s node %s (address %s)", + vnode.State, vnode.Name, h) } nodesToTrim = append(nodesToTrim, vnode.Name) } @@ -295,7 +317,7 @@ func (vcc VClusterCommands) trimNodesInCatalog(vdb *VCoordinationDatabase, var instructions []clusterOp // mark k-safety - if len(aliveHosts) < ksafetyThreshold { + if subscribingHostsCount < ksafetyThreshold { httpsMarkDesignKSafeOp, err := makeHTTPSMarkDesignKSafeOp(initiator, options.usePassword, options.UserName, options.Password, ksafeValueZero) @@ -385,7 +407,7 @@ func (vcc VClusterCommands) produceAddNodeInstructions(vdb *VCoordinationDatabas } nmaNetworkProfileOp := makeNMANetworkProfileOp(vdb.HostList) httpsCreateNodeOp, err := makeHTTPSCreateNodeOp(newHosts, initiatorHost, - usePassword, username, password, vdb, options.SCName) + usePassword, username, password, vdb, options.SCName, options.ComputeGroup) if err != nil { return instructions, err } @@ -413,14 +435,27 @@ func (vcc VClusterCommands) produceAddNodeInstructions(vdb *VCoordinationDatabas nil /*Sandbox name*/) nmaStartNewNodesOp := makeNMAStartNodeOpWithVDB(newHosts, options.StartUpConf, vdb) - httpsPollNodeStateOp, err := makeHTTPSPollNodeStateOp(newHosts, usePassword, username, password, options.TimeOut) - if err != nil { - return instructions, err + var pollNodeStateOp clusterOp + if options.ComputeGroup == "" { + // poll normally + httpsPollNodeStateOp, err := makeHTTPSPollNodeStateOp(newHosts, usePassword, username, password, options.TimeOut) + if err != nil { + return instructions, err + } + httpsPollNodeStateOp.cmdType = AddNodeCmd + pollNodeStateOp = &httpsPollNodeStateOp + } else { + // poll indirectly via nodes with catalog access + httpsPollComputeNodeStateOp, err := makeHTTPSPollComputeNodeStateOp(vdb.PrimaryUpNodes, newHosts, usePassword, + username, password, options.TimeOut) + if err != nil { + return instructions, err + } + pollNodeStateOp = &httpsPollComputeNodeStateOp } - httpsPollNodeStateOp.cmdType = AddNodeCmd instructions = append(instructions, &nmaStartNewNodesOp, - &httpsPollNodeStateOp, + pollNodeStateOp, ) return vcc.prepareAdditionalEonInstructions(vdb, options, instructions, @@ -447,7 +482,10 @@ func (vcc VClusterCommands) prepareAdditionalEonInstructions(vdb *VCoordinationD return instructions, err } instructions = append(instructions, &httpsSyncCatalogOp) - if !*options.SkipRebalanceShards { + // Rebalancing shards after only adding compute nodes is pointless as compute nodes only + // have ephemeral subscriptions. However, it may be needed if real nodes were just trimmed. + // Only ignore the specified option if compute nodes were added with no trimming. + if !*options.SkipRebalanceShards && (options.ComputeGroup == "" || len(options.ExpectedNodeNames) != 0) { httpsRBSCShardsOp, err := makeHTTPSRebalanceSubclusterShardsOp( initiatorHost, usePassword, username, options.Password, options.SCName) if err != nil { diff --git a/vclusterops/coordinator_database.go b/vclusterops/coordinator_database.go index 020b9aa..7eed111 100644 --- a/vclusterops/coordinator_database.go +++ b/vclusterops/coordinator_database.go @@ -57,6 +57,7 @@ type VCoordinationDatabase struct { Ipv6 bool PrimaryUpNodes []string + ComputeNodes []string FirstStartAfterRevive bool } @@ -199,6 +200,7 @@ func (vdb *VCoordinationDatabase) copy(targetHosts []string) VCoordinationDataba LicensePathOnNode: vdb.LicensePathOnNode, Ipv6: vdb.Ipv6, PrimaryUpNodes: util.CopySlice(vdb.PrimaryUpNodes), + ComputeNodes: util.CopySlice(vdb.ComputeNodes), } if len(targetHosts) == 0 { diff --git a/vclusterops/create_db.go b/vclusterops/create_db.go index 5a4a41b..3ecc336 100644 --- a/vclusterops/create_db.go +++ b/vclusterops/create_db.go @@ -457,7 +457,7 @@ func (vcc VClusterCommands) produceCreateDBWorkerNodesInstructions( newNodeHosts := util.SliceDiff(hosts, bootstrapHost) if len(hosts) > 1 { httpsCreateNodeOp, err := makeHTTPSCreateNodeOp(newNodeHosts, bootstrapHost, - true /* use password auth */, options.UserName, options.Password, vdb, "") + true /* use password auth */, options.UserName, options.Password, vdb, "" /* subcluster */, "" /* compute group */) if err != nil { return instructions, err } diff --git a/vclusterops/https_create_node_op.go b/vclusterops/https_create_node_op.go index d9ab616..c9b4537 100644 --- a/vclusterops/https_create_node_op.go +++ b/vclusterops/https_create_node_op.go @@ -28,9 +28,15 @@ type httpsCreateNodeOp struct { RequestParams map[string]string } +// some reused parameters +const ( + createNodeSCNameParam = "subcluster" + createNodeCGNameParam = "compute-group" +) + func makeHTTPSCreateNodeOp(newNodeHosts []string, bootstrapHost []string, useHTTPPassword bool, userName string, httpsPassword *string, - vdb *VCoordinationDatabase, scName string) (httpsCreateNodeOp, error) { + vdb *VCoordinationDatabase, scName, computeGroupName string) (httpsCreateNodeOp, error) { op := httpsCreateNodeOp{} op.name = "HTTPSCreateNodeOp" op.description = "Create node in catalog" @@ -41,7 +47,10 @@ func makeHTTPSCreateNodeOp(newNodeHosts []string, bootstrapHost []string, op.RequestParams["data-prefix"] = vdb.DataPrefix + "/" + vdb.Name op.RequestParams["hosts"] = util.ArrayToString(newNodeHosts, ",") if scName != "" { - op.RequestParams["subcluster"] = scName + op.RequestParams[createNodeSCNameParam] = scName + } + if computeGroupName != "" { + op.RequestParams[createNodeCGNameParam] = computeGroupName } err := op.validateAndSetUsernameAndPassword(op.name, useHTTPPassword, userName, httpsPassword) @@ -75,6 +84,16 @@ func (op *httpsCreateNodeOp) updateQueryParams(execContext *opEngineExecContext) } op.RequestParams["broadcast"] = profile.Broadcast } + + // if the compute group doesn't exist yet, and the compute node is in a compute group + // of the default subcluster, the sc name is explicitly needed for the create CG DDL + cgName, ok := op.RequestParams[createNodeCGNameParam] + if ok && cgName != "" { + scName, ok := op.RequestParams[createNodeSCNameParam] + if !ok || scName == "" { + op.RequestParams[createNodeSCNameParam] = execContext.defaultSCName + } + } return nil } diff --git a/vclusterops/https_get_nodes_info_op.go b/vclusterops/https_get_nodes_info_op.go index 8b65bb6..29a524a 100644 --- a/vclusterops/https_get_nodes_info_op.go +++ b/vclusterops/https_get_nodes_info_op.go @@ -140,6 +140,7 @@ func (op *httpsGetNodesInfoOp) processResult(_ *opEngineExecContext) error { op.vdb.HostNodeMap = makeVHostNodeMap() op.vdb.HostList = []string{} op.vdb.PrimaryUpNodes = []string{} + op.vdb.ComputeNodes = []string{} op.vdb.UnboundNodes = []*VCoordinationNode{} for _, node := range nodesStates.NodeList { if node.Database != op.dbName { @@ -150,6 +151,8 @@ func (op *httpsGetNodesInfoOp) processResult(_ *opEngineExecContext) error { vnode := buildVnodeFromNodeStateInfo(node) if node.IsPrimary && node.State == util.NodeUpState { op.vdb.PrimaryUpNodes = append(op.vdb.PrimaryUpNodes, node.Address) + } else if node.State == util.NodeComputeState { + op.vdb.ComputeNodes = append(op.vdb.ComputeNodes, node.Address) } err := op.vdb.addNode(&vnode) if err != nil { diff --git a/vclusterops/https_poll_compute_node_state_op.go b/vclusterops/https_poll_compute_node_state_op.go new file mode 100644 index 0000000..6944a16 --- /dev/null +++ b/vclusterops/https_poll_compute_node_state_op.go @@ -0,0 +1,227 @@ +/* + (c) Copyright [2023-2024] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package vclusterops + +import ( + "errors" + "fmt" + "strings" + + "github.com/vertica/vcluster/vclusterops/util" +) + +type httpsPollComputeNodeStateOp struct { + opBase + opHTTPSBase + // Map of compute hosts to be added to whether or not they are the desired status yet + computeHostStatus map[string]bool + // The timeout for the entire operation (polling) + timeout int + // The timeout for each http request. Requests will be repeated if timeout hasn't been exceeded. + httpRequestTimeout int + // poll for nodes down: Set to true if nodes need to be polled to be down + checkDown bool +} + +func makeHTTPSPollComputeNodeStateOpHelper(hosts, computeHosts []string, + useHTTPPassword bool, userName string, httpsPassword *string) (httpsPollComputeNodeStateOp, error) { + op := httpsPollComputeNodeStateOp{} + op.name = "HTTPSPollComputeNodeStateOp" + op.hosts = hosts // should be 1+ hosts capable of retrieving accurate node states, e.g. primary up hosts + if len(op.hosts) < 1 { + return op, errors.New("polling compute node state requires at least one primary up host") + } + op.computeHostStatus = make(map[string]bool, len(computeHosts)) + for _, computeHost := range computeHosts { + op.computeHostStatus[computeHost] = false + } + op.useHTTPPassword = useHTTPPassword + op.httpRequestTimeout = defaultHTTPSRequestTimeoutSeconds + err := util.ValidateUsernameAndPassword(op.name, useHTTPPassword, userName) + if err != nil { + return op, err + } + op.userName = userName + op.httpsPassword = httpsPassword + return op, nil +} + +func makeHTTPSPollComputeNodeStateOp(hosts, computeHosts []string, + useHTTPPassword bool, userName string, + httpsPassword *string, timeout int) (httpsPollComputeNodeStateOp, error) { + op, err := makeHTTPSPollComputeNodeStateOpHelper(hosts, computeHosts, useHTTPPassword, userName, httpsPassword) + if err != nil { + return op, err + } + if timeout == 0 { + // using default value + op.timeout = util.GetEnvInt("NODE_STATE_POLLING_TIMEOUT", StartupPollingTimeout) + } else { + op.timeout = timeout + } + op.checkDown = false // poll for COMPUTE state (UP equivalent) + op.description = fmt.Sprintf("Wait for %d compute node(s) to reach COMPUTE state", len(computeHosts)) + return op, err +} + +//nolint:unused // for NYI stop node +func makeHTTPSPollComputeNodeStateDownOp(hosts, computeHosts []string, + useHTTPPassword bool, userName string, + httpsPassword *string) (httpsPollComputeNodeStateOp, error) { + op, err := makeHTTPSPollComputeNodeStateOpHelper(hosts, computeHosts, useHTTPPassword, userName, httpsPassword) + if err != nil { + return op, err + } + op.timeout = util.GetEnvInt("NODE_STATE_POLLING_TIMEOUT", StartupPollingTimeout) + op.checkDown = true + op.description = fmt.Sprintf("Wait for %d compute node(s) to go DOWN", len(hosts)) + return op, nil +} + +func (op *httpsPollComputeNodeStateOp) getPollingTimeout() int { + return util.Max(op.timeout, 0) +} + +func (op *httpsPollComputeNodeStateOp) setupClusterHTTPRequest(hosts []string) error { + for _, host := range hosts { + httpRequest := hostHTTPRequest{} + httpRequest.Method = GetMethod + httpRequest.Timeout = op.httpRequestTimeout + httpRequest.buildHTTPSEndpoint(util.NodesEndpoint) + if op.useHTTPPassword { + httpRequest.Password = op.httpsPassword + httpRequest.Username = op.userName + } + + op.clusterHTTPRequest.RequestCollection[host] = httpRequest + } + + return nil +} + +func (op *httpsPollComputeNodeStateOp) prepare(execContext *opEngineExecContext) error { + execContext.dispatcher.setup(op.hosts) + + return op.setupClusterHTTPRequest(op.hosts) +} + +func (op *httpsPollComputeNodeStateOp) execute(execContext *opEngineExecContext) error { + if err := op.runExecute(execContext); err != nil { + return err + } + + return op.processResult(execContext) +} + +func (op *httpsPollComputeNodeStateOp) finalize(_ *opEngineExecContext) error { + return nil +} + +func (op *httpsPollComputeNodeStateOp) checkStatusToString() string { + if op.checkDown { + return strings.ToLower(util.NodeDownState) + } + return "up (compute)" +} + +func (op *httpsPollComputeNodeStateOp) getRemainingHostsString() string { + var remainingHosts []string + for host, statusOk := range op.computeHostStatus { + if statusOk { + remainingHosts = append(remainingHosts, host) + } + } + return strings.Join(remainingHosts, ",") +} + +func (op *httpsPollComputeNodeStateOp) processResult(execContext *opEngineExecContext) error { + op.logger.PrintInfo("[%s] expecting %d %s host(s)", op.name, len(op.hosts), op.checkStatusToString()) + + err := pollState(op, execContext) + if err != nil { + // show the hosts that are not COMPUTE or DOWN + msg := fmt.Sprintf("the hosts [%s] are not in %s state after %d seconds, details: %s", + op.getRemainingHostsString(), op.checkStatusToString(), op.timeout, err) + op.logger.PrintError(msg) + return errors.New(msg) + } + return nil +} + +func (op *httpsPollComputeNodeStateOp) shouldStopPolling() (bool, error) { + if op.checkDown { + return op.shouldStopPollingForDown() + } + + for host, result := range op.clusterHTTPRequest.ResultCollection { + // when we get timeout error, we know that the host is unreachable/dead + if result.isTimeout() { + return true, fmt.Errorf("[%s] cannot connect to host %s, please check if the host is still alive", op.name, host) + } + + // We don't need to wait until timeout to determine if all nodes are up or not. + // If we find the wrong password for the HTTPS service on any hosts, we should fail immediately. + // We also need to let user know to wait until all nodes are up + if result.isPasswordAndCertificateError(op.logger) { + op.logger.PrintError("[%s] The credentials are incorrect. 'Catalog Sync' will not be executed.", + op.name) + return false, makePollNodeStateAuthenticationError(op.name, host) + } + if result.isPassing() { + // parse the /nodes endpoint response for all nodes, then look for the new ones + nodesInformation := nodesInfo{} + err := op.parseAndCheckResponse(host, result.content, &nodesInformation) + if err != nil { + op.logger.PrintError("[%s] fail to parse result on host %s, details: %s", + op.name, host, err) + return true, err + } + + // check which nodes have COMPUTE status + upNodeCount := 0 + for _, nodeInfo := range nodesInformation.NodeList { + _, ok := op.computeHostStatus[nodeInfo.Address] + if !ok { + // skip unrelated nodes + continue + } + if nodeInfo.State == util.NodeComputeState { + upNodeCount++ + op.computeHostStatus[nodeInfo.Address] = true + } else { + // it would be weird for a previously COMPUTE node to change status while we're still + // polling, but no reason not to use the updated value in case it differs. + op.computeHostStatus[nodeInfo.Address] = false + } + } + if upNodeCount == len(op.computeHostStatus) { + op.logger.PrintInfo("[%s] All nodes are %s", op.name, op.checkStatusToString()) + op.updateSpinnerStopMessage("all nodes are %s", op.checkStatusToString()) + return true, nil + } + // try the next host's result + op.logger.PrintInfo("[%s] %d host(s) up (compute)", op.name, upNodeCount) + op.updateSpinnerMessage("%d host(s) up (compute), expecting %d up (compute) host(s)", upNodeCount, len(op.computeHostStatus)) + } + } + // no host returned all new compute nodes as status COMPUTE, so keep polling + return false, nil +} + +func (op *httpsPollComputeNodeStateOp) shouldStopPollingForDown() (bool, error) { + // for NYI stop node + return true, fmt.Errorf("NYI") +} diff --git a/vclusterops/https_poll_node_state_op.go b/vclusterops/https_poll_node_state_op.go index 3a34589..f6e5a39 100644 --- a/vclusterops/https_poll_node_state_op.go +++ b/vclusterops/https_poll_node_state_op.go @@ -163,8 +163,7 @@ func (op *httpsPollNodeStateOp) shouldStopPolling() (bool, error) { if op.cmdType == StartDBCmd || op.cmdType == StartNodeCmd { op.logger.PrintError("[%s] The credentials are incorrect. 'Catalog Sync' will not be executed.", op.name) - return false, fmt.Errorf("[%s] wrong password/certificate for https service on host %s, but the nodes' startup have been in progress."+ - "Please use vsql to check the nodes' status and manually run sync_catalog vsql command 'select sync_catalog()'", op.name, host) + return false, makePollNodeStateAuthenticationError(op.name, host) } else if op.cmdType == CreateDBCmd { return true, fmt.Errorf("[%s] wrong password/certificate for https service on host %s", op.name, host) @@ -253,3 +252,8 @@ func (op *httpsPollNodeStateOp) shouldStopPollingForDown() (bool, error) { return true, nil } + +func makePollNodeStateAuthenticationError(opName, hostName string) error { + return fmt.Errorf("[%s] wrong password/certificate for https service on host %s, but the nodes' startup have been in progress. "+ + "Please use vsql to check the nodes' status and manually run sync_catalog vsql command 'select sync_catalog()'", opName, hostName) +} diff --git a/vclusterops/remove_node.go b/vclusterops/remove_node.go index 9b43d3e..d73fd87 100644 --- a/vclusterops/remove_node.go +++ b/vclusterops/remove_node.go @@ -395,7 +395,12 @@ func (vcc VClusterCommands) produceRemoveNodeInstructions(vdb *VCoordinationData usePassword := options.usePassword password := options.Password - if (len(vdb.HostList) - len(options.HostsToRemove)) < ksafetyThreshold { + // compute nodes don't count towards shard coverage + permanentHostsToRemove := util.SliceDiff(options.HostsToRemove, vdb.ComputeNodes) + + // adjust k-safety if this remove node operation drops too many full nodes + permanentHostCount := len(vdb.HostList) - len(vdb.ComputeNodes) + if (permanentHostCount - len(permanentHostsToRemove)) < ksafetyThreshold { httpsMarkDesignKSafeOp, e := makeHTTPSMarkDesignKSafeOp(initiatorHost, usePassword, username, password, ksafeValueZero) if e != nil { @@ -404,49 +409,22 @@ func (vcc VClusterCommands) produceRemoveNodeInstructions(vdb *VCoordinationData instructions = append(instructions, &httpsMarkDesignKSafeOp) } - err := vcc.produceMarkEphemeralNodeOps(&instructions, options.HostsToRemove, initiatorHost, - usePassword, username, password, vdb.HostNodeMap) - if err != nil { - return instructions, err - } - - // this is a copy of the original that only - // contains the hosts to remove. - v := vdb.copy(options.HostsToRemove) - if vdb.IsEon { - // we pass the set of subclusters of the nodes to remove. - err = vcc.produceRebalanceSubclusterShardsOps(&instructions, initiatorHost, v.getSCNames(), - usePassword, username, password) + // compute nodes cannot be marked as ephemeral + if len(permanentHostsToRemove) > 0 { + err := vcc.produceMarkEphemeralNodeOps(&instructions, permanentHostsToRemove, initiatorHost, + usePassword, username, password, vdb.HostNodeMap) if err != nil { return instructions, err } + } - // for Eon DB, we check whether all UP nodes (nodesToPollSubs) have subscriptions being ACTIVE after rebalance shards - // also wait for all REMOVING subscriptions are gone for the nodes to remove (nodesToRemove) - // Sandboxed nodes cannot be removed, so even if the database has sandboxes, - // polling subscriptions for the main cluster is enough - var nodesToPollSubs, nodesToRemove []string - if len(options.NodesToPullSubs) > 0 { - nodesToPollSubs = options.NodesToPullSubs - } else { - getMainClusterNodes(vdb, options, &nodesToPollSubs, &nodesToRemove) - } - - httpsPollSubscriptionStateOp, e := makeHTTPSPollSubscriptionStateOp(initiatorHost, - usePassword, username, password, &nodesToPollSubs, &nodesToRemove) - if e != nil { - return instructions, e - } - instructions = append(instructions, &httpsPollSubscriptionStateOp) - } else { - var httpsRBCOp httpsRebalanceClusterOp - httpsRBCOp, err = makeHTTPSRebalanceClusterOp(initiatorHost, usePassword, username, - password) - if err != nil { - return instructions, err - } - instructions = append(instructions, &httpsRBCOp) + // perform any rebalancing operations needed + err := vcc.produceRebalanceClusterOps(&instructions, permanentHostsToRemove, initiatorHost, usePassword, username, password, + vdb, options) + if err != nil { + return instructions, err } + // only remove secondary nodes from spread err = vcc.produceSpreadRemoveNodeOp(&instructions, options.HostsToRemove, usePassword, username, password, @@ -469,6 +447,9 @@ func (vcc VClusterCommands) produceRemoveNodeInstructions(vdb *VCoordinationData } instructions = append(instructions, &httpsReloadSpreadOp) + // this is a copy of the original that only + // contains the hosts to remove, including any compute nodes + v := vdb.copy(options.HostsToRemove) nmaHealthOp := makeNMAHealthOpSkipUnreachable(v.HostList) nmaDeleteDirectoriesOp, err := makeNMADeleteDirectoriesOp(&v, options.ForceDelete) if err != nil { @@ -503,6 +484,52 @@ func (vcc VClusterCommands) produceMarkEphemeralNodeOps(instructions *[]clusterO return nil } +func (vcc VClusterCommands) produceRebalanceClusterOps(instructions *[]clusterOp, + permanentHostsToRemove, initiatorHost []string, + usePassword bool, username string, password *string, + vdb *VCoordinationDatabase, options *VRemoveNodeOptions) error { + if vdb.IsEon { + // if all nodes being removed are compute nodes, we can skip rebalancing. + if len(permanentHostsToRemove) > 0 { + // we pass the set of subclusters of the permanent nodes to remove. + v := vdb.copy(permanentHostsToRemove) + err := vcc.produceRebalanceSubclusterShardsOps(instructions, initiatorHost, v.getSCNames(), + usePassword, username, password) + if err != nil { + return err + } + } + + // For Eon DB, we check whether all UP nodes (nodesToPollSubs) have ACTIVE subscriptions after rebalancing shards. + // Also wait til all REMOVING subscriptions are gone for the nodes to remove (nodesToRemove). + // Sandboxed nodes cannot be removed, so even if the database has sandboxes, + // polling subscriptions for the main cluster is enough. + // This excludes compute nodes. + var nodesToPollSubs, nodesToRemove []string + if len(options.NodesToPullSubs) > 0 { + nodesToPollSubs = options.NodesToPullSubs + } else { + getMainClusterNodes(vdb, options, &nodesToPollSubs, &nodesToRemove) + } + + httpsPollSubscriptionStateOp, e := makeHTTPSPollSubscriptionStateOp(initiatorHost, + usePassword, username, password, &nodesToPollSubs, &nodesToRemove) + if e != nil { + return e + } + *instructions = append(*instructions, &httpsPollSubscriptionStateOp) + } else { + var httpsRBCOp httpsRebalanceClusterOp + httpsRBCOp, err := makeHTTPSRebalanceClusterOp(initiatorHost, usePassword, username, + password) + if err != nil { + return err + } + *instructions = append(*instructions, &httpsRBCOp) + } + return nil +} + // produceRebalanceSubclusterShardsOps gets a slice of subclusters and for each of them // produces an HTTPSRebalanceSubclusterShardsOp. func (vcc VClusterCommands) produceRebalanceSubclusterShardsOps(instructions *[]clusterOp, initiatorHost, scNames []string, diff --git a/vclusterops/revive_db.go b/vclusterops/revive_db.go index ef6a0d3..b354bdb 100644 --- a/vclusterops/revive_db.go +++ b/vclusterops/revive_db.go @@ -265,13 +265,18 @@ func (vcc VClusterCommands) VReviveDatabase(options *VReviveDatabaseOptions) (db return dbInfo, &vdb, fmt.Errorf("fail to revive database %w", err) } nmaVDB := clusterOpEngine.execContext.nmaVDatabase - for h, vnode := range nmaVDB.HostNodeMap { - _, ok := vdb.HostNodeMap[h] - if !ok { - continue + // collect nodes indexed by node name, in case node address has changed. + nodeMap := make(map[string]*VCoordinationNode) + for _, node := range vdb.HostNodeMap { + nodeMap[node.Name] = node + } + // update vdb info + for _, vnode := range nmaVDB.HostNodeMap { + if node, exists := nodeMap[vnode.Name]; exists { + node.Address = vnode.Address + node.Subcluster = vnode.Subcluster.Name + node.Sandbox = vnode.Subcluster.SandboxName } - vdb.HostNodeMap[h].Subcluster = vnode.Subcluster.Name - vdb.HostNodeMap[h].Sandbox = vnode.Subcluster.SandboxName } // fill vdb with VReviveDatabaseOptions information vdb.Name = options.DBName diff --git a/vclusterops/util/defaults.go b/vclusterops/util/defaults.go index da08508..8362aac 100644 --- a/vclusterops/util/defaults.go +++ b/vclusterops/util/defaults.go @@ -41,6 +41,7 @@ const ( DefaultControlSetSize = -1 NodeUpState = "UP" NodeDownState = "DOWN" + NodeComputeState = "COMPUTE" NodeUnknownState = "UNKNOWN" // this is for sandbox only SuppressHelp = "SUPPRESS_HELP" MainClusterSandbox = ""