Sync from server repo (7057bbc327d)

commands/cmd_add_node.go

@@ -123,6 +123,12 @@ func (c *CmdAddNode) setLocalFlags(cmd *cobra.Command) {
 		"",
 		"Comma-separated list of node names that exist in the cluster",
 	)
+	cmd.Flags().IntVar(
+		&c.addNodeOptions.TimeOut,
+		"add-node-timeout",
+		util.GetEnvInt("NODE_STATE_POLLING_TIMEOUT", util.DefaultTimeoutSeconds),
+		"The timeout to wait for the nodes to add",
+	)
 }
 
 func (c *CmdAddNode) Parse(inputArgv []string, logger vlog.Printer) error {

commands/cmd_create_db.go

@@ -208,7 +208,7 @@ func (c *CmdCreateDB) setLocalFlags(cmd *cobra.Command) {
 	cmd.Flags().IntVar(
 		&c.createDBOptions.TimeoutNodeStartupSeconds,
 		"startup-timeout",
-		util.DefaultTimeoutSeconds,
+		util.GetEnvInt("NODE_STATE_POLLING_TIMEOUT", util.DefaultTimeoutSeconds),
 		"The timeout in seconds to wait for the nodes to start",
 	)
 }

commands/cmd_list_all_nodes.go

@@ -119,6 +119,11 @@ func (c *CmdListAllNodes) Run(vcc vclusterops.ClusterCommands) error {
 
 	c.writeCmdOutputToFile(globals.file, bytes, vcc.GetLog())
 	vcc.LogInfo("Node states: ", "nodeStates", string(bytes))
+	// if writing into stdout, add a new line
+	// otherwise, the successful message may be wrapped into the same line of the node state output
+	if c.output == "" {
+		fmt.Println("")
+	}
 	vcc.DisplayInfo("Successfully listed all nodes")
 	return nil
 }

commands/cmd_restart_node.go

@@ -104,7 +104,7 @@ func (c *CmdStartNodes) setLocalFlags(cmd *cobra.Command) {
 	cmd.Flags().IntVar(
 		&c.startNodesOptions.StatePollingTimeout,
 		"timeout",
-		util.DefaultTimeoutSeconds,
+		util.GetEnvInt("NODE_STATE_POLLING_TIMEOUT", util.DefaultTimeoutSeconds),
 		"The timeout (in seconds) to wait for polling node state operation",
 	)
 
@@ -167,6 +167,12 @@ func (c *CmdStartNodes) Run(vcc vclusterops.ClusterCommands) error {
 		return err
 	}
 
+	// all nodes unreachable, nothing need to be done.
+	if len(options.Nodes) == 0 {
+		vcc.DisplayInfo("No reachable nodes to start")
+		return nil
+	}
+
 	var hostToStart []string
 	for _, ip := range options.Nodes {
 		hostToStart = append(hostToStart, ip)

commands/cmd_start_db.go

@@ -221,6 +221,7 @@ func (c *CmdStartDB) Run(vcc vclusterops.ClusterCommands) error {
 	}
 	dbConfig, readConfigErr := readConfig()
 	if readConfigErr == nil {
+		options.ReadFromConfig = true
 		if options.Sandbox != util.MainClusterSandbox || options.MainCluster {
 			options.RawHosts = filterInputHosts(options, dbConfig)
 		}
@@ -238,6 +239,12 @@ func (c *CmdStartDB) Run(vcc vclusterops.ClusterCommands) error {
 		return err
 	}
 
+	// all nodes unreachable
+	if len(options.Hosts) == 0 {
+		vcc.DisplayInfo("No reachable nodes to start database %s", options.DBName)
+		return nil
+	}
+
 	msg := fmt.Sprintf("Started database %s", options.DBName)
 	if options.Sandbox != "" {
 		sandboxMsg := fmt.Sprintf(" on sandbox %s", options.Sandbox)
@@ -253,12 +260,7 @@ func (c *CmdStartDB) Run(vcc vclusterops.ClusterCommands) error {
 
 	// for Eon database, update config file to fill nodes' subcluster information
 	if readConfigErr == nil && options.IsEon {
-		// write db info to vcluster config file
-		vdb.FirstStartAfterRevive = false
-		err = writeConfig(vdb, true /*forceOverwrite*/)
-		if err != nil {
-			vcc.DisplayWarning("fail to update config file, details: %s", err)
-		}
+		c.UpdateConfigFileForEon(vdb, vcc)
 	}
 
 	// write config parameters to vcluster config param file
@@ -270,6 +272,15 @@ func (c *CmdStartDB) Run(vcc vclusterops.ClusterCommands) error {
 	return nil
 }
 
+func (c *CmdStartDB) UpdateConfigFileForEon(vdb *vclusterops.VCoordinationDatabase, vcc vclusterops.ClusterCommands) {
+	// write db info to vcluster config file
+	vdb.FirstStartAfterRevive = false
+	err := writeConfig(vdb, true /*forceOverwrite*/)
+	if err != nil {
+		vcc.DisplayWarning("fail to update config file, details: %s", err)
+	}
+}
+
 // SetDatabaseOptions will assign a vclusterops.DatabaseOptions instance to the one in CmdStartDB
 func (c *CmdStartDB) SetDatabaseOptions(opt *vclusterops.DatabaseOptions) {
 	c.startDBOptions.DatabaseOptions = *opt

commands/cmd_start_subcluster.go

@@ -133,6 +133,12 @@ func (c *CmdStartSubcluster) Run(vcc vclusterops.ClusterCommands) error {
 		return err
 	}
 
+	// all nodes unreachable, nothing need to be done.
+	if len(options.Nodes) == 0 {
+		vcc.DisplayInfo("No reachable nodes to start in subcluster %s", options.SCName)
+		return nil
+	}
+
 	vcc.DisplayInfo("Successfully started subcluster %s for database %s",
 		options.SCName, options.DBName)
 

commands/vcluster_config.go

@@ -193,13 +193,19 @@ func writeConfig(vdb *vclusterops.VCoordinationDatabase, forceOverwrite bool) er
 		return fmt.Errorf("configuration file path is empty")
 	}
 
+	return WriteConfigToPath(vdb, dbOptions.ConfigPath, forceOverwrite)
+}
+
+func WriteConfigToPath(vdb *vclusterops.VCoordinationDatabase,
+	configPath string,
+	forceOverwrite bool) error {
 	dbConfig, err := readVDBToDBConfig(vdb)
 	if err != nil {
 		return err
 	}
 
 	// update db config with the given database info
-	err = dbConfig.write(dbOptions.ConfigPath, forceOverwrite)
+	err = dbConfig.write(configPath, forceOverwrite)
 	if err != nil {
 		return err
 	}

vclusterops/add_node.go

@@ -46,6 +46,9 @@ type VAddNodeOptions struct {
 	// Names of the existing nodes in the cluster. This option can be
 	// used to remove partially added nodes from catalog.
 	ExpectedNodeNames []string
+
+	// timeout for polling nodes in seconds when we add Nodes
+	TimeOut int
 }
 
 func VAddNodeOptionsFactory() VAddNodeOptions {
@@ -60,6 +63,10 @@ func (options *VAddNodeOptions) setDefaultValues() {
 	options.DatabaseOptions.setDefaultValues()
 
 	options.SkipRebalanceShards = new(bool)
+
+	// try to retrieve the timeout from the environment variable
+	// otherwise, set the default value (300 seconds) to the timeout
+	options.TimeOut = util.GetEnvInt("NODE_STATE_POLLING_TIMEOUT", util.DefaultTimeoutSeconds)
 }
 
 func (options *VAddNodeOptions) validateEonOptions() error {
@@ -70,7 +77,7 @@ func (options *VAddNodeOptions) validateEonOptions() error {
 }
 
 func (options *VAddNodeOptions) validateRequiredOptions(logger vlog.Printer) error {
-	err := options.validateBaseOptions(commandAddNode, logger)
+	err := options.validateBaseOptions(AddNodeCmd, logger)
 	if err != nil {
 		return err
 	}
@@ -398,10 +405,11 @@ func (vcc VClusterCommands) produceAddNodeInstructions(vdb *VCoordinationDatabas
 		vdb /*db configurations retrieved from a running db*/)
 
 	nmaStartNewNodesOp := makeNMAStartNodeOpWithVDB(newHosts, options.StartUpConf, vdb)
-	httpsPollNodeStateOp, err := makeHTTPSPollNodeStateOp(newHosts, usePassword, username, password)
+	httpsPollNodeStateOp, err := makeHTTPSPollNodeStateOp(newHosts, usePassword, username, password, options.TimeOut)
 	if err != nil {
 		return instructions, err
 	}
+	httpsPollNodeStateOp.cmdType = AddNodeCmd
 	instructions = append(instructions,
 		&nmaStartNewNodesOp,
 		&httpsPollNodeStateOp,

vclusterops/add_subcluster.go

@@ -67,7 +67,7 @@ func (options *VAddSubclusterOptions) setDefaultValues() {
 }
 
 func (options *VAddSubclusterOptions) validateRequiredOptions(logger vlog.Printer) error {
-	err := options.validateBaseOptions(commandAddSubcluster, logger)
+	err := options.validateBaseOptions(AddSubclusterCmd, logger)
 	if err != nil {
 		return err
 	}

vclusterops/alter_subcluster_type.go

@@ -72,7 +72,7 @@ func (options *VAlterSubclusterTypeOptions) validateParseOptions(logger vlog.Pri
 		return err
 	}
 
-	err = options.validateAuthOptions(commandAlterSubclusterType, logger)
+	err = options.validateAuthOptions(AlterSubclusterTypeCmd.CmdString(), logger)
 	if err != nil {
 		return err
 	}
@@ -89,7 +89,7 @@ func (options *VAlterSubclusterTypeOptions) validateParseOptions(logger vlog.Pri
 	if !options.SCType.IsValid() {
 		return fmt.Errorf("invalid subcluster type: must be 'primary' or 'secondary'")
 	}
-	return options.validateBaseOptions(commandAlterSubclusterType, logger)
+	return options.validateBaseOptions(AlterSubclusterTypeCmd, logger)
 }
 
 // analyzeOptions will modify some options based on what is chosen

vclusterops/cluster_op.go

@@ -277,6 +277,7 @@ func (op *opBase) setupSpinner() {
 			StopFailCharacter: "✘",
 			StopFailMessage:   "failed",
 			StopFailColors:    []string{"fgRed"},
+			Writer:            op.logger.Writer, // if nil, writing to stdout
 		}
 		spinner, err := yacspin.New(cfg)
 		if err != nil {

vclusterops/cluster_op_engine.go

@@ -55,6 +55,12 @@ func (opEngine *VClusterOpEngine) runWithExecContext(logger vlog.Printer, execCo
 		}
 	}
 
+	// display warning if any unreachable hosts detected
+	if len(opEngine.execContext.unreachableHosts) > 0 {
+		logger.DisplayWarning("Unreachable host(s) detected, please check the NMA connectivity in %v",
+			opEngine.execContext.unreachableHosts)
+	}
+
 	return nil
 }
 

vclusterops/cmd_type.go

@@ -0,0 +1,86 @@
+package vclusterops
+
+type CmdType int
+
+const (
+	// command types
+	CreateDBCmd CmdType = iota
+	DropDBCmd
+	StopDBCmd
+	StartDBCmd
+	AddNodeCmd
+	RemoveNodeCmd
+	StartNodeCmd
+	StopNodeCmd
+	RestartNodeCmd
+	AddSubclusterCmd
+	RemoveSubclusterCmd
+	StopSubclusterCmd
+	StartSubclusterCmd
+	SandboxSCCmd
+	UnsandboxSCCmd
+	ShowRestorePointsCmd
+	InstallPackagesCmd
+	ConfigRecoverCmd
+	ManageConnectionDrainingCmd
+	SetConfigurationParameterCmd
+	GetConfigurationParameterCmd
+	ReplicationStartCmd
+	PromoteSandboxToMainCmd
+	FetchNodesDetailsCmd
+	AlterSubclusterTypeCmd
+	RenameScCmd
+	ReIPCmd
+	ScrutinizeCmd
+	CreateDBSyncCat
+	StartDBSyncCat
+	StopDBSyncCat
+	StopSCSyncCat
+	AddNodeSyncCat
+	StartNodeSyncCat
+	RemoveNodeSyncCat
+)
+
+var cmdStringMap = map[CmdType]string{
+	CreateDBCmd:                  "create_db",
+	DropDBCmd:                    "drop_db",
+	StopDBCmd:                    "stop_db",
+	StartDBCmd:                   "start_db",
+	AddNodeCmd:                   "add_node",
+	RemoveNodeCmd:                "remove_node",
+	StartNodeCmd:                 "start_node",
+	StopNodeCmd:                  "stop_node",
+	RestartNodeCmd:               "restart_node",
+	AddSubclusterCmd:             "add_subcluster",
+	RemoveSubclusterCmd:          "remove_subcluster",
+	StopSubclusterCmd:            "stop_subcluster",
+	StartSubclusterCmd:           "start_subcluster",
+	SandboxSCCmd:                 "sandbox_subcluster",
+	UnsandboxSCCmd:               "unsandbox_subcluster",
+	ShowRestorePointsCmd:         "show_restore_points",
+	InstallPackagesCmd:           "install_packages",
+	ConfigRecoverCmd:             "manage_config_recover",
+	ManageConnectionDrainingCmd:  "manage_connection_draining",
+	SetConfigurationParameterCmd: "set_configuration_parameter",
+	ReplicationStartCmd:          "replication_start",
+	PromoteSandboxToMainCmd:      "promote_sandbox_to_main",
+	FetchNodesDetailsCmd:         "fetch_nodes_details",
+	AlterSubclusterTypeCmd:       "alter_subcluster_type",
+	RenameScCmd:                  "rename_subcluster",
+	ReIPCmd:                      "re_ip",
+	ScrutinizeCmd:                "scrutinize",
+	CreateDBSyncCat:              "create_db_sync_cat",
+	StartDBSyncCat:               "start_db_sync_cat",
+	StopDBSyncCat:                "stop_db_sync_cat",
+	StopSCSyncCat:                "stop_sc_sync_cat",
+	AddNodeSyncCat:               "add_node_sync_cat",
+	StartNodeSyncCat:             "start_node_sync_cat",
+	RemoveNodeSyncCat:            "remove_node_sync_cat",
+}
+
+func (cmd CmdType) CmdString() string {
+	if str, ok := cmdStringMap[cmd]; ok {
+		return str
+	}
+	return "unknown_operation"
+}

vclusterops/create_db.go

@@ -98,15 +98,14 @@ func (options *VCreateDatabaseOptions) setDefaultValues() {
 
 func (options *VCreateDatabaseOptions) validateRequiredOptions(logger vlog.Printer) error {
 	// validate base options
-	err := options.validateBaseOptions(commandCreateDB, logger)
+	err := options.validateBaseOptions(CreateDBCmd, logger)
 	if err != nil {
 		return err
 	}
 
 	// validate required parameters with default values
 	if options.Password == nil {
 		options.Password = new(string)
-		*options.Password = ""
 		logger.Info("no password specified, using none")
 	}
 
@@ -431,11 +430,12 @@ func (vcc VClusterCommands) produceCreateDBBootstrapInstructions(
 
 	nmaStartNodeOp := makeNMAStartNodeOp(bootstrapHost, options.StartUpConf)
 
-	httpsPollBootstrapNodeStateOp, err := makeHTTPSPollNodeStateOpWithTimeoutAndCommand(bootstrapHost, true, /* useHTTPPassword */
-		options.UserName, options.Password, options.TimeoutNodeStartupSeconds, CreateDBCmd)
+	httpsPollBootstrapNodeStateOp, err := makeHTTPSPollNodeStateOp(bootstrapHost, true, /* useHTTPPassword */
+		options.UserName, options.Password, options.TimeoutNodeStartupSeconds)
 	if err != nil {
 		return instructions, err
 	}
+	httpsPollBootstrapNodeStateOp.cmdType = CreateDBCmd
 
 	instructions = append(instructions,
 		&nmaStartNodeOp,
@@ -508,11 +508,12 @@ func (vcc VClusterCommands) produceAdditionalCreateDBInstructions(vdb *VCoordina
 	username := options.UserName
 
 	if !options.SkipStartupPolling {
-		httpsPollNodeStateOp, err := makeHTTPSPollNodeStateOpWithTimeoutAndCommand(hosts, true, username, options.Password,
-			options.TimeoutNodeStartupSeconds, CreateDBCmd)
+		httpsPollNodeStateOp, err := makeHTTPSPollNodeStateOp(hosts, true, username, options.Password,
+			options.TimeoutNodeStartupSeconds)
 		if err != nil {
 			return instructions, err
 		}
+		httpsPollNodeStateOp.cmdType = CreateDBCmd
 		instructions = append(instructions, &httpsPollNodeStateOp)
 	}
 

vclusterops/fetch_database.go

@@ -39,7 +39,7 @@ func VRecoverConfigOptionsFactory() VFetchCoordinationDatabaseOptions {
 }
 
 func (options *VFetchCoordinationDatabaseOptions) validateParseOptions(logger vlog.Printer) error {
-	return options.validateBaseOptions(commandConfigRecover, logger)
+	return options.validateBaseOptions(ConfigRecoverCmd, logger)
 }
 
 func (options *VFetchCoordinationDatabaseOptions) analyzeOptions() error {

vclusterops/fetch_node_state.go

@@ -112,12 +112,6 @@ func (vcc VClusterCommands) VFetchNodeState(options *VFetchNodeStateOptions) ([]
 			}
 		}
 
-		// display warning if any unreachable hosts detected
-		if len(clusterOpEngine.execContext.unreachableHosts) > 0 {
-			vcc.DisplayWarning("hosts %v are unreachable, please check the NMA connectivity in the hosts",
-				clusterOpEngine.execContext.unreachableHosts)
-		}
-
 		return nodeStates, nil
 	}
 

vclusterops/fetch_nodes_details.go

@@ -85,7 +85,7 @@ func (options *VFetchNodesDetailsOptions) setDefaultValues() {
 }
 
 func (options *VFetchNodesDetailsOptions) validateParseOptions(logger vlog.Printer) error {
-	err := options.validateBaseOptions(commandFetchNodesDetails, logger)
+	err := options.validateBaseOptions(FetchNodesDetailsCmd, logger)
 	if err != nil {
 		return err
 	}