Sync from server repo (64f73e7ebf4)

commands/cluster_command_launcher.go

@@ -100,6 +100,7 @@ func constructCmds(_ vlog.Printer) []ClusterCommand {
 		makeCmdReIP(),
 		makeCmdReviveDB(),
 		makeCmdShowRestorePoints(),
+		makeCmdInstallPackages(),
 		// sc-scope cmds
 		makeCmdAddSubcluster(),
 		makeCmdRemoveSubcluster(),

commands/cmd_install_packages.go

@@ -0,0 +1,124 @@
+/*
+ (c) Copyright [2023] Open Text.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ You may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package commands
+
+import (
+	"flag"
+
+	"github.com/vertica/vcluster/vclusterops"
+	"github.com/vertica/vcluster/vclusterops/util"
+	"github.com/vertica/vcluster/vclusterops/vlog"
+)
+
+/* CmdInstallPackages
+ *
+ * Parses arguments for VInstallPackagesOptions to pass down to
+ * VInstallPackages.
+ *
+ * Implements ClusterCommand interface
+ */
+
+type CmdInstallPackages struct {
+	CmdBase
+	installPkgOpts *vclusterops.VInstallPackagesOptions
+}
+
+func makeCmdInstallPackages() *CmdInstallPackages {
+	newCmd := &CmdInstallPackages{}
+
+	// parser, used to parse command-line flags
+	newCmd.parser = flag.NewFlagSet("install_packages", flag.ExitOnError)
+	installPkgOpts := vclusterops.VInstallPackagesOptionsFactory()
+
+	// required flags
+	installPkgOpts.DBName = newCmd.parser.String("db-name", "", "The name of the database to install packages in")
+
+	// optional flags
+	installPkgOpts.Password = newCmd.parser.String("password", "", util.GetOptionalFlagMsg("Database password in single quotes"))
+	newCmd.hostListStr = newCmd.parser.String("hosts", "", util.GetOptionalFlagMsg("Comma-separated list of hosts in database."))
+	newCmd.ipv6 = newCmd.parser.Bool("ipv6", false, util.GetOptionalFlagMsg("Used to specify the hosts are IPv6 hosts"))
+	installPkgOpts.HonorUserInput = newCmd.parser.Bool("honor-user-input", false,
+		util.GetOptionalFlagMsg("Forcefully use the user's input instead of reading the options from "+vclusterops.ConfigFileName))
+	installPkgOpts.ConfigDirectory = newCmd.parser.String("config-directory", "",
+		util.GetOptionalFlagMsg("Directory where "+vclusterops.ConfigFileName+" is located"))
+	installPkgOpts.ForceReinstall = newCmd.parser.Bool("force-reinstall", false,
+		util.GetOptionalFlagMsg("Install the packages, even if they are already installed."))
+
+	newCmd.installPkgOpts = &installPkgOpts
+
+	newCmd.parser.Usage = func() {
+		util.SetParserUsage(newCmd.parser, "install_packages")
+	}
+
+	return newCmd
+}
+
+func (c *CmdInstallPackages) CommandType() string {
+	return "install_packages"
+}
+
+func (c *CmdInstallPackages) Parse(inputArgv []string, logger vlog.Printer) error {
+	c.argv = inputArgv
+	err := c.ValidateParseArgv(c.CommandType(), logger)
+	if err != nil {
+		return err
+	}
+
+	// for some options, we do not want to use their default values,
+	// if they are not provided in cli,
+	// reset the value of those options to nil
+	if !util.IsOptionSet(c.parser, "password") {
+		c.installPkgOpts.Password = nil
+	}
+	if !util.IsOptionSet(c.parser, "ipv6") {
+		c.CmdBase.ipv6 = nil
+	}
+	if !util.IsOptionSet(c.parser, "config-directory") {
+		c.installPkgOpts.ConfigDirectory = nil
+	}
+
+	return c.validateParse()
+}
+
+// all validations of the arguments should go in here
+func (c *CmdInstallPackages) validateParse() error {
+	return c.ValidateParseBaseOptions(&c.installPkgOpts.DatabaseOptions)
+}
+
+func (c *CmdInstallPackages) Analyze(_ vlog.Printer) error {
+	return nil
+}
+
+func (c *CmdInstallPackages) Run(vcc vclusterops.VClusterCommands) error {
+	options := c.installPkgOpts
+
+	// get config from vertica_cluster.yaml
+	config, err := options.GetDBConfig(vcc)
+	if err != nil {
+		return err
+	}
+	options.Config = config
+
+	var status *vclusterops.InstallPackageStatus
+	status, err = vcc.VInstallPackages(options)
+	if err != nil {
+		vcc.Log.Error(err, "failed to install the packages")
+		return err
+	}
+
+	vcc.Log.PrintInfo("Installed the packages:\n%v", status.Packages)
+	return nil
+}

commands/cmd_revive_db.go

@@ -53,6 +53,8 @@ func makeCmdReviveDB() *CmdReviveDB {
 		"The (1-based) index of the restore point in the restore archive to restore from"))
 	reviveDBOptions.RestorePoint.ID = newCmd.parser.String("restore-point-id", "", util.GetOptionalFlagMsg(
 		"The identifier of the restore point in the restore archive to restore from"))
+	reviveDBOptions.ConfigDirectory = newCmd.parser.String("config-directory", "",
+		util.GetOptionalFlagMsg("Directory where "+vclusterops.ConfigFileName+" is located"))
 
 	newCmd.reviveDBOptions = &reviveDBOptions
 
@@ -79,6 +81,9 @@ func (c *CmdReviveDB) Parse(inputArgv []string, logger vlog.Printer) error {
 	if !util.IsOptionSet(c.parser, "ipv6") {
 		c.CmdBase.ipv6 = nil
 	}
+	if !util.IsOptionSet(c.parser, "config-directory") {
+		c.reviveDBOptions.ConfigDirectory = nil
+	}
 
 	return c.validateParse(logger)
 }
@@ -113,7 +118,7 @@ func (c *CmdReviveDB) Analyze(logger vlog.Printer) error {
 
 func (c *CmdReviveDB) Run(vcc vclusterops.VClusterCommands) error {
 	vcc.Log.V(1).Info("Called method Run()")
-	dbInfo, err := vcc.VReviveDatabase(c.reviveDBOptions)
+	dbInfo, vdb, err := vcc.VReviveDatabase(c.reviveDBOptions)
 	if err != nil {
 		vcc.Log.Error(err, "fail to revive database", "DBName", *c.reviveDBOptions.DBName)
 		return err
@@ -124,6 +129,11 @@ func (c *CmdReviveDB) Run(vcc vclusterops.VClusterCommands) error {
 		return nil
 	}
 
+	err = vdb.WriteClusterConfig(c.reviveDBOptions.ConfigDirectory, vcc.Log)
+	if err != nil {
+		vcc.Log.PrintWarning("fail to write config file, details: %s", err)
+	}
+
 	vcc.Log.PrintInfo("Successfully revived database %s", *c.reviveDBOptions.DBName)
 
 	return nil

vclusterops/cluster_config.go

@@ -149,6 +149,22 @@ func (c *ClusterConfig) getCommunalStorageLocation(dbName string) (communalStora
 	return dbConfig.CommunalStorageLocation, nil
 }
 
+func (c *ClusterConfig) removeDatabaseFromConfigFile(dbName, configDirectory string, logger vlog.Printer) error {
+	configFilePath := filepath.Join(configDirectory, ConfigFileName)
+
+	// back up the old config file
+	err := backupConfigFile(configFilePath, logger)
+	if err != nil {
+		return err
+	}
+
+	// remove the target database from the cluster config
+	// and overwrite the config file
+	delete(*c, dbName)
+
+	return c.WriteConfig(configFilePath)
+}
+
 func (c *DatabaseConfig) getHosts() []string {
 	var hostList []string
 
@@ -207,7 +223,7 @@ func backupConfigFile(configFilePath string, logger vlog.Printer) error {
 	return nil
 }
 
-func removeConfigFile(configDirectory string, logger vlog.Printer) error {
+func RemoveConfigFile(configDirectory string, logger vlog.Printer) error {
 	configFilePath := filepath.Join(configDirectory, ConfigFileName)
 	configBackupPath := filepath.Join(configDirectory, ConfigBackupName)
 

vclusterops/coordinator_database.go

@@ -439,9 +439,24 @@ func (vdb *VCoordinationDatabase) WriteClusterConfig(configDir *string, logger v
 		nodeConfig.Name = vnode.Name
 		nodeConfig.Address = vnode.Address
 		nodeConfig.Subcluster = vnode.Subcluster
-		nodeConfig.CatalogPath = vdb.CatalogPrefix
-		nodeConfig.DataPath = vdb.DataPrefix
-		nodeConfig.DepotPath = vdb.DepotPrefix
+
+		// VER-91869 will replace the path prefixes with full paths
+		if vdb.CatalogPrefix == "" {
+			nodeConfig.CatalogPath = util.GetPathPrefix(vnode.CatalogPath)
+		} else {
+			nodeConfig.CatalogPath = vdb.CatalogPrefix
+		}
+		if vdb.DataPrefix == "" && len(vnode.StorageLocations) > 0 {
+			nodeConfig.DataPath = util.GetPathPrefix(vnode.StorageLocations[0])
+		} else {
+			nodeConfig.DataPath = vdb.DataPrefix
+		}
+		if vdb.IsEon && vdb.DepotPrefix == "" {
+			nodeConfig.DepotPath = util.GetPathPrefix(vnode.DepotPath)
+		} else {
+			nodeConfig.DepotPath = vdb.DepotPrefix
+		}
+
 		dbConfig.Nodes = append(dbConfig.Nodes, &nodeConfig)
 	}
 	dbConfig.IsEon = vdb.IsEon

vclusterops/create_db.go

@@ -647,7 +647,8 @@ func (vcc *VClusterCommands) produceAdditionalCreateDBInstructions(vdb *VCoordin
 	}
 
 	if !*options.SkipPackageInstall {
-		httpsInstallPackagesOp, err := makeHTTPSInstallPackagesOp(vcc.Log, bootstrapHost, true, username, options.Password)
+		httpsInstallPackagesOp, err := makeHTTPSInstallPackagesOp(vcc.Log, bootstrapHost, true, username, options.Password,
+			false /* forceReinstall */, true /* verbose */)
 		if err != nil {
 			return instructions, err
 		}

vclusterops/drop_db.go

@@ -110,11 +110,12 @@ func (vcc *VClusterCommands) VDropDatabase(options *VDropDatabaseOptions) error
 		return fmt.Errorf("fail to drop database: %w", runError)
 	}
 
-	// if the database is successfully dropped, the config file will be removed
+	// if the database is successfully dropped, the database will be removed from the config file
 	// if failed to remove it, we will ask users to manually do it
-	err = removeConfigFile(configDir, vcc.Log)
+	err = clusterConfig.removeDatabaseFromConfigFile(vdb.Name, configDir, vcc.Log)
 	if err != nil {
-		vcc.Log.PrintWarning("Fail to remove the config file(s), please manually clean up under directory %s", configDir)
+		vcc.Log.PrintWarning("Fail to remove the database information from config file, "+
+			"please manually clean up under directory %s. Details: %v", configDir, err)
 	}
 
 	return nil

vclusterops/https_get_up_nodes_op.go

@@ -30,6 +30,7 @@ const (
 	StopDBCmd
 	ScrutinizeCmd
 	DBAddSubclusterCmd
+	InstallPackageCmd
 )
 
 type CommandType int

vclusterops/https_install_packages_op.go

@@ -18,6 +18,7 @@ package vclusterops
 import (
 	"errors"
 	"fmt"
+	"strconv"
 
 	"github.com/vertica/vcluster/vclusterops/util"
 	"github.com/vertica/vcluster/vclusterops/vlog"
@@ -26,15 +27,20 @@ import (
 type httpsInstallPackagesOp struct {
 	opBase
 	opHTTPSBase
+	verbose        bool // Include verbose output about package install status
+	forceReinstall bool
+	status         InstallPackageStatus // Filled in once the op completes
 }
 
 func makeHTTPSInstallPackagesOp(logger vlog.Printer, hosts []string, useHTTPPassword bool,
-	userName string, httpsPassword *string,
+	userName string, httpsPassword *string, forceReinstall bool, verbose bool,
 ) (httpsInstallPackagesOp, error) {
 	op := httpsInstallPackagesOp{}
 	op.name = "HTTPSInstallPackagesOp"
 	op.logger = logger.WithName(op.name)
 	op.hosts = hosts
+	op.verbose = verbose
+	op.forceReinstall = forceReinstall
 
 	err := util.ValidateUsernameAndPassword(op.name, useHTTPPassword, userName)
 	if err != nil {
@@ -55,13 +61,24 @@ func (op *httpsInstallPackagesOp) setupClusterHTTPRequest(hosts []string) error
 			httpRequest.Password = op.httpsPassword
 			httpRequest.Username = op.userName
 		}
+		httpRequest.QueryParams = map[string]string{
+			"force-install": strconv.FormatBool(op.forceReinstall),
+		}
 		op.clusterHTTPRequest.RequestCollection[host] = httpRequest
 	}
 
 	return nil
 }
 
 func (op *httpsInstallPackagesOp) prepare(execContext *opEngineExecContext) error {
+	// If no hosts passed in, we will find the hosts from execute-context
+	if len(op.hosts) == 0 {
+		if len(execContext.upHosts) == 0 {
+			return fmt.Errorf(`[%s] Cannot find any up hosts in OpEngineExecContext`, op.name)
+		}
+		// use first up host to execute https post request
+		op.hosts = []string{execContext.upHosts[0]}
+	}
 	execContext.dispatcher.setup(op.hosts)
 
 	return op.setupClusterHTTPRequest(op.hosts)
@@ -80,7 +97,8 @@ func (op *httpsInstallPackagesOp) finalize(_ *opEngineExecContext) error {
 }
 
 /*
-	httpsInstallPackagesResponse example:
+The response from the package endpoint, which are encoded in the next two
+structs, will look like this:
 
 {'packages': [
 
@@ -94,9 +112,22 @@ func (op *httpsInstallPackagesOp) finalize(_ *opEngineExecContext) error {
 	             },
 	           ...
 	           ]
-	}
+}
 */
-type httpsInstallPackagesResponse map[string][]map[string]string
+
+// InstallPackageStatus provides status for each package install attempted.
+type InstallPackageStatus struct {
+	Packages []PackageStatus `json:"packages"`
+}
+
+// PackageStatus has install status for a single package.
+type PackageStatus struct {
+	// Name of the package this status is for
+	PackageName string `json:"package_name"`
+	// One word outcome of the install status:
+	// Skipped, Success or Failure
+	InstallStatus string `json:"install_status"`
+}
 
 func (op *httpsInstallPackagesOp) processResult(_ *opEngineExecContext) error {
 	var allErrs error
@@ -109,21 +140,25 @@ func (op *httpsInstallPackagesOp) processResult(_ *opEngineExecContext) error {
 			continue
 		}
 
-		var responseObj httpsInstallPackagesResponse
-		err := op.parseAndCheckResponse(host, result.content, &responseObj)
-
+		err := op.parseAndCheckResponse(host, result.content, &op.status)
 		if err != nil {
 			allErrs = errors.Join(allErrs, err)
 			continue
 		}
 
-		installedPackages, ok := responseObj["packages"]
-		if !ok {
-			err = fmt.Errorf(`[%s] response does not contain field "packages"`, op.name)
+		if len(op.status.Packages) == 0 {
+			err = fmt.Errorf(`[%s] response does not have status for any packages`, op.name)
 			allErrs = errors.Join(allErrs, err)
 		}
 
-		op.logger.PrintInfo("[%s] installed packages: %v", op.name, installedPackages)
+		// Only print out status if verbose output was requested. Otherwise,
+		// just write status to the log.
+		msg := fmt.Sprintf("[%s] installation status of packages: %v", op.name, op.status.Packages)
+		if op.verbose {
+			op.logger.PrintInfo(msg)
+		} else {
+			op.logger.V(1).Info(msg)
+		}
 	}
 	return allErrs
 }