Sync from server repo (c953b125e28)

go.mod

@@ -3,6 +3,7 @@ module github.com/vertica/vcluster
 go 1.20
 
 require (
+	github.com/deckarep/golang-set/v2 v2.3.1
 	github.com/go-logr/logr v1.2.4
 	github.com/go-logr/zapr v1.2.4
 	github.com/stretchr/testify v1.8.2

go.sum

@@ -8,6 +8,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deckarep/golang-set/v2 v2.3.1 h1:vjmkvJt/IV27WXPyYQpAh4bRyWJc5Y435D17XQ9QU5A=
+github.com/deckarep/golang-set/v2 v2.3.1/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=
 github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=

vclusterops/create_db.go

@@ -28,33 +28,40 @@ import (
 // A good rule of thumb is to use normal strings unless you need nil.
 // Normal strings are easier and safer to use in Go.
 type VCreateDatabaseOptions struct {
-	// part 1: basic db info
+	/* part 1: basic db info */
 	DatabaseOptions
-	Policy            *string
-	SQLFile           *string
+	Policy            *string // database restart policy
+	SQLFile           *string // SQL file to run (as dbadmin) immediately on database creation
 	LicensePathOnNode *string // required to be a fully qualified path
-	// part 2: eon db info
-	ShardCount               *int
-	DepotSize                *string // like 10G
-	GetAwsCredentialsFromEnv *bool
+
+	/* part 2: eon db info */
+
+	ShardCount               *int    // number of shards in the database"
+	DepotSize                *string // depot size with two supported formats: % and KMGT, e.g., 50% or 10G
+	GetAwsCredentialsFromEnv *bool   // whether get AWS credentials from environmental variables
 	// part 3: optional info
-	ForceCleanupOnFailure     *bool
-	ForceRemovalAtCreation    *bool
-	SkipPackageInstall        *bool
-	TimeoutNodeStartupSeconds *int
-	// part 4: new params originally in installer generated admintools.conf, now in create db op
-	Broadcast          *bool
-	P2p                *bool
-	LargeCluster       *int
-	ClientPort         *int // for internal QA test only, do not abuse
-	SpreadLogging      *bool
-	SpreadLoggingLevel *int
-	// part 5: other params
-	SkipStartupPolling *bool
-	ConfigDirectory    *string
-	GenerateHTTPCerts  *bool
-
-	// hidden options (which cache information only)
+	ForceCleanupOnFailure     *bool // whether force remove existing directories on failure
+	ForceRemovalAtCreation    *bool // whether force remove existing directories before creating the database
+	SkipPackageInstall        *bool // whether skip package installation
+	TimeoutNodeStartupSeconds *int  // timeout in seconds for polling node start up state
+
+	/* part 3: new params originally in installer generated admintools.conf, now in create db op */
+
+	Broadcast          *bool // configure Spread to use UDP broadcast traffic between nodes on the same subnet
+	P2p                *bool // configure Spread to use point-to-point communication between all Vertica nodes
+	LargeCluster       *int  // whether enables a large cluster layout
+	ClientPort         *int  // for internal QA test only, do not abuse
+	SpreadLogging      *bool // whether enable spread logging
+	SpreadLoggingLevel *int  // spread logging level
+
+	/* part 4: other params */
+
+	SkipStartupPolling *bool // whether skip startup polling
+	GenerateHTTPCerts  *bool // whether generate http certificates
+
+	/* hidden options (which cache information only) */
+
+	// the host used for bootstrapping
 	bootstrapHost []string
 }
 
@@ -279,6 +286,10 @@ func validateDepotSize(size string) (bool, error) {
 
 func (opt *VCreateDatabaseOptions) validateEonOptions() error {
 	if *opt.CommunalStorageLocation != "" {
+		err := util.ValidateCommunalStorageLocation(*opt.CommunalStorageLocation)
+		if err != nil {
+			return err
+		}
 		if *opt.DepotPrefix == "" {
 			return fmt.Errorf("must specify a depot path with commual storage location")
 		}

vclusterops/drop_db.go

@@ -26,7 +26,7 @@ import (
 // Normal strings are easier and safer to use in Go.
 type VDropDatabaseOptions struct {
 	VCreateDatabaseOptions
-	ForceDelete *bool
+	ForceDelete *bool // whether force delete directories
 }
 
 func VDropDatabaseOptionsFactory() VDropDatabaseOptions {

vclusterops/https_poll_node_state_op.go

@@ -152,19 +152,6 @@ func (op *HTTPSPollNodeStateOp) processResult(execContext *OpEngineExecContext)
 	return nil
 }
 
-// the following structs only hosts necessary information for this op
-type NodeInfo struct {
-	Address string `json:"address"`
-	// vnode name, e.g., v_dbname_node0001
-	Name        string `json:"name"`
-	State       string `json:"state"`
-	CatalogPath string `json:"catalog_path"`
-}
-
-type NodesInfo struct {
-	NodeList []NodeInfo `json:"node_list"`
-}
-
 func (op *HTTPSPollNodeStateOp) shouldStopPolling() (bool, error) {
 	upNodeCount := 0
 

vclusterops/nma_stage_dc_tables_op.go

@@ -0,0 +1,103 @@
+/*
+ (c) Copyright [2023] Open Text.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ You may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package vclusterops
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/vertica/vcluster/vclusterops/vlog"
+)
+
+type NMAStageDCTablesOp struct {
+	ScrutinizeOpBase
+}
+
+type stageDCTablesRequestData struct {
+	CatalogPath string `json:"catalog_path"`
+}
+
+type stageDCTablesResponseData struct {
+	Name string `json:"name"`
+}
+
+func makeNMAStageDCTablesOp(logger vlog.Printer,
+	id string,
+	hosts []string,
+	hostNodeNameMap map[string]string,
+	hostCatPathMap map[string]string) (NMAStageDCTablesOp, error) {
+	// base members
+	op := NMAStageDCTablesOp{}
+	op.name = "NMAStageDCTablesOp"
+	op.logger = logger.WithName(op.name)
+	op.hosts = hosts
+
+	// scrutinize members
+	op.id = id
+	op.batch = scrutinizeBatchNormal
+	op.hostNodeNameMap = hostNodeNameMap
+	op.hostCatPathMap = hostCatPathMap
+	op.httpMethod = PostMethod
+	op.urlSuffix = "/data_collector"
+
+	// the caller is responsible for making sure hosts and maps match up exactly
+	err := validateHostMaps(hosts, hostNodeNameMap, hostCatPathMap)
+	return op, err
+}
+
+func (op *NMAStageDCTablesOp) setupRequestBody(hosts []string) error {
+	op.hostRequestBodyMap = make(map[string]string, len(hosts))
+	for _, host := range hosts {
+		stageDCTablesData := stageDCTablesRequestData{}
+		stageDCTablesData.CatalogPath = op.hostCatPathMap[host]
+
+		dataBytes, err := json.Marshal(stageDCTablesData)
+		if err != nil {
+			return fmt.Errorf("[%s] fail to marshal request data to JSON string, detail %w", op.name, err)
+		}
+
+		op.hostRequestBodyMap[host] = string(dataBytes)
+	}
+
+	return nil
+}
+
+func (op *NMAStageDCTablesOp) prepare(execContext *OpEngineExecContext) error {
+	err := op.setupRequestBody(op.hosts)
+	if err != nil {
+		return err
+	}
+	execContext.dispatcher.setup(op.hosts)
+
+	return op.setupClusterHTTPRequest(op.hosts)
+}
+
+func (op *NMAStageDCTablesOp) execute(execContext *OpEngineExecContext) error {
+	if err := op.runExecute(execContext); err != nil {
+		return err
+	}
+
+	return op.processResult(execContext)
+}
+
+func (op *NMAStageDCTablesOp) finalize(_ *OpEngineExecContext) error {
+	return nil
+}
+
+func (op *NMAStageDCTablesOp) processResult(_ *OpEngineExecContext) error {
+	fileList := make([]stageDCTablesResponseData, 0)
+	return processStagedFilesResult(&op.ScrutinizeOpBase, fileList)
+}

vclusterops/node_info.go

@@ -0,0 +1,44 @@
+/*
+ (c) Copyright [2023] Open Text.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ You may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package vclusterops
+
+import mapset "github.com/deckarep/golang-set/v2"
+
+// the following structs only hosts necessary information for this op
+type NodeInfo struct {
+	Address string `json:"address"`
+	// vnode name, e.g., v_dbname_node0001
+	Name        string `json:"name"`
+	State       string `json:"state"`
+	CatalogPath string `json:"catalog_path"`
+}
+
+type NodesInfo struct {
+	NodeList []NodeInfo `json:"node_list"`
+}
+
+// findHosts looks for hosts in a list of NodesInfo.
+// If found, return true; if not found, return false.
+func (nodesInfo *NodesInfo) findHosts(hosts []string) bool {
+	inputHostSet := mapset.NewSet(hosts...)
+
+	nodeAddrSet := mapset.NewSet[string]()
+	for _, n := range nodesInfo.NodeList {
+		nodeAddrSet.Add(n.Address)
+	}
+
+	return nodeAddrSet.Intersect(inputHostSet).Cardinality() > 0
+}

vclusterops/node_info_test.go

@@ -0,0 +1,53 @@
+/*
+ (c) Copyright [2023] Open Text.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ You may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package vclusterops
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFindHosts(t *testing.T) {
+	var nodesInfo NodesInfo
+
+	for i := 1; i <= 3; i++ {
+		var n NodeInfo
+		n.Address = fmt.Sprintf("vnode%d", i)
+		nodesInfo.NodeList = append(nodesInfo.NodeList, n)
+	}
+
+	// positive case: single input
+	found := nodesInfo.findHosts([]string{"vnode3"})
+	assert.True(t, found)
+
+	// positive case: input multiple hosts
+	found = nodesInfo.findHosts([]string{"vnode3", "vnode4"})
+	assert.True(t, found)
+
+	// negative case
+	found = nodesInfo.findHosts([]string{"vnode4"})
+	assert.False(t, found)
+
+	// negative case: input multiple hosts
+	found = nodesInfo.findHosts([]string{"vnode4", "vnode5"})
+	assert.False(t, found)
+
+	// negative case: empty input
+	found = nodesInfo.findHosts([]string{})
+	assert.False(t, found)
+}

vclusterops/re_ip.go

@@ -28,9 +28,12 @@ import (
 type VReIPOptions struct {
 	DatabaseOptions
 
+	// re-ip list
 	ReIPList []ReIPInfo
 
-	// hidden option
+	/* hidden option */
+
+	// whether trim re-ip list based on the catalog info
 	TrimReIPList bool
 }