[feat]curveadm: add export cmd for export the yaml that tools-v2 used

.gitignore

@@ -10,3 +10,4 @@ logs/*
 .build/
 .vscode/
 .CHANGELOG
+vendor/

Makefile

@@ -1,18 +1,21 @@
-.PHONY: build debug test tar
+.PHONY: build debug install test upload lint
 
 # go env
 GOPROXY     := "https://goproxy.cn,direct"
 GOOS        := $(if $(GOOS),$(GOOS),$(shell go env GOOS))
 GOARCH      := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH))
 CGO_LDFLAGS := "-static"
+CGO_CFLAGS  := "-D_LARGEFILE64_SOURCE"
 CC          := musl-gcc
 
 GOENV := GO111MODULE=on
 GOENV += GOPROXY=$(GOPROXY)
 GOENV += CC=$(CC)
-GOENV += CGO_ENABLED=1 CGO_LDFLAGS=$(CGO_LDFLAGS)
+GOENV += CGO_ENABLED=1 CGO_LDFLAGS=$(CGO_LDFLAGS) CGO_CFLAGS=$(CGO_CFLAGS)
 GOENV += GOOS=$(GOOS) GOARCH=$(GOARCH)
-
+GOLANGCILINT_VERSION ?= v1.50.0
+GOBIN := $(shell go env GOPATH)/bin
+GOBIN_GOLANGCILINT := $(shell which $(GOBIN)/golangci-lint)
 # go
 GO := go
 
@@ -22,6 +25,7 @@ OUTPUT := bin/curveadm
 # build flags
 LDFLAGS := -s -w
 LDFLAGS += -extldflags "-static -fpic"
+LDFLAGS += -X github.com/opencurve/curveadm/cli/cli.CommitId=$(shell git rev-parse --short HEAD)
 
 BUILD_FLAGS := -a
 BUILD_FLAGS += -trimpath
@@ -58,8 +62,16 @@ build:
 debug:
 	$(GOENV) $(GO) build -o $(OUTPUT) $(DEBUG_FLAGS) $(PACKAGES)
 
+install:
+	cp bin/curveadm ~/.curveadm/bin
+
 test:
 	$(GO_TEST) $(TEST_FLAGS) ./...
 
 upload:
 	@NOSCMD=$(NOSCMD) bash build/package/upload.sh $(VERSION)
+
+lint:
+	go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCILINT_VERSION)
+	$(GOBIN_GOLANGCILINT) run -v
+

cli/cli/cli.go

@@ -52,7 +52,6 @@ type CurveAdm struct {
 	pluginDir string
 	logDir    string
 	tempDir   string
-	dbpath    string
 	logpath   string
 	config    *configure.CurveAdmConfig
 
@@ -70,6 +69,8 @@ type CurveAdm struct {
 	clusterName         string // current cluster name
 	clusterTopologyData string // cluster topology
 	clusterPoolData     string // cluster pool
+	clusterType         string // cluster type like develop, production, etc.
+	monitor             storage.Monitor
 }
 
 /*
@@ -142,9 +143,9 @@ func (curveadm *CurveAdm) init() error {
 	// (4) Init error code
 	errno.Init(logpath)
 
-	// (5) New storage: create table in sqlite
-	dbpath := fmt.Sprintf("%s/curveadm.db", curveadm.dataDir)
-	s, err := storage.NewStorage(dbpath)
+	// (5) New storage: create table in sqlite/rqlite
+	dbUrl := config.GetDBUrl()
+	s, err := storage.NewStorage(dbUrl)
 	if err != nil {
 		log.Error("Init SQLite database failed",
 			log.Field("Error", err))
@@ -174,7 +175,13 @@ func (curveadm *CurveAdm) init() error {
 			log.Field("ClusterName", cluster.Name))
 	}
 
-	curveadm.dbpath = dbpath
+	// (8) Get monitor configure
+	monitor, err := s.GetMonitor(cluster.Id)
+	if err != nil {
+		log.Error("Get monitor failed", log.Field("Error", err))
+		return errno.ERR_GET_MONITOR_FAILED.E(err)
+	}
+
 	curveadm.logpath = logpath
 	curveadm.config = config
 	curveadm.in = os.Stdin
@@ -188,7 +195,8 @@ func (curveadm *CurveAdm) init() error {
 	curveadm.clusterName = cluster.Name
 	curveadm.clusterTopologyData = cluster.Topology
 	curveadm.clusterPoolData = cluster.Pool
-
+	curveadm.monitor = monitor
+	curveadm.clusterType = cluster.Type
 	return nil
 }
 
@@ -253,11 +261,11 @@ func (curveadm *CurveAdm) DataDir() string                   { return curveadm.d
 func (curveadm *CurveAdm) PluginDir() string                 { return curveadm.pluginDir }
 func (curveadm *CurveAdm) LogDir() string                    { return curveadm.logDir }
 func (curveadm *CurveAdm) TempDir() string                   { return curveadm.tempDir }
-func (curveadm *CurveAdm) DBPath() string                    { return curveadm.dbpath }
 func (curveadm *CurveAdm) LogPath() string                   { return curveadm.logpath }
 func (curveadm *CurveAdm) Config() *configure.CurveAdmConfig { return curveadm.config }
 func (curveadm *CurveAdm) SudoAlias() string                 { return curveadm.config.GetSudoAlias() }
 func (curveadm *CurveAdm) SSHTimeout() int                   { return curveadm.config.GetSSHTimeout() }
+func (curveadm *CurveAdm) Engine() string                    { return curveadm.config.GetEngine() }
 func (curveadm *CurveAdm) In() io.Reader                     { return curveadm.in }
 func (curveadm *CurveAdm) Out() io.Writer                    { return curveadm.out }
 func (curveadm *CurveAdm) Err() io.Writer                    { return curveadm.err }
@@ -269,6 +277,8 @@ func (curveadm *CurveAdm) ClusterUUId() string               { return curveadm.c
 func (curveadm *CurveAdm) ClusterName() string               { return curveadm.clusterName }
 func (curveadm *CurveAdm) ClusterTopologyData() string       { return curveadm.clusterTopologyData }
 func (curveadm *CurveAdm) ClusterPoolData() string           { return curveadm.clusterPoolData }
+func (curveadm *CurveAdm) ClusterType() string               { return curveadm.clusterType }
+func (curveadm *CurveAdm) Monitor() storage.Monitor          { return curveadm.monitor }
 
 func (curveadm *CurveAdm) GetHost(host string) (*hosts.HostConfig, error) {
 	if len(curveadm.Hosts()) == 0 {
@@ -377,6 +387,7 @@ func (curveadm *CurveAdm) ExecOptions() module.ExecOptions {
 		ExecInLocal:    false,
 		ExecSudoAlias:  curveadm.config.GetSudoAlias(),
 		ExecTimeoutSec: curveadm.config.GetTimeout(),
+		ExecWithEngine: curveadm.config.GetEngine(),
 	}
 }
 

cli/cli/version.go

@@ -25,5 +25,7 @@
 package cli
 
 var (
-	Version = "0.2.0"
+	Version = "0.3.0"
+
+	CommitId = "unknown"
 )

cli/command/client/map.go

@@ -23,10 +23,10 @@
 package client
 
 import (
-	"github.com/dustin/go-humanize"
 	"strconv"
 	"strings"
 
+	"github.com/dustin/go-humanize"
 	"github.com/fatih/color"
 	"github.com/opencurve/curveadm/cli/cli"
 	comm "github.com/opencurve/curveadm/internal/common"
@@ -41,9 +41,10 @@ import (
 
 const (
 	MAP_EXAMPLE = `Examples:
-  $ curveadm map user:/volume --host machine1 --create                 # Map volume which created by automatic
-  $ curveadm map user:/volume --host machine1 --size=10GB --create     # Map volume which size is 10GB and created by automatic
-  $ curveadm map user:/volume --host machine1 -c /path/to/client.yaml  # Map volume with specified configure file`
+  $ curveadm map user:/volume --host machine1 --create                  # Map volume which created by automatic
+  $ curveadm map user:/volume --host machine1 --size=10GiB --create     # Map volume which size is 10GiB and created by automatic
+  $ curveadm map user:/volume --host machine1 --create --poolset ssd    # Map volume created by automatic in poolset 'ssd'
+  $ curveadm map user:/volume --host machine1 -c /path/to/client.yaml   # Map volume with specified configure file`
 )
 
 var (
@@ -64,6 +65,7 @@ type mapOptions struct {
 	create      bool
 	filename    string
 	noExclusive bool
+	poolset     string
 }
 
 func ParseImage(image string) (user, name string, err error) {
@@ -159,6 +161,7 @@ func NewMapCommand(curveadm *cli.CurveAdm) *cobra.Command {
 	flags.BoolVar(&options.noExclusive, "no-exclusive", false, "Map volume non exclusive")
 	flags.StringVar(&options.size, "size", "10GiB", "Specify volume size")
 	flags.StringVarP(&options.filename, "conf", "c", "client.yaml", "Specify client configuration file")
+	flags.StringVar(&options.poolset, "poolset", "default", "Specify the poolset name")
 	return cmd
 }
 
@@ -181,6 +184,7 @@ func genMapPlaybook(curveadm *cli.CurveAdm,
 					Size:        size,
 					Create:      options.create,
 					NoExclusive: options.noExclusive,
+					Poolset:     options.poolset,
 				},
 				comm.KEY_CLIENT_HOST:              options.host, // for checker
 				comm.KEY_CHECK_KERNEL_MODULE_NAME: comm.KERNERL_MODULE_NBD,

cli/command/client/status.go

@@ -36,6 +36,7 @@ import (
 
 var (
 	GET_STATUS_PLAYBOOK_STEPS = []int{
+		playbook.INIT_CLIENT_STATUS,
 		playbook.GET_CLIENT_STATUS,
 	}
 )
@@ -57,6 +58,9 @@ func NewStatusCommand(curveadm *cli.CurveAdm) *cobra.Command {
 		DisableFlagsInUseLine: true,
 	}
 
+	flags := cmd.Flags()
+	flags.BoolVarP(&options.verbose, "verbose", "v", false, "Verbose output for status")
+
 	return cmd
 }
 
@@ -74,8 +78,13 @@ func genStatusPlaybook(curveadm *cli.CurveAdm,
 		pb.AddStep(&playbook.PlaybookStep{
 			Type:    step,
 			Configs: config,
+			Options: map[string]interface{}{
+				comm.KEY_CLIENT_STATUS_VERBOSE: options.verbose,
+			},
 			ExecOptions: playbook.ExecOptions{
-				SilentSubBar: true,
+				SilentSubBar:  true,
+				SilentMainBar: step == playbook.INIT_CLIENT_STATUS,
+				SkipError:     true,
 			},
 		})
 	}
@@ -113,11 +122,9 @@ func runStatus(curveadm *cli.CurveAdm, options statusOptions) error {
 	}
 
 	// 3) run playground
-	if err = pb.Run(); err != nil {
-		return err
-	}
+	err = pb.Run()
 
 	// 4) display service status
 	displayStatus(curveadm, clients, options)
-	return nil
+	return err
 }

cli/command/cluster/add.go

@@ -25,6 +25,7 @@
 package cluster
 
 import (
+	"github.com/google/uuid"
 	"github.com/opencurve/curveadm/cli/cli"
 	comm "github.com/opencurve/curveadm/internal/common"
 	"github.com/opencurve/curveadm/internal/configure/topology"
@@ -46,12 +47,18 @@ var (
 	CHECK_TOPOLOGY_PLAYBOOK_STEPS = []int{
 		playbook.CHECK_TOPOLOGY,
 	}
+	SUPPORTED_DEPLOY_TYPES = []string{
+		"production",
+		"test",
+		"develop",
+	}
 )
 
 type addOptions struct {
 	name        string
-	descriotion string
+	description string
 	filename    string
+	deployType  string
 }
 
 func NewAddCommand(curveadm *cli.CurveAdm) *cobra.Command {
@@ -62,6 +69,9 @@ func NewAddCommand(curveadm *cli.CurveAdm) *cobra.Command {
 		Short:   "Add cluster",
 		Args:    utils.ExactArgs(1),
 		Example: ADD_EXAMPLE,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return checkAddOptions(cmd)
+		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			options.name = args[0]
 			return runAdd(curveadm, options)
@@ -70,9 +80,9 @@ func NewAddCommand(curveadm *cli.CurveAdm) *cobra.Command {
 	}
 
 	flags := cmd.Flags()
-	flags.StringVarP(&options.descriotion, "description", "m", "", "Description for cluster")
+	flags.StringVarP(&options.description, "description", "m", "", "Description for cluster")
 	flags.StringVarP(&options.filename, "topology", "f", "", "Specify the path of topology file")
-
+	flags.StringVar(&options.deployType, "type", "develop", "Specify the type of cluster")
 	return cmd
 }
 
@@ -133,6 +143,19 @@ func checkTopology(curveadm *cli.CurveAdm, data string, options addOptions) erro
 	return pb.Run()
 }
 
+func checkAddOptions(cmd *cobra.Command) error {
+	deployType, err := cmd.Flags().GetString("deploy-type")
+	if err != nil {
+		return err
+	}
+	for _, t := range SUPPORTED_DEPLOY_TYPES {
+		if deployType == t {
+			return nil
+		}
+	}
+	return errno.ERR_UNSUPPORT_DEPLOY_TYPE.F("deploy type: %s", deployType)
+}
+
 func runAdd(curveadm *cli.CurveAdm, options addOptions) error {
 	// 1) check wether cluster already exist
 	name := options.name
@@ -161,7 +184,8 @@ func runAdd(curveadm *cli.CurveAdm, options addOptions) error {
 	}
 
 	// 4) insert cluster (with topology) into database
-	err = storage.InsertCluster(name, options.descriotion, data)
+	uuid := uuid.NewString()
+	err = storage.InsertCluster(name, uuid, options.description, data, options.deployType)
 	if err != nil {
 		return errno.ERR_INSERT_CLUSTER_FAILED.E(err)
 	}

cli/command/cluster/cmd.go

@@ -43,9 +43,9 @@ func NewClusterCommand(curveadm *cli.CurveAdm) *cobra.Command {
 		NewCheckoutCommand(curveadm),
 		NewListCommand(curveadm),
 		NewRemoveCommand(curveadm),
-		// TODO(P1): enable export and import
+		// TODO(P1): enable export
 		//NewExportCommand(curveadm),
-		//NewImportCommand(curveadm),
+		NewImportCommand(curveadm),
 	)
 	return cmd
 }