unify PUT and PROMOTE destination naming

* revise PROMOTE conventions - make it consistent w/ PUT
* CLI: update inline helps and docs
* fix python test, include separator
* with refactoring

Co-authored-by: Aaron Wilson <[email protected]>
Signed-off-by: Alex Aizman <[email protected]>

.golangci.yml

@@ -43,7 +43,7 @@ linters:
     # - dupl
     #
     # linters that maybe worth running from time to time:
-    # - gochecknoinits
+    # - gochecknoinits  - 'func init()'
     # - goconst
     # - errchkjson
     # - nilnil

ais/htrun.go

@@ -1238,7 +1238,7 @@ func (h *htrun) writeErrURL(w http.ResponseWriter, r *http.Request) {
 	if r.URL.Path == "/favicon.ico" || r.URL.Path == "favicon.ico" {
 		return
 	}
-	h.writeErrf(w, r, "invalid request: '%s %s'", r.Method, r.RequestURI)
+	h.writeErrf(w, r, "invalid request URI: '%s %s'", r.Method, r.RequestURI)
 }
 
 func (h *htrun) writeErrAct(w http.ResponseWriter, r *http.Request, action string) {

ais/proxy.go

@@ -595,6 +595,10 @@ func (p *proxy) httpbckget(w http.ResponseWriter, r *http.Request, dpq *dpq) {
 		p.writeErrf(w, r, cmn.FmtErrMorphUnmarshal, p.si, msg.Action, msg.Value, err)
 		return
 	}
+	if lsmsg.Prefix != "" && strings.Contains(lsmsg.Prefix, "../") {
+		p.writeErrf(w, r, "bad list-objects request: invalid prefix %q", lsmsg.Prefix)
+		return
+	}
 	bckArgs := bckInitArgs{p: p, w: w, r: r, msg: msg, perms: apc.AceObjLIST, bck: bck, dpq: dpq}
 	bckArgs.createAIS = false
 

ais/target.go

@@ -1043,7 +1043,7 @@ func (t *target) objhead(hdr http.Header, query url.Values, bck *meta.Bck, lom *
 		if v == "" {
 			return nil, false
 		}
-		name := cmn.PropToHeader(tag)
+		name := apc.PropToHeader(tag)
 		debug.Func(func() {
 			vv := hdr.Get(name)
 			debug.Assertf(vv == "", "not expecting duplications: %s=(%q, %q)", name, v, vv)

ais/tgttxn.go

@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/NVIDIA/aistore/api/apc"
@@ -898,6 +899,9 @@ func (t *target) promote(c *txnServerCtx, hdr http.Header) (string, error) {
 			err = fmt.Errorf(cmn.FmtErrMorphUnmarshal, t, c.msg.Action, c.msg.Value, err)
 			return "", err
 		}
+		if strings.Contains(prmMsg.ObjName, "../") || strings.Contains(prmMsg.ObjName, "~/") {
+			return "", fmt.Errorf("invalid object name or prefix %q", prmMsg.ObjName)
+		}
 		srcFQN := c.msg.Name
 		finfo, err := os.Stat(srcFQN)
 		if err != nil {

api/apc/headers.go

@@ -4,6 +4,8 @@
  */
 package apc
 
+import "strings"
+
 // For standard and provider-specific HTTP headers, see cmn/cos/const_http.go
 
 const HdrError = "Hdr-Error"
@@ -89,3 +91,19 @@ const (
 	HdrPromoteNamesHash = HeaderPrefix + "promote-names-hash"
 	HdrPromoteNamesNum  = HeaderPrefix + "promote-names-num"
 )
+
+//
+// convert prop name to HTTP header tag name
+//
+
+func PropToHeader(prop string) string {
+	if strings.HasPrefix(prop, HeaderPrefix) {
+		return prop
+	}
+	if prop[0] == '.' || prop[0] == '_' {
+		prop = prop[1:]
+	}
+	prop = strings.ReplaceAll(prop, ".", "-")
+	prop = strings.ReplaceAll(prop, "_", "-")
+	return HeaderPrefix + prop
+}

api/object.go

@@ -332,7 +332,7 @@ func HeadObject(bp BaseParams, bck cmn.Bck, object string, fltPresence int, sile
 	op.Cksum = op.ObjAttrs.FromHeader(hdr)
 	// second, all the rest
 	err = cmn.IterFields(op, func(tag string, field cmn.IterField) (error, bool) {
-		headerName := cmn.PropToHeader(tag)
+		headerName := apc.PropToHeader(tag)
 		// skip the missing ones
 		if _, ok := hdr[textproto.CanonicalMIMEHeaderKey(headerName)]; !ok {
 			return nil, false

cluster/target.go

@@ -121,7 +121,7 @@ type (
 	PromoteArgs struct {
 		DaemonID  string `json:"tid,omitempty"` // target ID
 		SrcFQN    string `json:"src,omitempty"` // source file or directory (must be absolute pathname)
-		ObjName   string `json:"obj,omitempty"` // destination object name
+		ObjName   string `json:"obj,omitempty"` // destination object name or prefix
 		Recursive bool   `json:"rcr,omitempty"` // recursively promote nested dirs
 		// once successfully promoted:
 		OverwriteDst bool `json:"ovw,omitempty"` // overwrite destination

cmd/cli/cli/bucket_hdlr.go

@@ -345,7 +345,7 @@ func showMisplacedAndMore(c *cli.Context) error {
 	f := func() error {
 		return checkObjectHealth(queryBcks)
 	}
-	return cmn.WaitForFunc(f, longClientTimeout)
+	return waitForFunc(f, longClientTimeout)
 }
 
 func mvBucketHandler(c *cli.Context) error {

cmd/cli/cli/const.go

@@ -252,19 +252,21 @@ const (
 
 	// Objects
 	objectArgument          = "BUCKET/OBJECT_NAME"
-	optionalObjArgument     = "BUCKET[/OBJECT_NAME]"
 	optionalObjectsArgument = "BUCKET[/OBJECT_NAME] ..."
-	shardArgument           = "BUCKET/SHARD_NAME"
-	optionalShardArgument   = "BUCKET[/SHARD_NAME]"
 	dstShardArgument        = bucketDstArgument + "/SHARD_NAME"
 
-	getObjectArgument   = optionalObjArgument + " [OUT_FILE|OUT_DIR|-]"
-	getShardArgument    = optionalShardArgument + " [OUT_FILE|OUT_DIR|-]"
-	putObjectArgument   = "[-|FILE|DIRECTORY[/PATTERN]] " + optionalObjArgument
-	putApndArchArgument = "[-|FILE|DIRECTORY[/PATTERN]] " + shardArgument
+	getObjectArgument = "BUCKET[/OBJECT_NAME] [OUT_FILE|OUT_DIR|-]"
 
-	promoteObjectArgument = "FILE|DIRECTORY[/PATTERN] " + optionalObjArgument
-	concatObjectArgument  = "FILE|DIRECTORY[/PATTERN] [ FILE|DIRECTORY[/PATTERN] ...] " + objectArgument
+	optionalPrefixArgument = "BUCKET[/OBJECT_NAME_or_PREFIX]"
+	putObjectArgument      = "[-|FILE|DIRECTORY[/PATTERN]] " + optionalPrefixArgument
+	promoteObjectArgument  = "FILE|DIRECTORY[/PATTERN] " + optionalPrefixArgument
+
+	shardArgument         = "BUCKET/SHARD_NAME"
+	optionalShardArgument = "BUCKET[/SHARD_NAME]"
+	putApndArchArgument   = "[-|FILE|DIRECTORY[/PATTERN]] " + shardArgument
+	getShardArgument      = optionalShardArgument + " [OUT_FILE|OUT_DIR|-]"
+
+	concatObjectArgument = "FILE|DIRECTORY[/PATTERN] [ FILE|DIRECTORY[/PATTERN] ...] " + objectArgument
 
 	renameObjectArgument = objectArgument + " NEW_OBJECT_NAME"
 

cmd/cli/cli/object_hdlr.go

@@ -107,21 +107,41 @@ var (
 	objectCmdPut = cli.Command{
 		Name: commandPut,
 		Usage: "PUT or APPEND one file, one directory, or multiple files and/or directories.\n" +
-			indent4 + "\t- use optional shell filename pattern (wildcard) to match/select multiple sources.\n" +
-			indent4 + "\tAssorted examples and usage options follow (and see docs/cli/object.md for more):\n" +
-			indent4 + "\t- upload matching files: 'ais put \"docs/*.md\" ais://abc/markdown/'\n" +
-			indent4 + "\t- (notice quotation marks and a forward slash after 'markdown/' destination);\n" +
-			indent4 + "\t- '--compute-checksum': use '--compute-checksum' to facilitate end-to-end protection;\n" +
-			indent4 + "\t- '--progress': progress bar, to show running counts and sizes of uploaded files;\n" +
-			indent4 + "\t- Ctrl-D: when writing directly from standard input use Ctrl-D to terminate;\n" +
-			indent4 + "\t- '--dry-run': see the results without making any changes.\n" +
-			indent4 + "\tNotes:\n" +
-			indent4 + "\t- to write or append to " + archExts + "-formatted objects (\"shards\"), use 'ais archive'",
+			indent1 + "Use optional shell filename PATTERN (wildcard) to match/select multiple sources.\n" +
+			indent1 + "Destination naming is consistent with 'ais object promote' command, whereby the optional OBJECT_NAME_or_PREFIX\n" +
+			indent1 + "becomes either a name, a prefix, or a virtual destination directory (if it ends with a forward '/').\n" +
+			indent1 + "Assorted examples and usage options follow (and see docs/cli/object.md for more):\n" +
+			indent1 + "\t- upload matching files: 'ais put \"docs/*.md\" ais://abc/markdown/'\n" +
+			indent1 + "\t- (notice quotation marks and a forward slash after 'markdown/' destination);\n" +
+			indent1 + "\t- '--compute-checksum': use '--compute-checksum' to facilitate end-to-end protection;\n" +
+			indent1 + "\t- '--progress': progress bar, to show running counts and sizes of uploaded files;\n" +
+			indent1 + "\t- Ctrl-D: when writing directly from standard input use Ctrl-D to terminate;\n" +
+			indent1 + "\t- '--dry-run': see the results without making any changes.\n" +
+			indent1 + "\tNotes:\n" +
+			indent1 + "\t- to write or append to " + archExts + "-formatted objects (\"shards\"), use 'ais archive'",
 		ArgsUsage:    putObjectArgument,
 		Flags:        append(objectCmdsFlags[commandPut], putObjCksumFlags...),
 		Action:       putHandler,
 		BashComplete: putPromApndCompletions,
 	}
+	objectCmdPromote = cli.Command{
+		Name: commandPromote,
+		Usage: "PROMOTE target-accessible files and directories.\n" +
+			indent1 + "The operation is intended for copying NFS and SMB shares mounted on any/all targets\n" +
+			indent1 + "but can be also used to copy local files (again, on any/all targets in the cluster).\n" +
+			indent1 + "Copied files and directories become regular stored objects that can be further listed and operated upon.\n" +
+			indent1 + "Destination naming is consistent with 'ais put' command, e.g.:\n" +
+			indent1 + "\t- 'promote /tmp/subdir/f1 ais://nnn'\t - ais://nnn/f1\n" +
+			indent1 + "\t- 'promote /tmp/subdir/f2 ais://nnn/aaa'\t - ais://nnn/aaa\n" +
+			indent1 + "\t- 'promote /tmp/subdir/f3 ais://nnn/aaa/'\t - ais://nnn/aaa/f3\n" +
+			indent1 + "\t- 'promote /tmp/subdir ais://nnn'\t - ais://nnn/f1, ais://nnn/f2, ais://nnn/f3\n" +
+			indent1 + "\t- 'promote /tmp/subdir ais://nnn/aaa/'\t - ais://nnn/aaa/f1, ais://nnn/aaa/f2, ais://nnn/aaa/f3\n" +
+			indent1 + "Other supported options follow below.",
+		ArgsUsage:    promoteObjectArgument,
+		Flags:        objectCmdsFlags[commandPromote],
+		Action:       promoteHandler,
+		BashComplete: putPromApndCompletions,
+	}
 
 	objectCmdSetCustom = cli.Command{
 		Name:      commandSetCustom,
@@ -138,6 +158,7 @@ var (
 			objectCmdGet,
 			bucketsObjectsCmdList,
 			objectCmdPut,
+			objectCmdPromote,
 			objectCmdSetCustom,
 			bucketObjCmdEvict,
 			makeAlias(showCmdObject, "", true, commandShow), // alias for `ais show`
@@ -157,14 +178,6 @@ var (
 				Action:       removeObjectHandler,
 				BashComplete: bucketCompletions(bcmplop{multiple: true, separator: true}),
 			},
-			{
-				Name:         commandPromote,
-				Usage:        "promote files and directories (i.e., replicate files and convert them to objects)",
-				ArgsUsage:    promoteObjectArgument,
-				Flags:        objectCmdsFlags[commandPromote],
-				Action:       promoteHandler,
-				BashComplete: putPromApndCompletions,
-			},
 			{
 				Name:      commandConcat,
 				Usage:     "concatenate multiple files and/or directories (with or without matching pattern) as a new single object",

cmd/cli/cli/storage_hdlr.go

@@ -435,7 +435,7 @@ func newBsummContext(c *cli.Context, units string, qbck cmn.QueryBcks, bckPresen
 // "slow" version of the bucket-summary (compare with `listBuckets` => `listBckTableWithSummary`)
 func (ctx *bsummCtx) slow() (xid string, res cmn.AllBsummResults, err error) {
 	if ctx.longWaitTime > 0 {
-		err = cmn.WaitForFunc(ctx.get, ctx.longWaitTime, ctx.longWaitPrompt)
+		err = waitForFunc(ctx.get, ctx.longWaitTime, ctx.longWaitPrompt)
 	} else {
 		err = ctx.get()
 	}

cmd/cli/cli/timeout.go

@@ -4,7 +4,10 @@
  */
 package cli
 
-import "time"
+import (
+	"fmt"
+	"time"
+)
 
 // see also: xact/api.go
 
@@ -32,3 +35,33 @@ const (
 	// job wait: start printing "."(s)
 	wasFast = refreshRateDefault
 )
+
+// execute a function in goroutine and wait for it to finish;
+// if the function runs longer than `timeLong`: return "please wait" error
+func waitForFunc(f func() error, timeLong time.Duration, prompts ...string) error {
+	var (
+		timer  = time.NewTimer(timeLong)
+		chDone = make(chan struct{}, 1)
+		err    error
+		prompt = "Please wait, the operation may take some time..."
+	)
+	if len(prompts) > 0 && prompts[0] != "" {
+		prompt = prompts[0]
+	}
+	go func() {
+		err = f()
+		chDone <- struct{}{}
+	}()
+loop:
+	for {
+		select {
+		case <-timer.C:
+			fmt.Println(prompt)
+		case <-chDone:
+			timer.Stop()
+			break loop
+		}
+	}
+
+	return err
+}

cmd/cli/test/promote.in

@@ -7,8 +7,8 @@ ais object promote /tmp/prm/obj ais://$BUCKET/one/one --delete-src=false --targe
 ls /tmp/prm/obj
 ais object promote /tmp/prm/discard ais://$BUCKET/two/ --delete-src=true  --target-id $RANDOM_TARGET
 ls /tmp/prm/discard // FAIL "such file or directory"
-ais object promote /tmp/prm/ ais://$BUCKET/three --delete-src=false --target-id $RANDOM_TARGET
-ais object promote /tmp/prm ais://$BUCKET/fourth --target-id $RANDOM_TARGET
+ais object promote /tmp/prm/ ais://$BUCKET/three/ --delete-src=false --target-id $RANDOM_TARGET
+ais object promote /tmp/prm ais://$BUCKET/fourth/ --target-id $RANDOM_TARGET
 ais object promote /tmp/prm ais://$BUCKET/fifth/ --delete-src=false --target-id $RANDOM_TARGET
 ais object promote /tmp/prm/ ais://$BUCKET/sixth/ --target-id $RANDOM_TARGET
 ais bucket ls ais://$BUCKET

cmn/jsp/file.go

@@ -7,7 +7,6 @@ package jsp
 
 import (
 	"errors"
-	"flag"
 	"io"
 	"os"
 
@@ -86,11 +85,9 @@ func Load(filepath string, v any, opts Options) (checksum *cos.Cksum, err error)
 	}
 	if errors.Is(err, &cos.ErrBadCksum{}) {
 		if errRm := os.Remove(filepath); errRm == nil {
-			if flag.Parsed() {
-				nlog.Errorf("bad checksum: removing %s", filepath)
-			}
-		} else if flag.Parsed() {
-			nlog.Errorf("bad checksum: failed to remove %s: %v", filepath, errRm)
+			cos.Errorf("jsp: %v, removed %q", err, filepath)
+		} else {
+			cos.Errorf("jsp: %v, failed to remove %q: %v", err, filepath, errRm)
 		}
 	}
 	return

cmn/ver_const.go

@@ -7,6 +7,8 @@ package cmn
 
 import "github.com/NVIDIA/aistore/cmn/jsp"
 
+const GitHubHome = "https://github.com/NVIDIA/aistore"
+
 // ========================== IMPORTANT NOTE ==============================
 //
 // - (major.minor) version indicates the current version of AIS software
@@ -31,8 +33,8 @@ const (
 )
 
 const (
-	MetaverSmap  = 2 // Smap (cluster map) formatting version (jsp)
-	MetaverBMD   = 2 // BMD (bucket metadata) --/-- (jsp)
+	MetaverSmap  = 2 // Smap (cluster map) formatting version a.k.a. meta-version (see cluster/meta/jsp.go)
+	MetaverBMD   = 2 // BMD (bucket metadata) --/--
 	MetaverRMD   = 1 // Rebalance MD (jsp)
 	MetaverVMD   = 1 // Volume MD (jsp)
 	MetaverEtlMD = 1 // ETL MD (jsp)