multi-object PUT (variations); traling slash

* destination prefix vs destination virtual directory
* name with a trailing slash
* CLI: usability

Signed-off-by: Alex Aizman <[email protected]>

ais/htrun.go

@@ -1257,8 +1257,8 @@ func (h *htrun) writeErrActf(w http.ResponseWriter, r *http.Request, action stri
 
 // also, validatePrefix
 func (h *htrun) isValidObjname(w http.ResponseWriter, r *http.Request, name string) bool {
-	if err := cmn.ValidateObjname(name); err != nil {
-		h.writeErr(w, r, err)
+	if cos.IsLastB(name, filepath.Separator) || strings.Contains(name, "../") {
+		h.writeErrf(w, r, "invalid object name %q", name)
 		return false
 	}
 	return true

ais/http.go

@@ -6,7 +6,6 @@ package ais
 
 import (
 	"net/http"
-	"strings"
 
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
@@ -30,7 +29,7 @@ var g global
 func handlePub(path string, handler func(http.ResponseWriter, *http.Request)) {
 	for _, v := range allHTTPverbs {
 		g.netServ.pub.muxers[v].HandleFunc(path, handler)
-		if !strings.HasSuffix(path, "/") {
+		if !cos.IsLastB(path, '/') {
 			g.netServ.pub.muxers[v].HandleFunc(path+"/", handler)
 		}
 	}
@@ -39,7 +38,7 @@ func handlePub(path string, handler func(http.ResponseWriter, *http.Request)) {
 func handleControl(path string, handler func(http.ResponseWriter, *http.Request)) {
 	for _, v := range allHTTPverbs {
 		g.netServ.control.muxers[v].HandleFunc(path, handler)
-		if !strings.HasSuffix(path, "/") {
+		if !cos.IsLastB(path, '/') {
 			g.netServ.control.muxers[v].HandleFunc(path+"/", handler)
 		}
 	}
@@ -48,7 +47,7 @@ func handleControl(path string, handler func(http.ResponseWriter, *http.Request)
 func handleData(path string, handler func(http.ResponseWriter, *http.Request)) {
 	for _, v := range allHTTPverbs {
 		g.netServ.data.muxers[v].HandleFunc(path, handler)
-		if !strings.HasSuffix(path, "/") {
+		if !cos.IsLastB(path, '/') {
 			g.netServ.data.muxers[v].HandleFunc(path+"/", handler)
 		}
 	}

ais/target.go

@@ -745,6 +745,9 @@ func (t *target) httpobjput(w http.ResponseWriter, r *http.Request, apireq *apiR
 		started = time.Now().UnixNano()
 		t2tput  = isT2TPut(r.Header)
 	)
+	if !t.isValidObjname(w, r, lom.ObjName) {
+		return
+	}
 	if apireq.dpq.ptime == "" && !t2tput {
 		t.writeErrf(w, r, "%s: %s(obj) is expected to be redirected or replicated", t.si, r.Method)
 		return
@@ -893,6 +896,9 @@ func (t *target) httpobjpost(w http.ResponseWriter, r *http.Request, apireq *api
 	}
 
 	lom := cluster.AllocLOM(apireq.items[1])
+	if !t.isValidObjname(w, r, lom.ObjName) {
+		return
+	}
 	err = lom.InitBck(apireq.bck.Bucket())
 	if err == nil {
 		err = t.objMv(lom, msg)
@@ -1074,6 +1080,9 @@ func (t *target) httpobjpatch(w http.ResponseWriter, r *http.Request, apireq *ap
 	}
 	lom := cluster.AllocLOM(apireq.items[1] /*objName*/)
 	defer cluster.FreeLOM(lom)
+	if !t.isValidObjname(w, r, lom.ObjName) {
+		return
+	}
 	if err := lom.InitBck(apireq.bck.Bucket()); err != nil {
 		t.writeErr(w, r, err)
 		return

cluster/linit.go

@@ -81,9 +81,6 @@ func (lom *LOM) InitBck(bck *cmn.Bck) (err error) {
 	if err = lom.bck.InitFast(g.t.Bowner()); err != nil {
 		return
 	}
-	if err = cmn.ValidateObjname(lom.ObjName); err != nil {
-		return
-	}
 	lom.md.uname = lom.bck.MakeUname(lom.ObjName)
 	lom.mi, lom.digest, err = fs.Hrw(lom.md.uname)
 	if err != nil {

cluster/meta/hrw.go

@@ -29,9 +29,12 @@ func (smap *Smap) HrwName2T(uname string) (*Snode, error) {
 func (smap *Smap) HrwMultiHome(uname string) (si *Snode, netName string, err error) {
 	digest := xxhash.Checksum64S(cos.UnsafeB(uname), cos.MLCG32)
 	si, err = smap.HrwHash2T(digest)
+	if err != nil {
+		return nil, cmn.NetPublic, err
+	}
 	l := len(si.PubExtra)
-	if l == 0 || err != nil {
-		return si, cmn.NetPublic, err
+	if l == 0 {
+		return si, cmn.NetPublic, nil
 	}
 	i := robin.Add(1) % uint64(l+1)
 	if i == 0 {

cmd/authn/hserv.go

@@ -7,7 +7,6 @@ package main
 import (
 	"fmt"
 	"net/http"
-	"strings"
 	"time"
 
 	"github.com/NVIDIA/aistore/api/apc"
@@ -76,7 +75,7 @@ rerr:
 
 func (h *hserv) registerHandler(path string, handler func(http.ResponseWriter, *http.Request)) {
 	h.mux.HandleFunc(path, handler)
-	if !strings.HasSuffix(path, "/") {
+	if !cos.IsLastB(path, '/') {
 		h.mux.HandleFunc(path+"/", handler)
 	}
 }

cmd/cli/cli/arch_hdlr.go

@@ -289,7 +289,7 @@ func putApndArchHandler(c *cli.Context) (err error) {
 	// multi-file cases
 	//
 	if !a.appendOnly && !a.appendOrPut {
-		warn := fmt.Sprintf("multi-file 'archive put' operation requires either %s or %s (command line)",
+		warn := fmt.Sprintf("multi-file 'archive put' operation requires either %s or %s option",
 			qflprn(archAppendOnlyFlag), qflprn(archAppendOrPutFlag))
 		actionWarn(c, warn)
 		if flagIsSet(c, yesFlag) {
@@ -303,7 +303,7 @@ func putApndArchHandler(c *cli.Context) (err error) {
 	}
 
 	// archpath
-	if a.archpath != "" && !strings.HasSuffix(a.archpath, "/") {
+	if a.archpath != "" && !cos.IsLastB(a.archpath, '/') {
 		if !flagIsSet(c, yesFlag) {
 			warn := fmt.Sprintf("no trailing filepath separator in: '%s=%s'", qflprn(archpathFlag), a.archpath)
 			actionWarn(c, warn)

cmd/cli/cli/const.go

@@ -778,7 +778,7 @@ var (
 
 	continueOnErrorFlag = cli.BoolFlag{
 		Name:  "cont-on-err",
-		Usage: "keep running archiving xaction in presence of errors in a any given multi-object transaction",
+		Usage: "keep running archiving xaction (job) in presence of errors in a any given multi-object transaction",
 	}
 	// end archive
 

cmd/cli/cli/object_hdlr.go

@@ -107,13 +107,16 @@ var (
 	objectCmdPut = cli.Command{
 		Name: commandPut,
 		Usage: "PUT or APPEND one file, one directory, or multiple files and/or directories.\n" +
-			indent4 + "\t- use optional shell filename pattern (wildcard) to match/select multiple sources, for example:\n" +
-			indent4 + "\t\t$ ais put 'docs/*.md' ais://abc/markdown/  # notice single quotes\n" +
-			indent4 + "\t- '--compute-checksum' to facilitate end-to-end protection;\n" +
-			indent4 + "\t- progress bar via '--progress' to show runtime execution (uploaded files count and size);\n" +
-			indent4 + "\t- when writing directly from standard input use Ctrl-D to terminate;\n" +
-			indent4 + "\t- '--archpath' to APPEND to an existing " + archExts + "-formatted object (\"shard\");\n" +
-			indent4 + "\t(tip: use '--dry-run' to see the results without making any changes)",
+			indent4 + "\t- use optional shell filename pattern (wildcard) to match/select multiple sources.\n" +
+			indent4 + "\tAssorted examples and usage options follow (and see docs/cli/object.md for more):\n" +
+			indent4 + "\t- upload matching files: 'ais put \"docs/*.md\" ais://abc/markdown/'\n" +
+			indent4 + "\t- (notice quotation marks and a forward slash after 'markdown/' destination);\n" +
+			indent4 + "\t- '--compute-checksum': use '--compute-checksum' to facilitate end-to-end protection;\n" +
+			indent4 + "\t- '--progress': progress bar, to show running counts and sizes of uploaded files;\n" +
+			indent4 + "\t- Ctrl-D: when writing directly from standard input use Ctrl-D to terminate;\n" +
+			indent4 + "\t- '--dry-run': see the results without making any changes.\n" +
+			indent4 + "\tNotes:\n" +
+			indent4 + "\t- to write or append to " + archExts + "-formatted objects (\"shards\"), use 'ais archive'",
 		ArgsUsage:    putObjectArgument,
 		Flags:        append(objectCmdsFlags[commandPut], putObjCksumFlags...),
 		Action:       putHandler,
@@ -271,66 +274,92 @@ func removeObjectHandler(c *cli.Context) (err error) {
 	return multiobjArg(c, commandRemove)
 }
 
-func putHandler(c *cli.Context) (err error) {
-	// main PUT switch
+// main PUT handler: cases 1 through 4
+func putHandler(c *cli.Context) error {
 	var a putargs
-	if err = a.parse(c, true /*empty dst oname*/); err != nil {
-		return
+	if err := a.parse(c, true /*empty dst oname*/); err != nil {
+		return err
 	}
 	if flagIsSet(c, dryRunFlag) {
 		dryRunCptn(c)
 	}
+
+	// 1. one file
 	if a.srcIsRegular() {
 		debug.Assert(a.src.abspath != "")
+		if cos.IsLastB(a.dst.oname, '/') {
+			a.dst.oname += a.src.arg
+		}
 		if err := putRegular(c, a.dst.bck, a.dst.oname, a.src.abspath, a.src.finfo); err != nil {
 			return err
 		}
 		actionDone(c, fmt.Sprintf("%s %q => %s\n", a.verb(), a.src.arg, a.dst.bck.Cname(a.dst.oname)))
 		return nil
 	}
-	// multi-file cases
+
+	// 2. multi-file list & range
 	incl := flagIsSet(c, inclSrcDirNameFlag)
 	switch {
 	case len(a.src.fdnames) > 0:
+		var s string
+		if len(a.src.fdnames) > 1 {
+			s = " ..."
+		}
+		if ok := warnMultiSrcDstPrefix(c, &a, fmt.Sprintf("from [%s%s]", a.src.fdnames[0], s)); !ok {
+			return nil
+		}
 		// a) csv of files and/or directories (names) embedded into the first arg, e.g. "f1[,f2...]" dst-bucket[/prefix]
 		// b) csv from '--list' flag
 		return verbList(c, &a, a.src.fdnames, a.dst.bck, a.dst.oname /*virt subdir*/, incl)
 	case a.pt != nil:
+		if ok := warnMultiSrcDstPrefix(c, &a, fmt.Sprintf("matching '%s'", a.src.tmpl)); !ok {
+			return nil
+		}
 		// a) range via the first arg, e.g. "/tmp/www/test{0..2}{0..2}.txt" dst-bucket/www
 		// b) range and prefix from the parsed '--template'
 		var trimPrefix string
 		if !incl {
 			trimPrefix = rangeTrimPrefix(a.pt)
 		}
 		return verbRange(c, &a, a.pt, a.dst.bck, trimPrefix, a.dst.oname, incl)
-	case a.src.stdin:
+	}
+
+	// 3. STDIN
+	if a.src.stdin {
 		return putStdin(c, &a)
-	default: // one directory
-		var ndir int
-
-		if a.dst.oname != "" {
-			warn := fmt.Sprintf("'%s' will be used as a destination name prefix for all files from '%s'",
-				a.dst.oname, a.src.arg)
-			actionWarn(c, warn)
-			if _, err := archive.Mime(a.dst.oname, ""); err == nil {
-				warn := fmt.Sprintf("did you want to use 'archive put' instead, with %q as the destination?",
-					a.dst.oname)
-				actionWarn(c, warn)
-			}
-			if !flagIsSet(c, yesFlag) {
-				if ok := confirm(c, "Proceed anyway?"); !ok {
-					return
-				}
-			}
-		}
+	}
 
-		fobjs, err := lsFobj(c, a.src.abspath, "", a.dst.oname, &ndir, a.src.recurs, incl)
-		if err != nil {
-			return err
+	// 4. directory
+	var ndir int
+	if ok := warnMultiSrcDstPrefix(c, &a, fmt.Sprintf("from '%s' directory", a.src.arg)); !ok {
+		return nil
+	}
+	fobjs, err := lsFobj(c, a.src.abspath, "", a.dst.oname, &ndir, a.src.recurs, incl)
+	if err != nil {
+		return err
+	}
+	debug.Assert(ndir == 1)
+	return verbFobjs(c, &a, fobjs, a.dst.bck, ndir, a.src.recurs)
+}
+
+func warnMultiSrcDstPrefix(c *cli.Context, a *putargs, from string) bool {
+	if a.dst.oname == "" || cos.IsLastB(a.dst.oname, '/') {
+		return true
+	}
+	warn := fmt.Sprintf("'%s' will be used as the destination name prefix for all files %s",
+		a.dst.oname, from)
+	actionWarn(c, warn)
+	if _, err := archive.Mime(a.dst.oname, ""); err == nil {
+		warn := fmt.Sprintf("did you want to use 'archive put' instead, with %q as the destination shard?",
+			a.dst.oname)
+		actionWarn(c, warn)
+	}
+	if !flagIsSet(c, yesFlag) {
+		if ok := confirm(c, "Proceed anyway?"); !ok {
+			return false
 		}
-		debug.Assert(ndir == 1)
-		return verbFobjs(c, &a, fobjs, a.dst.bck, ndir, a.src.recurs)
 	}
+	return true
 }
 
 func putStdin(c *cli.Context, a *putargs) error {

cmd/cli/cli/utils.go

@@ -844,12 +844,12 @@ func actionCptn(c *cli.Context, prefix, msg string) {
 	}
 }
 
+func dryRunHeader() string {
+	return fcyan("[DRY RUN]")
+}
+
 func dryRunCptn(c *cli.Context) {
-	const (
-		dryRunHeader      = "[DRY RUN]"
-		dryRunExplanation = "with no modifications to the cluster"
-	)
-	fmt.Fprintln(c.App.Writer, fcyan(dryRunHeader)+" "+dryRunExplanation)
+	fmt.Fprintln(c.App.Writer, dryRunHeader()+" with no modifications to the cluster")
 }
 
 //////////////

cmd/cli/cli/verbfobj.go

@@ -62,7 +62,11 @@ func verbFobjs(c *cli.Context, wop wop, fobjs []fobj, bck cmn.Bck, ndir int, rec
 		return fmt.Errorf("no files to %s (check source name and formatting, see examples)", wop.verb())
 	}
 
-	cptn := fmt.Sprintf("%s %d file%s", wop.verb(), l, cos.Plural(l))
+	var cptn string
+	if flagIsSet(c, dryRunFlag) {
+		cptn = dryRunHeader() + " "
+	}
+	cptn += fmt.Sprintf("%s %d file%s", wop.verb(), l, cos.Plural(l))
 	cptn += ndir2tag(ndir, recurs)
 	cptn += fmt.Sprintf(" => %s", wop.dest())
 

cmd/cli/cli/yap.go

@@ -22,6 +22,7 @@ type (
 	here struct {
 		arg     string
 		abspath string
+		tmpl    string
 		finfo   os.FileInfo
 		fdnames []string // files and directories (names)
 		isdir   bool
@@ -71,7 +72,14 @@ var (
 
 func (*putargs) verb() string { return "PUT" }
 
-func (a *putargs) dest() string       { return a.dst.bck.Cname("") }
+func (a *putargs) dest() string {
+	if a.dst.oname == "" {
+		return a.dst.bck.Cname("")
+	}
+	// if len(a.src.fdnames) < 2 {
+	return a.dst.bck.Cname(a.dst.oname)
+}
+
 func (a *putargs) srcIsRegular() bool { return a.src.finfo != nil && !a.src.isdir }
 
 func (a *putargs) parse(c *cli.Context, emptyDstOnameOK bool) (err error) {
@@ -105,11 +113,9 @@ func (a *putargs) parse(c *cli.Context, emptyDstOnameOK bool) (err error) {
 			return
 		}
 		// optional template to select local source(s)
-		var (
-			pt   cos.ParsedTemplate
-			tmpl = parseStrFlag(c, templateFlag)
-		)
-		pt, err = cos.NewParsedTemplate(tmpl)
+		var pt cos.ParsedTemplate
+		a.src.tmpl = parseStrFlag(c, templateFlag)
+		pt, err = cos.NewParsedTemplate(a.src.tmpl)
 		if err == nil {
 			a.pt = &pt
 		}

cmd/cli/go.mod

@@ -4,7 +4,7 @@ go 1.21
 
 // direct
 require (
-	github.com/NVIDIA/aistore v1.3.22-0.20231201003409-3ecd430019b0
+	github.com/NVIDIA/aistore v1.3.22-0.20231206174342-7ad18b6fb196
 	github.com/fatih/color v1.16.0
 	github.com/json-iterator/go v1.1.12
 	github.com/onsi/ginkgo v1.16.5

cmd/cli/go.sum

@@ -1,7 +1,7 @@
 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/NVIDIA/aistore v1.3.22-0.20231201003409-3ecd430019b0 h1:mgnlUcAFvCn4OUS5uYhgvPryCYluEQmOLnfXDyh4Te8=
-github.com/NVIDIA/aistore v1.3.22-0.20231201003409-3ecd430019b0/go.mod h1:cOTgDt5fVCQOB+rnvYZgVFRF3dEzPqu8f22F3F+Yvtg=
+github.com/NVIDIA/aistore v1.3.22-0.20231206174342-7ad18b6fb196 h1:cJPWF48JTT94VwovBs7PBpa8eLCPQ5K2pU90xOVApSg=
+github.com/NVIDIA/aistore v1.3.22-0.20231206174342-7ad18b6fb196/go.mod h1:cOTgDt5fVCQOB+rnvYZgVFRF3dEzPqu8f22F3F+Yvtg=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=

cmn/bck.go

@@ -283,14 +283,6 @@ func (b *Bck) ValidateName() (err error) {
 	return
 }
 
-// related, inlined
-func ValidateObjname(s string) error {
-	if !strings.Contains(s, "../") {
-		return nil
-	}
-	return fmt.Errorf("invalid object name %q", s)
-}
-
 // ditto
 func ValidatePrefix(s string) error {
 	if !strings.Contains(s, "../") {