Merge branch 'release-2.01.0' into stable

Dockerfile

@@ -66,7 +66,7 @@ FROM alpine:3.15.0 as base
 RUN apk add --no-cache libc6-compat
 
 FROM base as dev
-ARG GolangVersion=1.17.5
+ARG GolangVersion=1.18
 RUN apk add --no-cache               \
                        bind-tools    \
                        curl          \

README.md

@@ -62,14 +62,25 @@ To kick off development activities:
 
 * [Host shell] docker-compose up -d dev
 * [Host shell] docker-compose exec dev sh
+
+To build all the images:
 * [`dev` /src#] make
+
+To clear out prior deamon runs and run the daemons in the background:
 * [`dev` /src#] rm -rf /tmp/ickptDB
 * [`dev` /src#] ickpt/ickpt ickpt/dev.conf &
 * [`dev` /src#] imgr/imgr imgr/dev.conf &
+* [`dev` /src#] idestroy/idestroy iclient/dev.conf &
 * [`dev` /src#] imgr/mkmount.sh -fs
 * [`dev` /src#] iclient/iclient iclient/dev.conf &
 
-Note that `imgr` and `iclient` will be logging to $StdOut in this example launching. Each of the above can be terminated by delivering a SIGINT or SIGTERM to their processes.
+Notes:
+* `idestroy` step will fail with `httpGETResponse.Status unexpected: 404 Not Found` if this is the fist iteration through 
+* To relaunch the daemons without reformatting:
+  * Skip the `rm -rf /tmp/ickptDB` and `idestroy...` steps
+  * Pass `-s` rather than `-fs` to `imgr/mkmount.sh`
+* The daemons will be logging to $StdOut in this example launching
+* Each of the above daemons may be terminated by delivering a SIGINT or SIGTERM to their processes
 
 For a more appropriate environment in which to perform functional testing, the `docker-compose.yml` file my also be used to launch the suite of Docker Containers:
 
@@ -78,7 +89,7 @@ For a more appropriate environment in which to perform functional testing, the `
 
 At this point, you have a separate Docker Container running each of the ProxyFS services. Inside the `iclient` Docker Container, the `/mnt` directory will be the FUSE mount point to your ProxyFS file system.
 
-Both the `imgr` and `iclient` services provide an HTTP endpoint that, due to the port mapping specified in `docker-compose.yml`, should be reachable from a browser in the host:
+For either the up cases (i.e. `dev` or `iclient`), both the `imgr` and `iclient` services provide an HTTP endpoint that, due to the port mapping specified in `docker-compose.yml`, should be reachable from a browser in the host:
 
 * `imgr` - http://localhost:15346/
 * `iclient` - http://localhost:15347/

benchmark.fio

@@ -1,5 +1,4 @@
-; Job/Section Selection: --section={1MiB_write|1MiB_read|4KiB_write|4KiB_read}
-; Number of Threads:  --numjobs=<N>  (defaults to 1)
+; Job/Section Selection: --section={write_1|read_1|write_2|read_2|write_4|read_4|write_8|read_8|write_16|read_16}
 ;
 ; Suggested sequence:
 ;
@@ -9,23 +8,56 @@
 ;  Unmount file system
 ;  Restart file server
 ;  Remount file system
-;  fio --section=1MiB_write --numjobs=<N> benchmark.fio
+;  fio --section=write_1 benchmark.fio
 ;  Record results
 ;  Unmount file system
 ;  Restart file server
 ;  Remount file system
-;  fio --section=1MiB_read --numjobs=<N> benchmark.fio
+;  fio --section=read_1 benchmark.fio
 ;  Record results
 ;  Erase all files in file system
 ;  Unmount file system
 ;  Restart file server
 ;  Remount file system
-;  fio --section=4KiB_write --numjobs=<N> benchmark.fio
+;  fio --section=write_2 benchmark.fio
 ;  Record results
 ;  Unmount file system
 ;  Restart file server
 ;  Remount file system
-;  fio --section=4KiB_read --numjobs=<N> benchmark.fio
+;  fio --section=read_2 benchmark.fio
+;  Record results
+;  Erase all files in file system
+;  Unmount file system
+;  Restart file server
+;  Remount file system
+;  fio --section=write_4 benchmark.fio
+;  Record results
+;  Unmount file system
+;  Restart file server
+;  Remount file system
+;  fio --section=read_4 benchmark.fio
+;  Record results
+;  Erase all files in file system
+;  Unmount file system
+;  Restart file server
+;  Remount file system
+;  fio --section=write_8 benchmark.fio
+;  Record results
+;  Unmount file system
+;  Restart file server
+;  Remount file system
+;  fio --section=read_8 benchmark.fio
+;  Record results
+;  Erase all files in file system
+;  Unmount file system
+;  Restart file server
+;  Remount file system
+;  fio --section=write_16 benchmark.fio
+;  Record results
+;  Unmount file system
+;  Restart file server
+;  Remount file system
+;  fio --section=read_16 benchmark.fio
 ;  Record results
 ;  Erase all files in file system
 ;  Unmount file system
@@ -38,20 +70,46 @@ filename_format=fio.$jobnum.$filenum
 group_reporting
 iodepth=1
 ioengine=psync
+direct=1
+blocksize=64Ki
 size=4Gi
 
-[1MiB_write]
-blocksize=1Mi
+[write_1]
+readwrite=write
+numjobs=1
+
+[read_1]
+readwrite=read
+numjobs=1
+
+[write_2]
+readwrite=write
+numjobs=2
+
+[read_2]
+readwrite=read
+numjobs=2
+
+[write_4]
+readwrite=write
+numjobs=4
+
+[read_4]
+readwrite=read
+numjobs=4
+
+[write_8]
 readwrite=write
+numjobs=8
 
-[1MiB_read]
-blocksize=1Mi
+[read_8]
 readwrite=read
+numjobs=8
 
-[4KiB_write]
-blocksize=4Ki
+[write_16]
 readwrite=write
+numjobs=16
 
-[4KiB_read]
-blocksize=4Ki
+[read_16]
 readwrite=read
+numjobs=16

conf/api.go

@@ -237,6 +237,7 @@ func (confMap ConfMap) UpdateFromFile(confFilePath string) (err error) {
 			if confFileBytesLineOffsetStart < confFileBytesOffset {
 				currentLine = string(confFileBytes[confFileBytesLineOffsetStart:confFileBytesOffset])
 
+				currentLine = strings.TrimRight(currentLine, "\r")   // Trim trailing CR (if present)
 				currentLine = strings.SplitN(currentLine, ";", 2)[0] // Trim comment after ';'
 				currentLine = strings.SplitN(currentLine, "#", 2)[0] // Trim comment after '#'
 				currentLine = strings.Trim(currentLine, " \t")       // Trim leading & trailing spaces & tabs

conf/api_test.go

@@ -52,9 +52,9 @@ func TestMain(m *testing.M) {
 
 	tempFile2Name = tempFile2.Name()
 
-	io.WriteString(tempFile2, "; A comment on it's own line\n")
-	io.WriteString(tempFile2, "[TestNamespace:Test_-_Section] ; A comment at the end of a line\n")
-	io.WriteString(tempFile2, "Test_-_Option =\n")
+	io.WriteString(tempFile2, "; A comment on it's own line\r\n")
+	io.WriteString(tempFile2, "[TestNamespace:Test_-_Section] ; A comment at the end of a line\r\n")
+	io.WriteString(tempFile2, "Test_-_Option =\r\n")
 
 	tempFile2.Close()
 

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/google/btree v1.0.1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
+	golang.org/x/sys v0.0.0-20220315194320-039c03cc5b86
 )
 
 require (

go.sum

@@ -24,8 +24,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220315194320-039c03cc5b86 h1:A9i04dxx7Cribqbs8jf3FQLogkL/CV2YN7hj9KWJCkc=
+golang.org/x/sys v0.0.0-20220315194320-039c03cc5b86/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

ickpt/main.go

@@ -18,13 +18,15 @@ import (
 
 	"github.com/NVIDIA/proxyfs/conf"
 	"github.com/NVIDIA/proxyfs/ickpt/ickptpkg"
+	"github.com/NVIDIA/proxyfs/utils"
 )
 
 func main() {
 	var (
-		confMap    conf.ConfMap
-		err        error
-		signalChan chan os.Signal
+		confMap            conf.ConfMap
+		configMapJSONified string
+		err                error
+		signalChan         chan os.Signal
 	)
 
 	if len(os.Args) < 2 {
@@ -44,8 +46,12 @@ func main() {
 		os.Exit(1)
 	}
 
+	configMapJSONified = utils.JSONify(confMap, true)
+
 	// Start
 
+	fmt.Printf("Calling ickptpkg.Start(confMap)...with confMap:\n%s\n", configMapJSONified)
+
 	err = ickptpkg.Start(confMap)
 	if nil != err {
 		fmt.Fprintf(os.Stderr, "ickptpkg.Start(confMap) failed: %v\n", err)
@@ -65,6 +71,8 @@ func main() {
 
 	// Stop
 
+	fmt.Printf("Calling ickptpkg.Stop()...\n")
+
 	err = ickptpkg.Stop()
 	if nil != err {
 		fmt.Fprintf(os.Stderr, "ickptpkg.Stop() failed: %v\n", err)

iclient/dev.conf

@@ -43,5 +43,6 @@ LogFilePath:
 LogToConsole:                     true
 TraceEnabled:                     false
 FUSELogEnabled:                   false
+RetryRPCLogEnabled:               false
 HTTPServerIPAddr:                 dev
 HTTPServerPort:                   15347

iclient/iclient.conf

@@ -43,5 +43,6 @@ LogFilePath:
 LogToConsole:                     true
 TraceEnabled:                     false
 FUSELogEnabled:                   false
+RetryRPCLogEnabled:               false
 HTTPServerIPAddr:                 iclient
 HTTPServerPort:                   15347

iclient/iclientpkg/api.go

@@ -49,6 +49,7 @@
 //  LogToConsole:                     true
 //  TraceEnabled:                     false
 //  FUSELogEnabled:                   false
+//  RetryRPCLogEnabled:               false
 //  HTTPServerIPAddr:                                # Defaults to 0.0.0.0 (i.e. all interfaces)
 //  HTTPServerPort:                                  # Defaults to disabling the embedded HTTP Server
 //