Merge branch 'release/2.04.0' into stable

Dockerfile

@@ -4,7 +4,7 @@
 # To build this image:
 #
 #   docker build                                      \
-#          --target {base|dev|build|deploy}           \
+#          --target {base|buildable|dev|build|deploy} \
 #          [--build-arg GolangVersion=<X.YY.Z>]       \
 #          [--build-arg MakeTarget={|all|ci|minimal}] \
 #          [-t <repository>[:<tag>]]                  .
@@ -47,7 +47,6 @@
 #          [--cap-add SYS_ADMIN]                                      \
 #          [--device /dev/fuse]                                       \
 #          [--mount src="$(pwd)",target="/src",type=bind]             \
-#          [--env DISPLAY=<hostOrIP>:<displayNumber>[.<screenNumber]] \
 #          <image id>|<repository>[:<tag>]
 #
 #   Notes:
@@ -64,28 +63,37 @@
 #       1) bind mounts the context into /src in the container
 #       2) /src will be a read-write'able equivalent to the context dir
 #       3) only useful for --target dev
-#     --env DISPLAY: tells Docker to set ENV DISPLAY for X apps (e.g. wireshark)
-
-FROM alpine:3.17 as base
-RUN apk add --no-cache libc6-compat
-
-FROM base as dev
-ARG GolangVersion=1.19.4
-RUN apk add --no-cache               \
-                       bind-tools    \
-                       curl          \
-                       fio           \
-                       fuse          \
-                       gcc           \
-                       git           \
-                       jq            \
-                       libc-dev      \
-                       make          \
-                       tar           \
-                       terminus-font \
-                       wireshark
-ENV LIBGL_ALWAYS_INDIRECT=1
-ENV XDG_RUNTIME_DIR="/tmp/runtime-root"
+
+FROM ubuntu:22.04 as base
+
+RUN    apt-get update \
+    && apt-get upgrade -y
+
+ARG TimeZone=America/Los_Angeles
+RUN ln -snf /usr/share/zoneinfo/${TimeZone} /etc/localtime
+RUN echo ${TimeZone} > /etc/timezone
+
+RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y tzdata
+
+FROM base as buildable
+
+RUN    apt-get update \
+    && apt-get install -y \
+                        build-essential \
+                        curl \
+                        dnsutils \
+                        fuse \
+                        git \
+                        iputils-ping \
+                        jq \
+                        make \
+                        protobuf-compiler \
+                        s3cmd \
+                        tree \
+                        vim \
+                        wget
+
+ARG GolangVersion=1.22.0
 ENV GolangBasename="go${GolangVersion}.linux-amd64.tar.gz"
 ENV GolangURL="https://golang.org/dl/${GolangBasename}"
 WORKDIR /tmp
@@ -96,21 +104,35 @@ RUN git clone https://github.com/go-delve/delve
 WORKDIR /tmp/delve
 RUN go build github.com/go-delve/delve/cmd/dlv
 RUN cp dlv /usr/local/go/bin/.
+WORKDIR /
+
+RUN echo '#!/bin/bash'                  >  /root/.bashrc_additions
+RUN echo 'export PS1="\w$ "'            >> /root/.bashrc_additions
+RUN echo 'export GOPATH=${HOME}/go'     >> /root/.bashrc_additions
+RUN echo 'export GOBIN=${GOPATH}/bin'   >> /root/.bashrc_additions
+RUN echo 'export PATH=${GOBIN}:${PATH}' >> /root/.bashrc_additions
+RUN echo 'go env -w CGO_ENABLED=1'      >> /root/.bashrc_additions
+
+RUN echo ""                      >> /root/.bashrc
+RUN echo ". ~/.bashrc_additions" >> /root/.bashrc
+
+FROM buildable as dev
+
 VOLUME /src
 WORKDIR /src
 RUN git config --global --add safe.directory /src
 
-FROM dev as build
+FROM buildable as build
+
 ARG MakeTarget
 COPY . /clone
 WORKDIR /clone
+RUN git config --global --add safe.directory /clone
 RUN make clean
 RUN make $MakeTarget
 
-FROM base as deploy
-RUN apk add --no-cache      \
-                       curl \
-                       fuse
+FROM buildable as deploy
+
 COPY --from=build /clone/iauth/iauth-swift/iauth-swift.so ./
 COPY --from=build /clone/ickpt/ickpt                      ./
 COPY --from=build /clone/ickpt/ickpt.conf                 ./

README.md

@@ -61,7 +61,7 @@ To instantiate either a development or test environment, a `docker-compose.yml`
 To kick off development activities:
 
 * [Host shell] docker-compose up -d dev
-* [Host shell] docker-compose exec dev sh
+* [Host shell] docker-compose exec dev bash
 
 To build all the images:
 * [`dev` /src#] make
@@ -80,7 +80,7 @@ Notes:
   * Skip the `rm -rf /tmp/ickptDB` and `idestroy...` steps
   * Pass `-s` rather than `-fs` to `imgr/mkmount.sh`
 * The daemons will be logging to $StdOut in this example launching
-* Each of the above daemons may be terminated by delivering a SIGINT or SIGTERM to their processes
+* Each of the above daemons may be terminated by delivering a SIGINT or SIGTERM to their corresponding process
 
 For a more appropriate environment in which to perform functional testing, the `docker-compose.yml` file my also be used to launch the suite of Docker Containers:
 

docker-compose.yml

@@ -59,6 +59,7 @@ services:
       - SYS_ADMIN
     devices:
       - '/dev/fuse'
+    privileged: true
     expose:
       - 33123                                 # ICKPT.Port
       - 32356                                 # IMGR.RetryRPCPort
@@ -144,6 +145,7 @@ services:
       - SYS_ADMIN
     devices:
       - '/dev/fuse'
+    privileged: true
     expose:
       - 15347                                 # ICLIENT.HTTPServerPort
     ports:

go.mod

@@ -1,14 +1,16 @@
 module github.com/NVIDIA/proxyfs
 
-go 1.19
+go 1.21
+
+toolchain go1.21.6
 
 require (
-	github.com/NVIDIA/fission v0.0.0-20221207000636-f8dd12d86072
+	github.com/NVIDIA/fission v0.0.0-20240131203418-9fb5956a9e99
 	github.com/NVIDIA/sortedmap v0.0.0-20221206223250-5aaf17459438
 	github.com/google/btree v1.0.1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.1
-	golang.org/x/sys v0.3.0
+	golang.org/x/sys v0.13.0
 )
 
 require (

go.sum

@@ -1,13 +1,7 @@
-github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 h1:hMAAyAeYB1T1DnxqdDZzjWeTDz/hL0ZGFhz3uQyH1nQ=
-github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8/go.mod h1:GPbuJvLD4QWiHPS6vivLzh+XMAx6va0Aucm6ipa5S0I=
 github.com/NVIDIA/cstruct v0.0.0-20221206222058-cbc877f192d5 h1:u9jQ3H8RSqtqTu7b+oACnChS7HsVpbV46/qBJPVCBjU=
 github.com/NVIDIA/cstruct v0.0.0-20221206222058-cbc877f192d5/go.mod h1:s/x+WQqgkcAbDPltLPIs0AwqxY7W6UG0RGhuMVy8tds=
-github.com/NVIDIA/fission v0.0.0-20220110231326-e35d84a12929 h1:iFIrEXsx1JGcepyqoMPo9IHqMFbe799HAOpxVeykQks=
-github.com/NVIDIA/fission v0.0.0-20220110231326-e35d84a12929/go.mod h1:9wVslsyxZaBvW/ollg7JLxJOxKb+Ik2KH1WVs1nicMA=
-github.com/NVIDIA/fission v0.0.0-20221207000636-f8dd12d86072 h1:ablyovtDtsJkcUc9SRJ5YTCiIPbx77Fb+KwFAdunphc=
-github.com/NVIDIA/fission v0.0.0-20221207000636-f8dd12d86072/go.mod h1:OWPp+2S3hQCved6NTfpAFeGjibx8dKKpBDdiU12ZrrE=
-github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 h1:mDx/maO8psu+pHQqEDoL15WTj/BAAnu/sKSeOVR8wZI=
-github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k=
+github.com/NVIDIA/fission v0.0.0-20240131203418-9fb5956a9e99 h1:/8vH86v/JpLiP0EQuNfmxXJum4WOEZGAr1r83hcu1Us=
+github.com/NVIDIA/fission v0.0.0-20240131203418-9fb5956a9e99/go.mod h1:inroKY56+F8ByUjfG2kO+gUQJR6GRNn8lpzuZxp1sGI=
 github.com/NVIDIA/sortedmap v0.0.0-20221206223250-5aaf17459438 h1:0TZfWYaOMyrK2aiyf5KcX9rxFyVapg8m1zd8AWBefwE=
 github.com/NVIDIA/sortedmap v0.0.0-20221206223250-5aaf17459438/go.mod h1:uTjxdPdujmeJm/c1BNApnnX4KW+9nYmt0L6Ijft1+M4=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -34,11 +28,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664 h1:v1W7bwXHsnLLloWYTVEdvGvA7BHMeBYsPcF0GLDxIRs=
-golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=

iclient/iclient.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 Usage="$(basename "$0") - Ask swift/imgr to format and serve testvol... and then sleep
 where:
@@ -30,7 +30,7 @@ AuthToken=""
 while [ "$AuthToken" == "" ]
 do
   sleep 1
-  AuthToken=`curl -v -s -H "X-Auth-User: test:tester" -H "X-Auth-Key: testing" swift:8080/auth/v1.0 2>&1 | awk /"X-Auth-Token:"/'{print $3}'`
+  AuthToken=`curl -v -s -H "X-Auth-User: test:tester" -H "X-Auth-Key: testing" swift:8080/auth/v1.0 2>&1 | awk /"X-Auth-Token:"/'{print $3}' | tr -d '\r'`
 done
 
 curl -v -s -H "X-Auth-Token: $AuthToken" swift:8080/v1/AUTH_test/con -X PUT

iclient/iclientpkg/fission.go

@@ -2700,8 +2700,9 @@ func (dummy *globalsStruct) DoInit(inHeader *fission.InHeader, initIn *fission.I
 		MaxWrite:             globals.config.FUSEMaxWrite,
 		TimeGran:             0, // accept default
 		MaxPages:             globals.config.FUSEMaxPages,
-		Padding:              0,
-		Unused:               [8]uint32{0, 0, 0, 0, 0, 0, 0, 0},
+		MapAlignment:         0, // accept default
+		Flags2:               0,
+		Unused:               [7]uint32{0, 0, 0, 0, 0, 0, 0},
 	}
 
 	errno = 0

imgr/mkmount.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 Usage="$(basename "$0") - Ask swift/imgr to format and serve testvol
 where:
@@ -30,7 +30,7 @@ AuthToken=""
 while [ "$AuthToken" == "" ]
 do
   sleep 1
-  AuthToken=`curl -v -s -H "X-Auth-User: test:tester" -H "X-Auth-Key: testing" swift:8080/auth/v1.0 2>&1 | awk /"X-Auth-Token:"/'{print $3}'`
+  AuthToken=`curl -v -s -H "X-Auth-User: test:tester" -H "X-Auth-Key: testing" swift:8080/auth/v1.0 2>&1 | awk /"X-Auth-Token:"/'{print $3}' | tr -d '\r'`
 done
 
 echo

release_notes.md

@@ -1,5 +1,13 @@
 # ProxyFS Release Notes
 
+## 2.04.0 (March 1, 2024)
+
+### Notes:
+
+Picked up latest version of [Package Fission](https://github.com/NVIDIA/fission). Commensurate with
+this, also upgraded various dependencies to be at least Ubuntu 22.04 and Golang 1.19 now required by
+[Package Fission](https://github.com/NVIDIA/fission).
+
 ## 2.03.0 (December 6, 2022)
 
 ### Bug Fixes: