v0.9.0 (#258) (#309)

* feat: added taskmanager to speedup loading metadata

* feature: added `pprof` flag to disable running pprof server always

* feat: writeAt feature on files, fix #259 (#260)

* feat: implement #170 (file/rename)

* feat: implement #262 (dir/rename) (#263) 

* feat: rename with move feature

* feat: add dir & file rename REST apis (#264)

* feat: add rest api tests for rename (#267)

* feat: add file seek api, #271 (#272)

* feat: #273, writeAt with truncate (#274)

* fix: allow pod to open if any file metadata gets corrupted

* feat: windows tests (#283)

* fix: #285  (broken ws server)

* feat: #229, #230 (v2 login support and use dfs.API)

* fix: kv `/present` moved to `/entry/present`

* feat: add pod/open-async api (#291)

* feat: metadata & content encryption (#294)

* feat: swagger integration

* fix: allow passing version to download.sh

* feat: remove unnecessary metadata fields, json tags changed to camelCase #298, #303

* fix: remove password from pod requests (#306)

* fix: ReceiveInfo does not require podName now #281 

* feat: `make swagger` added

* fix: swagger notations

* feat: add dependabot

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: weekly
+      day: sunday

.github/workflows/go.yml

@@ -48,5 +48,4 @@ jobs:
       if: matrix.os == 'ubuntu-latest'
       run: make test-race
     - name: Test
-      if: matrix.os != 'windows-latest'
       run: make test

.goreleaser.yaml

@@ -148,9 +148,12 @@ builds:
       - amd64
 
 archives:
-  - id: binary
+  - id: binary-version
     name_template: '{{ .Binary }}_v{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}{{ with .Mips }}_{{ . }}{{ end }}{{ if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}'
     format: binary
+  - id: binary
+    name_template: '{{ .Binary }}_{{ .Os }}_{{ .Arch }}'
+    format: binary
   - id: zip
     name_template: '{{ .Binary }}_v{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}{{ with .Mips }}_{{ . }}{{ end }}{{ if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}'
     format: zip

Dockerfile

@@ -4,16 +4,18 @@ WORKDIR /usr/fairos
 COPY go.mod go.sum /usr/fairos/
 RUN go mod download
 COPY . /usr/fairos/
-RUN apk add --update --no-cache make gcc git musl-dev libc-dev linux-headers bash #skipcq: DOK-DL3018
-RUN make binary
+
+#skipcq: DOK-DL3018
+RUN apk add --update --no-cache make gcc git musl-dev libc-dev linux-headers bash \
+    && make binary
 
 FROM alpine:3.15
 
 ARG CONFIG
 ENV CONFIG=$CONFIG
 
-RUN addgroup -g 10000 fds
-RUN adduser -u 10000 -G fds -h /home/fds -D fds
+RUN addgroup -g 10000 fds \
+    && adduser -u 10000 -G fds -h /home/fds -D fds
 USER fds
 RUN if [ -n "$CONFIG" ]; then  echo -e "$CONFIG" > ~/.dfs.yaml; fi
 EXPOSE 9090

Makefile

@@ -29,6 +29,11 @@ lint: linter
 linter:
 	test -f $(GOLANGCI_LINT) || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$($(GO) env GOPATH)/bin $(GOLANGCI_LINT_VERSION)
 
+.PHONY: swagger
+swagger:
+	which swag || ( echo "install swag for your system from https://github.com/swaggo/swag" && exit 1)
+	swag init -g ./cmd/server.go -d cmd/dfs,pkg/api,cmd/common,pkg/dir,pkg/file,pkg/pod,pkg/user,pkg/collection -o ./swagger
+
 .PHONY: vet
 vet:
 	$(GO) vet ./...
@@ -51,7 +56,7 @@ githooks:
 
 .PHONY: protobuftools
 protobuftools:
-	which protoc || ( echo "install protoc for your system from https://github.com/protocolbuffers/protobuf/releases" && exit 1)
+	which protoac || ( echo "install protoc for your system from https://github.com/protocolbuffers/protobuf/releases" && exit 1)
 	which $(GOGOPROTOBUF) || ( cd /tmp && GO111MODULE=on $(GO) get -u github.com/gogo/protobuf/$(GOGOPROTOBUF)@$(GOGOPROTOBUF_VERSION) )
 
 .PHONY: protobuf

README.md

@@ -154,7 +154,7 @@ Flags:
       --postageBlockId string   the postage block used to store the data in bee
       --pprofPort string        pprof port (default ":9091")
       --rpc string              rpc endpoint for ens network. xDai for mainnet | Goerli for testnet | local fdp-play rpc endpoint for play
-
+      --swag                    should run swagger-ui
 Global Flags:
       --beeApi string      full bee api endpoint (default "localhost:1633")
       --config string      config file (default "/Users/sabyasachipatra/.dfs.yaml")
@@ -179,3 +179,42 @@ To make binaries for all platforms run this command
 
 `./generate-exe.sh`
 
+### Generate swagger docs
+
+#### Install swag
+
+```
+go install github.com/swaggo/swag/cmd/swag@latest
+```
+
+#### Generate
+
+```
+swag init -g cmd/server.go -d cmd/dfs,pkg/api,cmd/common,pkg/dir,pkg/file,pkg/pod,pkg/user,pkg/collection -o ./swagger
+```
+
+### Running swagger-ui
+
+By default, swagger-ui is disabled. To run swagger-ui we run the `server` command with `--swag` flag
+
+```
+$ dfs server --swag
+```
+
+This should run the dfs server along with swagger-ui, available at `http://localhost:9090/swagger/index.html` assuming 
+server is running on default `9090` port on your localhost
+
+### Running fairOS on goerli testnet and swarm mainnet
+
+we need to set `network` configuration in the config file as testnet and bee configuration should point to a bee running
+on mainnet 
+
+```
+network: "testnet"
+bee:
+  bee-api-endpoint: http://localhost:1633 # bee running on mainnet  
+  postage-batch-id: <BATCH>
+  is-gateway-proxy: false
+```
+
+

cmd/common/request.go

@@ -16,50 +16,77 @@ limitations under the License.
 
 package common
 
-type UserRequest struct {
-	UserName string `json:"user_name,omitempty"`
+type UserSignupRequest struct {
+	UserName string `json:"userName,omitempty"`
 	Password string `json:"password,omitempty"`
-	Address  string `json:"address,omitempty"`
 	Mnemonic string `json:"mnemonic,omitempty"`
 }
 
+type UserLoginRequest struct {
+	UserName string `json:"userName,omitempty"`
+	Password string `json:"password,omitempty"`
+}
+
 type PodRequest struct {
-	PodName       string `json:"pod_name,omitempty"`
+	PodName       string `json:"podName,omitempty"`
 	Password      string `json:"password,omitempty"`
 	Reference     string `json:"reference,omitempty"`
-	SharedPodName string `json:"shared_pod_name,omitempty"`
+	SharedPodName string `json:"sharedPodName,omitempty"`
+}
+
+type PodShareRequest struct {
+	PodName       string `json:"podName,omitempty"`
+	SharedPodName string `json:"sharedPodName,omitempty"`
+}
+
+type PodReceiveRequest struct {
+	PodName       string `json:"podName,omitempty"`
+	Reference     string `json:"sharingRef,omitempty"`
+	SharedPodName string `json:"sharedPodName,omitempty"`
 }
 
 type FileSystemRequest struct {
-	PodName       string `json:"pod_name,omitempty"`
-	DirectoryPath string `json:"dir_path,omitempty"`
-	DirectoryName string `json:"dir_name,omitempty"`
-	FilePath      string `json:"file_path,omitempty"`
-	FileName      string `json:"file_name,omitempty"`
-	Destination   string `json:"dest_user,omitempty"`
+	PodName       string `json:"podName,omitempty"`
+	DirectoryPath string `json:"dirPath,omitempty"`
+	DirectoryName string `json:"dirName,omitempty"`
+	FilePath      string `json:"filePath,omitempty"`
+	FileName      string `json:"fileName,omitempty"`
+	Destination   string `json:"destUser,omitempty"`
+}
+
+type RenameRequest struct {
+	PodName string `json:"podName,omitempty"`
+	OldPath string `json:"oldPath,omitempty"`
+	NewPath string `json:"newPath,omitempty"`
+}
+
+type FileReceiveRequest struct {
+	PodName          string `json:"podName,omitempty"`
+	SharingReference string `json:"sharingRef,omitempty"`
+	DirectoryPath    string `json:"dirPath,omitempty"`
 }
 
 type KVRequest struct {
-	PodName     string `json:"pod_name,omitempty"`
-	TableName   string `json:"table_name,omitempty"`
-	IndexType   string `json:"index_type,omitempty"`
+	PodName     string `json:"podName,omitempty"`
+	TableName   string `json:"tableName,omitempty"`
+	IndexType   string `json:"indexType,omitempty"`
 	Key         string `json:"key,omitempty"`
 	Value       string `json:"value,omitempty"`
-	StartPrefix string `json:"start_prefix,omitempty"`
-	EndPrefix   string `json:"end_prefix,omitempty"`
+	StartPrefix string `json:"startPrefix,omitempty"`
+	EndPrefix   string `json:"endPrefix,omitempty"`
 	Limit       string `json:"limit,omitempty"`
 	Memory      string `json:"memory,omitempty"`
 }
 
 type DocRequest struct {
-	PodName       string `json:"pod_name,omitempty"`
-	TableName     string `json:"table_name,omitempty"`
+	PodName       string `json:"podName,omitempty"`
+	TableName     string `json:"tableName,omitempty"`
 	ID            string `json:"id,omitempty"`
 	Document      string `json:"doc,omitempty"`
 	SimpleIndex   string `json:"si,omitempty"`
 	CompoundIndex string `json:"ci,omitempty"`
 	Expression    string `json:"expr,omitempty"`
 	Mutable       bool   `json:"mutable,omitempty"`
 	Limit         string `json:"limit,omitempty"`
-	FileName      string `json:"file_name,omitempty"`
+	FileName      string `json:"fileName,omitempty"`
 }

cmd/common/websocket_request.go

@@ -3,16 +3,18 @@ package common
 import (
 	"bytes"
 	"encoding/json"
-	"net/http"
 )
 
 type Event string
 
 var (
 	UserSignup         Event = "/user/signup"
+	UserSignupV2       Event = "/user/signupV2"
 	UserLogin          Event = "/user/login"
+	UserLoginV2        Event = "/user/loginV2"
 	UserImport         Event = "/user/import"
 	UserPresent        Event = "/user/present"
+	UserPresentV2      Event = "/user/presentV2"
 	UserIsLoggedin     Event = "/user/isloggedin"
 	UserLogout         Event = "/user/logout"
 	UserExport         Event = "/user/export"
@@ -30,6 +32,7 @@ var (
 	PodReceiveInfo     Event = "/pod/receiveinfo"
 	DirIsPresent       Event = "/dir/present"
 	DirMkdir           Event = "/dir/mkdir"
+	DirRename          Event = "/dir/rename"
 	DirRmdir           Event = "/dir/rmdir"
 	DirLs              Event = "/dir/ls"
 	DirStat            Event = "/dir/stat"
@@ -39,6 +42,7 @@ var (
 	FileUploadStream   Event = "/file/upload/stream"
 	FileShare          Event = "/file/share"
 	FileReceive        Event = "/file/receive"
+	FileRename         Event = "/file/rename"
 	FileReceiveInfo    Event = "/file/receiveinfo"
 	FileDelete         Event = "/file/delete"
 	FileStat           Event = "/file/stat"
@@ -47,6 +51,7 @@ var (
 	KVOpen             Event = "/kv/open"
 	KVDelete           Event = "/kv/delete"
 	KVCount            Event = "/kv/count"
+	KVEntryPresent     Event = "/kv/entry/present"
 	KVEntryPut         Event = "/kv/entry/put"
 	KVEntryGet         Event = "/kv/entry/get"
 	KVEntryDelete      Event = "/kv/entry/del"
@@ -69,70 +74,59 @@ var (
 )
 
 type WebsocketRequest struct {
+	Id     string      `json:"_id"`
 	Event  Event       `json:"event"`
 	Params interface{} `json:"params,omitempty"`
 }
 
 type FileRequest struct {
-	PodName   string `json:"pod_name,omitempty"`
-	TableName string `json:"table_name,omitempty"`
-	DirPath   string `json:"dir_path,omitempty"`
-	BlockSize string `json:"block_size,omitempty"`
-	FileName  string `json:"file_name,omitempty"`
+	PodName       string `json:"podName,omitempty"`
+	TableName     string `json:"tableName,omitempty"`
+	DirPath       string `json:"dirPath,omitempty"`
+	BlockSize     string `json:"blockSize,omitempty"`
+	FileName      string `json:"fileName,omitempty"`
+	ContentLength string `json:"contentLength,omitempty"`
+	Compression   string `json:"compression,omitempty"`
+	Overwrite     bool   `json:"overwrite,omitempty"`
 }
 
 type FileDownloadRequest struct {
-	PodName  string `json:"pod_name,omitempty"`
-	Filepath string `json:"file_path,omitempty"`
+	PodName  string `json:"podName,omitempty"`
+	Filepath string `json:"filePath,omitempty"`
 }
 
 type WebsocketResponse struct {
-	Event      Event       `json:"event"`
-	StatusCode int         `json:"code"`
-	Params     interface{} `json:"params,omitempty"`
-	header     http.Header
-	buf        bytes.Buffer
+	Id          string      `json:"_id"`
+	Event       Event       `json:"event"`
+	Params      interface{} `json:"params,omitempty"`
+	StatusCode  int         `json:"code,omitempty"`
+	buf         bytes.Buffer
+	contentType string
 }
 
 func NewWebsocketResponse() *WebsocketResponse {
-	return &WebsocketResponse{
-		header: map[string][]string{},
-	}
-}
-
-func (w *WebsocketResponse) Header() http.Header {
-	return w.header
+	return &WebsocketResponse{}
 }
 
 func (w *WebsocketResponse) Write(bytes []byte) (int, error) {
-	if w.Header().Get("Content-Type") == "application/json; charset=utf-8" ||
-		w.Header().Get("Content-Type") == "application/json" {
-		body := map[string]interface{}{}
-		err := json.Unmarshal(bytes, &body)
-		if err != nil {
-			return 0, err
-		}
-		w.Params = body
-		return len(bytes), nil
-	}
-	if w.Header().Get("Content-Length") != "" || w.Header().Get("Content-Length") != "0" {
-		return w.buf.Write(bytes)
-	}
-	return 0, nil
+	return w.buf.Write(bytes)
 }
 
-func (w *WebsocketResponse) WriteHeader(statusCode int) {
-	w.StatusCode = statusCode
+func (w *WebsocketResponse) WriteJson(bytes []byte) (int, error) {
+	w.contentType = "json"
+	body := map[string]interface{}{}
+	err := json.Unmarshal(bytes, &body)
+	if err != nil {
+		return 0, err
+	}
+	w.Params = body
+	return len(bytes), nil
 }
 
 func (w *WebsocketResponse) Marshal() []byte {
-	if w.Header().Get("Content-Type") == "application/json; charset=utf-8" ||
-		w.Header().Get("Content-Type") == "application/json" {
+	if w.contentType == "json" {
 		data, _ := json.Marshal(w)
 		return data
 	}
-	if w.Header().Get("Content-Length") != "" {
-		return w.buf.Bytes()
-	}
-	return nil
+	return w.buf.Bytes()
 }

cmd/dfs-cli/cmd/filesystem.go

@@ -123,7 +123,7 @@ func statFileOrDirectory(podName, statElement string) {
 			fmt.Println("Mo. Time	  : ", time.Unix(accTime, 0).String())
 			fmt.Println("Ac. Time	  : ", time.Unix(modTime, 0).String())
 			for _, b := range resp.Blocks {
-				blkStr := fmt.Sprintf("%s, 0x%s, %s bytes, %s bytes", b.Name, b.Reference, b.Size, b.CompressedSize)
+				blkStr := fmt.Sprintf("0x%s, %s bytes, %s bytes", b.Reference, b.Size, b.CompressedSize)
 				fmt.Println(blkStr)
 			}
 		} else {
@@ -302,7 +302,6 @@ func fileReceiveInfo(podName, sharingRef string) {
 	fmt.Println("NumberOfBlocks : ", resp.NumberOfBlocks)
 	fmt.Println("ContentType    : ", resp.ContentType)
 	fmt.Println("Compression    : ", resp.Compression)
-	fmt.Println("PodName        : ", resp.PodName)
 	fmt.Println("Sender         : ", resp.Sender)
 	fmt.Println("Receiver       : ", resp.Receiver)
 	fmt.Println("SharedTime     : ", shTime)