websocket (#162)

* websocket initialised, user and pod commands added (#152)

* addressing #119, uploading same file multiple times

* addressing #127, server requires postageBlockId to run

* addressing #119, backup old file in case same name

* code cleanup

* addressing #117 and #118, issues related to pod deletion (#1)

* addressing #117 and #118, issues related to pod deletion

* fix reploading deleted file

* fix #124, handle removed root directory

* fixed #19, pod deletion requires password

* mod tidy (#146)

* mod tidy

* manually restore vendor

* manually restore vendor

* fix #86, shared pod removed from list

* websocket initialised, user and pod commands added

* fixing lint

* fixing lint

* file related events added

* download working

* most of the kv and doc commands added

* loadcsv, loadjson, indexjson

* minor type changes

* whitelist origins

* add stream functionality

* add streaming for loadcsv and loadjson

* require centent_length for upload stream

* lint fixes

* fix #165 (#166)

* cli can connect to remote dfs server (#164)

* cli can connect to remote dfs server

* lint fixes

* bump version

cmd/common/websocket_request.go

@@ -0,0 +1,138 @@
+package common
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+)
+
+type Event string
+
+var (
+	UserSignup         Event = "/user/signup"
+	UserLogin          Event = "/user/login"
+	UserImport         Event = "/user/import"
+	UserPresent        Event = "/user/present"
+	UserIsLoggedin     Event = "/user/isloggedin"
+	UserLogout         Event = "/user/logout"
+	UserExport         Event = "/user/export"
+	UserDelete         Event = "/user/delete"
+	UserStat           Event = "/user/stat"
+	PodNew             Event = "/pod/new"
+	PodOpen            Event = "/pod/open"
+	PodClose           Event = "/pod/close"
+	PodSync            Event = "/pod/sync"
+	PodDelete          Event = "/pod/delete"
+	PodLs              Event = "/pod/ls"
+	PodStat            Event = "/pod/stat"
+	PodShare           Event = "/pod/share"
+	PodReceive         Event = "/pod/receive"
+	PodReceiveInfo     Event = "/pod/receiveinfo"
+	DirIsPresent       Event = "/dir/present"
+	DirMkdir           Event = "/dir/mkdir"
+	DirRmdir           Event = "/dir/rmdir"
+	DirLs              Event = "/dir/ls"
+	DirStat            Event = "/dir/stat"
+	FileDownload       Event = "/file/download"
+	FileDownloadStream Event = "/file/download/stream"
+	FileUpload         Event = "/file/upload"
+	FileUploadStream   Event = "/file/upload/stream"
+	FileShare          Event = "/file/share"
+	FileReceive        Event = "/file/receive"
+	FileReceiveInfo    Event = "/file/receiveinfo"
+	FileDelete         Event = "/file/delete"
+	FileStat           Event = "/file/stat"
+	KVCreate           Event = "/kv/new"
+	KVList             Event = "/kv/ls"
+	KVOpen             Event = "/kv/open"
+	KVDelete           Event = "/kv/delete"
+	KVCount            Event = "/kv/count"
+	KVEntryPut         Event = "/kv/entry/put"
+	KVEntryGet         Event = "/kv/entry/get"
+	KVEntryDelete      Event = "/kv/entry/del"
+	KVLoadCSV          Event = "/kv/loadcsv"
+	KVLoadCSVStream    Event = "/kv/loadcsv/stream"
+	KVSeek             Event = "/kv/seek"
+	KVSeekNext         Event = "/kv/seek/next"
+	DocCreate          Event = "/doc/new"
+	DocList            Event = "/doc/ls"
+	DocOpen            Event = "/doc/open"
+	DocCount           Event = "/doc/count"
+	DocDelete          Event = "/doc/delete"
+	DocFind            Event = "/doc/find"
+	DocEntryPut        Event = "/doc/entry/put"
+	DocEntryGet        Event = "/doc/entry/get"
+	DocEntryDel        Event = "/doc/entry/del"
+	DocLoadJson        Event = "/doc/loadjson"
+	DocLoadJsonStream  Event = "/doc/loadjson/stream"
+	DocIndexJson       Event = "/doc/indexjson"
+)
+
+type WebsocketRequest struct {
+	Event  Event       `json:"event"`
+	Params interface{} `json:"params,omitempty"`
+}
+
+type FileRequest struct {
+	PodName   string `json:"pod_name,omitempty"`
+	TableName string `json:"table_name,omitempty"`
+	DirPath   string `json:"dir_path,omitempty"`
+	BlockSize string `json:"block_size,omitempty"`
+	FileName  string `json:"file_name,omitempty"`
+}
+
+type FileDownloadRequest struct {
+	PodName  string `json:"pod_name,omitempty"`
+	Filepath string `json:"file_path,omitempty"`
+}
+
+type WebsocketResponse struct {
+	Event      Event       `json:"event"`
+	StatusCode int         `json:"code"`
+	Params     interface{} `json:"params,omitempty"`
+	header     http.Header
+	buf        bytes.Buffer
+}
+
+func NewWebsocketResponse() *WebsocketResponse {
+	return &WebsocketResponse{
+		header: map[string][]string{},
+	}
+}
+
+func (w *WebsocketResponse) Header() http.Header {
+	return w.header
+}
+
+func (w *WebsocketResponse) Write(bytes []byte) (int, error) {
+	if w.Header().Get("Content-Type") == "application/json; charset=utf-8" ||
+		w.Header().Get("Content-Type") == "application/json" {
+		body := map[string]interface{}{}
+		err := json.Unmarshal(bytes, &body)
+		if err != nil {
+			return 0, err
+		}
+		w.Params = body
+		return len(bytes), nil
+	}
+	if w.Header().Get("Content-Length") != "" || w.Header().Get("Content-Length") != "0" {
+		return w.buf.Write(bytes)
+	}
+	return 0, nil
+}
+
+func (w *WebsocketResponse) WriteHeader(statusCode int) {
+	w.StatusCode = statusCode
+}
+
+func (w *WebsocketResponse) Marshal() []byte {
+	if w.Header().Get("Content-Type") == "application/json; charset=utf-8" ||
+		w.Header().Get("Content-Type") == "application/json" {
+		data, _ := json.Marshal(w)
+		return data
+	}
+	if w.Header().Get("Content-Length") != "" {
+		return w.buf.Bytes()
+	}
+	return nil
+}

cmd/dfs-cli/cmd/fdfs-api.go

@@ -47,13 +47,13 @@ type FdfsClient struct {
 	cookie *http.Cookie
 }
 
-func NewFdfsClient(host, port string) (*FdfsClient, error) {
+func NewFdfsClient(fdfsServer string) (*FdfsClient, error) {
 	client, err := createHTTPClient()
 	if err != nil {
 		return nil, err
 	}
 	return &FdfsClient{
-		url:    fmt.Sprintf("http://" + host + ":" + port),
+		url:    fdfsServer,
 		client: client,
 	}, nil
 }
@@ -84,24 +84,15 @@ func (s *FdfsClient) CheckConnection() bool {
 	if err != nil {
 		return false
 	}
+	defer response.Body.Close()
 	req.Close = true
 
 	if response.StatusCode != http.StatusOK {
 		return false
 	}
 
-	data, err := ioutil.ReadAll(response.Body)
-	if err != nil {
-		return false
-	}
-	err = response.Body.Close()
-	if err != nil {
-		return false
-	}
-	if !strings.HasPrefix(string(data), "FairOS-dfs") {
-		return false
-	}
-	return true
+	_, err = ioutil.ReadAll(response.Body)
+	return err == nil
 }
 
 func (s *FdfsClient) postReq(method, urlPath string, jsonBytes []byte) ([]byte, error) {

cmd/dfs-cli/cmd/pod.go

@@ -48,8 +48,10 @@ func podNew(podName string) {
 }
 
 func deletePod(podName string) {
+	password := getPassword()
 	delPod := common.PodRequest{
-		PodName: podName,
+		PodName:  podName,
+		Password: password,
 	}
 	jsonData, err := json.Marshal(delPod)
 	if err != nil {

cmd/dfs-cli/cmd/prompt.go

@@ -109,7 +109,7 @@ type Message struct {
 // NewPrompt spawns dfs-client and checks if the it is connected to it.
 func NewPrompt() {
 	var err error
-	fdfsAPI, err = NewFdfsClient(fdfsHost, fdfsPort)
+	fdfsAPI, err = NewFdfsClient(fdfsServer)
 	if err != nil {
 		fmt.Println("could not create fdfs client")
 		os.Exit(1)

cmd/dfs-cli/cmd/root.go

@@ -29,9 +29,8 @@ import (
 )
 
 var (
-	cfgFile  string
-	fdfsHost string
-	fdfsPort string
+	cfgFile    string
+	fdfsServer string
 )
 
 // rootCmd represents the base command when called without any subcommands
@@ -42,8 +41,7 @@ var rootCmd = &cobra.Command{
 
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Println("version    : ", dfs.Version)
-		fmt.Println("fdfsHost   : ", fdfsHost)
-		fmt.Println("fdfsPort   : ", fdfsPort)
+		fmt.Println("fdfsServer   : ", fdfsServer)
 		NewPrompt()
 		initPrompt()
 	},
@@ -88,8 +86,20 @@ func init() {
 	// when this action is called directly.
 	rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
 
-	rootCmd.PersistentFlags().StringVar(&fdfsHost, "fdfsHost", "127.0.0.1", "fdfs host")
-	rootCmd.PersistentFlags().StringVar(&fdfsPort, "fdfsPort", "9090", "fdfs port")
+	rootCmd.PersistentFlags().String("fdfsHost", "127.0.0.1", "fdfs host")
+	rootCmd.PersistentFlags().String("fdfsPort", "9090", "fdfs port")
+
+	rootCmd.PersistentFlags().StringVar(&fdfsServer, "fdfsServer", "http://localhost:9090", "fdfs server api endpoint")
+
+	if err := rootCmd.PersistentFlags().MarkDeprecated("fdfsHost", "run --fdfsServer, fdfs server api endpoint"); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	if err := rootCmd.PersistentFlags().MarkDeprecated("fdfsPort", "run --fdfsServer, fdfs server api endpoint"); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
 }
 
 // initConfig reads in config file and ENV variables if set.

cmd/dfs/cmd/server.go

@@ -23,7 +23,6 @@ import (
 	"strings"
 
 	dfs "github.com/fairdatasociety/fairOS-dfs"
-
 	"github.com/fairdatasociety/fairOS-dfs/pkg/api"
 	"github.com/fairdatasociety/fairOS-dfs/pkg/logging"
 	"github.com/gorilla/mux"
@@ -161,8 +160,11 @@ func startHttpService(logger logging.Logger) {
 			return
 		}
 	})
-
 	apiVersion := "v1"
+
+	wsRouter := router.PathPrefix("/ws/" + apiVersion).Subrouter()
+	wsRouter.HandleFunc("/", handler.WebsocketHandler)
+
 	baseRouter := router.PathPrefix("/" + apiVersion).Subrouter()
 	baseRouter.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
 		_, err := fmt.Fprintln(w, "User-agent: *\nDisallow: /")

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/gorilla/mux v1.7.4
 	github.com/gorilla/securecookie v1.1.1
+	github.com/gorilla/websocket v1.4.2
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/klauspost/pgzip v1.2.5
 	github.com/mattn/go-tty v0.0.3 // indirect

pkg/api/doc_indexjson.go

@@ -49,7 +49,7 @@ func (h *Handler) DocIndexJsonHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	podName := docReq.TableName
+	podName := docReq.PodName
 	if podName == "" {
 		h.logger.Errorf("doc indexjson: \"pod_name\" argument missing")
 		jsonhttp.BadRequest(w, "doc indexjson: \"pod_name\" argument missing")

pkg/api/doc_new.go

@@ -116,5 +116,5 @@ func (h *Handler) DocCreateHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	jsonhttp.OK(w, "document db created")
+	jsonhttp.Created(w, "document db created")
 }

pkg/api/file_download.go

@@ -83,7 +83,7 @@ func (h *Handler) FileDownloadHandler(w http.ResponseWriter, r *http.Request) {
 	if err != nil {
 		h.logger.Errorf("download: %v", err)
 		w.Header().Set("Content-Type", " application/json")
-		jsonhttp.InternalServerError(w, "stat dir: "+err.Error())
+		jsonhttp.InternalServerError(w, "download: "+err.Error())
 	}
 	_ = reader.Close()
 }

pkg/api/kv_new.go

@@ -101,5 +101,5 @@ func (h *Handler) KVCreateHandler(w http.ResponseWriter, r *http.Request) {
 		jsonhttp.InternalServerError(w, "kv create: "+err.Error())
 		return
 	}
-	jsonhttp.OK(w, "kv store created")
+	jsonhttp.Created(w, "kv store created")
 }

pkg/api/login_middleware.go

@@ -25,7 +25,7 @@ import (
 )
 
 // LoginMiddleware is a middleware that gets called before executing any of the protected handlers.
-// this check if a there is a valid session id the request and then alows the request handler to
+// this check if a there is a valid session id the request and then allows the request handler to
 // proceed for execution.
 func (h *Handler) LoginMiddleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {