diff --git a/README.md b/README.md index 0bfe2f1b..e6434911 100644 --- a/README.md +++ b/README.md @@ -31,12 +31,50 @@ The user can share files in his pod with any other user just like in other centr Pod creation is cheap. A user can create multiple pods and use it to organise his data. for ex: Personal-Pod, Applications-Pod etc. ### How to run FairOS-dfs? -Download the latest release from https://github.com/fairDataSociety/fairOS-dfs/releases. +Run the following command to download the latest release + +``` +curl -o- https://raw.githubusercontent.com/fairDataSociety/fairOS-dfs/master/download.sh | bash +``` +``` +wget -qO- https://raw.githubusercontent.com/fairDataSociety/fairOS-dfs/master/download.sh | bash +``` + +Or download the latest release from https://github.com/fairDataSociety/fairOS-dfs/releases. Or use Docker to run the project https://docs.fairos.fairdatasociety.org/docs/fairOS-dfs/docker-installation. Or build the latest version with the instruction https://docs.fairos.fairdatasociety.org/docs/fairOS-dfs/manual-installation. +### Configure FairOS-dfs +To get the most out of your FairOS-dfs it is important that you configure FairOS-dfs for your specific use case! + +##### Configuration for Bee +``` +bee: + bee-api-endpoint: http://localhost:1633 + bee-debug-api-endpoint: http://localhost:1635 + postage-batch-id: "" +``` + +##### Configuration for FairOS-dfs +``` +dfs: + data-dir: /Users/fairos/.fairOS/dfs + ports: + http-port: :9090 + pprof-port: :9091 +``` + +#### Other configuration +``` +cookie-domain: api.fairos.io +cors-allowed-origins: [] +verbosity: trace +``` + +Run `dfs config` to see all configurations + ### Introduction to Key Value Store over Swarm [![](https://j.gifs.com/6XZwvl.gif)](https://gateway.ethswarm.org/access/130dcf7d01442836bc14c8c38db32ebfc4d5771c28677438b6a2a2a078bd1414) @@ -53,3 +91,4 @@ https://docs.fairos.fairdatasociety.org/docs/fairOS-dfs/cli-reference To make binaries for all platforms run this command `./generate-exe.sh` + diff --git a/cmd/dfs-cli/cmd/prompt.go b/cmd/dfs-cli/cmd/prompt.go index 99c1a10b..6d12a215 100644 --- a/cmd/dfs-cli/cmd/prompt.go +++ b/cmd/dfs-cli/cmd/prompt.go @@ -136,6 +136,53 @@ func changeLivePrefix() (string, bool) { return currentPrompt, true } +var userSuggestions = []prompt.Suggest{ + {Text: "new", Description: "create a new user"}, + {Text: "del", Description: "delete a existing user"}, + {Text: "login", Description: "login to a existing user"}, + {Text: "logout", Description: "logout from a logged in user"}, + {Text: "present", Description: "is user present"}, + {Text: "export ", Description: "exports the user"}, + {Text: "import ", Description: "imports the user"}, + {Text: "stat ", Description: "shows information about a user"}, +} + +var podSuggestions = []prompt.Suggest{ + {Text: "new", Description: "create a new pod for a user"}, + {Text: "del", Description: "delete a existing pod of a user"}, + {Text: "open", Description: "open to a existing pod of a user"}, + {Text: "close", Description: "close a already opened pod of a user"}, + {Text: "ls", Description: "list all the existing pods of a user"}, + {Text: "stat", Description: "show the metadata of a pod of a user"}, + {Text: "sync", Description: "sync the pod from swarm"}, +} + +var kvSuggestions = []prompt.Suggest{ + {Text: "new", Description: "create new key value store"}, + {Text: "delete", Description: "delete the key value store"}, + {Text: "ls", Description: "lists all the key value stores"}, + {Text: "open", Description: "open already created key value store"}, + {Text: "get", Description: "get value from key"}, + {Text: "put", Description: "put key and value in kv store"}, + {Text: "del", Description: "delete key and value from the store"}, + {Text: "loadcsv", Description: "loads the csv file in to kv store"}, + {Text: "seek", Description: "seek to the given start prefix"}, + {Text: "getnext", Description: "get the next element"}, +} + +var docSuggestions = []prompt.Suggest{ + {Text: "new", Description: "creates a new document store"}, + {Text: "delete", Description: "deletes a document store"}, + {Text: "open", Description: "open the document store"}, + {Text: "ls", Description: "list all document dbs"}, + {Text: "count", Description: "count the docs in the table satisfying the expression"}, + {Text: "find", Description: "find the docs in the table satisfying the expression and limit"}, + {Text: "put", Description: "insert a json document in to document store"}, + {Text: "get", Description: "get the document having the id from the store"}, + {Text: "del", Description: "delete the document having the id from the store"}, + {Text: "loadjson", Description: "load the json file in to the newly created document db"}, +} + var suggestions = []prompt.Suggest{ {Text: "user new", Description: "create a new user"}, {Text: "user del", Description: "delete a existing user"}, @@ -149,7 +196,7 @@ var suggestions = []prompt.Suggest{ {Text: "pod del", Description: "delete a existing pod of a user"}, {Text: "pod open", Description: "open to a existing pod of a user"}, {Text: "pod close", Description: "close a already opened pod of a user"}, - {Text: "pod ls", Description: "list all the existing pods of auser"}, + {Text: "pod ls", Description: "list all the existing pods of a user"}, {Text: "pod stat", Description: "show the metadata of a pod of a user"}, {Text: "pod sync", Description: "sync the pod from swarm"}, {Text: "kv new", Description: "create new key value store"}, @@ -187,10 +234,20 @@ var suggestions = []prompt.Suggest{ } func completer(in prompt.Document) []prompt.Suggest { - w := in.GetWordBeforeCursor() - if w == "" { + w := in.Text + if w == "" || len(strings.Split(w, " ")) >= 3 { return []prompt.Suggest{} } + + if strings.HasPrefix(in.TextBeforeCursor(), "user") { + return prompt.FilterHasPrefix(userSuggestions, in.GetWordBeforeCursor(), true) + } else if strings.HasPrefix(in.TextBeforeCursor(), "pod") { + return prompt.FilterHasPrefix(podSuggestions, in.GetWordBeforeCursor(), true) + } else if strings.HasPrefix(in.TextBeforeCursor(), "kv") { + return prompt.FilterHasPrefix(kvSuggestions, in.GetWordBeforeCursor(), true) + } else if strings.HasPrefix(in.TextBeforeCursor(), "doc") { + return prompt.FilterHasPrefix(docSuggestions, in.GetWordBeforeCursor(), true) + } return prompt.FilterHasPrefix(suggestions, w, true) } diff --git a/cmd/dfs-cli/cmd/root.go b/cmd/dfs-cli/cmd/root.go index 301af5cb..516c16c7 100644 --- a/cmd/dfs-cli/cmd/root.go +++ b/cmd/dfs-cli/cmd/root.go @@ -82,10 +82,6 @@ func init() { defaultConfig := filepath.Join(home, ".fairOS/dfs-cli.yml") rootCmd.PersistentFlags().StringVar(&cfgFile, "config", defaultConfig, "config file") - // Cobra also supports local flags, which will only run - // when this action is called directly. - rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") - rootCmd.PersistentFlags().String("fdfsHost", "127.0.0.1", "fdfs host") rootCmd.PersistentFlags().String("fdfsPort", "9090", "fdfs port") diff --git a/cmd/dfs-cli/cmd/user.go b/cmd/dfs-cli/cmd/user.go index 603d788c..cbd09e97 100644 --- a/cmd/dfs-cli/cmd/user.go +++ b/cmd/dfs-cli/cmd/user.go @@ -201,7 +201,7 @@ func presentUser(userName string) { fmt.Println("user present: ", err) return } - var resp api.UserPresentResponse + var resp api.PresentResponse err = json.Unmarshal(data, &resp) if err != nil { fmt.Println("import user: ", err) diff --git a/cmd/dfs/cmd/config.go b/cmd/dfs/cmd/config.go new file mode 100644 index 00000000..652dae36 --- /dev/null +++ b/cmd/dfs/cmd/config.go @@ -0,0 +1,49 @@ +package cmd + +import ( + "github.com/spf13/cobra" + "gopkg.in/yaml.v2" +) + +var ( + optionCORSAllowedOrigins = "cors-allowed-origins" + optionDFSDataDir = "dfs.data-dir" + optionDFSHttpPort = "dfs.ports.http-port" + optionDFSPprofPort = "dfs.ports.pprof-port" + optionVerbosity = "verbosity" + optionBeeApi = "bee.bee-api-endpoint" + optionBeeDebugApi = "bee.bee-debug-api-endpoint" + optionBeePostageBatchId = "bee.postage-batch-id" + optionCookieDomain = "cookie-domain" + + defaultCORSAllowedOrigins = []string{} + defaultDFSHttpPort = ":9090" + defaultDFSPprofPort = ":9091" + defaultVerbosity = "trace" + defaultBeeApi = "http://localhost:1633" + defaultBeeDebugApi = "http://localhost:1635" + defaultCookieDomain = "api.fairos.io" +) + +var configCmd = &cobra.Command{ + Use: "config", + Short: "Print default or provided configuration in yaml format", + RunE: func(cmd *cobra.Command, args []string) (err error) { + + if len(args) > 0 { + return cmd.Help() + } + + d := config.AllSettings() + ym, err := yaml.Marshal(d) + if err != nil { + return err + } + cmd.Println(string(ym)) + return nil + }, +} + +func init() { + rootCmd.AddCommand(configCmd) +} diff --git a/cmd/dfs/cmd/root.go b/cmd/dfs/cmd/root.go index eefe6b23..6aadc4a2 100644 --- a/cmd/dfs/cmd/root.go +++ b/cmd/dfs/cmd/root.go @@ -21,30 +21,53 @@ import ( "os" "path/filepath" - homedir "github.com/mitchellh/go-homedir" + "github.com/mitchellh/go-homedir" "github.com/spf13/cobra" "github.com/spf13/viper" ) var ( - cfgFile string - beeHost string - beePort string - verbosity string - dataDir string + defaultDir = filepath.Join(".fairOS", "dfs") + defaultConfig = ".dfs.yaml" + + cfgFile string + beeApi string + beeDebugApi string + verbosity string + dataDir string + + dataDirPath string + config = viper.New() ) // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "dfs", Short: "Decentralised file system over Swarm(https://ethswarm.org/)", - Long: `dfs is the file system layer of internetOS. It is a thin layer over Swarm. -It adds features to Swarm that is required by the internetOS to parallelize computation of data. + Long: `dfs is the file system layer of fairOS. It is a thin layer over Swarm. +It adds features to Swarm that is required by the fairOS to parallelize computation of data. It manages the metadata of directories and files created and expose them to higher layers. It can also be used as a standalone personal, decentralised drive over the internet`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if err := config.BindPFlag(optionDFSDataDir, cmd.Flags().Lookup("dataDir")); err != nil { + return err + } + if err := config.BindPFlag(optionBeeApi, cmd.Flags().Lookup("beeApi")); err != nil { + return err + } + if err := config.BindPFlag(optionBeeDebugApi, cmd.Flags().Lookup("beeDebugApi")); err != nil { + return err + } + if err := config.BindPFlag(optionVerbosity, cmd.Flags().Lookup("verbosity")); err != nil { + return err + } - //Run: func(cmd *cobra.Command, args []string) { - //}, + dataDir = config.GetString(optionDFSDataDir) + beeApi = config.GetString(optionBeeApi) + beeDebugApi = config.GetString(optionBeeDebugApi) + verbosity = config.GetString(optionVerbosity) + return nil + }, } // Execute adds all child commands to the root command and sets flags appropriately. @@ -76,29 +99,41 @@ func init() { fmt.Println(err) os.Exit(1) } - // Here you will define your flags and configuration settings. // Cobra supports persistent flags, which, if defined here, // will be global for your application. - defaultConfig := filepath.Join(home, ".fairOS/dfs.yml") - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", defaultConfig, "config file") - - // Cobra also supports local flags, which will only run - // when this action is called directly. - rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") - - defaultDataDir := filepath.Join(home, ".fairOS/dfs") - rootCmd.PersistentFlags().StringVar(&dataDir, "dataDir", defaultDataDir, "store data in this dir") - rootCmd.PersistentFlags().StringVar(&beeHost, "beeHost", "127.0.0.1", "bee host") - rootCmd.PersistentFlags().StringVar(&beePort, "beePort", "1633", "bee port") - rootCmd.PersistentFlags().StringVar(&verbosity, "verbosity", "5", "verbosity level") + configPath := filepath.Join(home, defaultConfig) + + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", configPath, "config file") + + dataDirPath = filepath.Join(home, defaultDir) + rootCmd.PersistentFlags().String("dataDir", dataDirPath, "store data in this dir") + rootCmd.PersistentFlags().String("beeApi", "localhost:1633", "full bee api endpoint") + rootCmd.PersistentFlags().String("beeDebugApi", "localhost:1635", "full bee debug api endpoint") + rootCmd.PersistentFlags().String("verbosity", "trace", "verbosity level") + + rootCmd.PersistentFlags().String("beeHost", "127.0.0.1", "bee host") + rootCmd.PersistentFlags().String("beePort", "1633", "bee port") + if err := rootCmd.PersistentFlags().MarkDeprecated("beeHost", "run --beeApi, full bee api endpoint"); err != nil { + fmt.Println(err) + os.Exit(1) + } + if err := rootCmd.PersistentFlags().MarkDeprecated("beePort", "run --beeApi, full bee api endpoint"); err != nil { + fmt.Println(err) + os.Exit(1) + } } // initConfig reads in config file and ENV variables if set. func initConfig() { if cfgFile != "" { + // check file stat + if _, err := os.Stat(cfgFile); err != nil { + // if there is no configFile, write it + writeConfig() + } // Use config file from the flag. - viper.SetConfigFile(cfgFile) + config.SetConfigFile(cfgFile) } else { // Find home dir. home, err := homedir.Dir() @@ -106,16 +141,37 @@ func initConfig() { fmt.Println(err) os.Exit(1) } + // check file stat + cfgFile = filepath.Join(home, defaultConfig) + if _, err := os.Stat(cfgFile); err != nil { + // if there is no configFile, write it + writeConfig() + } - // Search config in home dir with name ".dfs" (without extension). - viper.AddConfigPath(home) - viper.SetConfigName(".fairOS/dfs") + config.SetConfigFile(cfgFile) } - viper.AutomaticEnv() // read in environment variables that match - + config.AutomaticEnv() // read in environment variables that match // If a config file is found, read it in. - if err := viper.ReadInConfig(); err == nil { - fmt.Println("Using config file:", viper.ConfigFileUsed()) + if err := config.ReadInConfig(); err == nil { + fmt.Println("Using config file:", config.ConfigFileUsed()) + } +} + +func writeConfig() { + c := viper.New() + c.Set(optionCORSAllowedOrigins, defaultCORSAllowedOrigins) + c.Set(optionDFSDataDir, dataDirPath) + c.Set(optionDFSHttpPort, defaultDFSHttpPort) + c.Set(optionDFSPprofPort, defaultDFSPprofPort) + c.Set(optionVerbosity, defaultVerbosity) + c.Set(optionBeeApi, defaultBeeApi) + c.Set(optionBeeDebugApi, defaultBeeDebugApi) + c.Set(optionBeePostageBatchId, "") + c.Set(optionCookieDomain, defaultCookieDomain) + + if err := c.WriteConfigAs(cfgFile); err != nil { + fmt.Println("failed to write config file") + os.Exit(1) } } diff --git a/cmd/dfs/cmd/server.go b/cmd/dfs/cmd/server.go index 2691306c..804c6106 100644 --- a/cmd/dfs/cmd/server.go +++ b/cmd/dfs/cmd/server.go @@ -17,6 +17,7 @@ limitations under the License. package cmd import ( + "encoding/hex" "fmt" "io/ioutil" "net/http" @@ -46,11 +47,43 @@ var serverCmd = &cobra.Command{ Short: "starts a HTTP server for the dfs", Long: `Serves all the dfs commands through an HTTP server so that the upper layers can consume it.`, + PreRunE: func(cmd *cobra.Command, args []string) error { + if err := config.BindPFlag(optionDFSHttpPort, cmd.Flags().Lookup("httpPort")); err != nil { + return err + } + if err := config.BindPFlag(optionDFSPprofPort, cmd.Flags().Lookup("pprofPort")); err != nil { + return err + } + if err := config.BindPFlag(optionCookieDomain, cmd.Flags().Lookup("cookieDomain")); err != nil { + return err + } + if err := config.BindPFlag(optionCORSAllowedOrigins, cmd.Flags().Lookup("cors-origins")); err != nil { + return err + } + if err := config.BindPFlag(optionBeePostageBatchId, cmd.Flags().Lookup("postageBlockId")); err != nil { + return err + } + return nil + }, Run: func(cmd *cobra.Command, args []string) { + httpPort = config.GetString(optionDFSHttpPort) + pprofPort = config.GetString(optionDFSPprofPort) + cookieDomain = config.GetString(optionCookieDomain) + postageBlockId = config.GetString(optionBeePostageBatchId) + corsOrigins = config.GetStringSlice(optionCORSAllowedOrigins) + verbosity = config.GetString(optionVerbosity) if postageBlockId == "" { _ = cmd.Help() fmt.Println("\npostageBlockId is required to run server") return + } else if len(postageBlockId) != 64 { + fmt.Println("\npostageBlockId is invalid") + return + } + _, err := hex.DecodeString(postageBlockId) + if err != nil { + fmt.Println("\npostageBlockId is invalid") + return } var logger logging.Logger @@ -75,15 +108,15 @@ can consume it.`, logger.Info("configuration values") logger.Info("version : ", dfs.Version) logger.Info("dataDir : ", dataDir) - logger.Info("beeHost : ", beeHost) - logger.Info("beePort : ", beePort) + logger.Info("beeApi : ", beeApi) + logger.Info("beeDebugApi : ", beeDebugApi) logger.Info("verbosity : ", verbosity) logger.Info("httpPort : ", httpPort) logger.Info("pprofPort : ", pprofPort) logger.Info("cookieDomain : ", cookieDomain) logger.Info("postageBlockId : ", postageBlockId) logger.Info("corsOrigins : ", corsOrigins) - hdlr, err := api.NewHandler(dataDir, beeHost, beePort, cookieDomain, postageBlockId, logger) + hdlr, err := api.NewHandler(dataDir, beeApi, beeDebugApi, cookieDomain, postageBlockId, corsOrigins, logger) if err != nil { logger.Error(err.Error()) return @@ -94,11 +127,11 @@ can consume it.`, } func init() { - serverCmd.Flags().StringVar(&httpPort, "httpPort", "9090", "http port") - serverCmd.Flags().StringVar(&pprofPort, "pprofPort", "9091", "pprof port") - serverCmd.Flags().StringVar(&cookieDomain, "cookieDomain", "api.fairos.io", "the domain to use in the cookie") - serverCmd.Flags().StringVar(&postageBlockId, "postageBlockId", "", "the postage block used to store the data in bee") - serverCmd.Flags().StringSliceVar(&corsOrigins, "cors-origins", []string{}, "allow CORS headers for the given origins") + serverCmd.Flags().String("httpPort", defaultDFSHttpPort, "http port") + serverCmd.Flags().String("pprofPort", defaultDFSPprofPort, "pprof port") + serverCmd.Flags().String("cookieDomain", defaultCookieDomain, "the domain to use in the cookie") + serverCmd.Flags().String("postageBlockId", "", "the postage block used to store the data in bee") + serverCmd.Flags().StringSlice("cors-origins", defaultCORSAllowedOrigins, "allow CORS headers for the given origins") rootCmd.AddCommand(serverCmd) } @@ -122,7 +155,12 @@ func startHttpService(logger logging.Logger) { logger.Errorf("error in API /: ", err) return } - _, err = fmt.Fprintln(w, beeHost+":"+beePort) + _, err = fmt.Fprintln(w, beeApi) + if err != nil { + logger.Errorf("error in API /: ", err) + return + } + _, err = fmt.Fprintln(w, beeDebugApi) if err != nil { logger.Errorf("error in API /: ", err) return @@ -195,6 +233,7 @@ func startHttpService(logger logging.Logger) { baseRouter.HandleFunc("/pod/receiveinfo", handler.PodReceiveInfoHandler).Methods("GET") podRouter := baseRouter.PathPrefix("/pod/").Subrouter() podRouter.Use(handler.LoginMiddleware) + podRouter.HandleFunc("/present", handler.PodPresentHandler).Methods("GET") podRouter.HandleFunc("/new", handler.PodCreateHandler).Methods("POST") podRouter.HandleFunc("/open", handler.PodOpenHandler).Methods("POST") podRouter.HandleFunc("/close", handler.PodCloseHandler).Methods("POST") @@ -233,8 +272,10 @@ func startHttpService(logger logging.Logger) { kvRouter.HandleFunc("/open", handler.KVOpenHandler).Methods("POST") kvRouter.HandleFunc("/count", handler.KVCountHandler).Methods("POST") kvRouter.HandleFunc("/delete", handler.KVDeleteHandler).Methods("DELETE") + kvRouter.HandleFunc("/present", handler.KVPresentHandler).Methods("GET") kvRouter.HandleFunc("/entry/put", handler.KVPutHandler).Methods("POST") kvRouter.HandleFunc("/entry/get", handler.KVGetHandler).Methods("GET") + kvRouter.HandleFunc("/entry/get-data", handler.KVGetDataHandler).Methods("GET") kvRouter.HandleFunc("/entry/del", handler.KVDelHandler).Methods("DELETE") kvRouter.HandleFunc("/loadcsv", handler.KVLoadCSVHandler).Methods("POST") kvRouter.HandleFunc("/seek", handler.KVSeekHandler).Methods("POST") @@ -274,7 +315,7 @@ func startHttpService(logger logging.Logger) { // starting the pprof server go func() { logger.Infof("fairOS-dfs pprof listening on port: %v", pprofPort) - err := http.ListenAndServe("localhost:"+pprofPort, nil) + err := http.ListenAndServe("localhost"+pprofPort, nil) if err != nil { logger.Errorf("pprof listenAndServe: %v ", err.Error()) return @@ -282,7 +323,7 @@ func startHttpService(logger logging.Logger) { }() logger.Infof("fairOS-dfs API server listening on port: %v", httpPort) - err := http.ListenAndServe(":"+httpPort, handler) + err := http.ListenAndServe(httpPort, handler) if err != nil { logger.Errorf("http listenAndServe: %v ", err.Error()) return diff --git a/download.sh b/download.sh new file mode 100755 index 00000000..702e28d1 --- /dev/null +++ b/download.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash + +{ + +GH_README="https://github.com/fairDataSociety/fairOS-dfs#how-to-run-fairos-dfs" + +dfs_has() { + type "$1" > /dev/null 2>&1 +} + +dfs_echo() { + command printf %s\\n "$*" 2>/dev/null +} + +dfs_download() { + if ! dfs_has "curl"; then + dfs_echo "Error: you need to have wget installed and in your path. Use brew (mac) or apt (unix) to install curl" + exit 1 + fi + + if ! dfs_has "wget"; then + dfs_echo "Error: you need to have wget installed and in your path. Use brew (mac) or apt (unix) to install wget" + exit 1 + fi + + eval curl -s https://api.github.com/repos/fairDataSociety/fairOS-dfs/releases/latest \ +| grep "$1" \ +| cut -d : -f 2,3 \ +| tr -d \" \ +| wget -qi - + +} + +install_dfs() { + BIN_NAME="dfs-" + + if [[ "$OSTYPE" == "linux-gnu" ]]; then + DETECTED_OS="linux" # TODO (Test) + elif [[ "$OSTYPE" == "darwin"* ]]; then + DETECTED_OS="mac" + elif [[ "$OSTYPE" == "cygwin" ]]; then + DETECTED_OS="linux" # TODO (Test) + elif [[ "$OSTYPE" == "msys" ]]; then + DETECTED_OS="windows" + elif [[ "$OSTYPE" == "win32" ]]; then + DETECTED_OS="windows" # TODO (Test) + elif [[ "$OSTYPE" == "freebsd"* ]]; then + DETECTED_OS="linux" # TODO (Test) + else + dfs_echo "Error: unable to detect operating system. Please install manually by referring to $GH_README" + exit 1 + fi + + ARCH=$(uname -m) + + echo " /@@@@@@ /@@ /@@@@@@ /@@@@@@ /@@ /@@@@@@" + echo " /@@__ @@ |__/ /@@__ @@ /@@__ @@ | @@ /@@__ @@" + echo "| @@ \__//@@@@@@ /@@ /@@@@@@ | @@ \ @@| @@ \__/ /@@@@@@@| @@ \__//@@@@@@@" + echo "| @@@@ |____ @@| @@ /@@__ @@| @@ | @@| @@@@@@ /@@@@@@ /@@__ @@| @@@@ /@@_____/" + echo "| @@_/ /@@@@@@@| @@| @@ \__/| @@ | @@ \____ @@|______/| @@ | @@| @@_/ | @@@@@@" + echo "| @@ /@@__ @@| @@| @@ | @@ | @@ /@@ \ @@ | @@ | @@| @@ \____ @@" + echo "| @@ | @@@@@@@| @@| @@ | @@@@@@/| @@@@@@/ | @@@@@@@| @@ /@@@@@@@/" + echo "|__/ \_______/|__/|__/ \______/ \______/ \_______/|__/ |_______/" + + echo "========== FairOs-dfs Installation ==========" + echo "Detected OS: $DETECTED_OS" + echo "Detected Architecture: $ARCH" + echo "=====================================================" + + if [[ "$ARCH" == "arm64" && $DETECTED_OS == "mac" ]]; then + BIN_NAME="dfs-darwin-amd64" + dfs_echo $BIN_NAME + elif [[ "$ARCH" == "x86_64" && $DETECTED_OS == "windows" ]]; then + BIN_NAME="dfs-windows-amd64.exe" + dfs_echo $BIN_NAME + elif [[ "$ARCH" == "x86_32" && $DETECTED_OS == "windows" ]]; then + BIN_NAME="dfs-windows-386.exe" + dfs_echo $BIN_NAME + elif [[ "$ARCH" == "arm64" && $DETECTED_OS == "linux" ]]; then + BIN_NAME="dfs-linux-arm64.exe" + dfs_echo $BIN_NAME + elif [[ "$ARCH" == "x86_32" && $DETECTED_OS == "linux" ]]; then + BIN_NAME="dfs-linux-386.exe" + dfs_echo $BIN_NAME + elif [[ "$ARCH" == "x86_64" && $DETECTED_OS == "linux" ]]; then + BIN_NAME="dfs-linux-amd64.exe" + dfs_echo $BIN_NAME + elif [[ "$ARCH" == "amd64" && $DETECTED_OS == "linux" ]]; then + BIN_NAME="dfs-linux-amd64.exe" + dfs_echo $BIN_NAME + else + dfs_echo "Error: unable to detect architecture. Please install manually by referring to $GH_README" + exit 1 + fi + + dfs_download $BIN_NAME +} + +install_dfs + +} \ No newline at end of file diff --git a/go.mod b/go.mod index f41a1f14..e827b029 100644 --- a/go.mod +++ b/go.mod @@ -28,5 +28,6 @@ require ( github.com/tyler-smith/go-bip39 v1.0.2 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf + gopkg.in/yaml.v2 v2.3.0 resenje.org/jsonhttp v0.2.0 ) diff --git a/openapi/dfs.yaml b/openapi/dfs.yaml index 7ec4e590..e1a19b51 100644 --- a/openapi/dfs.yaml +++ b/openapi/dfs.yaml @@ -1,7 +1,7 @@ openapi: 3.0.0 info: - version: 0.6.0 + version: 0.7.3 title: FairOS-dfs API description: 'A list of the currently provided Interfaces to interact with FairOS decentralised file system(dfs), implementing user, pod, file system, key value store and document store' @@ -56,7 +56,7 @@ paths: schema: oneOf: - $ref: '#/components/schemas/UserSignupResponse' - - $ref: '#/components/schemas/UserSignupResponseWithMenonic' + - $ref: '#/components/schemas/UserSignupResponseWithMnemonic' '400': $ref: '#/components/responses/400' '500': @@ -106,8 +106,8 @@ paths: summary: 'Import user' description: 'Import a user from one dfs server to another and subsequently logs in returning the authentication cookie `fairOS-dfs`. The cookie needs to be sent n the request header in all subsequent requests. - Importing is usefull when a user switches machines or lost a old machine which was running his dfs server. - Imput contains a form object containing the login, password, ethereum address and optionally mnemonic. + Importing is useful when a user switches machines or lost a old machine which was running his dfs server. + Input contains a form object containing the login, password, ethereum address and optionally mnemonic. If mnemonic is provided a user will be created while importing.' tags: - User @@ -568,8 +568,12 @@ paths: properties: pod_name: $ref: '#/components/schemas/PodName' + password: + $ref: '#/components/schemas/Password' required: - pod_name + - password + responses: '200': description: 'Ok' @@ -586,7 +590,7 @@ paths: '/pod/ls': get: summary: 'List pod' - description: 'Lista all pods of a user' + description: 'Lists all pods of a user' tags: - Pod parameters: @@ -649,8 +653,40 @@ paths: '500': $ref: '#/components/responses/500' - '/dir/mkdir': + '/pod/present': get: + summary: 'Pod Present' + description: 'Is Pod present' + tags: + - Pod + parameters: + - in: cookie + name: fairOS-dfs + schema: + type: string + - in: query + name: fields + schema: + type: object + properties: + pod_name: + $ref: '#/components/schemas/PodName' + required: + - pod_name + responses: + '200': + description: 'Ok' + content: + application/json: + schema: + $ref: '#/components/schemas/DirPresentResponse' + '400': + $ref: '#/components/responses/400' + '500': + $ref: '#/components/responses/500' + + '/dir/mkdir': + post: summary: 'Make dir' description: 'make a new directory inside a pod' tags: @@ -686,7 +722,7 @@ paths: $ref: '#/components/responses/500' '/dir/rmdir': - get: + delete: summary: 'Remove dir' description: 'remove a directory inside a pod' tags: @@ -833,7 +869,7 @@ paths: $ref: '#/components/responses/500' '/file/upload': - get: + post: summary: 'Upload File' description: 'upload a file to dfs' tags: @@ -921,8 +957,44 @@ paths: '500': $ref: '#/components/responses/500' + post: + summary: 'Download file' + description: 'Download a file from the pod tp the local dir' + tags: + - File System + parameters: + - in: cookie + name: fairOS-dfs + schema: + type: string + requestBody: + content: + application/json: + schema: + type: object + properties: + pod_name: + $ref: '#/components/schemas/PodName' + file_path: + $ref: '#/components/schemas/DirPath' + required: + - pod_name + - file_path + responses: + '200': + description: 'Ok' + content: + application/octet-stream: + schema: + type: string + format: binary + '400': + $ref: '#/components/responses/400' + '500': + $ref: '#/components/responses/500' + '/file/share': - get: + post: summary: 'Share file' description: 'Share a file with another user' tags: @@ -1244,10 +1316,9 @@ paths: '200': description: 'Ok' content: - application/octet-stream: + application/json: schema: - type: string - format: binary + $ref: '#/components/schemas/KVCount' '400': $ref: '#/components/responses/400' '500': @@ -1375,6 +1446,51 @@ paths: '500': $ref: '#/components/responses/500' + '/kv/entry/get-data': + get: + summary: 'Get Value' + description: 'Get value given a key' + tags: + - Key Value Store + parameters: + - in: cookie + name: fairOS-dfs + schema: + type: string + - in: query + name: fields + schema: + type: object + properties: + pod_name: + $ref: '#/components/schemas/PodName' + table_name: + $ref: '#/components/schemas/KVTableName' + key: + $ref: '#/components/schemas/KVKey' + format: + type: string + enum: [ string, byte-string ] + required: + - pod_name + - table_name + - key + responses: + '200': + description: 'Ok' + content: + application/json: + schema: + properties: + keys: + $ref: '#/components/schemas/KVgetResponseKeys' + values: + $ref: '#/components/schemas/KVgetResponseValues' + '400': + $ref: '#/components/responses/400' + '500': + $ref: '#/components/responses/500' + '/kv/entry/del': delete: summary: 'Delete Value' @@ -1544,6 +1660,44 @@ paths: '500': $ref: '#/components/responses/500' + '/kv/present': + get: + summary: 'Key Present' + description: 'Is Key present' + tags: + - Key Value Store + parameters: + - in: cookie + name: fairOS-dfs + schema: + type: string + - in: query + name: fields + schema: + type: object + properties: + pod_name: + $ref: '#/components/schemas/PodName' + table_name: + $ref: '#/components/schemas/KVTableName' + key: + $ref: '#/components/schemas/KVKey' + required: + - pod_name + - table_name + - key + responses: + '200': + description: 'Ok' + content: + application/json: + schema: + $ref: '#/components/schemas/DirPresentResponse' + '400': + $ref: '#/components/responses/400' + '500': + $ref: '#/components/responses/500' + '/doc/new': post: summary: 'Create DocumentDB' @@ -1701,7 +1855,7 @@ paths: '/doc/delete': delete: summary: 'Delete DocumentDB' - description: 'Delete the given ocument DB and all its documents and indexes' + description: 'Delete the given document DB and all its documents and indexes' tags: - Document DB parameters: @@ -2052,7 +2206,7 @@ components: $ref: '#/components/schemas/UserName' password: $ref: '#/components/schemas/Password' - address: + mnemonic: $ref: '#/components/schemas/Mnemonic' @@ -2361,6 +2515,17 @@ components: type: string example: "kv_table1" + KVCount: + type: object + properties: + count: + type: integer + table_name: + $ref: '#/components/schemas/KVTableName' + required: + - file_name + - reference + KVMemoryTable: type: string example: "true" @@ -2394,7 +2559,6 @@ components: type: string example: first_name=string,age=number,tags=map - DocTableName: type: string example: "doc_table1" diff --git a/pkg/api/handler.go b/pkg/api/handler.go index 86851e86..29f7a37b 100644 --- a/pkg/api/handler.go +++ b/pkg/api/handler.go @@ -24,15 +24,18 @@ import ( type Handler struct { dfsAPI *dfs.DfsAPI logger logging.Logger + + whitelistedOrigins []string } -func NewHandler(dataDir, beeHost, beePort, cookieDomain, postageBlockId string, logger logging.Logger) (*Handler, error) { - api, err := dfs.NewDfsAPI(dataDir, beeHost, beePort, cookieDomain, postageBlockId, logger) +func NewHandler(dataDir, beeApi, beeDebugApi, cookieDomain, postageBlockId string, whitelistedOrigins []string, logger logging.Logger) (*Handler, error) { + api, err := dfs.NewDfsAPI(dataDir, beeApi, beeDebugApi, cookieDomain, postageBlockId, logger) if err != nil { return nil, dfs.ErrBeeClient } return &Handler{ - dfsAPI: api, - logger: logger, + dfsAPI: api, + logger: logger, + whitelistedOrigins: whitelistedOrigins, }, nil } diff --git a/pkg/api/kv_put_get_del.go b/pkg/api/kv_put_get_del.go index 784022c3..dc6fd8dc 100644 --- a/pkg/api/kv_put_get_del.go +++ b/pkg/api/kv_put_get_del.go @@ -18,10 +18,11 @@ package api import ( "encoding/json" + "fmt" "net/http" "github.com/fairdatasociety/fairOS-dfs/cmd/common" - + "github.com/fairdatasociety/fairOS-dfs/pkg/collection" "github.com/fairdatasociety/fairOS-dfs/pkg/cookie" "resenje.org/jsonhttp" ) @@ -31,6 +32,11 @@ type KVResponse struct { Values []byte `json:"values"` } +type KVResponseRaw struct { + Keys []string `json:"keys,omitempty"` + Values string `json:"values"` +} + // KVPutHandler is the api handler to insert a key and value in to the kv table // it takes three arguments // - table_name: the name of the kv table @@ -104,7 +110,8 @@ func (h *Handler) KVPutHandler(w http.ResponseWriter, r *http.Request) { } // KVGetHandler is the api handler to get a value from the kv table -// it takes two arguments +// it takes three arguments +// - pod_name: the name of the pod // - table_name: the name of the kv table // - key: the key string func (h *Handler) KVGetHandler(w http.ResponseWriter, r *http.Request) { @@ -163,6 +170,10 @@ func (h *Handler) KVGetHandler(w http.ResponseWriter, r *http.Request) { columns, data, err := h.dfsAPI.KVGet(sessionId, podName, name, key) if err != nil { h.logger.Errorf("kv get: %v", err) + if err == collection.ErrEntryNotFound { + jsonhttp.NotFound(w, "kv get: "+err.Error()) + return + } jsonhttp.InternalServerError(w, "kv get: "+err.Error()) return } @@ -179,6 +190,112 @@ func (h *Handler) KVGetHandler(w http.ResponseWriter, r *http.Request) { jsonhttp.OK(w, &resp) } +// KVGetDataHandler is the api handler to get a value from the kv table +// it takes four arguments +// - pod_name: the name of the pod +// - table_name: the name of the kv table +// - key: the key string +// - format: whether the data should be string or byte-string +func (h *Handler) KVGetDataHandler(w http.ResponseWriter, r *http.Request) { + keys, ok := r.URL.Query()["pod_name"] + if !ok || len(keys[0]) < 1 { + h.logger.Errorf("kv get: \"pod_name\" argument missing") + jsonhttp.BadRequest(w, "kv get: \"pod_name\" argument missing") + return + } + podName := keys[0] + if podName == "" { + h.logger.Errorf("kv get: \"pod_name\" argument missing") + jsonhttp.BadRequest(w, "kv get: \"pod_name\" argument missing") + return + } + + keys, ok = r.URL.Query()["table_name"] + if !ok || len(keys[0]) < 1 { + h.logger.Errorf("kv get: \"table_name\" argument missing") + jsonhttp.BadRequest(w, "kv get: \"table_name\" argument missing") + return + } + name := keys[0] + if name == "" { + h.logger.Errorf("kv get: \"table_name\" argument missing") + jsonhttp.BadRequest(w, "kv get: \"table_name\" argument missing") + return + } + + keys, ok = r.URL.Query()["key"] + if !ok || len(keys[0]) < 1 { + h.logger.Errorf("kv get: \"sharing_ref\" argument missing") + jsonhttp.BadRequest(w, "kv get: \"sharing_ref\" argument missing") + return + } + key := keys[0] + if key == "" { + h.logger.Errorf("kv get: \"key\" argument missing") + jsonhttp.BadRequest(w, "kv get: \"key\" argument missing") + return + } + + formats, ok := r.URL.Query()["format"] + if !ok || len(formats[0]) < 1 { + h.logger.Errorf("kv get: \"format\" argument missing") + jsonhttp.BadRequest(w, "kv get: \"format\" argument missing") + return + } + format := formats[0] + if format == "" { + h.logger.Errorf("kv get: \"format\" argument missing") + jsonhttp.BadRequest(w, "kv get: \"format\" argument missing") + return + } + + if format != "string" && format != "byte-string" { + h.logger.Errorf("kv get: \"format\" argument is unknown") + jsonhttp.BadRequest(w, "kv get: \"format\" argument is unknown") + return + } + + // get values from cookie + sessionId, err := cookie.GetSessionIdFromCookie(r) + if err != nil { + h.logger.Errorf("kv get: invalid cookie: %v", err) + jsonhttp.BadRequest(w, ErrInvalidCookie) + return + } + if sessionId == "" { + h.logger.Errorf("kv get: \"cookie-id\" parameter missing in cookie") + jsonhttp.BadRequest(w, "kv get: \"cookie-id\" parameter missing in cookie") + return + } + + columns, data, err := h.dfsAPI.KVGet(sessionId, podName, name, key) + if err != nil { + h.logger.Errorf("kv get: %v", err) + if err == collection.ErrEntryNotFound { + jsonhttp.NotFound(w, "kv get: "+err.Error()) + return + } + jsonhttp.InternalServerError(w, "kv get: "+err.Error()) + return + } + + var resp KVResponseRaw + if columns != nil { + resp.Keys = columns + } else { + resp.Keys = []string{key} + } + + if format == "string" { + resp.Values = string(data) + } else { + resp.Values = fmt.Sprintf("%v", data) + } + + w.Header().Set("Content-Type", "application/json") + jsonhttp.OK(w, &resp) +} + // KVDelHandler is the api handler to delete a key and value from the kv table // it takes two arguments // - table_name: the name of the kv table @@ -242,3 +359,75 @@ func (h *Handler) KVDelHandler(w http.ResponseWriter, r *http.Request) { } jsonhttp.OK(w, "key deleted") } + +// KVPresentHandler is the api handler to check if a value exists in the kv table +// it takes three arguments +// - pod_name: the name of the pod +// - table_name: the name of the kv table +// - key: the key string +func (h *Handler) KVPresentHandler(w http.ResponseWriter, r *http.Request) { + keys, ok := r.URL.Query()["pod_name"] + if !ok || len(keys[0]) < 1 { + h.logger.Errorf("kv get: \"pod_name\" argument missing") + jsonhttp.BadRequest(w, "kv get: \"pod_name\" argument missing") + return + } + podName := keys[0] + if podName == "" { + h.logger.Errorf("kv get: \"pod_name\" argument missing") + jsonhttp.BadRequest(w, "kv get: \"pod_name\" argument missing") + return + } + + keys, ok = r.URL.Query()["table_name"] + if !ok || len(keys[0]) < 1 { + h.logger.Errorf("kv get: \"table_name\" argument missing") + jsonhttp.BadRequest(w, "kv get: \"table_name\" argument missing") + return + } + name := keys[0] + if name == "" { + h.logger.Errorf("kv get: \"table_name\" argument missing") + jsonhttp.BadRequest(w, "kv get: \"table_name\" argument missing") + return + } + + keys, ok = r.URL.Query()["key"] + if !ok || len(keys[0]) < 1 { + h.logger.Errorf("kv get: \"sharing_ref\" argument missing") + jsonhttp.BadRequest(w, "kv get: \"sharing_ref\" argument missing") + return + } + key := keys[0] + if key == "" { + h.logger.Errorf("kv get: \"key\" argument missing") + jsonhttp.BadRequest(w, "kv get: \"key\" argument missing") + return + } + + // get values from cookie + sessionId, err := cookie.GetSessionIdFromCookie(r) + if err != nil { + h.logger.Errorf("kv get: invalid cookie: %v", err) + jsonhttp.BadRequest(w, ErrInvalidCookie) + return + } + if sessionId == "" { + h.logger.Errorf("kv get: \"cookie-id\" parameter missing in cookie") + jsonhttp.BadRequest(w, "kv get: \"cookie-id\" parameter missing in cookie") + return + } + w.Header().Set("Content-Type", "application/json") + + _, _, err = h.dfsAPI.KVGet(sessionId, podName, name, key) + if err != nil { + jsonhttp.OK(w, &PresentResponse{ + Present: false, + }) + return + } + + jsonhttp.OK(w, &PresentResponse{ + Present: true, + }) +} diff --git a/pkg/api/pod_present.go b/pkg/api/pod_present.go new file mode 100644 index 00000000..935ceded --- /dev/null +++ b/pkg/api/pod_present.go @@ -0,0 +1,48 @@ +package api + +import ( + "net/http" + + "github.com/fairdatasociety/fairOS-dfs/pkg/cookie" + + "resenje.org/jsonhttp" +) + +// PodPresentHandler is the api handler to check if a pod is present +// it takes pod_name as query parameter +func (h *Handler) PodPresentHandler(w http.ResponseWriter, r *http.Request) { + keys, ok := r.URL.Query()["pod_name"] + if !ok || len(keys[0]) < 1 { + h.logger.Errorf("doc ls: \"pod_name\" argument missing") + jsonhttp.BadRequest(w, "doc ls: \"pod_name\" argument missing") + return + } + podName := keys[0] + if podName == "" { + h.logger.Errorf("doc ls: \"pod_name\" argument missing") + jsonhttp.BadRequest(w, "doc ls: \"pod_name\" argument missing") + return + } + + // get values from cookie + sessionId, err := cookie.GetSessionIdFromCookie(r) + if err != nil { + h.logger.Errorf("pod open: invalid cookie: %v", err) + jsonhttp.BadRequest(w, ErrInvalidCookie) + return + } + if sessionId == "" { + h.logger.Errorf("pod open: \"cookie-id\" parameter missing in cookie") + jsonhttp.BadRequest(w, "pod open: \"cookie-id\" parameter missing in cookie") + return + } + if h.dfsAPI.IsPodExist(podName, sessionId) { + jsonhttp.OK(w, &PresentResponse{ + Present: true, + }) + } else { + jsonhttp.OK(w, &PresentResponse{ + Present: false, + }) + } +} diff --git a/pkg/api/user_login.go b/pkg/api/user_login.go index 4f9816f2..e63f8788 100644 --- a/pkg/api/user_login.go +++ b/pkg/api/user_login.go @@ -49,8 +49,8 @@ func (h *Handler) UserLoginHandler(w http.ResponseWriter, r *http.Request) { user := userReq.UserName password := userReq.Password if user == "" { - h.logger.Errorf("user login: \"user\" argument missing") - jsonhttp.BadRequest(w, "user login: \"user\" argument missing") + h.logger.Errorf("user login: \"user_name\" argument missing") + jsonhttp.BadRequest(w, "user login: \"user_name\" argument missing") return } if password == "" { diff --git a/pkg/api/user_present.go b/pkg/api/user_present.go index 27520496..f9f1c343 100644 --- a/pkg/api/user_present.go +++ b/pkg/api/user_present.go @@ -22,7 +22,7 @@ import ( "resenje.org/jsonhttp" ) -type UserPresentResponse struct { +type PresentResponse struct { Present bool `json:"present"` } @@ -47,11 +47,11 @@ func (h *Handler) UserPresentHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", " application/json") // check if user is present if h.dfsAPI.IsUserNameAvailable(user) { - jsonhttp.OK(w, &UserPresentResponse{ + jsonhttp.OK(w, &PresentResponse{ Present: true, }) } else { - jsonhttp.OK(w, &UserPresentResponse{ + jsonhttp.OK(w, &PresentResponse{ Present: false, }) } diff --git a/pkg/api/ws.go b/pkg/api/ws.go index b15b51dc..008ae48a 100644 --- a/pkg/api/ws.go +++ b/pkg/api/ws.go @@ -31,8 +31,13 @@ var ( func (h *Handler) WebsocketHandler(w http.ResponseWriter, r *http.Request) { upgrader := websocket.Upgrader{} // use default options upgrader.CheckOrigin = func(r *http.Request) bool { - // TODO: whitelist origins - return true + origin := r.Header.Get("Origin") + for _, v := range h.whitelistedOrigins { + if origin == v { + return true + } + } + return false } conn, err := upgrader.Upgrade(w, r, nil) if err != nil { diff --git a/pkg/blockstore/bee/client.go b/pkg/blockstore/bee/client.go index 3586ef64..05c13cd9 100644 --- a/pkg/blockstore/bee/client.go +++ b/pkg/blockstore/bee/client.go @@ -54,9 +54,8 @@ const ( ) type BeeClient struct { - host string - port string url string + debugUrl string client *http.Client hasher *bmtlegacy.Hasher chunkCache *lru.Cache @@ -75,7 +74,7 @@ type bytesPostResponse struct { } // NewBeeClient creates a new client which connects to the Swarm bee node to access the Swarm network. -func NewBeeClient(host, port, postageBlockId string, logger logging.Logger) *BeeClient { +func NewBeeClient(apiUrl, debugApiUrl, postageBlockId string, logger logging.Logger) *BeeClient { p := bmtlegacy.NewTreePool(hashFunc, swarm.Branches, bmtlegacy.PoolSize) cache, err := lru.New(chunkCacheSize) if err != nil { @@ -91,9 +90,8 @@ func NewBeeClient(host, port, postageBlockId string, logger logging.Logger) *Bee } return &BeeClient{ - host: host, - port: port, - url: fmt.Sprintf("http://" + host + ":" + port), + url: apiUrl, + debugUrl: debugApiUrl, client: createHTTPClient(), hasher: bmtlegacy.New(p), chunkCache: cache, @@ -462,7 +460,7 @@ func (s *BeeClient) GetNewPostageBatch() error { to := time.Now() s.logger.Infof("Trying to get new postage batch id") path := filepath.Join(postageBatchUrl, "10000000/20") - fullUrl := fmt.Sprintf(s.url + path) + fullUrl := fmt.Sprintf(s.debugUrl + path) req, err := http.NewRequest(http.MethodPost, fullUrl, nil) if err != nil { return err diff --git a/pkg/dfs/api.go b/pkg/dfs/api.go index 1ab577da..fec98fba 100644 --- a/pkg/dfs/api.go +++ b/pkg/dfs/api.go @@ -31,8 +31,8 @@ type DfsAPI struct { } // NewDfsAPI is the main entry point for the df controller. -func NewDfsAPI(dataDir, host, port, cookieDomain, postageBlockId string, logger logging.Logger) (*DfsAPI, error) { - c := bee.NewBeeClient(host, port, postageBlockId, logger) +func NewDfsAPI(dataDir, apiUrl, debugApiUrl, cookieDomain, postageBlockId string, logger logging.Logger) (*DfsAPI, error) { + c := bee.NewBeeClient(apiUrl, debugApiUrl, postageBlockId, logger) if !c.CheckConnection() { return nil, ErrBeeClient } diff --git a/pkg/dfs/pod_api.go b/pkg/dfs/pod_api.go index 89ff4203..36e929aa 100644 --- a/pkg/dfs/pod_api.go +++ b/pkg/dfs/pod_api.go @@ -245,3 +245,11 @@ func (d *DfsAPI) PodReceive(sessionId string, ref utils.Reference) (*pod.Info, e return ui.GetPod().ReceivePod(ref) } + +func (d *DfsAPI) IsPodExist(podName, sessionId string) bool { + ui := d.users.GetLoggedInUserInfo(sessionId) + if ui == nil { + return false + } + return ui.GetPod().IsPodPresent(podName) +} diff --git a/pkg/file/upload.go b/pkg/file/upload.go index bbd1780c..5a63e1e3 100644 --- a/pkg/file/upload.go +++ b/pkg/file/upload.go @@ -65,69 +65,86 @@ func (f *File) Upload(fd io.Reader, podFileName string, fileSize int64, blockSiz refMap := make(map[int]*BlockInfo) refMapMu := sync.RWMutex{} var contentBytes []byte - for { - data := make([]byte, blockSize, blockSize+1024) - r, err := reader.Read(data) - totalLength += uint64(r) - if err != nil { - if err == io.EOF { - if totalLength < uint64(fileSize) { - return fmt.Errorf("invalid file length of file data received") + wg.Add(1) + go func() { + var mainErr error + for { + if mainErr != nil { + errC <- mainErr + wg.Done() + return + } + data := make([]byte, blockSize, blockSize+1024) + r, err := reader.Read(data) + totalLength += uint64(r) + if err != nil { + if err == io.EOF { + if totalLength < uint64(fileSize) { + errC <- fmt.Errorf("invalid file length of file data received") + return + } + wg.Done() + break } - break + errC <- err + return } - return err - } - // determine the content type from the first 512 bytes of the file - if len(contentBytes) < 512 { - contentBytes = append(contentBytes, data[:r]...) - if len(contentBytes) >= 512 { - cBytes := bytes.NewReader(contentBytes[:512]) - cReader := bufio.NewReader(cBytes) - meta.ContentType = f.getContentType(cReader) + // determine the content type from the first 512 bytes of the file + if len(contentBytes) < 512 { + contentBytes = append(contentBytes, data[:r]...) + if len(contentBytes) >= 512 { + cBytes := bytes.NewReader(contentBytes[:512]) + cReader := bufio.NewReader(cBytes) + meta.ContentType = f.getContentType(cReader) + } } - } - wg.Add(1) - worker <- true - go func(counter, size int) { - blockName := fmt.Sprintf("block-%05d", counter) - defer func() { - <-worker - wg.Done() - f.logger.Info("done uploading block ", blockName) - }() - - f.logger.Info("Uploading ", blockName) - // compress the data - uploadData := data[:size] - if compression != "" { - uploadData, err = compress(data[:size], compression, blockSize) - if err != nil { - errC <- err + wg.Add(1) + worker <- true + go func(counter, size int) { + blockName := fmt.Sprintf("block-%05d", counter) + defer func() { + <-worker + wg.Done() + if mainErr != nil { + f.logger.Error("failed uploading block ", blockName) + return + } + f.logger.Info("done uploading block ", blockName) + }() + + f.logger.Info("Uploading ", blockName) + // compress the data + uploadData := data[:size] + if compression != "" { + uploadData, err = compress(data[:size], compression, blockSize) + if err != nil { + mainErr = err + return + } } - } - addr, uploadErr := f.client.UploadBlob(uploadData, true, true) - if uploadErr != nil { - errC <- uploadErr - return - } - fileBlock := &BlockInfo{ - Name: blockName, - Size: uint32(size), - CompressedSize: uint32(len(uploadData)), - Reference: utils.NewReference(addr), - } + addr, uploadErr := f.client.UploadBlob(uploadData, true, true) + if uploadErr != nil { + mainErr = uploadErr + return + } + fileBlock := &BlockInfo{ + Name: blockName, + Size: uint32(size), + CompressedSize: uint32(len(uploadData)), + Reference: utils.NewReference(addr), + } - refMapMu.Lock() - defer refMapMu.Unlock() - refMap[counter] = fileBlock - }(i, r) + refMapMu.Lock() + defer refMapMu.Unlock() + refMap[counter] = fileBlock + }(i, r) - i++ - } + i++ + } + }() go func() { wg.Wait() diff --git a/pkg/pod/utils.go b/pkg/pod/utils.go index 0fd1e257..a27d3bbb 100644 --- a/pkg/pod/utils.go +++ b/pkg/pod/utils.go @@ -33,6 +33,25 @@ func (p *Pod) IsPodOpened(podName string) bool { return false } +func (p *Pod) IsPodPresent(podName string) bool { + podName, err := CleanPodName(podName) + if err != nil { + return false + } + // check if pods is present and get free index + pods, sharedPods, err := p.loadUserPods() + if err != nil { + return false + } + if p.checkIfPodPresent(pods, podName) { + return true + } + if p.checkIfSharedPodPresent(sharedPods, podName) { + return true + } + return false +} + func (*Pod) GetPath(inode *d.Inode) string { if inode != nil { return inode.Meta.Path diff --git a/vendor/modules.txt b/vendor/modules.txt index 676393e3..6ecb3b4d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -329,6 +329,7 @@ gopkg.in/ini.v1 # gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/natefinch/npipe.v2 # gopkg.in/yaml.v2 v2.3.0 +## explicit gopkg.in/yaml.v2 # resenje.org/jsonhttp v0.2.0 ## explicit diff --git a/version.go b/version.go index 99cbbc09..66db20e1 100644 --- a/version.go +++ b/version.go @@ -5,7 +5,7 @@ package dfs var ( - version = "0.7.1" // manually set semantic version number + version = "0.7.3" // manually set semantic version number commit string // automatically set git commit hash Version = func() string {