diff --git a/README.md b/README.md index f1ac6cb88f..a7156888b9 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,8 @@ In this document, we use the default configuration, which has the following beha The GraphQL endpoint can be used with a GraphQL client (e.g., Altair) to conveniently perform requests (`query`, `mutation`) and obtain schema introspection. +Read more about the configuration [here](./docs/config.md). + ## External port binding By default the HTTP API and P2P network will use localhost. If you want to expose the ports externally you need to specify the addresses in the config or command line parameters. diff --git a/cli/backup_export.go b/cli/backup_export.go index 9e8d1c056e..b905bdf9c7 100644 --- a/cli/backup_export.go +++ b/cli/backup_export.go @@ -38,7 +38,7 @@ Example: export data for the 'Users' collection: defradb client export --collection Users user_data.json`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) if !isValidExportFormat(format) { return ErrInvalidExportFormat diff --git a/cli/backup_import.go b/cli/backup_import.go index 35af345a0a..56f1907643 100644 --- a/cli/backup_import.go +++ b/cli/backup_import.go @@ -24,7 +24,7 @@ Example: import data to the database: defradb client import user_data.json`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) return store.BasicImport(cmd.Context(), args[0]) }, } diff --git a/cli/cli.go b/cli/cli.go index b7d5e05ec7..4cdb8c443b 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -16,14 +16,13 @@ package cli import ( "github.com/spf13/cobra" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/logging" ) var log = logging.MustNewLogger("cli") // NewDefraCommand returns the root command instanciated with its tree of subcommands. -func NewDefraCommand(cfg *config.Config) *cobra.Command { +func NewDefraCommand() *cobra.Command { p2p_collection := MakeP2PCollectionCommand() p2p_collection.AddCommand( MakeP2PCollectionAddCommand(), @@ -83,12 +82,12 @@ func NewDefraCommand(cfg *config.Config) *cobra.Command { tx := MakeTxCommand() tx.AddCommand( - MakeTxCreateCommand(cfg), - MakeTxCommitCommand(cfg), - MakeTxDiscardCommand(cfg), + MakeTxCreateCommand(), + MakeTxCommitCommand(), + MakeTxDiscardCommand(), ) - collection := MakeCollectionCommand(cfg) + collection := MakeCollectionCommand() collection.AddCommand( MakeCollectionGetCommand(), MakeCollectionListDocIDsCommand(), @@ -98,7 +97,7 @@ func NewDefraCommand(cfg *config.Config) *cobra.Command { MakeCollectionDescribeCommand(), ) - client := MakeClientCommand(cfg) + client := MakeClientCommand() client.AddCommand( MakeDumpCommand(), MakeRequestCommand(), @@ -111,13 +110,12 @@ func NewDefraCommand(cfg *config.Config) *cobra.Command { collection, ) - root := MakeRootCommand(cfg) + root := MakeRootCommand() root.AddCommand( client, - MakeStartCommand(cfg), - MakeServerDumpCmd(cfg), + MakeStartCommand(), + MakeServerDumpCmd(), MakeVersionCommand(), - MakeInitCommand(cfg), ) return root diff --git a/cli/client.go b/cli/client.go index 8866294f69..532712e8f8 100644 --- a/cli/client.go +++ b/cli/client.go @@ -12,11 +12,9 @@ package cli import ( "github.com/spf13/cobra" - - "github.com/sourcenetwork/defradb/config" ) -func MakeClientCommand(cfg *config.Config) *cobra.Command { +func MakeClientCommand() *cobra.Command { var txID uint64 var cmd = &cobra.Command{ Use: "client", @@ -24,13 +22,16 @@ func MakeClientCommand(cfg *config.Config) *cobra.Command { Long: `Interact with a DefraDB node. Execute queries, add schema types, obtain node info, etc.`, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := loadConfig(cfg); err != nil { + if err := setContextRootDir(cmd); err != nil { + return err + } + if err := setContextConfig(cmd); err != nil { return err } - if err := setTransactionContext(cmd, cfg, txID); err != nil { + if err := setContextTransaction(cmd, txID); err != nil { return err } - return setStoreContext(cmd, cfg) + return setContextStore(cmd) }, } cmd.PersistentFlags().Uint64Var(&txID, "tx", 0, "Transaction ID") diff --git a/cli/collection.go b/cli/collection.go index 996af66f9a..b15a627cb4 100644 --- a/cli/collection.go +++ b/cli/collection.go @@ -16,11 +16,10 @@ import ( "github.com/spf13/cobra" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/datastore" ) -func MakeCollectionCommand(cfg *config.Config) *cobra.Command { +func MakeCollectionCommand() *cobra.Command { var txID uint64 var name string var schemaRoot string @@ -31,16 +30,19 @@ func MakeCollectionCommand(cfg *config.Config) *cobra.Command { Long: `Create, read, update, and delete documents within a collection.`, PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { // cobra does not chain pre run calls so we have to run them again here - if err := loadConfig(cfg); err != nil { + if err := setContextRootDir(cmd); err != nil { return err } - if err := setTransactionContext(cmd, cfg, txID); err != nil { + if err := setContextConfig(cmd); err != nil { return err } - if err := setStoreContext(cmd, cfg); err != nil { + if err := setContextTransaction(cmd, txID); err != nil { return err } - store := mustGetStoreContext(cmd) + if err := setContextStore(cmd); err != nil { + return err + } + store := mustGetContextStore(cmd) var col client.Collection var cols []client.Collection diff --git a/cli/collection_create.go b/cli/collection_create.go index 82e1e5db09..efeee61494 100644 --- a/cli/collection_create.go +++ b/cli/collection_create.go @@ -40,7 +40,7 @@ Example: create from stdin `, Args: cobra.RangeArgs(0, 1), RunE: func(cmd *cobra.Command, args []string) error { - col, ok := tryGetCollectionContext(cmd) + col, ok := tryGetContextCollection(cmd) if !ok { return cmd.Usage() } diff --git a/cli/collection_delete.go b/cli/collection_delete.go index dcd7c9d872..d1f945d9ae 100644 --- a/cli/collection_delete.go +++ b/cli/collection_delete.go @@ -31,7 +31,7 @@ Example: delete by filter defradb client collection delete --name User --filter '{ "_gte": { "points": 100 } }' `, RunE: func(cmd *cobra.Command, args []string) error { - col, ok := tryGetCollectionContext(cmd) + col, ok := tryGetContextCollection(cmd) if !ok { return cmd.Usage() } diff --git a/cli/collection_describe.go b/cli/collection_describe.go index 0e2e36e718..19719329f1 100644 --- a/cli/collection_describe.go +++ b/cli/collection_describe.go @@ -36,9 +36,9 @@ Example: view collection by version id defradb client collection describe --version bae123 `, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) - col, ok := tryGetCollectionContext(cmd) + col, ok := tryGetContextCollection(cmd) if ok { return writeJSON(cmd, col.Definition()) } diff --git a/cli/collection_get.go b/cli/collection_get.go index d753e0a8db..55c84d6289 100644 --- a/cli/collection_get.go +++ b/cli/collection_get.go @@ -28,7 +28,7 @@ Example: `, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - col, ok := tryGetCollectionContext(cmd) + col, ok := tryGetContextCollection(cmd) if !ok { return cmd.Usage() } diff --git a/cli/collection_list_doc_ids.go b/cli/collection_list_doc_ids.go index d7009cb300..7112a88817 100644 --- a/cli/collection_list_doc_ids.go +++ b/cli/collection_list_doc_ids.go @@ -26,7 +26,7 @@ Example: defradb client collection docIDs --name User `, RunE: func(cmd *cobra.Command, args []string) error { - col, ok := tryGetCollectionContext(cmd) + col, ok := tryGetContextCollection(cmd) if !ok { return cmd.Usage() } diff --git a/cli/collection_update.go b/cli/collection_update.go index 9fd2deed3f..42354948a9 100644 --- a/cli/collection_update.go +++ b/cli/collection_update.go @@ -38,7 +38,7 @@ Example: update by docIDs `, Args: cobra.RangeArgs(0, 1), RunE: func(cmd *cobra.Command, args []string) error { - col, ok := tryGetCollectionContext(cmd) + col, ok := tryGetContextCollection(cmd) if !ok { return cmd.Usage() } diff --git a/cli/config.go b/cli/config.go new file mode 100644 index 0000000000..bb57a8cb3d --- /dev/null +++ b/cli/config.go @@ -0,0 +1,185 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package cli + +import ( + "os" + "path/filepath" + "strings" + + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "github.com/sourcenetwork/defradb/logging" +) + +const ( + configStoreBadger = "badger" + configStoreMemory = "memory" + configLogFormatJSON = "json" + configLogFormatCSV = "csv" + configLogLevelInfo = "info" + configLogLevelDebug = "debug" + configLogLevelError = "error" + configLogLevelFatal = "fatal" +) + +// configPaths are config keys that will be made relative to the rootdir +var configPaths = []string{ + "datastore.badger.path", + "api.pubkeypath", + "api.privkeypath", +} + +// configFlags is a mapping of config keys to cli flags to bind to. +var configFlags = map[string]string{ + "log.level": "loglevel", + "log.output": "logoutput", + "log.format": "logformat", + "log.stacktrace": "logtrace", + "log.nocolor": "lognocolor", + "api.address": "url", + "datastore.maxtxnretries": "max-txn-retries", + "datastore.store": "store", + "datastore.badger.valuelogfilesize": "valuelogfilesize", + "net.peers": "peers", + "net.p2paddresses": "p2paddr", + "net.p2pdisabled": "no-p2p", + "api.allowed-origins": "allowed-origins", + "api.pubkeypath": "pubkeypath", + "api.privkeypath": "privkeypath", +} + +// defaultConfig returns a new config with default values. +func defaultConfig() *viper.Viper { + cfg := viper.New() + + cfg.AutomaticEnv() + cfg.SetEnvPrefix("DEFRA") + cfg.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_")) + + cfg.SetConfigName("config") + cfg.SetConfigType("yaml") + + cfg.SetDefault("datastore.badger.path", "data") + cfg.SetDefault("net.pubSubEnabled", true) + cfg.SetDefault("net.relay", false) + cfg.SetDefault("log.caller", false) + + return cfg +} + +// createConfig writes the default config file if one does not exist. +func createConfig(rootdir string, flags *pflag.FlagSet) error { + cfg := defaultConfig() + cfg.AddConfigPath(rootdir) + + if err := bindConfigFlags(cfg, flags); err != nil { + return err + } + // make sure rootdir exists + if err := os.MkdirAll(rootdir, 0755); err != nil { + return err + } + err := cfg.SafeWriteConfig() + // error type is known and shouldn't be wrapped + // + //nolint:errorlint + if _, ok := err.(viper.ConfigFileAlreadyExistsError); ok { + return nil + } + return err +} + +// loadConfig returns a new config with values from the config in the given rootdir. +func loadConfig(rootdir string, flags *pflag.FlagSet) (*viper.Viper, error) { + cfg := defaultConfig() + cfg.AddConfigPath(rootdir) + + // attempt to read the existing config + err := cfg.ReadInConfig() + // error type is known and shouldn't be wrapped + // + //nolint:errorlint + if _, ok := err.(viper.ConfigFileNotFoundError); err != nil && !ok { + return nil, err + } + // bind cli flags to config keys + if err := bindConfigFlags(cfg, flags); err != nil { + return nil, err + } + + // make paths relative to the rootdir + for _, key := range configPaths { + path := cfg.GetString(key) + if path != "" && !filepath.IsAbs(path) { + cfg.Set(key, filepath.Join(rootdir, path)) + } + } + + logCfg := loggingConfig(cfg.Sub("log")) + logCfg.OverridesByLoggerName = make(map[string]logging.Config) + + // apply named logging overrides + for key := range cfg.GetStringMap("log.overrides") { + logCfg.OverridesByLoggerName[key] = loggingConfig(cfg.Sub("log.overrides." + key)) + } + logging.SetConfig(logCfg) + + return cfg, nil +} + +// bindConfigFlags binds the set of cli flags to config values. +func bindConfigFlags(cfg *viper.Viper, flags *pflag.FlagSet) error { + for key, flag := range configFlags { + err := cfg.BindPFlag(key, flags.Lookup(flag)) + if err != nil { + return err + } + } + return nil +} + +// loggingConfig returns a new logging config from the given config. +func loggingConfig(cfg *viper.Viper) logging.Config { + var level int8 + switch value := cfg.GetString("level"); value { + case configLogLevelDebug: + level = logging.Debug + case configLogLevelInfo: + level = logging.Info + case configLogLevelError: + level = logging.Error + case configLogLevelFatal: + level = logging.Fatal + default: + level = logging.Info + } + + var format logging.EncoderFormat + switch value := cfg.GetString("format"); value { + case configLogFormatJSON: + format = logging.JSON + case configLogFormatCSV: + format = logging.CSV + default: + format = logging.CSV + } + + return logging.Config{ + Level: logging.NewLogLevelOption(level), + EnableStackTrace: logging.NewEnableStackTraceOption(cfg.GetBool("stacktrace")), + DisableColor: logging.NewDisableColorOption(cfg.GetBool("nocolor")), + EncoderFormat: logging.NewEncoderFormatOption(format), + OutputPaths: []string{cfg.GetString("output")}, + EnableCaller: logging.NewEnableCallerOption(cfg.GetBool("caller")), + } +} diff --git a/cli/config_test.go b/cli/config_test.go new file mode 100644 index 0000000000..210743477c --- /dev/null +++ b/cli/config_test.go @@ -0,0 +1,61 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package cli + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateConfig(t *testing.T) { + rootdir := t.TempDir() + err := createConfig(rootdir, NewDefraCommand().PersistentFlags()) + require.NoError(t, err) + + // ensure no errors when config already exists + err = createConfig(rootdir, NewDefraCommand().PersistentFlags()) + require.NoError(t, err) + + assert.FileExists(t, filepath.Join(rootdir, "config.yaml")) +} + +func TestLoadConfigNotExist(t *testing.T) { + rootdir := t.TempDir() + cfg, err := loadConfig(rootdir, NewDefraCommand().PersistentFlags()) + require.NoError(t, err) + + assert.Equal(t, 5, cfg.GetInt("datastore.maxtxnretries")) + + assert.Equal(t, filepath.Join(rootdir, "data"), cfg.GetString("datastore.badger.path")) + assert.Equal(t, 1<<30, cfg.GetInt("datastore.badger.valuelogfilesize")) + assert.Equal(t, "badger", cfg.GetString("datastore.store")) + + assert.Equal(t, "127.0.0.1:9181", cfg.GetString("api.address")) + assert.Equal(t, []string{}, cfg.GetStringSlice("api.allowed-origins")) + assert.Equal(t, "", cfg.GetString("api.pubkeypath")) + assert.Equal(t, "", cfg.GetString("api.privkeypath")) + + assert.Equal(t, false, cfg.GetBool("net.p2pdisabled")) + assert.Equal(t, []string{"/ip4/127.0.0.1/tcp/9171"}, cfg.GetStringSlice("net.p2paddresses")) + assert.Equal(t, true, cfg.GetBool("net.pubsubenabled")) + assert.Equal(t, false, cfg.GetBool("net.relay")) + assert.Equal(t, []string{}, cfg.GetStringSlice("net.peers")) + + assert.Equal(t, "info", cfg.GetString("log.level")) + assert.Equal(t, false, cfg.GetBool("log.stacktrace")) + assert.Equal(t, "csv", cfg.GetString("log.format")) + assert.Equal(t, "stderr", cfg.GetString("log.output")) + assert.Equal(t, false, cfg.GetBool("log.nocolor")) + assert.Equal(t, false, cfg.GetBool("log.caller")) +} diff --git a/cli/index_create.go b/cli/index_create.go index 099eb7e7a6..bfe5ec64c2 100644 --- a/cli/index_create.go +++ b/cli/index_create.go @@ -37,7 +37,7 @@ Example: create a named index for 'Users' collection on 'name' field: defradb client index create --collection Users --fields name --name UsersByName`, ValidArgs: []string{"collection", "fields", "name"}, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var fields []client.IndexedFieldDescription for _, name := range fieldsArg { diff --git a/cli/index_drop.go b/cli/index_drop.go index 03639fb277..96f007268d 100644 --- a/cli/index_drop.go +++ b/cli/index_drop.go @@ -28,7 +28,7 @@ Example: drop the index 'UsersByName' for 'Users' collection: defradb client index create --collection Users --name UsersByName`, ValidArgs: []string{"collection", "name"}, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) col, err := store.GetCollectionByName(cmd.Context(), collectionArg) if err != nil { diff --git a/cli/index_list.go b/cli/index_list.go index 92ada3e007..bf1fd21251 100644 --- a/cli/index_list.go +++ b/cli/index_list.go @@ -30,7 +30,7 @@ Example: show all index for 'Users' collection: defradb client index list --collection Users`, ValidArgs: []string{"collection"}, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) switch { case collectionArg != "": diff --git a/cli/init.go b/cli/init.go deleted file mode 100644 index f9af1850b7..0000000000 --- a/cli/init.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package cli - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/sourcenetwork/defradb/config" - "github.com/sourcenetwork/defradb/errors" -) - -/* -The `init` command initializes the configuration file and root directory. - -It covers three possible situations: -- root dir doesn't exist -- root dir exists and doesn't contain a config file -- root dir exists and contains a config file -*/ -func MakeInitCommand(cfg *config.Config) *cobra.Command { - var reinitialize bool - var cmd = &cobra.Command{ - Use: "init", - Short: "Initialize DefraDB's root directory and configuration file", - Long: `Initialize a directory for configuration and data at the given path. -Passed flags will be persisted in the stored configuration.`, - // Load a default configuration, considering env. variables and CLI flags. - PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { - if err := cfg.LoadWithRootdir(false); err != nil { - return errors.Wrap("failed to load configuration", err) - } - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - if config.FolderExists(cfg.Rootdir) { - if cfg.ConfigFileExists() { - if reinitialize { - if err := cfg.DeleteConfigFile(); err != nil { - return err - } - if err := cfg.WriteConfigFile(); err != nil { - return err - } - } else { - log.FeedbackError( - cmd.Context(), - fmt.Sprintf( - "Configuration file already exists at %v. Consider using --reinitialize", - cfg.ConfigFilePath(), - ), - ) - } - } else { - if err := cfg.WriteConfigFile(); err != nil { - return errors.Wrap("failed to create configuration file", err) - } - } - } else { - if err := cfg.CreateRootDirAndConfigFile(); err != nil { - return err - } - } - return nil - }, - } - - cmd.Flags().BoolVar( - &reinitialize, "reinitialize", false, - "Reinitialize the configuration file", - ) - - return cmd -} diff --git a/cli/p2p_collection_add.go b/cli/p2p_collection_add.go index dedae0a358..8a867e6abb 100644 --- a/cli/p2p_collection_add.go +++ b/cli/p2p_collection_add.go @@ -31,7 +31,7 @@ Example: add multiple collections `, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - p2p := mustGetP2PContext(cmd) + p2p := mustGetContextP2P(cmd) var collectionIDs []string for _, id := range strings.Split(args[0], ",") { diff --git a/cli/p2p_collection_getall.go b/cli/p2p_collection_getall.go index 10d98582c6..8a005df801 100644 --- a/cli/p2p_collection_getall.go +++ b/cli/p2p_collection_getall.go @@ -22,7 +22,7 @@ func MakeP2PCollectionGetAllCommand() *cobra.Command { This is the list of collections of the node that are synchronized on the pubsub network.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - p2p := mustGetP2PContext(cmd) + p2p := mustGetContextP2P(cmd) cols, err := p2p.GetAllP2PCollections(cmd.Context()) if err != nil { diff --git a/cli/p2p_collection_remove.go b/cli/p2p_collection_remove.go index 8aa0b5b7df..7def06e779 100644 --- a/cli/p2p_collection_remove.go +++ b/cli/p2p_collection_remove.go @@ -31,7 +31,7 @@ Example: remove multiple collections `, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - p2p := mustGetP2PContext(cmd) + p2p := mustGetContextP2P(cmd) var collectionIDs []string for _, id := range strings.Split(args[0], ",") { diff --git a/cli/p2p_replicator_delete.go b/cli/p2p_replicator_delete.go index 6cc2ddf785..debd0ac280 100644 --- a/cli/p2p_replicator_delete.go +++ b/cli/p2p_replicator_delete.go @@ -32,7 +32,7 @@ Example: `, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - p2p := mustGetP2PContext(cmd) + p2p := mustGetContextP2P(cmd) var info peer.AddrInfo if err := json.Unmarshal([]byte(args[0]), &info); err != nil { diff --git a/cli/p2p_replicator_getall.go b/cli/p2p_replicator_getall.go index 4bdf6e8487..0e5549fea6 100644 --- a/cli/p2p_replicator_getall.go +++ b/cli/p2p_replicator_getall.go @@ -25,7 +25,7 @@ Example: defradb client p2p replicator getall `, RunE: func(cmd *cobra.Command, args []string) error { - p2p := mustGetP2PContext(cmd) + p2p := mustGetContextP2P(cmd) reps, err := p2p.GetAllReplicators(cmd.Context()) if err != nil { diff --git a/cli/p2p_replicator_set.go b/cli/p2p_replicator_set.go index 5d9c712a82..29109a920a 100644 --- a/cli/p2p_replicator_set.go +++ b/cli/p2p_replicator_set.go @@ -32,7 +32,7 @@ Example: `, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - p2p := mustGetP2PContext(cmd) + p2p := mustGetContextP2P(cmd) var info peer.AddrInfo if err := json.Unmarshal([]byte(args[0]), &info); err != nil { diff --git a/cli/request.go b/cli/request.go index 56e33d7c4a..d5e37e79a3 100644 --- a/cli/request.go +++ b/cli/request.go @@ -45,7 +45,7 @@ with the database more conveniently. To learn more about the DefraDB GraphQL Query Language, refer to https://docs.source.network.`, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var request string switch { diff --git a/cli/root.go b/cli/root.go index 729b638f02..e4ba349f76 100644 --- a/cli/root.go +++ b/cli/root.go @@ -11,14 +11,10 @@ package cli import ( - "context" - "github.com/spf13/cobra" - - "github.com/sourcenetwork/defradb/config" ) -func MakeRootCommand(cfg *config.Config) *cobra.Command { +func MakeRootCommand() *cobra.Command { var cmd = &cobra.Command{ SilenceUsage: true, Use: "defradb", @@ -28,81 +24,108 @@ func MakeRootCommand(cfg *config.Config) *cobra.Command { Start a DefraDB node, interact with a local or remote node, and much more. `, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return loadConfig(cfg) + if err := setContextRootDir(cmd); err != nil { + return err + } + return setContextConfig(cmd) }, } cmd.PersistentFlags().String( - "rootdir", "", - "Directory for data and configuration to use (default: $HOME/.defradb)", + "rootdir", + "", + "Directory for persistent data (default: $HOME/.defradb)", ) - err := cfg.BindFlag(config.RootdirKey, cmd.PersistentFlags().Lookup("rootdir")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind rootdir", err) - } cmd.PersistentFlags().String( - "loglevel", cfg.Log.Level, + "loglevel", + "info", "Log level to use. Options are debug, info, error, fatal", ) - err = cfg.BindFlag("log.level", cmd.PersistentFlags().Lookup("loglevel")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind log.loglevel", err) - } - - cmd.PersistentFlags().StringArray( - "logger", []string{}, - "Override logger parameters. Usage: --logger ,level=,output=,...", - ) - err = cfg.BindFlag("log.logger", cmd.PersistentFlags().Lookup("logger")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind log.logger", err) - } cmd.PersistentFlags().String( - "logoutput", cfg.Log.Output, + "logoutput", + "stderr", "Log output path", ) - err = cfg.BindFlag("log.output", cmd.PersistentFlags().Lookup("logoutput")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind log.output", err) - } cmd.PersistentFlags().String( - "logformat", cfg.Log.Format, + "logformat", + "csv", "Log format to use. Options are csv, json", ) - err = cfg.BindFlag("log.format", cmd.PersistentFlags().Lookup("logformat")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind log.format", err) - } cmd.PersistentFlags().Bool( - "logtrace", cfg.Log.Stacktrace, + "logtrace", + false, "Include stacktrace in error and fatal logs", ) - err = cfg.BindFlag("log.stacktrace", cmd.PersistentFlags().Lookup("logtrace")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind log.stacktrace", err) - } cmd.PersistentFlags().Bool( - "lognocolor", cfg.Log.NoColor, + "lognocolor", + false, "Disable colored log output", ) - err = cfg.BindFlag("log.nocolor", cmd.PersistentFlags().Lookup("lognocolor")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind log.nocolor", err) - } cmd.PersistentFlags().String( - "url", cfg.API.Address, + "url", + "127.0.0.1:9181", "URL of HTTP endpoint to listen on or connect to", ) - err = cfg.BindFlag("api.address", cmd.PersistentFlags().Lookup("url")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind api.address", err) - } + + cmd.PersistentFlags().StringArray( + "peers", + []string{}, + "List of peers to connect to", + ) + + cmd.PersistentFlags().Int( + "max-txn-retries", + 5, + "Specify the maximum number of retries per transaction", + ) + + cmd.PersistentFlags().String( + "store", + "badger", + "Specify the datastore to use (supported: badger, memory)", + ) + + cmd.PersistentFlags().Int( + "valuelogfilesize", + 1<<30, + "Specify the datastore value log file size (in bytes). In memory size will be 2*valuelogfilesize", + ) + + cmd.PersistentFlags().StringSlice( + "p2paddr", + []string{"/ip4/127.0.0.1/tcp/9171"}, + "Listen addresses for the p2p network (formatted as a libp2p MultiAddr)", + ) + + cmd.PersistentFlags().Bool( + "no-p2p", + false, + "Disable the peer-to-peer network synchronization system", + ) + + cmd.PersistentFlags().StringArray( + "allowed-origins", + []string{}, + "List of origins to allow for CORS requests", + ) + + cmd.PersistentFlags().String( + "pubkeypath", + "", + "Path to the public key for tls", + ) + + cmd.PersistentFlags().String( + "privkeypath", + "", + "Path to the private key for tls", + ) return cmd } diff --git a/cli/schema_add.go b/cli/schema_add.go index b93427a883..f987d062df 100644 --- a/cli/schema_add.go +++ b/cli/schema_add.go @@ -36,7 +36,7 @@ Example: add from stdin: Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var schema string switch { diff --git a/cli/schema_describe.go b/cli/schema_describe.go index 72d8eda474..7d8306e93b 100644 --- a/cli/schema_describe.go +++ b/cli/schema_describe.go @@ -39,7 +39,7 @@ Example: view a single schema by version id defradb client schema describe --version bae123 `, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var schemas []client.SchemaDescription switch { diff --git a/cli/schema_migration_down.go b/cli/schema_migration_down.go index 52ca2a2ac6..1d7622257c 100644 --- a/cli/schema_migration_down.go +++ b/cli/schema_migration_down.go @@ -41,7 +41,7 @@ Example: migrate from stdin `, Args: cobra.RangeArgs(0, 1), RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var srcData []byte switch { diff --git a/cli/schema_migration_reload.go b/cli/schema_migration_reload.go index d04aebed65..4266b3ec3f 100644 --- a/cli/schema_migration_reload.go +++ b/cli/schema_migration_reload.go @@ -22,7 +22,7 @@ func MakeSchemaMigrationReloadCommand() *cobra.Command { Short: "Reload the schema migrations within DefraDB", Long: `Reload the schema migrations within DefraDB`, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) lens := store.LensRegistry() if tx, ok := cmd.Context().Value(txContextKey).(datastore.Txn); ok { diff --git a/cli/schema_migration_set.go b/cli/schema_migration_set.go index 66809db6a7..f7b32103b9 100644 --- a/cli/schema_migration_set.go +++ b/cli/schema_migration_set.go @@ -42,7 +42,7 @@ Example: add from stdin: Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`, Args: cobra.RangeArgs(2, 3), RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var lensCfgJson string switch { diff --git a/cli/schema_migration_set_registry.go b/cli/schema_migration_set_registry.go index 4303c99c9e..cc5098afae 100644 --- a/cli/schema_migration_set_registry.go +++ b/cli/schema_migration_set_registry.go @@ -32,7 +32,7 @@ Example: set from an argument string: Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`, Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) decoder := json.NewDecoder(strings.NewReader(args[1])) decoder.DisallowUnknownFields() diff --git a/cli/schema_migration_up.go b/cli/schema_migration_up.go index e4f4399181..577b87d4c7 100644 --- a/cli/schema_migration_up.go +++ b/cli/schema_migration_up.go @@ -41,7 +41,7 @@ Example: migrate from stdin `, Args: cobra.RangeArgs(0, 1), RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var srcData []byte switch { diff --git a/cli/schema_patch.go b/cli/schema_patch.go index bfc5f56c93..23f425396d 100644 --- a/cli/schema_patch.go +++ b/cli/schema_patch.go @@ -44,7 +44,7 @@ Example: patch from stdin: To learn more about the DefraDB GraphQL Schema Language, refer to https://docs.source.network.`, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var patch string switch { diff --git a/cli/schema_set_active.go b/cli/schema_set_active.go index 4f19ef12aa..2b13713461 100644 --- a/cli/schema_set_active.go +++ b/cli/schema_set_active.go @@ -22,7 +22,7 @@ func MakeSchemaSetActiveCommand() *cobra.Command { those without it (if they share the same schema root).`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) return store.SetActiveSchemaVersion(cmd.Context(), args[0]) }, } diff --git a/cli/server_dump.go b/cli/server_dump.go index c57299dafc..6d2b2cefc6 100644 --- a/cli/server_dump.go +++ b/cli/server_dump.go @@ -13,25 +13,25 @@ package cli import ( "github.com/spf13/cobra" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/db" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/node" ) -func MakeServerDumpCmd(cfg *config.Config) *cobra.Command { +func MakeServerDumpCmd() *cobra.Command { cmd := &cobra.Command{ Use: "server-dump", Short: "Dumps the state of the entire database", RunE: func(cmd *cobra.Command, _ []string) error { + cfg := mustGetContextConfig(cmd) log.FeedbackInfo(cmd.Context(), "Dumping DB state...") - if cfg.Datastore.Store == config.DatastoreMemory { + if cfg.GetString("datastore.store") != configStoreBadger { return errors.New("server-side dump is only supported for the Badger datastore") } storeOpts := []node.StoreOpt{ - node.WithPath(cfg.Datastore.Badger.Path), - node.WithInMemory(cfg.Datastore.Store == config.DatastoreMemory), + node.WithPath(cfg.GetString("datastore.badger.path")), + node.WithInMemory(cfg.GetString("datastore.store") != configStoreMemory), } rootstore, err := node.NewStore(storeOpts...) if err != nil { diff --git a/cli/start.go b/cli/start.go index f3e12fb8a7..d4e789cbc6 100644 --- a/cli/start.go +++ b/cli/start.go @@ -11,18 +11,15 @@ package cli import ( - "context" "fmt" "os" "os/signal" "path/filepath" - "strings" "syscall" "github.com/libp2p/go-libp2p/core/peer" "github.com/spf13/cobra" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/db" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/http" @@ -31,60 +28,64 @@ import ( "github.com/sourcenetwork/defradb/node" ) -func MakeStartCommand(cfg *config.Config) *cobra.Command { +func MakeStartCommand() *cobra.Command { var cmd = &cobra.Command{ Use: "start", Short: "Start a DefraDB node", Long: "Start a DefraDB node.", // Load the root config if it exists, otherwise create it. PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { - if err := loadConfig(cfg); err != nil { + if err := setContextRootDir(cmd); err != nil { return err } - if !cfg.ConfigFileExists() { - return createConfig(cfg) + rootdir := mustGetContextRootDir(cmd) + if err := createConfig(rootdir, cmd.Root().PersistentFlags()); err != nil { + return err } - return nil + return setContextConfig(cmd) }, RunE: func(cmd *cobra.Command, args []string) error { + cfg := mustGetContextConfig(cmd) + dbOpts := []db.Option{ db.WithUpdateEvents(), - db.WithMaxRetries(cfg.Datastore.MaxTxnRetries), + db.WithMaxRetries(cfg.GetInt("datastore.MaxTxnRetries")), } netOpts := []net.NodeOpt{ - net.WithListenAddresses(cfg.Net.P2PAddresses...), - net.WithEnablePubSub(cfg.Net.PubSubEnabled), - net.WithEnableRelay(cfg.Net.RelayEnabled), + net.WithListenAddresses(cfg.GetStringSlice("net.p2pAddresses")...), + net.WithEnablePubSub(cfg.GetBool("net.pubSubEnabled")), + net.WithEnableRelay(cfg.GetBool("net.relayEnabled")), } serverOpts := []http.ServerOpt{ - http.WithAddress(cfg.API.Address), - http.WithAllowedOrigins(cfg.API.AllowedOrigins...), - http.WithTLSCertPath(cfg.API.PubKeyPath), - http.WithTLSKeyPath(cfg.API.PrivKeyPath), + http.WithAddress(cfg.GetString("api.address")), + http.WithAllowedOrigins(cfg.GetStringSlice("api.allowed-origins")...), + http.WithTLSCertPath(cfg.GetString("api.pubKeyPath")), + http.WithTLSKeyPath(cfg.GetString("api.privKeyPath")), } storeOpts := []node.StoreOpt{ - node.WithPath(cfg.Datastore.Badger.Path), - node.WithInMemory(cfg.Datastore.Store == config.DatastoreMemory), + node.WithPath(cfg.GetString("datastore.badger.path")), + node.WithInMemory(cfg.GetString("datastore.store") == configStoreMemory), } var peers []peer.AddrInfo - if cfg.Net.Peers != "" { - addrs, err := netutils.ParsePeers(strings.Split(cfg.Net.Peers, ",")) + if val := cfg.GetStringSlice("net.peers"); len(val) > 0 { + addrs, err := netutils.ParsePeers(val) if err != nil { - return errors.Wrap(fmt.Sprintf("failed to parse bootstrap peers %v", cfg.Net.Peers), err) + return errors.Wrap(fmt.Sprintf("failed to parse bootstrap peers %s", val), err) } peers = addrs } - if cfg.Datastore.Store == "badger" { + if cfg.GetString("datastore.store") != configStoreMemory { // It would be ideal to not have the key path tied to the datastore. // Running with memory store mode will always generate a random key. // Adding support for an ephemeral mode and moving the key to the // config would solve both of these issues. - key, err := loadOrGeneratePrivateKey(filepath.Join(cfg.Rootdir, "data", "key")) + rootdir := mustGetContextRootDir(cmd) + key, err := loadOrGeneratePrivateKey(filepath.Join(rootdir, "data", "key")) if err != nil { return err } @@ -97,7 +98,7 @@ func MakeStartCommand(cfg *config.Config) *cobra.Command { node.WithDatabaseOpts(dbOpts...), node.WithNetOpts(netOpts...), node.WithServerOpts(serverOpts...), - node.WithDisableP2P(cfg.Net.P2PDisabled), + node.WithDisableP2P(cfg.GetBool("net.p2pDisabled")), } n, err := node.NewNode(cmd.Context(), opts...) @@ -130,103 +131,5 @@ func MakeStartCommand(cfg *config.Config) *cobra.Command { }, } - cmd.Flags().String( - "peers", cfg.Net.Peers, - "List of peers to connect to", - ) - err := cfg.BindFlag("net.peers", cmd.Flags().Lookup("peers")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind net.peers", err) - } - - cmd.Flags().Int( - "max-txn-retries", cfg.Datastore.MaxTxnRetries, - "Specify the maximum number of retries per transaction", - ) - err = cfg.BindFlag("datastore.maxtxnretries", cmd.Flags().Lookup("max-txn-retries")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind datastore.maxtxnretries", err) - } - - cmd.Flags().String( - "store", cfg.Datastore.Store, - "Specify the datastore to use (supported: badger, memory)", - ) - err = cfg.BindFlag("datastore.store", cmd.Flags().Lookup("store")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind datastore.store", err) - } - - cmd.Flags().Var( - &cfg.Datastore.Badger.ValueLogFileSize, "valuelogfilesize", - "Specify the datastore value log file size (in bytes). In memory size will be 2*valuelogfilesize", - ) - err = cfg.BindFlag("datastore.badger.valuelogfilesize", cmd.Flags().Lookup("valuelogfilesize")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind datastore.badger.valuelogfilesize", err) - } - - cmd.Flags().StringSlice( - "p2paddr", cfg.Net.P2PAddresses, - "Listen addresses for the p2p network (formatted as a libp2p MultiAddr)", - ) - err = cfg.BindFlag("net.p2paddresses", cmd.Flags().Lookup("p2paddr")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind net.p2paddress", err) - } - - cmd.Flags().Bool( - "no-p2p", cfg.Net.P2PDisabled, - "Disable the peer-to-peer network synchronization system", - ) - err = cfg.BindFlag("net.p2pdisabled", cmd.Flags().Lookup("no-p2p")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind net.p2pdisabled", err) - } - - cmd.Flags().Bool( - "tls", cfg.API.TLS, - "Enable serving the API over https", - ) - err = cfg.BindFlag("api.tls", cmd.Flags().Lookup("tls")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind api.tls", err) - } - - cmd.Flags().StringArray( - "allowed-origins", cfg.API.AllowedOrigins, - "List of origins to allow for CORS requests", - ) - err = cfg.BindFlag("api.allowed-origins", cmd.Flags().Lookup("allowed-origins")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind api.allowed-origins", err) - } - - cmd.Flags().String( - "pubkeypath", cfg.API.PubKeyPath, - "Path to the public key for tls", - ) - err = cfg.BindFlag("api.pubkeypath", cmd.Flags().Lookup("pubkeypath")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind api.pubkeypath", err) - } - - cmd.Flags().String( - "privkeypath", cfg.API.PrivKeyPath, - "Path to the private key for tls", - ) - err = cfg.BindFlag("api.privkeypath", cmd.Flags().Lookup("privkeypath")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind api.privkeypath", err) - } - - cmd.Flags().String( - "email", cfg.API.Email, - "Email address used by the CA for notifications", - ) - err = cfg.BindFlag("api.email", cmd.Flags().Lookup("email")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind api.email", err) - } return cmd } diff --git a/cli/tx_commit.go b/cli/tx_commit.go index 260a274a08..f7ef112988 100644 --- a/cli/tx_commit.go +++ b/cli/tx_commit.go @@ -15,22 +15,23 @@ import ( "github.com/spf13/cobra" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/http" ) -func MakeTxCommitCommand(cfg *config.Config) *cobra.Command { +func MakeTxCommitCommand() *cobra.Command { var cmd = &cobra.Command{ Use: "commit [id]", Short: "Commit a DefraDB transaction.", Long: `Commit a DefraDB transaction.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) (err error) { + cfg := mustGetContextConfig(cmd) + id, err := strconv.ParseUint(args[0], 10, 64) if err != nil { return err } - tx, err := http.NewTransaction(cfg.API.Address, id) + tx, err := http.NewTransaction(cfg.GetString("api.address"), id) if err != nil { return err } diff --git a/cli/tx_create.go b/cli/tx_create.go index 987a784077..da239b6943 100644 --- a/cli/tx_create.go +++ b/cli/tx_create.go @@ -14,11 +14,10 @@ import ( "github.com/spf13/cobra" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/datastore" ) -func MakeTxCreateCommand(cfg *config.Config) *cobra.Command { +func MakeTxCreateCommand() *cobra.Command { var concurrent bool var readOnly bool var cmd = &cobra.Command{ diff --git a/cli/tx_discard.go b/cli/tx_discard.go index 351f919f53..0a980a63f5 100644 --- a/cli/tx_discard.go +++ b/cli/tx_discard.go @@ -15,22 +15,23 @@ import ( "github.com/spf13/cobra" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/http" ) -func MakeTxDiscardCommand(cfg *config.Config) *cobra.Command { +func MakeTxDiscardCommand() *cobra.Command { var cmd = &cobra.Command{ Use: "discard [id]", Short: "Discard a DefraDB transaction.", Long: `Discard a DefraDB transaction.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) (err error) { + cfg := mustGetContextConfig(cmd) + id, err := strconv.ParseUint(args[0], 10, 64) if err != nil { return err } - tx, err := http.NewTransaction(cfg.API.Address, id) + tx, err := http.NewTransaction(cfg.GetString("api.address"), id) if err != nil { return err } diff --git a/cli/utils.go b/cli/utils.go index b8b102c0c8..caeb282606 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -18,9 +18,9 @@ import ( "github.com/libp2p/go-libp2p/core/crypto" "github.com/spf13/cobra" + "github.com/spf13/viper" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/http" ) @@ -28,6 +28,10 @@ import ( type contextKey string var ( + // cfgContextKey is the context key for the config. + cfgContextKey = contextKey("cfg") + // rootDirContextKey is the context key for the root directory. + rootDirContextKey = contextKey("rootDir") // txContextKey is the context key for the datastore.Txn // // This will only be set if a transaction id is specified. @@ -46,33 +50,61 @@ var ( colContextKey = contextKey("col") ) -// mustGetStoreContext returns the store for the current command context. +// mustGetContextStore returns the store for the current command context. // // If a store is not set in the current context this function panics. -func mustGetStoreContext(cmd *cobra.Command) client.Store { +func mustGetContextStore(cmd *cobra.Command) client.Store { return cmd.Context().Value(storeContextKey).(client.Store) } -// mustGetP2PContext returns the p2p implementation for the current command context. +// mustGetContextP2P returns the p2p implementation for the current command context. // // If a p2p implementation is not set in the current context this function panics. -func mustGetP2PContext(cmd *cobra.Command) client.P2P { +func mustGetContextP2P(cmd *cobra.Command) client.P2P { return cmd.Context().Value(dbContextKey).(client.P2P) } -// tryGetCollectionContext returns the collection for the current command context +// mustGetContextConfig returns the config for the current command context. +// +// If a config is not set in the current context this function panics. +func mustGetContextConfig(cmd *cobra.Command) *viper.Viper { + return cmd.Context().Value(cfgContextKey).(*viper.Viper) +} + +// mustGetContextRootDir returns the rootdir for the current command context. +// +// If a rootdir is not set in the current context this function panics. +func mustGetContextRootDir(cmd *cobra.Command) string { + return cmd.Context().Value(rootDirContextKey).(string) +} + +// tryGetContextCollection returns the collection for the current command context // and a boolean indicating if the collection was set. -func tryGetCollectionContext(cmd *cobra.Command) (client.Collection, bool) { +func tryGetContextCollection(cmd *cobra.Command) (client.Collection, bool) { col, ok := cmd.Context().Value(colContextKey).(client.Collection) return col, ok } -// setTransactionContext sets the transaction for the current command context. -func setTransactionContext(cmd *cobra.Command, cfg *config.Config, txId uint64) error { +// setContextConfig sets teh config for the current command context. +func setContextConfig(cmd *cobra.Command) error { + rootdir := mustGetContextRootDir(cmd) + flags := cmd.Root().PersistentFlags() + cfg, err := loadConfig(rootdir, flags) + if err != nil { + return err + } + ctx := context.WithValue(cmd.Context(), cfgContextKey, cfg) + cmd.SetContext(ctx) + return nil +} + +// setContextTransaction sets the transaction for the current command context. +func setContextTransaction(cmd *cobra.Command, txId uint64) error { if txId == 0 { return nil } - tx, err := http.NewTransaction(cfg.API.Address, txId) + cfg := mustGetContextConfig(cmd) + tx, err := http.NewTransaction(cfg.GetString("api.address"), txId) if err != nil { return err } @@ -81,9 +113,10 @@ func setTransactionContext(cmd *cobra.Command, cfg *config.Config, txId uint64) return nil } -// setStoreContext sets the store for the current command context. -func setStoreContext(cmd *cobra.Command, cfg *config.Config) error { - db, err := http.NewClient(cfg.API.Address) +// setContextStore sets the store for the current command context. +func setContextStore(cmd *cobra.Command) error { + cfg := mustGetContextConfig(cmd) + db, err := http.NewClient(cfg.GetString("api.address")) if err != nil { return err } @@ -97,22 +130,22 @@ func setStoreContext(cmd *cobra.Command, cfg *config.Config) error { return nil } -// loadConfig loads the rootDir containing the configuration file, -// otherwise warn about it and load a default configuration. -func loadConfig(cfg *config.Config) error { - if err := cfg.LoadRootDirFromFlagOrDefault(); err != nil { +// setContextRootDir sets the rootdir for the current command context. +func setContextRootDir(cmd *cobra.Command) error { + rootdir, err := cmd.Root().PersistentFlags().GetString("rootdir") + if err != nil { return err } - return cfg.LoadWithRootdir(cfg.ConfigFileExists()) -} - -// createConfig creates the config directories and writes -// the current config to a file. -func createConfig(cfg *config.Config) error { - if config.FolderExists(cfg.Rootdir) { - return cfg.WriteConfigFile() + home, err := os.UserHomeDir() + if err != nil { + return err + } + if rootdir == "" { + rootdir = filepath.Join(home, ".defradb") } - return cfg.CreateRootDirAndConfigFile() + ctx := context.WithValue(cmd.Context(), rootDirContextKey, rootdir) + cmd.SetContext(ctx) + return nil } // loadOrGeneratePrivateKey loads the private key from the given path diff --git a/cli/view_add.go b/cli/view_add.go index 44fa6d348e..9c7d42b723 100644 --- a/cli/view_add.go +++ b/cli/view_add.go @@ -34,7 +34,7 @@ Example: add from an argument string: Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`, Args: cobra.RangeArgs(2, 4), RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) query := args[0] sdl := args[1] diff --git a/cmd/defradb/main.go b/cmd/defradb/main.go index 2406885a76..e827177132 100644 --- a/cmd/defradb/main.go +++ b/cmd/defradb/main.go @@ -15,12 +15,11 @@ import ( "os" "github.com/sourcenetwork/defradb/cli" - "github.com/sourcenetwork/defradb/config" ) // Execute adds all child commands to the root command and sets flags appropriately. func main() { - defraCmd := cli.NewDefraCommand(config.DefaultConfig()) + defraCmd := cli.NewDefraCommand() if err := defraCmd.Execute(); err != nil { // this error is okay to discard because cobra // logs any errors encountered during execution diff --git a/cmd/genclidocs/main.go b/cmd/genclidocs/main.go index f556c26d20..a9a6f198c4 100644 --- a/cmd/genclidocs/main.go +++ b/cmd/genclidocs/main.go @@ -21,7 +21,6 @@ import ( "github.com/spf13/cobra/doc" "github.com/sourcenetwork/defradb/cli" - "github.com/sourcenetwork/defradb/config" ) var path string @@ -33,7 +32,7 @@ func init() { func main() { flag.Parse() - defraCmd := cli.NewDefraCommand(config.DefaultConfig()) + defraCmd := cli.NewDefraCommand() defraCmd.DisableAutoGenTag = true if err := os.MkdirAll(path, os.ModePerm); err != nil { diff --git a/cmd/gendocs/main.go b/cmd/gendocs/main.go index 44901b0faf..4c1f7dcf9a 100644 --- a/cmd/gendocs/main.go +++ b/cmd/gendocs/main.go @@ -16,13 +16,11 @@ package main import ( "os" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/tests/gen/cli" ) func main() { - conf := config.DefaultConfig() - gendocsCmd := cli.MakeGenDocCommand(conf) + gendocsCmd := cli.MakeGenDocCommand() if err := gendocsCmd.Execute(); err != nil { // this error is okay to discard because cobra // logs any errors encountered during execution diff --git a/cmd/genmanpages/main.go b/cmd/genmanpages/main.go index 1a9b43df7c..ecdc78762d 100644 --- a/cmd/genmanpages/main.go +++ b/cmd/genmanpages/main.go @@ -22,7 +22,6 @@ import ( "github.com/spf13/cobra/doc" "github.com/sourcenetwork/defradb/cli" - "github.com/sourcenetwork/defradb/config" ) const defaultPerm os.FileMode = 0o777 @@ -41,7 +40,7 @@ func init() { func main() { flag.Parse() - defraCmd := cli.NewDefraCommand(config.DefaultConfig()) + defraCmd := cli.NewDefraCommand() if err := os.MkdirAll(dir, defaultPerm); err != nil { log.Fatal("Failed to create directory", err) diff --git a/config/config.go b/config/config.go deleted file mode 100644 index 52c93c8372..0000000000 --- a/config/config.go +++ /dev/null @@ -1,704 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -/* -Package config provides the central point for DefraDB's configuration and related facilities. - -[Config] embeds component-specific config structs. Each config struct can have a function providing -default options, a method providing test configurations, a method for validation, a method handling -deprecated fields (e.g. with warnings). This is extensible. - -The 'root directory' is where the configuration file and data of a DefraDB instance exists. -It is specified as a global flag `defradb --rootdir path/to/somewhere. - -Some packages of DefraDB provide their own configuration approach (logging, node). -For each, a way to go from top-level configuration to package-specific configuration is provided. - -Parameters are determined by, in order of least importance: defaults, configuration file, env. variables, and then CLI -flags. That is, CLI flags can override everything else. - -For example `DEFRA_DATASTORE_BADGER_PATH` matches [Config.Datastore.Badger.Path] and in the config file: - - datastore: - badger: - path: /tmp/badger - -This implementation does not support online modification of configuration. - -How to use, e.g. without using a rootdir: - - cfg := config.DefaultConfig() - cfg.NetConfig.P2PDisabled = true // as example - err := cfg.LoadWithRootdir(false) - if err != nil { - ... -*/ -package config - -import ( - "bytes" - "encoding/json" - "fmt" - "net" - "path/filepath" - "strconv" - "strings" - "text/template" - - "github.com/mitchellh/mapstructure" - ma "github.com/multiformats/go-multiaddr" - "github.com/spf13/pflag" - "github.com/spf13/viper" - "golang.org/x/net/idna" - - badgerds "github.com/sourcenetwork/defradb/datastore/badger/v4" - "github.com/sourcenetwork/defradb/logging" -) - -var log = logging.MustNewLogger("config") - -const ( - DefaultAPIEmail = "example@example.com" - RootdirKey = "rootdircli" - defraEnvPrefix = "DEFRA" - logLevelDebug = "debug" - logLevelInfo = "info" - logLevelError = "error" - logLevelFatal = "fatal" - DatastoreMemory = "memory" -) - -// Config is DefraDB's main configuration struct, embedding component-specific config structs. -type Config struct { - Datastore *DatastoreConfig - API *APIConfig - Net *NetConfig - Log *LoggingConfig - Rootdir string - v *viper.Viper -} - -// DefaultConfig returns the default configuration (or panics). -func DefaultConfig() *Config { - cfg := &Config{ - Datastore: defaultDatastoreConfig(), - API: defaultAPIConfig(), - Net: defaultNetConfig(), - Log: defaultLogConfig(), - Rootdir: "", - v: viper.New(), - } - - cfg.v.SetEnvPrefix(defraEnvPrefix) - cfg.v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) - - cfg.v.SetConfigName(DefaultConfigFileName) - cfg.v.SetConfigType(configType) - - cfg.Persist() - - return cfg -} - -// Persist persists manually set config parameters to the viper config. -func (cfg *Config) Persist() { - // Load new values in viper. - b, err := cfg.toBytes() - if err != nil { - panic(err) - } - if err = cfg.v.ReadConfig(bytes.NewReader(b)); err != nil { - panic(NewErrReadingConfigFile(err)) - } -} - -// LoadWithRootdir loads a Config with parameters from defaults, config file, environment variables, and CLI flags. -// It loads from config file when `fromFile` is true, otherwise it loads directly from a default configuration. -// Use on a Config struct already loaded with default values from DefaultConfig(). -// To be executed once at the beginning of the program. -func (cfg *Config) LoadWithRootdir(withRootdir bool) error { - // Use default logging configuration here, so that - // we can log errors in a consistent way even in the case of early failure. - defaultLogCfg := defaultLogConfig() - if err := defaultLogCfg.load(); err != nil { - return err - } - - if err := cfg.LoadRootDirFromFlagOrDefault(); err != nil { - return err - } - - if withRootdir { - if err := cfg.v.ReadInConfig(); err != nil { - return NewErrReadingConfigFile(err) - } - } - - cfg.v.AutomaticEnv() - - if err := cfg.paramsPreprocessing(); err != nil { - return err - } - // We load the viper configuration in the Config struct. - if err := cfg.v.Unmarshal(cfg, viper.DecodeHook(mapstructure.TextUnmarshallerHookFunc())); err != nil { - return NewErrLoadingConfig(err) - } - if err := cfg.validate(); err != nil { - return err - } - if err := cfg.load(); err != nil { - return err - } - - return nil -} - -func (cfg *Config) LoadRootDirFromFlagOrDefault() error { - if cfg.Rootdir == "" { - rootdir := cfg.v.GetString(RootdirKey) - if rootdir != "" { - return cfg.setRootdir(rootdir) - } - - return cfg.setRootdir(DefaultRootDir()) - } - - return nil -} - -func (cfg *Config) setRootdir(rootdir string) error { - var err error - if rootdir == "" { - return NewErrInvalidRootDir(rootdir) - } - // using absolute rootdir for robustness. - cfg.Rootdir, err = filepath.Abs(rootdir) - if err != nil { - return err - } - cfg.v.AddConfigPath(cfg.Rootdir) - return nil -} - -func (cfg *Config) validate() error { - if err := cfg.Datastore.validate(); err != nil { - return NewErrFailedToValidateConfig(err) - } - if err := cfg.API.validate(); err != nil { - return NewErrFailedToValidateConfig(err) - } - if err := cfg.Net.validate(); err != nil { - return NewErrFailedToValidateConfig(err) - } - if err := cfg.Log.validate(); err != nil { - return NewErrFailedToValidateConfig(err) - } - return nil -} - -func (cfg *Config) paramsPreprocessing() error { - // We prefer using absolute paths, relative to the rootdir. - if !filepath.IsAbs(cfg.v.GetString("datastore.badger.path")) { - cfg.v.Set("datastore.badger.path", filepath.Join(cfg.Rootdir, cfg.v.GetString("datastore.badger.path"))) - } - privKeyPath := cfg.v.GetString("api.privkeypath") - if privKeyPath != "" && !filepath.IsAbs(privKeyPath) { - cfg.v.Set("api.privkeypath", filepath.Join(cfg.Rootdir, privKeyPath)) - } - pubKeyPath := cfg.v.GetString("api.pubkeypath") - if pubKeyPath != "" && !filepath.IsAbs(pubKeyPath) { - cfg.v.Set("api.pubkeypath", filepath.Join(cfg.Rootdir, pubKeyPath)) - } - - // log.logger configuration as a string - logloggerAsStringSlice := cfg.v.GetStringSlice("log.logger") - if logloggerAsStringSlice != nil { - cfg.v.Set("log.logger", strings.Join(logloggerAsStringSlice, ";")) - } - - // Expand the passed in `~` if it wasn't expanded properly by the shell. - // That can happen when the parameters are passed from outside of a shell. - if err := expandHomeDir(&cfg.API.PrivKeyPath); err != nil { - return err - } - if err := expandHomeDir(&cfg.API.PubKeyPath); err != nil { - return err - } - - var bs ByteSize - if err := bs.Set(cfg.v.GetString("datastore.badger.valuelogfilesize")); err != nil { - return err - } - cfg.Datastore.Badger.ValueLogFileSize = bs - - return nil -} - -func (cfg *Config) load() error { - if err := cfg.Log.load(); err != nil { - return err - } - return nil -} - -// DatastoreConfig configures datastores. -type DatastoreConfig struct { - Store string - Memory MemoryConfig - Badger BadgerConfig - MaxTxnRetries int -} - -// BadgerConfig configures Badger's on-disk / filesystem mode. -type BadgerConfig struct { - Path string - ValueLogFileSize ByteSize - *badgerds.Options -} - -// MemoryConfig configures of Badger's memory mode. -type MemoryConfig struct { - Size uint64 -} - -func defaultDatastoreConfig() *DatastoreConfig { - // create a copy of the default badger options - opts := badgerds.DefaultOptions - return &DatastoreConfig{ - Store: "badger", - Badger: BadgerConfig{ - Path: "data", - ValueLogFileSize: 1 * GiB, - Options: &opts, - }, - MaxTxnRetries: 5, - } -} - -func (dbcfg DatastoreConfig) validate() error { - switch dbcfg.Store { - case "badger", "memory": - default: - return NewErrInvalidDatastoreType(dbcfg.Store) - } - return nil -} - -// APIConfig configures the API endpoints. -type APIConfig struct { - Address string - TLS bool - AllowedOrigins []string `mapstructure:"allowed-origins"` - PubKeyPath string - PrivKeyPath string - Email string -} - -func defaultAPIConfig() *APIConfig { - return &APIConfig{ - Address: "localhost:9181", - TLS: false, - AllowedOrigins: []string{}, - Email: DefaultAPIEmail, - } -} - -func (apicfg *APIConfig) validate() error { - if apicfg.Address == "" { - return ErrInvalidDatabaseURL - } - - if apicfg.Address == "localhost" || net.ParseIP(apicfg.Address) != nil { //nolint:goconst - return ErrMissingPortNumber - } - - if isValidDomainName(apicfg.Address) { - return nil - } - - host, _, err := net.SplitHostPort(apicfg.Address) - if err != nil { - return NewErrInvalidDatabaseURL(err) - } - if host == "localhost" { - return nil - } - if net.ParseIP(host) == nil { - return ErrNoPortWithDomain - } - - return nil -} - -func isValidDomainName(domain string) bool { - asciiDomain, err := idna.Registration.ToASCII(domain) - if err != nil { - return false - } - return asciiDomain == domain -} - -// AddressToURL provides the API address as URL. -func (apicfg *APIConfig) AddressToURL() string { - if apicfg.TLS { - return fmt.Sprintf("https://%s", apicfg.Address) - } - return fmt.Sprintf("http://%s", apicfg.Address) -} - -// NetConfig configures aspects of network and peer-to-peer. -type NetConfig struct { - P2PAddresses []string - P2PDisabled bool - Peers string - PubSubEnabled bool `mapstructure:"pubsub"` - RelayEnabled bool `mapstructure:"relay"` -} - -func defaultNetConfig() *NetConfig { - return &NetConfig{ - P2PAddresses: []string{"/ip4/127.0.0.1/tcp/9171"}, - P2PDisabled: false, - Peers: "", - PubSubEnabled: true, - RelayEnabled: false, - } -} - -func (netcfg *NetConfig) validate() error { - for _, addr := range netcfg.P2PAddresses { - _, err := ma.NewMultiaddr(addr) - if err != nil { - return NewErrInvalidP2PAddress(err, addr) - } - } - if len(netcfg.Peers) > 0 { - peers := strings.Split(netcfg.Peers, ",") - maddrs := make([]ma.Multiaddr, len(peers)) - for i, addr := range peers { - addr, err := ma.NewMultiaddr(addr) - if err != nil { - return NewErrInvalidBootstrapPeers(err, netcfg.Peers) - } - maddrs[i] = addr - } - } - return nil -} - -// LogConfig configures output and logger. -type LoggingConfig struct { - Level string - Stacktrace bool - Format string - Output string // logging actually supports multiple output paths, but here only one is supported - Caller bool - NoColor bool - Logger string - NamedOverrides map[string]*NamedLoggingConfig -} - -// NamedLoggingConfig is a named logging config, used for named overrides of the default config. -type NamedLoggingConfig struct { - Name string - LoggingConfig -} - -func defaultLogConfig() *LoggingConfig { - return &LoggingConfig{ - Level: logLevelInfo, - Stacktrace: false, - Format: "csv", - Output: "stderr", - Caller: false, - NoColor: false, - Logger: "", - NamedOverrides: make(map[string]*NamedLoggingConfig), - } -} - -// validate ensures that the logging config is valid. -func (logcfg *LoggingConfig) validate() error { - /* - `loglevel` is either a single value, or a single value with comma-separated list of key=value pairs, for which - the key is the name of the logger and the value is the log level, each logger name is unique, and value is valid. - - `--loglevels ,=,...` - */ - kvs := []map[string]string{} - validLevel := func(level string) bool { - for _, l := range []string{ - logLevelDebug, - logLevelInfo, - logLevelError, - logLevelFatal, - } { - if l == level { - return true - } - } - return false - } - ensureUniqueKeys := func(kvs []map[string]string) error { - keys := make(map[string]bool) - for _, kv := range kvs { - for k := range kv { - if keys[k] { - return NewErrDuplicateLoggerName(k) - } - keys[k] = true - } - } - return nil - } - - parts := strings.Split(logcfg.Level, ",") - if len(parts) > 0 { - if !validLevel(parts[0]) { - return NewErrInvalidLogLevel(parts[0]) - } - for _, kv := range parts[1:] { - parsedKV, err := parseKV(kv) - if err != nil { - return err - } - // ensure each value is a valid loglevel validLevel - if !validLevel(parsedKV[1]) { - return NewErrInvalidLogLevel(parsedKV[1]) - } - kvs = append(kvs, map[string]string{parsedKV[0]: parsedKV[1]}) - } - if err := ensureUniqueKeys(kvs); err != nil { - return err - } - } - - // logger: expect format like: `net,nocolor=true,level=debug;config,output=stdout,level=info` - if len(logcfg.Logger) != 0 { - namedconfigs := strings.Split(logcfg.Logger, ";") - for _, c := range namedconfigs { - parts := strings.Split(c, ",") - if len(parts) < 2 { - return NewErrLoggerConfig("unexpected format (expected: `module,key=value;module,key=value;...`") - } - if parts[0] == "" { - return ErrLoggerNameEmpty - } - for _, pair := range parts[1:] { - parsedKV, err := parseKV(pair) - if err != nil { - return err - } - if !isLowercaseAlpha(parsedKV[0]) { - return NewErrInvalidLoggerName(parsedKV[0]) - } - switch parsedKV[0] { - case "format", "output", "nocolor", "stacktrace", "caller": //nolint:goconst - // valid logger parameters - case "level": //nolint:goconst - // ensure each value is a valid loglevel validLevel - if !validLevel(parsedKV[1]) { - return NewErrInvalidLogLevel(parsedKV[1]) - } - default: - return NewErrUnknownLoggerParameter(parsedKV[0]) - } - } - } - } - - return nil -} - -func (logcfg *LoggingConfig) load() error { - // load loglevel - parts := strings.Split(logcfg.Level, ",") - if len(parts) > 0 { - logcfg.Level = parts[0] - } - if len(parts) > 1 { - for _, kv := range parts[1:] { - parsedKV := strings.Split(kv, "=") - if len(parsedKV) != 2 { - return NewErrInvalidLogLevel(kv) - } - c, err := logcfg.GetOrCreateNamedLogger(parsedKV[0]) - if err != nil { - return NewErrCouldNotObtainLoggerConfig(err, parsedKV[0]) - } - c.Level = parsedKV[1] - } - } - - // load logger - // e.g. `net,nocolor=true,level=debug;config,output=stdout,level=info` - // logger has higher priority over loglevel whenever both touch the same parameters - if len(logcfg.Logger) != 0 { - s := strings.Split(logcfg.Logger, ";") - for _, v := range s { - vs := strings.Split(v, ",") - override, err := logcfg.GetOrCreateNamedLogger(vs[0]) - if err != nil { - return NewErrCouldNotObtainLoggerConfig(err, vs[0]) - } - override.Name = vs[0] - for _, v := range vs[1:] { - parsedKV := strings.Split(v, "=") - if len(parsedKV) != 2 { - return NewErrNotProvidedAsKV(v) - } - switch param := strings.ToLower(parsedKV[0]); param { - case "level": // string - override.Level = parsedKV[1] - case "format": // string - override.Format = parsedKV[1] - case "output": // string - override.Output = parsedKV[1] - case "stacktrace": // bool - if override.Stacktrace, err = strconv.ParseBool(parsedKV[1]); err != nil { - return NewErrCouldNotParseType(err, "bool") - } - case "nocolor": // bool - if override.NoColor, err = strconv.ParseBool(parsedKV[1]); err != nil { - return NewErrCouldNotParseType(err, "bool") - } - case "caller": // bool - if override.Caller, err = strconv.ParseBool(parsedKV[1]); err != nil { - return NewErrCouldNotParseType(err, "bool") - } - default: - return NewErrUnknownLoggerParameter(param) - } - } - } - } - - c, err := logcfg.toLoggerConfig() - if err != nil { - return err - } - logging.SetConfig(c) - return nil -} - -func convertLoglevel(level string) (logging.LogLevel, error) { - switch level { - case logLevelDebug: - return logging.Debug, nil - case logLevelInfo: - return logging.Info, nil - case logLevelError: - return logging.Error, nil - case logLevelFatal: - return logging.Fatal, nil - default: - return logging.LogLevel(0), NewErrInvalidLogLevel(level) - } -} - -// Exports the logging config to the logging library's config. -func (logcfg LoggingConfig) toLoggerConfig() (logging.Config, error) { - loglevel, err := convertLoglevel(logcfg.Level) - if err != nil { - return logging.Config{}, err - } - - var encfmt logging.EncoderFormat - switch logcfg.Format { - case "json": - encfmt = logging.JSON - case "csv": - encfmt = logging.CSV - default: - return logging.Config{}, NewErrInvalidLogFormat(logcfg.Format) - } - - // handle logger named overrides - overrides := make(map[string]logging.Config) - for name, cfg := range logcfg.NamedOverrides { - c, err := cfg.toLoggerConfig() - if err != nil { - return logging.Config{}, NewErrOverrideConfigConvertFailed(err, name) - } - overrides[name] = c - } - - c := logging.Config{ - Level: logging.NewLogLevelOption(loglevel), - EnableStackTrace: logging.NewEnableStackTraceOption(logcfg.Stacktrace), - DisableColor: logging.NewDisableColorOption(logcfg.NoColor), - EncoderFormat: logging.NewEncoderFormatOption(encfmt), - OutputPaths: []string{logcfg.Output}, - EnableCaller: logging.NewEnableCallerOption(logcfg.Caller), - OverridesByLoggerName: overrides, - } - return c, nil -} - -// this is a copy that doesn't deep copy the NamedOverrides map -// copy is handled by runtime "pass-by-value" -func (logcfg LoggingConfig) copy() LoggingConfig { - logcfg.NamedOverrides = make(map[string]*NamedLoggingConfig) - return logcfg -} - -// GetOrCreateNamedLogger returns a named logger config, or creates a default one if it doesn't exist. -func (logcfg *LoggingConfig) GetOrCreateNamedLogger(name string) (*NamedLoggingConfig, error) { - if name == "" { - return nil, ErrLoggerNameEmpty - } - if namedCfg, exists := logcfg.NamedOverrides[name]; exists { - return namedCfg, nil - } - // create default and save to overrides - namedCfg := &NamedLoggingConfig{ - Name: name, - LoggingConfig: logcfg.copy(), - } - logcfg.NamedOverrides[name] = namedCfg - - return namedCfg, nil -} - -// BindFlag binds a CLI flag to a config key. -func (cfg *Config) BindFlag(key string, flag *pflag.Flag) error { - return cfg.v.BindPFlag(key, flag) -} - -// ToJSON serializes the config to a JSON byte array. -func (c *Config) ToJSON() ([]byte, error) { - jsonbytes, err := json.Marshal(c) - if err != nil { - return []byte{}, NewErrConfigToJSONFailed(err) - } - return jsonbytes, nil -} - -// String serializes the config to a JSON string. -func (c *Config) String() string { - jsonbytes, err := c.ToJSON() - if err != nil { - return fmt.Sprintf("failed to convert config to string: %s", err) - } - return string(jsonbytes) -} - -func (c *Config) toBytes() ([]byte, error) { - tmpl := template.New("configTemplate") - configTemplate, err := tmpl.Parse(defaultConfigTemplate) - if err != nil { - return nil, NewErrConfigTemplateFailed(err) - } - var buffer bytes.Buffer - if err := configTemplate.Execute(&buffer, c); err != nil { - return nil, NewErrConfigTemplateFailed(err) - } - return buffer.Bytes(), nil -} diff --git a/config/config_test.go b/config/config_test.go deleted file mode 100644 index 859cdc5e04..0000000000 --- a/config/config_test.go +++ /dev/null @@ -1,435 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package config - -import ( - "encoding/json" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" -) - -var envVarsDifferent = map[string]string{ - "DEFRA_DATASTORE_STORE": "memory", - "DEFRA_DATASTORE_BADGER_PATH": "defra_data", - "DEFRA_API_ADDRESS": "localhost:9999", - "DEFRA_NET_P2PDISABLED": "true", - "DEFRA_NET_P2PADDRESSES": "/ip4/0.0.0.0/tcp/9876", - "DEFRA_NET_PUBSUB": "false", - "DEFRA_NET_RELAY": "false", - "DEFRA_LOG_LEVEL": "error", - "DEFRA_LOG_STACKTRACE": "true", - "DEFRA_LOG_FORMAT": "json", -} - -var envVarsInvalid = map[string]string{ - "DEFRA_DATASTORE_STORE": "^=+()&**()*(&))", - "DEFRA_DATASTORE_BADGER_PATH": "^=+()&**()*(&))", - "DEFRA_API_ADDRESS": "^=+()&**()*(&))", - "DEFRA_NET_P2PDISABLED": "^=+()&**()*(&))", - "DEFRA_NET_P2PADDRESSES": "^=+()&**()*(&))", - "DEFRA_NET_PUBSUB": "^=+()&**()*(&))", - "DEFRA_NET_RELAY": "^=+()&**()*(&))", - "DEFRA_LOG_LEVEL": "^=+()&**()*(&))", - "DEFRA_LOG_STACKTRACE": "^=+()&**()*(&))", - "DEFRA_LOG_FORMAT": "^=+()&**()*(&))", -} - -func FixtureEnvKeyValue(t *testing.T, key, value string) { - t.Helper() - os.Setenv(key, value) - t.Cleanup(func() { - os.Unsetenv(key) - }) -} - -func FixtureEnvVars(t *testing.T, envVars map[string]string) { - t.Helper() - for k, v := range envVars { - os.Setenv(k, v) - } - t.Cleanup(func() { - for k := range envVars { - os.Unsetenv(k) - } - }) -} - -func TestConfigValidateBasic(t *testing.T) { - cfg := DefaultConfig() - assert.NoError(t, cfg.validate()) - - err := cfg.validate() - assert.NoError(t, err) - // asserting equality of some unlikely-to-change default values - assert.Equal(t, "stderr", cfg.Log.Output) - assert.Equal(t, "csv", cfg.Log.Format) - assert.Equal(t, false, cfg.API.TLS) - assert.Equal(t, false, cfg.Net.RelayEnabled) -} - -func TestLoadIncorrectValuesFromConfigFile(t *testing.T) { - var cfg *Config - - testcases := []struct { - setter func() - err error - }{ - { - setter: func() { - cfg.Datastore.Store = "antibadger" - }, - err: ErrInvalidDatastoreType, - }, - { - setter: func() { - cfg.Log.Level = "antilevel" - }, - err: ErrInvalidLogLevel, - }, - { - setter: func() { - cfg.Log.Format = "antiformat" - }, - - err: ErrInvalidLogFormat, - }, - } - - for _, tc := range testcases { - cfg = DefaultConfig() - err := cfg.setRootdir(t.TempDir()) - assert.NoError(t, err) - tc.setter() - err = cfg.WriteConfigFile() - assert.NoError(t, err) - err = cfg.LoadWithRootdir(true) - assert.ErrorIs(t, err, tc.err) - } -} - -func TestJSONSerialization(t *testing.T) { - cfg := DefaultConfig() - var m map[string]any - - b, errSerialize := cfg.ToJSON() - errUnmarshal := json.Unmarshal(b, &m) - - assert.NoError(t, errUnmarshal) - assert.NoError(t, errSerialize) - for k, v := range m { - if k != "Rootdir" { // Rootdir is not serialized - assert.NotEmpty(t, v) - } - } -} - -func TestLoadValidationDefaultsConfigFileEnv(t *testing.T) { - tmpdir := t.TempDir() - cfg := DefaultConfig() - err := cfg.setRootdir(tmpdir) - assert.NoError(t, err) - FixtureEnvVars(t, envVarsDifferent) - errWriteConfig := cfg.WriteConfigFile() - - errLoad := cfg.LoadWithRootdir(true) - - assert.NoError(t, errWriteConfig) - assert.NoError(t, errLoad) - assert.Equal(t, "localhost:9999", cfg.API.Address) - assert.Equal(t, filepath.Join(tmpdir, "defra_data"), cfg.Datastore.Badger.Path) -} - -func TestLoadDefaultsEnv(t *testing.T) { - cfg := DefaultConfig() - FixtureEnvVars(t, envVarsDifferent) - - err := cfg.LoadWithRootdir(false) - - assert.NoError(t, err) - assert.Equal(t, "localhost:9999", cfg.API.Address) - assert.Equal(t, filepath.Join(cfg.Rootdir, "defra_data"), cfg.Datastore.Badger.Path) -} - -func TestEnvVariablesAllConsidered(t *testing.T) { - cfg := DefaultConfig() - FixtureEnvVars(t, envVarsDifferent) - - err := cfg.LoadWithRootdir(false) - - assert.NoError(t, err) - assert.Equal(t, "localhost:9999", cfg.API.Address) - assert.Equal(t, filepath.Join(cfg.Rootdir, "defra_data"), cfg.Datastore.Badger.Path) - assert.Equal(t, "memory", cfg.Datastore.Store) - assert.Equal(t, true, cfg.Net.P2PDisabled) - assert.Equal(t, []string{"/ip4/0.0.0.0/tcp/9876"}, cfg.Net.P2PAddresses) - assert.Equal(t, false, cfg.Net.PubSubEnabled) - assert.Equal(t, false, cfg.Net.RelayEnabled) - assert.Equal(t, "error", cfg.Log.Level) - assert.Equal(t, true, cfg.Log.Stacktrace) - assert.Equal(t, "json", cfg.Log.Format) -} - -func TestLoadNonExistingConfigFile(t *testing.T) { - cfg := DefaultConfig() - err := cfg.setRootdir(t.TempDir()) - assert.NoError(t, err) - err = cfg.LoadWithRootdir(true) - assert.ErrorIs(t, err, ErrReadingConfigFile) -} - -func TestLoadInvalidConfigFile(t *testing.T) { - cfg := DefaultConfig() - tmpdir := t.TempDir() - - errWrite := os.WriteFile( - filepath.Join(tmpdir, DefaultConfigFileName), - []byte("{"), - 0644, - ) - assert.NoError(t, errWrite) - - err := cfg.setRootdir(tmpdir) - assert.NoError(t, err) - errLoad := cfg.LoadWithRootdir(true) - assert.ErrorIs(t, errLoad, ErrReadingConfigFile) -} - -func TestInvalidEnvVars(t *testing.T) { - cfg := DefaultConfig() - FixtureEnvVars(t, envVarsInvalid) - - err := cfg.LoadWithRootdir(false) - - assert.ErrorIs(t, err, ErrLoadingConfig) -} - -func TestCreateAndLoadCustomConfig(t *testing.T) { - testdir := t.TempDir() - - cfg := DefaultConfig() - err := cfg.setRootdir(testdir) - assert.NoError(t, err) - // a few valid but non-default changes - cfg.Net.PubSubEnabled = false - cfg.Log.Level = "fatal" - - err = cfg.CreateRootDirAndConfigFile() - assert.NoError(t, err) - - assert.True(t, cfg.ConfigFileExists()) - - // check that the config file loads properly - cfg2 := DefaultConfig() - err = cfg2.setRootdir(testdir) - assert.NoError(t, err) - err = cfg2.LoadWithRootdir(true) - assert.NoError(t, err) - assert.Equal(t, cfg.Net.PubSubEnabled, cfg2.Net.PubSubEnabled) - assert.Equal(t, cfg.Log.Level, cfg2.Log.Level) -} - -func TestLoadValidationEnvLoggingConfig(t *testing.T) { - FixtureEnvKeyValue(t, "DEFRA_LOG_LEVEL", "debug,net=info,log=error,cli=fatal") - cfg := DefaultConfig() - err := cfg.LoadWithRootdir(false) - assert.NoError(t, err) - assert.Equal(t, "debug", cfg.Log.Level) - for _, override := range cfg.Log.NamedOverrides { - switch override.Name { - case "net": - assert.Equal(t, "info", override.Level) - case "log": - assert.Equal(t, "error", override.Level) - case "cli": - assert.Equal(t, "fatal", override.Level) - default: - t.Fatal("unexpected named override") - } - } -} - -func TestLoadValidationEnvLoggerConfig(t *testing.T) { - FixtureEnvKeyValue(t, "DEFRA_LOG_LOGGER", "net,nocolor=true,level=debug;config,output=stdout,level=info") - cfg := DefaultConfig() - err := cfg.LoadWithRootdir(false) - assert.NoError(t, err) - for _, override := range cfg.Log.NamedOverrides { - switch override.Name { - case "net": - assert.Equal(t, true, override.NoColor) - assert.Equal(t, "debug", override.Level) - case "config": - assert.Equal(t, "info", override.Level) - assert.Equal(t, "stdout", override.Output) - default: - t.Fatal("unexpected named override") - } - } -} - -func TestLoadValidationEnvLoggerConfigInvalid(t *testing.T) { - // logging config parameter not provided as = pair - FixtureEnvKeyValue(t, "DEFRA_LOG_LOGGER", "net,nocolor,true,level,debug;config,output,stdout,level,info") - cfg := DefaultConfig() - err := cfg.LoadWithRootdir(false) - assert.ErrorIs(t, err, ErrFailedToValidateConfig) - - // invalid logger names - FixtureEnvKeyValue(t, "DEFRA_LOG_LOGGER", "13;2134;™¡£¡™£∞¡™∞¡™£¢;1234;1") - cfg = DefaultConfig() - err = cfg.LoadWithRootdir(false) - assert.ErrorIs(t, err, ErrFailedToValidateConfig) -} - -func TestLoadValidationLoggerConfigFromEnvExhaustive(t *testing.T) { - FixtureEnvKeyValue(t, "DEFRA_LOG_LOGGER", "net,nocolor=true,level=debug;config,output=stdout,caller=false;logging,stacktrace=true,format=json") - cfg := DefaultConfig() - err := cfg.LoadWithRootdir(false) - assert.NoError(t, err) - for _, override := range cfg.Log.NamedOverrides { - switch override.Name { - case "net": - assert.Equal(t, true, override.NoColor) - assert.Equal(t, "debug", override.Level) - case "config": - assert.Equal(t, "stdout", override.Output) - assert.Equal(t, false, override.Caller) - case "logging": - assert.Equal(t, true, override.Stacktrace) - assert.Equal(t, "json", override.Format) - default: - t.Fatal("unexpected named override") - } - } -} - -func TestLoadValidationLoggerConfigFromEnvUnknownParam(t *testing.T) { - FixtureEnvKeyValue(t, "DEFRA_LOG_LOGGER", "net,unknown=true,level=debug") - cfg := DefaultConfig() - err := cfg.LoadWithRootdir(false) - assert.ErrorIs(t, err, ErrUnknownLoggerParameter) -} - -func TestLoadValidationInvalidDatastoreConfig(t *testing.T) { - FixtureEnvKeyValue(t, "DEFRA_DATASTORE_STORE", "antibadger") - cfg := DefaultConfig() - err := cfg.LoadWithRootdir(false) - assert.ErrorIs(t, err, ErrInvalidDatastoreType) -} - -func TestValidationLogger(t *testing.T) { - testCases := []struct { - input string - expectedErr error - }{ - {"node,level=debug,output=stdout", nil}, - {"node,level=fatal,format=csv", nil}, - {"node,level=warn", ErrInvalidLogLevel}, - {"node,level=debug;cli,", ErrNotProvidedAsKV}, - {"node,level", ErrNotProvidedAsKV}, - - {";", ErrInvalidLoggerConfig}, - {";;", ErrInvalidLoggerConfig}, - {",level=debug", ErrLoggerNameEmpty}, - {"node,bar=baz", ErrUnknownLoggerParameter}, // unknown parameter - {"m,level=debug,output-json", ErrNotProvidedAsKV}, // key-value pair with invalid separator - {"myModule,level=debug,extraPart", ErrNotProvidedAsKV}, // additional part after last key-value pair - {"myModule,=myValue", ErrNotProvidedAsKV}, // empty key - {",k=v", ErrLoggerNameEmpty}, // empty module - {";foo", ErrInvalidLoggerConfig}, // empty module name - {"k=v", ErrInvalidLoggerConfig}, // missing module - {"debug,net=,log=error,cli=fatal", ErrNotProvidedAsKV}, // empty value - - } - - for _, tc := range testCases { - cfg := DefaultConfig() - cfg.Log.Logger = tc.input - t.Log(tc.input) - err := cfg.validate() - assert.ErrorIs(t, err, tc.expectedErr) - } -} - -func TestValidationInvalidEmptyAPIAddress(t *testing.T) { - cfg := DefaultConfig() - cfg.API.Address = "" - err := cfg.validate() - assert.ErrorIs(t, err, ErrInvalidDatabaseURL) -} - -func TestValidationNetConfigPeers(t *testing.T) { - cfg := DefaultConfig() - cfg.Net.Peers = "/ip4/127.0.0.1/udp/1234,/ip4/7.7.7.7/tcp/4242/p2p/QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N" - err := cfg.validate() - assert.NoError(t, err) -} - -func TestValidationInvalidNetConfigPeers(t *testing.T) { - cfg := DefaultConfig() - cfg.Net.Peers = "&(*^(*&^(*&^(*&^))), mmmmh,123123" - err := cfg.validate() - assert.ErrorIs(t, err, ErrFailedToValidateConfig) -} - -func TestValidationInvalidLoggingConfig(t *testing.T) { - cfg := DefaultConfig() - cfg.Log.Level = "546578" - cfg.Log.Format = "*&)*&" - err := cfg.validate() - assert.ErrorIs(t, err, ErrInvalidLogLevel) -} - -func TestValidationAddressBasicIncomplete(t *testing.T) { - cfg := DefaultConfig() - cfg.API.Address = "localhost" - err := cfg.validate() - assert.ErrorIs(t, err, ErrFailedToValidateConfig) -} - -func TestValidationAddressLocalhostValid(t *testing.T) { - cfg := DefaultConfig() - cfg.API.Address = "localhost:9876" - err := cfg.validate() - assert.NoError(t, err) -} - -func TestValidationAddress0000Incomplete(t *testing.T) { - cfg := DefaultConfig() - cfg.API.Address = "0.0.0.0" - err := cfg.validate() - assert.ErrorIs(t, err, ErrFailedToValidateConfig) -} - -func TestValidationAddress0000Valid(t *testing.T) { - cfg := DefaultConfig() - cfg.API.Address = "0.0.0.0:9876" - err := cfg.validate() - assert.NoError(t, err) -} - -func TestValidationAddressDomainWithSubdomainValidWithTLSCorrectPortIsInvalid(t *testing.T) { - cfg := DefaultConfig() - cfg.API.Address = "sub.example.com:443" - cfg.API.TLS = true - err := cfg.validate() - assert.ErrorIs(t, err, ErrNoPortWithDomain) -} - -func TestValidationAddressDomainWithSubdomainWrongPortIsInvalid(t *testing.T) { - cfg := DefaultConfig() - cfg.API.Address = "sub.example.com:9876" - err := cfg.validate() - assert.ErrorIs(t, err, ErrNoPortWithDomain) -} diff --git a/config/config_utils.go b/config/config_utils.go deleted file mode 100644 index 46e5762570..0000000000 --- a/config/config_utils.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package config - -import ( - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - "unicode" -) - -type ByteSize uint64 - -const ( - B ByteSize = 1 - KiB = B << 10 - MiB = KiB << 10 - GiB = MiB << 10 - TiB = GiB << 10 - PiB = TiB << 10 -) - -// UnmarshalText calls Set on ByteSize with the given text -func (bs *ByteSize) UnmarshalText(text []byte) error { - return bs.Set(string(text)) -} - -// String returns the string formatted output of ByteSize -func (bs *ByteSize) String() string { - const unit = 1024 - bsInt := int64(*bs) - if bsInt < unit { - return fmt.Sprintf("%d", bsInt) - } - div, exp := int64(unit), 0 - for n := bsInt / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%d%ciB", bsInt/div, "KMGTP"[exp]) -} - -// Type returns the type as a string. -func (bs *ByteSize) Type() string { - return "ByteSize" -} - -// Set parses a string into ByteSize -func (bs *ByteSize) Set(s string) error { - digitString := "" - unit := "" - for _, char := range s { - if unicode.IsDigit(char) { - digitString += string(char) - } else { - unit += string(char) - } - } - digits, err := strconv.Atoi(digitString) - if err != nil { - return NewErrUnableToParseByteSize(err) - } - - switch strings.ToUpper(strings.Trim(unit, " ")) { - case "B": - *bs = ByteSize(digits) * B - case "KB", "KIB": - *bs = ByteSize(digits) * KiB - case "MB", "MIB": - *bs = ByteSize(digits) * MiB - case "GB", "GIB": - *bs = ByteSize(digits) * GiB - case "TB", "TIB": - *bs = ByteSize(digits) * TiB - case "PB", "PIB": - *bs = ByteSize(digits) * PiB - default: - *bs = ByteSize(digits) - } - - return nil -} - -// expandHomeDir expands paths if they were passed in as `~` rather than `${HOME}` -// converts `~/.defradb/certs/server.crt` to `/home/username/.defradb/certs/server.crt`. -func expandHomeDir(path *string) error { - if *path == "~" { - return ErrPathCannotBeHomeDir - } else if strings.HasPrefix(*path, "~/") { - homeDir, err := os.UserHomeDir() - if err != nil { - return NewErrUnableToExpandHomeDir(err) - } - - // Use strings.HasPrefix so we don't match paths like "/x/~/x/" - *path = filepath.Join(homeDir, (*path)[2:]) - } - - return nil -} - -func isLowercaseAlpha(s string) bool { - for i := 0; i < len(s); i++ { - c := s[i] - if c < 'a' || c > 'z' { - return false - } - } - return true -} - -func parseKV(kv string) ([]string, error) { - parsedKV := strings.Split(kv, "=") - if len(parsedKV) != 2 { - return nil, NewErrNotProvidedAsKV(kv) - } - if parsedKV[0] == "" || parsedKV[1] == "" { - return nil, NewErrNotProvidedAsKV(kv) - } - return parsedKV, nil -} diff --git a/config/config_utils_test.go b/config/config_utils_test.go deleted file mode 100644 index f9eea47c16..0000000000 --- a/config/config_utils_test.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package config - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestUnmarshalByteSize(t *testing.T) { - var bs ByteSize - - b := []byte("10") - err := bs.UnmarshalText(b) - assert.NoError(t, err) - assert.Equal(t, 10*B, bs) - - b = []byte("10B") - err = bs.UnmarshalText(b) - assert.NoError(t, err) - assert.Equal(t, 10*B, bs) - - b = []byte("10 B") - err = bs.UnmarshalText(b) - assert.NoError(t, err) - assert.Equal(t, 10*B, bs) - - kb := []byte("10KB") - err = bs.UnmarshalText(kb) - assert.NoError(t, err) - assert.Equal(t, 10*KiB, bs) - - kb = []byte("10KiB") - err = bs.UnmarshalText(kb) - assert.NoError(t, err) - assert.Equal(t, 10*KiB, bs) - - kb = []byte("10 kb") - err = bs.UnmarshalText(kb) - assert.NoError(t, err) - assert.Equal(t, 10*KiB, bs) - - mb := []byte("10MB") - err = bs.UnmarshalText(mb) - assert.NoError(t, err) - assert.Equal(t, 10*MiB, bs) - - mb = []byte("10MiB") - err = bs.UnmarshalText(mb) - assert.NoError(t, err) - assert.Equal(t, 10*MiB, bs) - - gb := []byte("10GB") - err = bs.UnmarshalText(gb) - assert.NoError(t, err) - assert.Equal(t, 10*GiB, bs) - - gb = []byte("10GiB") - err = bs.UnmarshalText(gb) - assert.NoError(t, err) - assert.Equal(t, 10*GiB, bs) - - tb := []byte("10TB") - err = bs.UnmarshalText(tb) - assert.NoError(t, err) - assert.Equal(t, 10*TiB, bs) - - tb = []byte("10TiB") - err = bs.UnmarshalText(tb) - assert.NoError(t, err) - assert.Equal(t, 10*TiB, bs) - - pb := []byte("10PB") - err = bs.UnmarshalText(pb) - assert.NoError(t, err) - assert.Equal(t, 10*PiB, bs) - - pb = []byte("10PiB") - err = bs.UnmarshalText(pb) - assert.NoError(t, err) - assert.Equal(t, 10*PiB, bs) - - eb := []byte("१") - err = bs.UnmarshalText(eb) - assert.ErrorIs(t, err, ErrUnableToParseByteSize) -} - -func TestByteSizeType(t *testing.T) { - var bs ByteSize - assert.Equal(t, "ByteSize", bs.Type()) -} - -func TestByteSizeToString(t *testing.T) { - b := 999 * B - assert.Equal(t, "999", b.String()) - - mb := 10 * MiB - assert.Equal(t, "10MiB", mb.String()) -} diff --git a/config/configfile.go b/config/configfile.go deleted file mode 100644 index f13b6695e1..0000000000 --- a/config/configfile.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package config - -import ( - "context" - _ "embed" - "fmt" - "os" - "path/filepath" -) - -const ( - DefaultConfigFileName = "config.yaml" - configType = "yaml" - defaultDirPerm = 0o700 - defaultConfigFilePerm = 0o644 -) - -// defaultConfigTemplate must reflect the Config struct in content and configuration. -// All parameters must be represented here, to support Viper's automatic environment variable handling. -// We embed the default config template for clarity and to avoid autoformatters from breaking it. -// -//go:embed configfile_yaml.gotmpl -var defaultConfigTemplate string - -func (cfg *Config) ConfigFilePath() string { - return filepath.Join(cfg.Rootdir, DefaultConfigFileName) -} - -func (cfg *Config) WriteConfigFile() error { - path := cfg.ConfigFilePath() - buffer, err := cfg.toBytes() - if err != nil { - return err - } - if err := os.WriteFile(path, buffer, defaultConfigFilePerm); err != nil { - return NewErrFailedToWriteFile(err, path) - } - log.FeedbackInfo(context.Background(), fmt.Sprintf("Created config file at %v", path)) - return nil -} - -func (cfg *Config) DeleteConfigFile() error { - if err := os.Remove(cfg.ConfigFilePath()); err != nil { - return NewErrFailedToRemoveConfigFile(err) - } - log.FeedbackInfo(context.Background(), fmt.Sprintf("Deleted config file at %v", cfg.ConfigFilePath())) - return nil -} - -func (cfg *Config) CreateRootDirAndConfigFile() error { - if err := os.MkdirAll(cfg.Rootdir, defaultDirPerm); err != nil { - return err - } - log.FeedbackInfo(context.Background(), fmt.Sprintf("Created DefraDB root directory at %v", cfg.Rootdir)) - if err := cfg.WriteConfigFile(); err != nil { - return err - } - return nil -} - -func (cfg *Config) ConfigFileExists() bool { - statInfo, err := os.Stat(cfg.ConfigFilePath()) - existsAsFile := (err == nil && !statInfo.IsDir()) - return existsAsFile -} - -func DefaultRootDir() string { - home, err := os.UserHomeDir() - if err != nil { - log.FatalE(context.Background(), "error determining user directory", err) - } - return filepath.Join(home, ".defradb") -} - -func FolderExists(folderPath string) bool { - statInfo, err := os.Stat(folderPath) - existsAsFolder := (err == nil && statInfo.IsDir()) - return existsAsFolder -} diff --git a/config/configfile_test.go b/config/configfile_test.go deleted file mode 100644 index 5f7aed26aa..0000000000 --- a/config/configfile_test.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package config - -import ( - "bytes" - "os" - "path/filepath" - "runtime" - "testing" - "text/template" - - "github.com/stretchr/testify/assert" -) - -func TestConfigTemplateSerialize(t *testing.T) { - var buffer bytes.Buffer - cfg := DefaultConfig() - tmpl := template.New("configTemplate") - configTemplate, err := tmpl.Parse(defaultConfigTemplate) - assert.NoError(t, err) - err = configTemplate.Execute(&buffer, cfg) - assert.NoError(t, err) - _, err = cfg.ToJSON() - assert.NoError(t, err) -} - -func TestConfigTemplateExecutes(t *testing.T) { - cfg := DefaultConfig() - var buffer bytes.Buffer - tmpl := template.New("configTemplate") - configTemplate, err := tmpl.Parse(defaultConfigTemplate) - assert.NoError(t, err) - err = configTemplate.Execute(&buffer, cfg) - assert.NoError(t, err) -} - -func TestWritesConfigFile(t *testing.T) { - cfg := DefaultConfig() - tmpdir := t.TempDir() - cfg.Rootdir = tmpdir - err := cfg.WriteConfigFile() - assert.NoError(t, err) - path := filepath.Join(tmpdir, DefaultConfigFileName) - _, err = os.Stat(path) - assert.Nil(t, err) -} - -func TestWritesConfigFileErroneousPath(t *testing.T) { - cfg := DefaultConfig() - cfg.Rootdir = filepath.Join(t.TempDir(), "////*&^^(*8769876////bar") - err := cfg.WriteConfigFile() - assert.Error(t, err) -} - -func TestReadConfigFileForLogger(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skipf("Test is not supported on windows as it leaks resources, see https://github.com/sourcenetwork/defradb/issues/2057") - } - - cfg := DefaultConfig() - tmpdir := t.TempDir() - cfg.Rootdir = tmpdir - cfg.Log.Caller = true - cfg.Log.Format = "json" - cfg.Log.Level = logLevelDebug - cfg.Log.NoColor = true - cfg.Log.Output = filepath.Join(tmpdir, "log.txt") - cfg.Log.Stacktrace = true - - err := cfg.WriteConfigFile() - assert.NoError(t, err) - - assert.True(t, cfg.ConfigFileExists()) - - cfgFromFile := DefaultConfig() - err = cfgFromFile.setRootdir(tmpdir) - assert.NoError(t, err) - err = cfgFromFile.LoadWithRootdir(true) - assert.NoError(t, err) - - assert.Equal(t, cfg.Log.Caller, cfgFromFile.Log.Caller) - assert.Equal(t, cfg.Log.Format, cfgFromFile.Log.Format) - assert.Equal(t, cfg.Log.Level, cfgFromFile.Log.Level) - assert.Equal(t, cfg.Log.NoColor, cfgFromFile.Log.NoColor) - assert.Equal(t, cfg.Log.Output, cfgFromFile.Log.Output) - assert.Equal(t, cfg.Log.Stacktrace, cfgFromFile.Log.Stacktrace) -} - -func TestReadConfigFileForDatastore(t *testing.T) { - tmpdir := t.TempDir() - - cfg := DefaultConfig() - err := cfg.setRootdir(tmpdir) - assert.NoError(t, err) - cfg.Datastore.Store = "badger" - cfg.Datastore.Badger.Path = "dataPath" - cfg.Datastore.Badger.ValueLogFileSize = 512 * MiB - - err = cfg.WriteConfigFile() - assert.NoError(t, err) - - configPath := filepath.Join(tmpdir, DefaultConfigFileName) - _, err = os.Stat(configPath) - assert.NoError(t, err) - - cfgFromFile := DefaultConfig() - err = cfgFromFile.setRootdir(tmpdir) - assert.NoError(t, err) - err = cfgFromFile.LoadWithRootdir(true) - assert.NoError(t, err) - - assert.Equal(t, cfg.Datastore.Store, cfgFromFile.Datastore.Store) - assert.Equal(t, filepath.Join(tmpdir, cfg.Datastore.Badger.Path), cfgFromFile.Datastore.Badger.Path) - assert.Equal(t, cfg.Datastore.Badger.ValueLogFileSize, cfgFromFile.Datastore.Badger.ValueLogFileSize) -} - -func TestConfigFileExists(t *testing.T) { - cfg := DefaultConfig() - err := cfg.setRootdir(t.TempDir()) - assert.NoError(t, err) - assert.False(t, cfg.ConfigFileExists()) - - err = cfg.WriteConfigFile() - assert.NoError(t, err) - assert.True(t, cfg.ConfigFileExists()) -} - -func TestConfigFileExistsErroneousPath(t *testing.T) { - cfg := DefaultConfig() - cfg.Rootdir = filepath.Join(t.TempDir(), "////*&^^(*8769876////bar") - assert.False(t, cfg.ConfigFileExists()) -} - -func TestDeleteConfigFile(t *testing.T) { - cfg := DefaultConfig() - tmpdir := t.TempDir() - cfg.Rootdir = tmpdir - err := cfg.WriteConfigFile() - assert.NoError(t, err) - - assert.True(t, cfg.ConfigFileExists()) - - err = cfg.DeleteConfigFile() - assert.NoError(t, err) - assert.False(t, cfg.ConfigFileExists()) -} diff --git a/config/configfile_yaml.gotmpl b/config/configfile_yaml.gotmpl deleted file mode 100644 index d789456a83..0000000000 --- a/config/configfile_yaml.gotmpl +++ /dev/null @@ -1,65 +0,0 @@ -# DefraDB configuration (YAML) - -# The default DefraDB directory is "$HOME/.defradb". It can be changed via the --rootdir CLI flag. -# Relative paths are interpreted as being rooted in the DefraDB directory. - -datastore: - # Store can be badger | memory - # badger: fast pure Go key-value store optimized for SSDs (https://github.com/dgraph-io/badger) - # memory: in-memory version of badger - store: {{ .Datastore.Store }} - badger: - # The path to the database data file(s). - path: {{ .Datastore.Badger.Path }} - # Maximum file size of the value log files. The in-memory file size will be 2*valuelogfilesize. - # Human friendly units can be used (ex: 500MB). - valuelogfilesize: {{ .Datastore.Badger.ValueLogFileSize }} - maxtxnretries: {{ .Datastore.MaxTxnRetries }} - # memory: - # size: {{ .Datastore.Memory.Size }} - -api: - # Address of the HTTP API to listen on or connect to - address: {{ .API.Address }} - # Whether the API server should listen over HTTPS - tls: {{ .API.TLS }} - # The list of origins a cross-domain request can be executed from. - # allowed-origins: {{ .API.AllowedOrigins }} - # The path to the public key file. Ignored if domains is set. - # pubkeypath: {{ .API.PubKeyPath }} - # The path to the private key file. Ignored if domains is set. - # privkeypath: {{ .API.PrivKeyPath }} - # Email address to let the CA (Let's Encrypt) send notifications via email when there are issues (optional). - # email: {{ .API.Email }} - -net: - # Whether the P2P is disabled - p2pdisabled: {{ .Net.P2PDisabled }} - # Listening address of the P2P network - p2paddresses: - {{ range $addr := .Net.P2PAddresses }} - - {{ . }} - {{ end }} - # Whether the node has pubsub enabled or not - pubsub: {{ .Net.PubSubEnabled }} - # Enable libp2p's Circuit relay transport protocol https://docs.libp2p.io/concepts/circuit-relay/ - relay: {{ .Net.RelayEnabled }} - # List of peers to boostrap with, specified as multiaddresses (https://docs.libp2p.io/concepts/addressing/) - peers: {{ .Net.Peers }} - -log: - # Log level. Options are debug, info, error, fatal - level: {{ .Log.Level }} - # Include stacktrace in error and fatal logs - stacktrace: {{ .Log.Stacktrace }} - # Supported log formats are json, csv - format: {{ .Log.Format }} - # Where the log output is written to - output: {{ .Log.Output }} - # Disable colored log output - nocolor: {{ .Log.NoColor }} - # Caller location in log output - caller: {{ .Log.Caller }} - # Provide specific named component logger configuration - # e.g. net,nocolor=true,level=debug;config,output=stdout,format=json - logger: {{ .Log.Logger }} \ No newline at end of file diff --git a/config/errors.go b/config/errors.go deleted file mode 100644 index 872b362b66..0000000000 --- a/config/errors.go +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package config - -import ( - "github.com/sourcenetwork/defradb/errors" -) - -const ( - errFailedToWriteFile string = "failed to write file" - errFailedToRemoveConfigFile string = "failed to remove config file" - errCannotBeHomeDir string = "path cannot be just ~ (home directory)" - errUnableToExpandHomeDir string = "unable to expand home directory" - errNoDatabaseURLProvided string = "no database URL provided" - errLoggingConfigNotObtained string = "could not get logging config" - errFailedToValidateConfig string = "failed to validate config" - errOverrideConfigConvertFailed string = "invalid override config" - errConfigToJSONFailed string = "failed to marshal Config to JSON" - errInvalidDatabaseURL string = "invalid database URL" - errInvalidRPCTimeout string = "invalid RPC timeout" - errInvalidRPCMaxConnectionIdle string = "invalid RPC MaxConnectionIdle" - errInvalidP2PAddress string = "invalid P2P address" - errInvalidRPCAddress string = "invalid RPC address" - errInvalidBootstrapPeers string = "invalid bootstrap peers" - errInvalidLogLevel string = "invalid log level" - errInvalidDatastoreType string = "invalid store type" - errInvalidLogFormat string = "invalid log format" - errInvalidNamedLoggerName string = "invalid named logger name" - errInvalidLoggerConfig string = "invalid logger config" - errConfigTemplateFailed string = "could not process config template" - errCouldNotObtainLoggerConfig string = "could not get named logger config" - errNotProvidedAsKV string = "logging config parameter was not provided as = pair" - errLoggerNameEmpty string = "logger name cannot be empty" - errCouldNotParseType string = "could not parse type" - errUnknownLoggerParameter string = "unknown logger parameter" - errInvalidLoggerName string = "invalid logger name" - errDuplicateLoggerName string = "duplicate logger name" - errReadingConfigFile string = "failed to read config" - errLoadingConfig string = "failed to load config" - errUnableToParseByteSize string = "unable to parse byte size" - errInvalidDatastorePath string = "invalid datastore path" - errMissingPortNumber string = "missing port number" - errNoPortWithDomain string = "cannot provide port with domain name" - errInvalidRootDir string = "invalid root directory" -) - -var ( - ErrFailedToWriteFile = errors.New(errFailedToWriteFile) - ErrFailedToRemoveConfigFile = errors.New(errFailedToRemoveConfigFile) - ErrPathCannotBeHomeDir = errors.New(errCannotBeHomeDir) - ErrUnableToExpandHomeDir = errors.New(errUnableToExpandHomeDir) - ErrNoDatabaseURLProvided = errors.New(errNoDatabaseURLProvided) - ErrInvalidDatabaseURL = errors.New(errInvalidDatabaseURL) - ErrLoggingConfigNotObtained = errors.New(errLoggingConfigNotObtained) - ErrFailedToValidateConfig = errors.New(errFailedToValidateConfig) - ErrInvalidRPCTimeout = errors.New(errInvalidRPCTimeout) - ErrInvalidRPCMaxConnectionIdle = errors.New(errInvalidRPCMaxConnectionIdle) - ErrInvalidP2PAddress = errors.New(errInvalidP2PAddress) - ErrInvalidRPCAddress = errors.New(errInvalidRPCAddress) - ErrInvalidBootstrapPeers = errors.New(errInvalidBootstrapPeers) - ErrInvalidLogLevel = errors.New(errInvalidLogLevel) - ErrInvalidDatastoreType = errors.New(errInvalidDatastoreType) - ErrOverrideConfigConvertFailed = errors.New(errOverrideConfigConvertFailed) - ErrInvalidLogFormat = errors.New(errInvalidLogFormat) - ErrConfigToJSONFailed = errors.New(errConfigToJSONFailed) - ErrInvalidNamedLoggerName = errors.New(errInvalidNamedLoggerName) - ErrConfigTemplateFailed = errors.New(errConfigTemplateFailed) - ErrCouldNotObtainLoggerConfig = errors.New(errCouldNotObtainLoggerConfig) - ErrNotProvidedAsKV = errors.New(errNotProvidedAsKV) - ErrLoggerNameEmpty = errors.New(errLoggerNameEmpty) - ErrCouldNotParseType = errors.New(errCouldNotParseType) - ErrUnknownLoggerParameter = errors.New(errUnknownLoggerParameter) - ErrInvalidLoggerName = errors.New(errInvalidLoggerName) - ErrDuplicateLoggerName = errors.New(errDuplicateLoggerName) - ErrReadingConfigFile = errors.New(errReadingConfigFile) - ErrLoadingConfig = errors.New(errLoadingConfig) - ErrUnableToParseByteSize = errors.New(errUnableToParseByteSize) - ErrInvalidLoggerConfig = errors.New(errInvalidLoggerConfig) - ErrorInvalidDatastorePath = errors.New(errInvalidDatastorePath) - ErrMissingPortNumber = errors.New(errMissingPortNumber) - ErrNoPortWithDomain = errors.New(errNoPortWithDomain) - ErrorInvalidRootDir = errors.New(errInvalidRootDir) -) - -func NewErrFailedToWriteFile(inner error, path string) error { - return errors.Wrap(errFailedToWriteFile, inner, errors.NewKV("path", path)) -} - -func NewErrFailedToRemoveConfigFile(inner error) error { - return errors.Wrap(errFailedToRemoveConfigFile, inner) -} - -func NewErrPathCannotBeHomeDir(inner error) error { - return errors.Wrap(errCannotBeHomeDir, inner) -} - -func NewErrUnableToExpandHomeDir(inner error) error { - return errors.Wrap(errUnableToExpandHomeDir, inner) -} - -func NewErrNoDatabaseURLProvided(inner error) error { - return errors.Wrap(errNoDatabaseURLProvided, inner) -} - -func NewErrInvalidDatabaseURL(inner error) error { - return errors.Wrap(errInvalidDatabaseURL, inner) -} - -func NewErrLoggingConfigNotObtained(inner error) error { - return errors.Wrap(errLoggingConfigNotObtained, inner) -} - -func NewErrFailedToValidateConfig(inner error) error { - return errors.Wrap(errFailedToValidateConfig, inner) -} - -func NewErrInvalidRPCTimeout(inner error, timeout string) error { - return errors.Wrap(errInvalidRPCTimeout, inner, errors.NewKV("timeout", timeout)) -} - -func NewErrInvalidRPCMaxConnectionIdle(inner error, timeout string) error { - return errors.Wrap(errInvalidRPCMaxConnectionIdle, inner, errors.NewKV("timeout", timeout)) -} - -func NewErrInvalidP2PAddress(inner error, address string) error { - return errors.Wrap(errInvalidP2PAddress, inner, errors.NewKV("address", address)) -} - -func NewErrInvalidRPCAddress(inner error, address string) error { - return errors.Wrap(errInvalidRPCAddress, inner, errors.NewKV("address", address)) -} - -func NewErrInvalidBootstrapPeers(inner error, peers string) error { - return errors.Wrap(errInvalidBootstrapPeers, inner, errors.NewKV("peers", peers)) -} - -func NewErrInvalidLogLevel(level string) error { - return errors.New(errInvalidLogLevel, errors.NewKV("level", level)) -} - -func NewErrInvalidDatastoreType(storeType string) error { - return errors.New(errInvalidDatastoreType, errors.NewKV("store_type", storeType)) -} - -func NewErrOverrideConfigConvertFailed(inner error, name string) error { - return errors.Wrap(errOverrideConfigConvertFailed, inner, errors.NewKV("name", name)) -} - -func NewErrInvalidLogFormat(format string) error { - return errors.New(errInvalidLogFormat, errors.NewKV("format", format)) -} - -func NewErrConfigToJSONFailed(inner error) error { - return errors.Wrap(errConfigToJSONFailed, inner) -} - -func NewErrInvalidNamedLoggerName(name string) error { - return errors.New(errInvalidNamedLoggerName, errors.NewKV("name", name)) -} - -func NewErrConfigTemplateFailed(inner error) error { - return errors.Wrap(errConfigTemplateFailed, inner) -} - -func NewErrCouldNotObtainLoggerConfig(inner error, name string) error { - return errors.Wrap(errCouldNotObtainLoggerConfig, inner, errors.NewKV("name", name)) -} - -func NewErrNotProvidedAsKV(kv string) error { - return errors.New(errNotProvidedAsKV, errors.NewKV("KV", kv)) -} - -func NewErrCouldNotParseType(inner error, name string) error { - return errors.Wrap(errCouldNotParseType, inner, errors.NewKV("name", name)) -} - -func NewErrUnknownLoggerParameter(name string) error { - return errors.New(errUnknownLoggerParameter, errors.NewKV("param", name)) -} - -func NewErrInvalidLoggerName(name string) error { - return errors.New(errInvalidLoggerName, errors.NewKV("name", name)) -} - -func NewErrDuplicateLoggerName(name string) error { - return errors.New(errDuplicateLoggerName, errors.NewKV("name", name)) -} - -func NewErrReadingConfigFile(inner error) error { - return errors.Wrap(errReadingConfigFile, inner) -} - -func NewErrLoadingConfig(inner error) error { - return errors.Wrap(errLoadingConfig, inner) -} - -func NewErrUnableToParseByteSize(err error) error { - return errors.Wrap(errUnableToParseByteSize, err) -} - -func NewErrLoggerConfig(s string) error { - return errors.New(errInvalidLoggerConfig, errors.NewKV("explanation", s)) -} - -func NewErrInvalidDatastorePath(path string) error { - return errors.New(errInvalidDatastorePath, errors.NewKV("path", path)) -} - -func NewErrInvalidRootDir(path string) error { - return errors.New(errInvalidRootDir, errors.NewKV("path", path)) -} diff --git a/docs/config.md b/docs/config.md new file mode 100644 index 0000000000..5f8985f71c --- /dev/null +++ b/docs/config.md @@ -0,0 +1,66 @@ +# DefraDB configuration (YAML) + +The default DefraDB directory is `$HOME/.defradb`. It can be changed via the --rootdir CLI flag. + +Relative paths are interpreted as being rooted in the DefraDB directory. + +## `datastore.store` + +Store can be badger or memory. Defaults to `badger`. + +- badger: fast pure Go key-value store optimized for SSDs (https://github.com/dgraph-io/badger) +- memory: in-memory version of badger + +## `datastore.maxtxnretries` + +The number of retries to make in the event of a transaction conflict. Defaults to `5`. + +Currently this is only used within the P2P system and will not affect operations initiated by users. + +## `datastore.badger.path` + +The path to the database data file(s). Defaults to `data`. + +## `datastore.badger.valuelogfilesize` + +Maximum file size of the value log files. + +## `api.address` + +Address of the HTTP API to listen on or connect to. Defaults to `127.0.0.1:9181`. + +## `api.allowed-origins` + +The list of origins a cross-domain request can be executed from. + +## `api.pubkeypath` + +The path to the public key file for TLS / HTTPS. + +## `api.privkeypath` + +The path to the private key file for TLS / HTTPS. + +## `net.p2pdisabled` + +Whether P2P networking is disabled. Defaults to `false`. + +## `net.p2paddresses` + +List of addresses for the P2P network to listen on. Defaults to `/ip4/127.0.0.1/tcp/9171`. + +## `net.pubsubenabled` + +Whether PubSub is enabled. Defaults to `true`. + +## `net.peers` + +List of peers to boostrap with, specified as multiaddresses. + +https://docs.libp2p.io/concepts/addressing/ + +## `net.relay` + +Enable libp2p's Circuit relay transport protocol. Defaults to `false`. + +https://docs.libp2p.io/concepts/circuit-relay/ \ No newline at end of file diff --git a/tests/clients/cli/wrapper_cli.go b/tests/clients/cli/wrapper_cli.go index 1f73b20e25..e8dc24270c 100644 --- a/tests/clients/cli/wrapper_cli.go +++ b/tests/clients/cli/wrapper_cli.go @@ -17,7 +17,6 @@ import ( "strings" "github.com/sourcenetwork/defradb/cli" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/datastore" ) @@ -67,7 +66,7 @@ func (w *cliWrapper) executeStream(ctx context.Context, args []string) (io.ReadC } args = append(args, "--url", w.address) - cmd := cli.NewDefraCommand(config.DefaultConfig()) + cmd := cli.NewDefraCommand() cmd.SetOut(stdOutWrite) cmd.SetErr(stdErrWrite) cmd.SetArgs(args) diff --git a/tests/gen/cli/gendocs.go b/tests/gen/cli/gendocs.go index b325fffd3f..ab714b4660 100644 --- a/tests/gen/cli/gendocs.go +++ b/tests/gen/cli/gendocs.go @@ -20,16 +20,15 @@ import ( "github.com/spf13/cobra" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/http" "github.com/sourcenetwork/defradb/tests/gen" ) const defaultBatchSize = 1000 -func MakeGenDocCommand(cfg *config.Config) *cobra.Command { +func MakeGenDocCommand() *cobra.Command { var demandJSON string - + var url string var cmd = &cobra.Command{ Use: "gendocs --demand ", Short: "Automatically generates documents for existing collections.", @@ -39,11 +38,7 @@ Example: The following command generates 100 User documents and 500 Device docum gendocs --demand '{"User": 100, "Device": 500 }'`, ValidArgs: []string{"demand"}, RunE: func(cmd *cobra.Command, args []string) error { - // cobra does not chain pre run calls so we have to run them again here - if err := loadConfig(cfg); err != nil { - return err - } - store, err := http.NewClient(cfg.API.Address) + store, err := http.NewClient(url) if err != nil { return err } @@ -101,15 +96,7 @@ Example: The following command generates 100 User documents and 500 Device docum }, } - cmd.PersistentFlags().String( - "url", cfg.API.Address, - "URL of HTTP endpoint to listen on or connect to", - ) - err := cfg.BindFlag("api.address", cmd.PersistentFlags().Lookup("url")) - if err != nil { - panic(err) - } - + cmd.Flags().StringVar(&url, "url", "localhost:9181", "URL of HTTP endpoint to listen on or connect to") cmd.Flags().StringVarP(&demandJSON, "demand", "d", "", "Documents' demand in JSON format") return cmd @@ -160,10 +147,3 @@ func colsToDefs(cols []client.Collection) []client.CollectionDefinition { } return colDefs } - -func loadConfig(cfg *config.Config) error { - if err := cfg.LoadRootDirFromFlagOrDefault(); err != nil { - return err - } - return cfg.LoadWithRootdir(cfg.ConfigFileExists()) -} diff --git a/tests/gen/cli/gendocs_test.go b/tests/gen/cli/gendocs_test.go index 1bd30297a6..35202ec9e0 100644 --- a/tests/gen/cli/gendocs_test.go +++ b/tests/gen/cli/gendocs_test.go @@ -37,7 +37,7 @@ func TestGendocsCmd_IfNoErrors_ReturnGenerationOutput(t *testing.T) { owner: User }`) - genDocsCmd := MakeGenDocCommand(getTestConfig(t)) + genDocsCmd := MakeGenDocCommand() outputBuf := bytes.NewBufferString("") genDocsCmd.SetOut(outputBuf) @@ -71,7 +71,7 @@ func TestGendocsCmd_IfInvalidDemandValue_ReturnError(t *testing.T) { name: String }`) - genDocsCmd := MakeGenDocCommand(getTestConfig(t)) + genDocsCmd := MakeGenDocCommand() genDocsCmd.SetArgs([]string{ "--demand", `{"User": invalid}`, "--url", strings.TrimPrefix(defra.server.URL, "http://"), @@ -90,7 +90,7 @@ func TestGendocsCmd_IfInvalidConfig_ReturnError(t *testing.T) { name: String }`) - genDocsCmd := MakeGenDocCommand(getTestConfig(t)) + genDocsCmd := MakeGenDocCommand() genDocsCmd.SetArgs([]string{ "--demand", `{"Unknown": 3}`, "--url", strings.TrimPrefix(defra.server.URL, "http://"), diff --git a/tests/gen/cli/util_test.go b/tests/gen/cli/util_test.go index 81d713955c..10bd98ca99 100644 --- a/tests/gen/cli/util_test.go +++ b/tests/gen/cli/util_test.go @@ -19,7 +19,6 @@ import ( "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" badgerds "github.com/sourcenetwork/defradb/datastore/badger/v4" "github.com/sourcenetwork/defradb/db" "github.com/sourcenetwork/defradb/errors" @@ -67,17 +66,6 @@ func start(ctx context.Context) (*defraInstance, error) { }, nil } -func getTestConfig(t *testing.T) *config.Config { - cfg := config.DefaultConfig() - cfg.Datastore.Store = "memory" - cfg.Net.P2PDisabled = true - cfg.Rootdir = t.TempDir() - cfg.Net.P2PAddresses = []string{"/ip4/127.0.0.1/tcp/0"} - cfg.API.Address = "127.0.0.1:0" - cfg.Persist() - return cfg -} - func startTestNode(t *testing.T) (*defraInstance, func()) { ctx := context.Background() di, err := start(ctx) diff --git a/tests/integration/net/order/tcp_test.go b/tests/integration/net/order/tcp_test.go index 8a419360d5..f80701c64c 100644 --- a/tests/integration/net/order/tcp_test.go +++ b/tests/integration/net/order/tcp_test.go @@ -16,16 +16,16 @@ import ( "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" + "github.com/sourcenetwork/defradb/net" testutils "github.com/sourcenetwork/defradb/tests/integration" ) // TestP2PWithSingleDocumentUpdatePerNode tests document syncing between two nodes with a single update per node func TestP2PWithSingleDocumentUpdatePerNode(t *testing.T) { test := P2PTestCase{ - NodeConfig: []*config.Config{ - randomNetworkingConfig(), - randomNetworkingConfig(), + NodeConfig: [][]net.NodeOpt{ + testutils.RandomNetworkingConfig()(), + testutils.RandomNetworkingConfig()(), }, NodePeers: map[int][]int{ 1: { @@ -74,9 +74,9 @@ func TestP2PWithSingleDocumentUpdatePerNode(t *testing.T) { // TestP2PWithMultipleDocumentUpdatesPerNode tests document syncing between two nodes with multiple updates per node. func TestP2PWithMultipleDocumentUpdatesPerNode(t *testing.T) { test := P2PTestCase{ - NodeConfig: []*config.Config{ - randomNetworkingConfig(), - randomNetworkingConfig(), + NodeConfig: [][]net.NodeOpt{ + testutils.RandomNetworkingConfig()(), + testutils.RandomNetworkingConfig()(), }, NodePeers: map[int][]int{ 1: { @@ -145,9 +145,9 @@ func TestP2FullPReplicator(t *testing.T) { require.NoError(t, err) test := P2PTestCase{ - NodeConfig: []*config.Config{ - randomNetworkingConfig(), - randomNetworkingConfig(), + NodeConfig: [][]net.NodeOpt{ + testutils.RandomNetworkingConfig()(), + testutils.RandomNetworkingConfig()(), }, NodeReplicators: map[int][]int{ 0: { diff --git a/tests/integration/net/order/utils.go b/tests/integration/net/order/utils.go index 178272b155..3ba5fc7f26 100644 --- a/tests/integration/net/order/utils.go +++ b/tests/integration/net/order/utils.go @@ -13,14 +13,12 @@ package order import ( "context" "fmt" - "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" coreDB "github.com/sourcenetwork/defradb/db" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/logging" @@ -50,7 +48,7 @@ const ( type P2PTestCase struct { Query string // Configuration parameters for each peer - NodeConfig []*config.Config + NodeConfig [][]net.NodeOpt // List of peers for each net. // Only peers with lower index than the node can be used in the list of peers. @@ -69,7 +67,12 @@ type P2PTestCase struct { ReplicatorResult map[int]map[string]map[string]any } -func setupDefraNode(t *testing.T, cfg *config.Config, seeds []string) (*net.Node, []client.DocID, error) { +func setupDefraNode( + t *testing.T, + opts []net.NodeOpt, + peers []string, + seeds []string, +) (*net.Node, []client.DocID, error) { ctx := context.Background() log.Info(ctx, "Building new memory store") @@ -92,40 +95,28 @@ func setupDefraNode(t *testing.T, cfg *config.Config, seeds []string) (*net.Node // init the P2P node var n *net.Node - log.Info(ctx, "Starting P2P node", logging.NewKV("P2P addresses", cfg.Net.P2PAddresses)) - n, err = net.NewNode( - ctx, - db, - net.WithListenAddresses(cfg.Net.P2PAddresses...), - net.WithEnablePubSub(cfg.Net.PubSubEnabled), - net.WithEnableRelay(cfg.Net.RelayEnabled), - ) + n, err = net.NewNode(ctx, db, opts...) if err != nil { return nil, nil, errors.Wrap("failed to start P2P node", err) } // parse peers and bootstrap - if len(cfg.Net.Peers) != 0 { - log.Info(ctx, "Parsing bootstrap peers", logging.NewKV("Peers", cfg.Net.Peers)) - addrs, err := netutils.ParsePeers(strings.Split(cfg.Net.Peers, ",")) + if len(peers) != 0 { + log.Info(ctx, "Parsing bootstrap peers", logging.NewKV("Peers", peers)) + addrs, err := netutils.ParsePeers(peers) if err != nil { - return nil, nil, errors.Wrap(fmt.Sprintf("failed to parse bootstrap peers %v", cfg.Net.Peers), err) + return nil, nil, errors.Wrap(fmt.Sprintf("failed to parse bootstrap peers %v", peers), err) } log.Info(ctx, "Bootstrapping with peers", logging.NewKV("Addresses", addrs)) n.Bootstrap(addrs) } + log.Info(ctx, "Starting P2P node", logging.NewKV("P2P addresses", n.PeerInfo().Addrs)) if err := n.Start(); err != nil { n.Close() return nil, nil, errors.Wrap("unable to start P2P listeners", err) } - var addresses []string - for _, addr := range n.ListenAddrs() { - addresses = append(addresses, addr.String()) - } - cfg.Net.P2PAddresses = addresses - return n, docIDs, nil } @@ -201,9 +192,8 @@ func executeTestCase(t *testing.T, test P2PTestCase) { for i, cfg := range test.NodeConfig { log.Info(ctx, fmt.Sprintf("Setting up node %d", i)) - cfg.Datastore.Badger.Path = t.TempDir() + var peerAddresses []string if peers, ok := test.NodePeers[i]; ok { - peerAddresses := []string{} for _, p := range peers { if p >= len(nodes) { log.Info(ctx, "cannot set a peer that hasn't been started. Skipping to next peer") @@ -215,9 +205,8 @@ func executeTestCase(t *testing.T, test P2PTestCase) { fmt.Sprintf("%s/p2p/%s", peerInfo.Addrs[0], peerInfo.ID), ) } - cfg.Net.Peers = strings.Join(peerAddresses, ",") } - n, d, err := setupDefraNode(t, cfg, test.SeedDocuments) + n, d, err := setupDefraNode(t, cfg, peerAddresses, test.SeedDocuments) require.NoError(t, err) if i == 0 { @@ -348,10 +337,3 @@ func executeTestCase(t *testing.T, test P2PTestCase) { n.DB.Close() } } - -func randomNetworkingConfig() *config.Config { - cfg := config.DefaultConfig() - cfg.Net.P2PAddresses = []string{"/ip4/127.0.0.1/tcp/0"} - cfg.Net.RelayEnabled = false - return cfg -} diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index 463d1bd653..4d48cb033b 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -14,8 +14,8 @@ import ( "time" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/logging" + "github.com/sourcenetwork/defradb/net" "github.com/sourcenetwork/defradb/tests/clients" "github.com/libp2p/go-libp2p/core/peer" @@ -523,10 +523,10 @@ func waitForSync( } func RandomNetworkingConfig() ConfigureNode { - return func() config.Config { - cfg := config.DefaultConfig() - cfg.Net.P2PAddresses = []string{"/ip4/127.0.0.1/tcp/0"} - cfg.Net.RelayEnabled = false - return *cfg + return func() []net.NodeOpt { + return []net.NodeOpt{ + net.WithListenAddresses("/ip4/127.0.0.1/tcp/0"), + net.WithEnableRelay(false), + } } } diff --git a/tests/integration/state.go b/tests/integration/state.go index ca795a2492..25a248413b 100644 --- a/tests/integration/state.go +++ b/tests/integration/state.go @@ -14,12 +14,11 @@ import ( "context" "testing" - "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/datastore" + "github.com/sourcenetwork/defradb/net" "github.com/sourcenetwork/defradb/tests/clients" ) @@ -53,14 +52,11 @@ type state struct { // These synchronisation channels allow async actions to track their completion. syncChans []chan struct{} - // The private keys for any nodes. - nodePrivateKeys []crypto.PrivKey - // The addresses of any nodes configured. nodeAddresses []peer.AddrInfo // The configurations for any nodes - nodeConfigs []config.Config + nodeConfigs [][]net.NodeOpt // The nodes active in this test. nodes []clients.Client @@ -108,9 +104,8 @@ func newState( allActionsDone: make(chan struct{}), subscriptionResultsChans: []chan func(){}, syncChans: []chan struct{}{}, - nodePrivateKeys: []crypto.PrivKey{}, nodeAddresses: []peer.AddrInfo{}, - nodeConfigs: []config.Config{}, + nodeConfigs: [][]net.NodeOpt{}, nodes: []clients.Client{}, dbPaths: []string{}, collections: [][]client.Collection{}, diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index a2cf6b16d6..628863a032 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -17,7 +17,7 @@ import ( "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" + "github.com/sourcenetwork/defradb/net" "github.com/sourcenetwork/defradb/tests/gen" "github.com/sourcenetwork/defradb/tests/predefined" ) @@ -55,7 +55,7 @@ type SetupComplete struct{} // Nodes may be explicitly referenced by index by other actions using `NodeID` properties. // If the action has a `NodeID` property and it is not specified, the action will be // effected on all nodes. -type ConfigureNode func() config.Config +type ConfigureNode func() []net.NodeOpt // Restart is an action that will close and then start all nodes. type Restart struct{} diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index a00330b93b..83f00f61d1 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -664,25 +664,18 @@ func restartNodes( continue } - key := s.nodePrivateKeys[i] - cfg := s.nodeConfigs[i] // We need to make sure the node is configured with its old address, otherwise // a new one may be selected and reconnnection to it will fail. var addresses []string for _, addr := range s.nodeAddresses[i].Addrs { addresses = append(addresses, addr.String()) } - cfg.Net.P2PAddresses = addresses + + nodeOpts := s.nodeConfigs[i] + nodeOpts = append(nodeOpts, net.WithListenAddresses(addresses...)) var n *net.Node - n, err = net.NewNode( - s.ctx, - db, - net.WithListenAddresses(cfg.Net.P2PAddresses...), - net.WithEnablePubSub(cfg.Net.PubSubEnabled), - net.WithEnableRelay(cfg.Net.RelayEnabled), - net.WithPrivateKey(key), - ) + n, err = net.NewNode(s.ctx, db, nodeOpts...) require.NoError(s.t, err) if err := n.Start(); err != nil { @@ -771,23 +764,17 @@ func configureNode( return } - cfg := action() db, path, err := setupDatabase(s) //disable change dector, or allow it? require.NoError(s.t, err) privateKey, _, err := crypto.GenerateKeyPair(crypto.Ed25519, 0) require.NoError(s.t, err) + nodeOpts := action() + nodeOpts = append(nodeOpts, net.WithPrivateKey(privateKey)) + var n *net.Node - log.Info(s.ctx, "Starting P2P node", logging.NewKV("P2P address", cfg.Net.P2PAddresses)) - n, err = net.NewNode( - s.ctx, - db, - net.WithListenAddresses(cfg.Net.P2PAddresses...), - net.WithEnablePubSub(cfg.Net.PubSubEnabled), - net.WithEnableRelay(cfg.Net.RelayEnabled), - net.WithPrivateKey(privateKey), - ) + n, err = net.NewNode(s.ctx, db, nodeOpts...) require.NoError(s.t, err) log.Info(s.ctx, "Starting P2P node", logging.NewKV("P2P address", n.PeerInfo())) @@ -797,8 +784,7 @@ func configureNode( } s.nodeAddresses = append(s.nodeAddresses, n.PeerInfo()) - s.nodeConfigs = append(s.nodeConfigs, cfg) - s.nodePrivateKeys = append(s.nodePrivateKeys, privateKey) + s.nodeConfigs = append(s.nodeConfigs, nodeOpts) c, err := setupClient(s, n) require.NoError(s.t, err)