From 9739882c71faa0bfeaad1415884b73ad9c6f6e13 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 12 Feb 2024 16:30:08 -0800 Subject: [PATCH] move config into cli package --- cli/client.go | 2 +- cli/collection.go | 2 +- cli/config.go | 179 ++++++++++++++++++++++++ cli/config_test.go | 49 +++++++ cli/root.go | 30 ++-- cli/server_dump.go | 9 +- cli/start.go | 16 +-- cli/utils.go | 75 +--------- config/config.go | 105 -------------- config/config.yaml | 79 ----------- config/config_test.go | 61 -------- tests/gen/cli/gendocs.go | 26 +--- tests/gen/cli/gendocs_test.go | 6 +- tests/gen/cli/util_test.go | 12 -- tests/integration/net/order/tcp_test.go | 20 +-- tests/integration/net/order/utils.go | 48 ++----- tests/integration/p2p.go | 12 +- tests/integration/state.go | 11 +- tests/integration/test_case.go | 4 +- tests/integration/utils2.go | 32 ++--- 20 files changed, 305 insertions(+), 473 deletions(-) create mode 100644 cli/config.go create mode 100644 cli/config_test.go delete mode 100644 config/config.go delete mode 100644 config/config.yaml delete mode 100644 config/config_test.go diff --git a/cli/client.go b/cli/client.go index 98f7b1ba49..26dfe3524f 100644 --- a/cli/client.go +++ b/cli/client.go @@ -22,7 +22,7 @@ func MakeClientCommand() *cobra.Command { Long: `Interact with a DefraDB node. Execute queries, add schema types, obtain node info, etc.`, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := setConfigContext(cmd); err != nil { + if err := setConfigContext(cmd, false); err != nil { return err } if err := setTransactionContext(cmd, txID); err != nil { diff --git a/cli/collection.go b/cli/collection.go index 5fdad1218a..2e19c940b5 100644 --- a/cli/collection.go +++ b/cli/collection.go @@ -30,7 +30,7 @@ func MakeCollectionCommand() *cobra.Command { Long: `Create, read, update, and delete documents within a collection.`, PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { // cobra does not chain pre run calls so we have to run them again here - if err := setConfigContext(cmd); err != nil { + if err := setConfigContext(cmd, false); err != nil { return err } if err := setTransactionContext(cmd, txID); err != nil { diff --git a/cli/config.go b/cli/config.go new file mode 100644 index 0000000000..6ca3218c11 --- /dev/null +++ b/cli/config.go @@ -0,0 +1,179 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package cli + +import ( + _ "embed" + "errors" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/sourcenetwork/defradb/logging" +) + +// configPaths are config keys that will be made relative to the rootdir +var configPaths = []string{ + "datastore.badger.path", + "api.pubkeypath", + "api.privkeypath", +} + +// configFlags is a mapping of config keys to cli flags to bind to. +var configFlags = map[string]string{ + "log.level": "loglevel", + "log.output": "logoutput", + "log.format": "logformat", + "log.stacktrace": "logtrace", + "log.nocolor": "lognocolor", + "api.address": "url", + "datastore.maxtxnretries": "max-txn-retries", + "datastore.badger.inMemory": "in-memory", + "datastore.badger.valuelogfilesize": "valuelogfilesize", + "net.peers": "peers", + "net.p2paddresses": "p2paddr", + "net.p2pdisabled": "no-p2p", + "api.allowed-origins": "allowed-origins", + "api.pubkeypath": "pubkeypath", + "api.privkeypath": "privkeypath", +} + +// defaultConfig returns a new config with default values. +func defaultConfig() *viper.Viper { + cfg := viper.New() + + cfg.AutomaticEnv() + cfg.SetEnvPrefix("DEFRA") + cfg.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_")) + + cfg.SetConfigName("config") + cfg.SetConfigType("yaml") + + cfg.SetDefault("datastore.maxTxnRetries", 5) + + cfg.SetDefault("datastore.badger.path", "data") + cfg.SetDefault("datastore.badger.inMemory", false) + cfg.SetDefault("datastore.badger.valueLogFileSize", 1<<30) + + cfg.SetDefault("api.address", "127.0.0.1:9181") + cfg.SetDefault("api.pubKeyPath", "") + cfg.SetDefault("api.privKeyPath", "") + cfg.SetDefault("api.allowed-origins", []string{}) + + cfg.SetDefault("net.p2pDisabled", false) + cfg.SetDefault("net.p2pAddresses", []any{"/ip4/127.0.0.1/tcp/9171"}) + cfg.SetDefault("net.pubSubEnabled", true) + cfg.SetDefault("net.relay", false) + cfg.SetDefault("net.peers", []string{}) + + cfg.SetDefault("log.level", "info") + cfg.SetDefault("log.stackTrace", true) + cfg.SetDefault("log.format", "csv") + cfg.SetDefault("log.output", "stderr") + cfg.SetDefault("log.noColor", false) + cfg.SetDefault("log.caller", false) + + return cfg +} + +// loadConfig returns a new config with values from the config in the given rootdir. +func loadConfig(rootdir string, cmd *cobra.Command, create bool) (*viper.Viper, error) { + // rootdir defaults to $HOME/.defradb + home, err := os.UserHomeDir() + if err != nil { + return nil, err + } + if rootdir == "" { + rootdir = filepath.Join(home, ".defradb") + } + + cfg := defaultConfig() + cfg.AddConfigPath(rootdir) + + // make sure rootdir exists + if err := os.MkdirAll(rootdir, 0755); err != nil { + return nil, err + } + // attempt to read the existing config + err = cfg.ReadInConfig() + if create && errors.As(err, &viper.ConfigFileNotFoundError{}) { + err = cfg.SafeWriteConfig() + } + if err != nil && !errors.As(err, &viper.ConfigFileNotFoundError{}) { + return nil, err + } + + // bind cli flags to config keys + for key, flag := range configFlags { + err := cfg.BindPFlag(key, cmd.Root().PersistentFlags().Lookup(flag)) + if err != nil { + return nil, err + } + } + + // make paths relative to the rootdir + for _, key := range configPaths { + path := cfg.GetString(key) + if path != "" && !filepath.IsAbs(path) { + cfg.Set(key, filepath.Join(rootdir, path)) + } + } + + logCfg := loggingConfig(cfg.Sub("log")) + logCfg.OverridesByLoggerName = make(map[string]logging.Config) + + // apply named logging overrides + for key := range cfg.GetStringMap("log.overrides") { + logCfg.OverridesByLoggerName[key] = loggingConfig(cfg.Sub("log.overrides." + key)) + } + logging.SetConfig(logCfg) + + return cfg, nil +} + +// loggingConfig returns a new logging config from the given config. +func loggingConfig(cfg *viper.Viper) logging.Config { + var level int8 + switch value := cfg.GetString("level"); value { + case "debug": + level = logging.Debug + case "info": + level = logging.Info + case "error": + level = logging.Error + case "fatal": + level = logging.Fatal + default: + level = logging.Info + } + + var format logging.EncoderFormat + switch value := cfg.GetString("format"); value { + case "json": //nolint:goconst + format = logging.JSON + case "csv": + format = logging.CSV + default: + format = logging.CSV + } + + return logging.Config{ + Level: logging.NewLogLevelOption(level), + EnableStackTrace: logging.NewEnableStackTraceOption(cfg.GetBool("stacktrace")), + DisableColor: logging.NewDisableColorOption(cfg.GetBool("nocolor")), + EncoderFormat: logging.NewEncoderFormatOption(format), + OutputPaths: []string{cfg.GetString("output")}, + EnableCaller: logging.NewEnableCallerOption(cfg.GetBool("caller")), + } +} diff --git a/cli/config_test.go b/cli/config_test.go new file mode 100644 index 0000000000..d87350093c --- /dev/null +++ b/cli/config_test.go @@ -0,0 +1,49 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package cli + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadConfigAndCreate(t *testing.T) { + rootdir := t.TempDir() + cfg, err := loadConfig(rootdir, NewDefraCommand(), true) + require.NoError(t, err) + + assert.FileExists(t, filepath.Join(rootdir, "config.yaml")) + assert.Equal(t, 5, cfg.GetInt("datastore.maxtxnretries")) + + assert.Equal(t, filepath.Join(rootdir, "data"), cfg.GetString("datastore.badger.path")) + assert.Equal(t, 1<<30, cfg.GetInt("datastore.badger.valuelogfilesize")) + + assert.Equal(t, "127.0.0.1:9181", cfg.GetString("api.address")) + assert.Equal(t, []string{}, cfg.GetStringSlice("api.allowed-origins")) + assert.Equal(t, "", cfg.GetString("api.pubkeypath")) + assert.Equal(t, "", cfg.GetString("api.privkeypath")) + + assert.Equal(t, false, cfg.GetBool("net.p2pdisabled")) + assert.Equal(t, []string{"/ip4/127.0.0.1/tcp/9171"}, cfg.GetStringSlice("net.p2paddresses")) + assert.Equal(t, true, cfg.GetBool("net.pubsubenabled")) + assert.Equal(t, false, cfg.GetBool("net.relay")) + assert.Equal(t, []string{}, cfg.GetStringSlice("net.peers")) + + assert.Equal(t, "info", cfg.GetString("log.level")) + assert.Equal(t, true, cfg.GetBool("log.stacktrace")) + assert.Equal(t, "csv", cfg.GetString("log.format")) + assert.Equal(t, "stderr", cfg.GetString("log.output")) + assert.Equal(t, false, cfg.GetBool("log.nocolor")) + assert.Equal(t, false, cfg.GetBool("log.caller")) +} diff --git a/cli/root.go b/cli/root.go index 2278c1962e..1126f1ff58 100644 --- a/cli/root.go +++ b/cli/root.go @@ -24,13 +24,13 @@ func MakeRootCommand() *cobra.Command { Start a DefraDB node, interact with a local or remote node, and much more. `, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return setConfigContext(cmd) + return setConfigContext(cmd, false) }, } cmd.PersistentFlags().String( "rootdir", - "$HOME/.defradb", + "", "Directory for data and configuration to use (default: $HOME/.defradb)", ) @@ -66,7 +66,7 @@ Start a DefraDB node, interact with a local or remote node, and much more. cmd.PersistentFlags().String( "url", - "localhost:9181", + "127.0.0.1:9181", "URL of HTTP endpoint to listen on or connect to", ) @@ -82,10 +82,10 @@ Start a DefraDB node, interact with a local or remote node, and much more. "Specify the maximum number of retries per transaction", ) - cmd.PersistentFlags().String( - "store", - "badger", - "Specify the datastore to use (supported: badger, memory)", + cmd.PersistentFlags().Bool( + "in-memory", + false, + "Enables the badger in memory only datastore.", ) cmd.PersistentFlags().Int( @@ -106,12 +106,6 @@ Start a DefraDB node, interact with a local or remote node, and much more. "Disable the peer-to-peer network synchronization system", ) - cmd.PersistentFlags().Bool( - "tls", - false, - "Enable serving the API over https", - ) - cmd.PersistentFlags().StringArray( "allowed-origins", []string{}, @@ -120,21 +114,15 @@ Start a DefraDB node, interact with a local or remote node, and much more. cmd.PersistentFlags().String( "pubkeypath", - "cert.pub", + "", "Path to the public key for tls", ) cmd.PersistentFlags().String( "privkeypath", - "cert.key", + "", "Path to the private key for tls", ) - cmd.PersistentFlags().String( - "email", - "example@example.com", - "Email address used by the CA for notifications", - ) - return cmd } diff --git a/cli/server_dump.go b/cli/server_dump.go index 86ac665df1..ae03f45d51 100644 --- a/cli/server_dump.go +++ b/cli/server_dump.go @@ -12,27 +12,26 @@ package cli import ( "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/sourcenetwork/defradb/db" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/node" ) -func MakeServerDumpCmd(cfg *viper.Viper) *cobra.Command { +func MakeServerDumpCmd() *cobra.Command { cmd := &cobra.Command{ Use: "server-dump", Short: "Dumps the state of the entire database", RunE: func(cmd *cobra.Command, _ []string) error { + cfg := mustGetConfigContext(cmd) log.FeedbackInfo(cmd.Context(), "Dumping DB state...") - datastore := cfg.GetString("datastore.store") - if datastore == "memory" { + if cfg.GetBool("datastore.badger.inMemory") { return errors.New("server-side dump is only supported for the Badger datastore") } storeOpts := []node.StoreOpt{ node.WithPath(cfg.GetString("datastore.badger.path")), - node.WithInMemory(datastore == "memory"), + node.WithInMemory(cfg.GetBool("datastore.badger.inMemory")), } rootstore, err := node.NewStore(storeOpts...) if err != nil { diff --git a/cli/start.go b/cli/start.go index 7b9598c857..f6b3b70cb6 100644 --- a/cli/start.go +++ b/cli/start.go @@ -15,12 +15,10 @@ import ( "os" "os/signal" "path/filepath" - "strings" "syscall" "github.com/libp2p/go-libp2p/core/peer" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/sourcenetwork/defradb/db" "github.com/sourcenetwork/defradb/errors" @@ -30,16 +28,18 @@ import ( "github.com/sourcenetwork/defradb/node" ) -func MakeStartCommand(cfg *viper.Viper) *cobra.Command { +func MakeStartCommand() *cobra.Command { var cmd = &cobra.Command{ Use: "start", Short: "Start a DefraDB node", Long: "Start a DefraDB node.", // Load the root config if it exists, otherwise create it. PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { - return setConfigContext(cmd) + return setConfigContext(cmd, true) }, RunE: func(cmd *cobra.Command, args []string) error { + cfg := mustGetConfigContext(cmd) + dbOpts := []db.Option{ db.WithUpdateEvents(), db.WithMaxRetries(cfg.GetInt("datastore.MaxTxnRetries")), @@ -60,19 +60,19 @@ func MakeStartCommand(cfg *viper.Viper) *cobra.Command { storeOpts := []node.StoreOpt{ node.WithPath(cfg.GetString("datastore.badger.path")), - node.WithInMemory(cfg.GetString("datastore.store") == "memory"), + node.WithInMemory(cfg.GetBool("datastore.badger.inMemory")), } var peers []peer.AddrInfo - if val := cfg.GetString("net.peers"); val != "" { - addrs, err := netutils.ParsePeers(strings.Split(val, ",")) + if val := cfg.GetStringSlice("net.peers"); len(val) > 0 { + addrs, err := netutils.ParsePeers(val) if err != nil { return errors.Wrap(fmt.Sprintf("failed to parse bootstrap peers %s", val), err) } peers = addrs } - if cfg.GetString("datastore.store") == "badger" { + if cfg.GetBool("datastore.badger.inMemory") { // It would be ideal to not have the key path tied to the datastore. // Running with memory store mode will always generate a random key. // Adding support for an ephemeral mode and moving the key to the diff --git a/cli/utils.go b/cli/utils.go index fc8c724b1e..6ec7096479 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -21,7 +21,6 @@ import ( "github.com/spf13/viper" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/http" ) @@ -78,80 +77,12 @@ func tryGetCollectionContext(cmd *cobra.Command) (client.Collection, bool) { } // setConfigContext sets teh config for the current command context. -func setConfigContext(cmd *cobra.Command) error { - rootdir, err := cmd.PersistentFlags().GetString("rootdir") +func setConfigContext(cmd *cobra.Command, create bool) error { + rootdir, err := cmd.Root().PersistentFlags().GetString("rootdir") if err != nil { return err } - cfg, err := config.LoadConfig(rootdir) - if err != nil { - return err - } - err = cfg.BindPFlag("log.level", cmd.PersistentFlags().Lookup("loglevel")) - if err != nil { - return err - } - err = cfg.BindPFlag("log.output", cmd.PersistentFlags().Lookup("logoutput")) - if err != nil { - return err - } - err = cfg.BindPFlag("log.format", cmd.PersistentFlags().Lookup("logformat")) - if err != nil { - return err - } - err = cfg.BindPFlag("log.stacktrace", cmd.PersistentFlags().Lookup("logtrace")) - if err != nil { - return err - } - err = cfg.BindPFlag("log.nocolor", cmd.PersistentFlags().Lookup("lognocolor")) - if err != nil { - return err - } - err = cfg.BindPFlag("api.address", cmd.PersistentFlags().Lookup("url")) - if err != nil { - return err - } - err = cfg.BindPFlag("net.peers", cmd.PersistentFlags().Lookup("peers")) - if err != nil { - return err - } - err = cfg.BindPFlag("datastore.maxtxnretries", cmd.PersistentFlags().Lookup("max-txn-retries")) - if err != nil { - return err - } - err = cfg.BindPFlag("datastore.store", cmd.PersistentFlags().Lookup("store")) - if err != nil { - return err - } - err = cfg.BindPFlag("datastore.badger.valuelogfilesize", cmd.PersistentFlags().Lookup("valuelogfilesize")) - if err != nil { - return err - } - err = cfg.BindPFlag("net.p2paddresses", cmd.PersistentFlags().Lookup("p2paddr")) - if err != nil { - return err - } - err = cfg.BindPFlag("net.p2pdisabled", cmd.PersistentFlags().Lookup("no-p2p")) - if err != nil { - return err - } - err = cfg.BindPFlag("api.tls", cmd.PersistentFlags().Lookup("tls")) - if err != nil { - return err - } - err = cfg.BindPFlag("api.allowed-origins", cmd.PersistentFlags().Lookup("allowed-origins")) - if err != nil { - return err - } - err = cfg.BindPFlag("api.pubkeypath", cmd.PersistentFlags().Lookup("pubkeypath")) - if err != nil { - return err - } - err = cfg.BindPFlag("api.privkeypath", cmd.PersistentFlags().Lookup("privkeypath")) - if err != nil { - return err - } - err = cfg.BindPFlag("api.email", cmd.PersistentFlags().Lookup("email")) + cfg, err := loadConfig(rootdir, cmd, create) if err != nil { return err } diff --git a/config/config.go b/config/config.go deleted file mode 100644 index f8298f1270..0000000000 --- a/config/config.go +++ /dev/null @@ -1,105 +0,0 @@ -package config - -import ( - "bytes" - _ "embed" - "os" - "path/filepath" - "strings" - - "github.com/sourcenetwork/defradb/logging" - "github.com/spf13/viper" -) - -const ( - // configName is the config file name - configName = "config.yaml" - // DefaultRoot is the name of the default root directory - DefaultRoot = "$HOME/.defradb" -) - -var ( - //go:embed config.yaml - defaultConfig []byte - // relativePathKeys are config keys that will be made relative to the rootdir - relativePathKeys = []string{ - "datastore.badger.path", - "api.pubkeypath", - "api.privkeypath", - } -) - -// WriteDefaultConfig writes the default config file to the given rootdir. -func WriteDefaultConfig(rootdir string) error { - if err := os.MkdirAll(rootdir, 0755); err != nil { - return err - } - return os.WriteFile(filepath.Join(rootdir, configName), defaultConfig, 0666) -} - -// LoadConfig returns a new config with values from the config in the given rootdir. -func LoadConfig(rootdir string) (*viper.Viper, error) { - cfg := viper.New() - - cfg.SetEnvPrefix("DEFRA") - cfg.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_")) - - cfg.SetConfigName(configName) - cfg.SetConfigType("yaml") - - cfg.AddConfigPath(rootdir) - cfg.AutomaticEnv() - - // load defaults first then merge persisted config - err := cfg.MergeConfig(bytes.NewBuffer(defaultConfig)) - if err != nil { - return nil, err - } - err = cfg.MergeInConfig() - if _, ok := err.(viper.ConfigFileNotFoundError); !ok && err != nil { - return nil, err - } - - // make paths relative to the rootdir - for _, key := range relativePathKeys { - path := cfg.GetString(key) - if !filepath.IsAbs(path) { - cfg.Set(key, filepath.Join(rootdir, path)) - } - } - - var level int8 - switch value := cfg.GetString("log.level"); value { - case "debug": - level = logging.Debug - case "info": - level = logging.Info - case "error": - level = logging.Error - case "fatal": - level = logging.Fatal - default: - level = logging.Info - } - - var format logging.EncoderFormat - switch value := cfg.GetString("log.format"); value { - case "json": - format = logging.JSON - case "csv": - format = logging.CSV - default: - format = logging.CSV - } - - logging.SetConfig(logging.Config{ - Level: logging.NewLogLevelOption(level), - EnableStackTrace: logging.NewEnableStackTraceOption(cfg.GetBool("log.stacktrace")), - DisableColor: logging.NewDisableColorOption(cfg.GetBool("log.nocolor")), - EncoderFormat: logging.NewEncoderFormatOption(format), - OutputPaths: []string{cfg.GetString("log.output")}, - EnableCaller: logging.NewEnableCallerOption(cfg.GetBool("log.caller")), - }) - - return cfg, nil -} diff --git a/config/config.yaml b/config/config.yaml deleted file mode 100644 index 04cfa93684..0000000000 --- a/config/config.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# DefraDB configuration (YAML) - -# The default DefraDB directory is "$HOME/.defradb". It can be changed via the --rootdir CLI flag. -# Relative paths are interpreted as being rooted in the DefraDB directory. - -datastore: - # Store can be badger | memory - # badger: fast pure Go key-value store optimized for SSDs (https://github.com/dgraph-io/badger) - # memory: in-memory version of badger - store: badger - - # Maximum number of times to retry failed transactions. - maxTxnRetries: 5 - - # badger specific settings - badger: - # The path to the database data file(s). - path: data - - # Maximum file size of the value log files. - # The in-memory file size will be 2*valuelogfilesize. - valueLogFileSize: 1073741824 - -api: - # Address of the HTTP API to listen on or connect to - address: localhost:9181 - - # The list of origins a cross-domain request can be executed from. - # allowed-origins: - # - localhost - - # Whether the API server should listen over HTTPS - tls: false - - # The path to the public key file. Ignored if domains is set. - pubKeyPath: cert.pub - - # The path to the private key file. Ignored if domains is set. - privKeyPath: cert.key - - # Email address to let the CA (Let's Encrypt) send notifications via email when there are issues (optional). - email: example@example.com - -net: - # Whether the P2P is disabled - p2pDisabled: false - - # Listening address of the P2P network - p2pAddresses: - - /ip4/127.0.0.1/tcp/9171 - - # Whether the node has pubsub enabled or not - pubSubEnabled: true - - # Enable libp2p's Circuit relay transport protocol https://docs.libp2p.io/concepts/circuit-relay/ - relay: false - - # List of peers to boostrap with, specified as multiaddresses (https://docs.libp2p.io/concepts/addressing/) - # peers: - # - /ip4/127.0.0.1/tcp/9171/QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N - -log: - # Log level. Options are debug, info, error, fatal - level: info - - # Include stack trace in error and fatal logs - stackTrace: true - - # Supported log formats are json, csv - format: csv - - # Where the log output is written to - output: stderr - - # Disable colored log output - noColor: false - - # Caller location in log output - caller: false diff --git a/config/config_test.go b/config/config_test.go deleted file mode 100644 index d4d1df7ecd..0000000000 --- a/config/config_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package config - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestLoadConfigNotExist(t *testing.T) { - rootdir := t.TempDir() - cfg, err := LoadConfig(rootdir) - require.NoError(t, err) - - assert.Equal(t, "badger", cfg.GetString("datastore.store")) - assert.Equal(t, 5, cfg.GetInt("datastore.maxtxnretries")) - - assert.Equal(t, filepath.Join(rootdir, "data"), cfg.GetString("datastore.badger.path")) - assert.Equal(t, 1073741824, cfg.GetInt("datastore.badger.valuelogfilesize")) - - assert.Equal(t, "localhost:9181", cfg.GetString("api.address")) - assert.Equal(t, false, cfg.GetBool("api.tls")) - assert.Equal(t, []string(nil), cfg.GetStringSlice("api.allowed-origins")) - assert.Equal(t, filepath.Join(rootdir, "cert.pub"), cfg.GetString("api.pubkeypath")) - assert.Equal(t, filepath.Join(rootdir, "cert.key"), cfg.GetString("api.privkeypath")) - assert.Equal(t, "example@example.com", cfg.GetString("api.email")) - - assert.Equal(t, false, cfg.GetBool("net.p2pdisabled")) - assert.Equal(t, []string{"/ip4/127.0.0.1/tcp/9171"}, cfg.GetStringSlice("net.p2paddresses")) - assert.Equal(t, true, cfg.GetBool("net.pubsubenabled")) - assert.Equal(t, false, cfg.GetBool("net.relay")) - assert.Equal(t, []string(nil), cfg.GetStringSlice("net.peers")) - - assert.Equal(t, "info", cfg.GetString("log.level")) - assert.Equal(t, true, cfg.GetBool("log.stacktrace")) - assert.Equal(t, "csv", cfg.GetString("log.format")) - assert.Equal(t, "stderr", cfg.GetString("log.output")) - assert.Equal(t, false, cfg.GetBool("log.nocolor")) - assert.Equal(t, false, cfg.GetBool("log.caller")) -} - -func TestLoadConfigExists(t *testing.T) { - rootdir := t.TempDir() - err := WriteDefaultConfig(rootdir) - require.NoError(t, err) - - _, err = LoadConfig(rootdir) - require.NoError(t, err) -} - -func TestWriteDefaultConfig(t *testing.T) { - rootdir := t.TempDir() - err := WriteDefaultConfig(rootdir) - require.NoError(t, err) - - data, err := os.ReadFile(filepath.Join(rootdir, configName)) - require.NoError(t, err) - assert.Equal(t, defaultConfig, data) -} diff --git a/tests/gen/cli/gendocs.go b/tests/gen/cli/gendocs.go index bb7d9e35d3..aa0267fcce 100644 --- a/tests/gen/cli/gendocs.go +++ b/tests/gen/cli/gendocs.go @@ -20,7 +20,6 @@ import ( "github.com/spf13/cobra" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/http" "github.com/sourcenetwork/defradb/tests/gen" ) @@ -29,7 +28,7 @@ const defaultBatchSize = 1000 func MakeGenDocCommand() *cobra.Command { var demandJSON string - + var url string var cmd = &cobra.Command{ Use: "gendocs --demand ", Short: "Automatically generates documents for existing collections.", @@ -39,11 +38,7 @@ Example: The following command generates 100 User documents and 500 Device docum gendocs --demand '{"User": 100, "Device": 500 }'`, ValidArgs: []string{"demand"}, RunE: func(cmd *cobra.Command, args []string) error { - // cobra does not chain pre run calls so we have to run them again here - if err := loadConfig(cfg); err != nil { - return err - } - store, err := http.NewClient(cfg.API.Address) + store, err := http.NewClient(url) if err != nil { return err } @@ -101,15 +96,7 @@ Example: The following command generates 100 User documents and 500 Device docum }, } - cmd.PersistentFlags().String( - "url", cfg.API.Address, - "URL of HTTP endpoint to listen on or connect to", - ) - err := cfg.BindFlag("api.address", cmd.PersistentFlags().Lookup("url")) - if err != nil { - panic(err) - } - + cmd.Flags().StringVar(&url, "url", "localhost:9181", "URL of HTTP endpoint to listen on or connect to") cmd.Flags().StringVarP(&demandJSON, "demand", "d", "", "Documents' demand in JSON format") return cmd @@ -160,10 +147,3 @@ func colsToDefs(cols []client.Collection) []client.CollectionDefinition { } return colDefs } - -func loadConfig(cfg *config.Config) error { - if err := cfg.LoadRootDirFromFlagOrDefault(); err != nil { - return err - } - return cfg.LoadWithRootdir(cfg.ConfigFileExists()) -} diff --git a/tests/gen/cli/gendocs_test.go b/tests/gen/cli/gendocs_test.go index 1bd30297a6..35202ec9e0 100644 --- a/tests/gen/cli/gendocs_test.go +++ b/tests/gen/cli/gendocs_test.go @@ -37,7 +37,7 @@ func TestGendocsCmd_IfNoErrors_ReturnGenerationOutput(t *testing.T) { owner: User }`) - genDocsCmd := MakeGenDocCommand(getTestConfig(t)) + genDocsCmd := MakeGenDocCommand() outputBuf := bytes.NewBufferString("") genDocsCmd.SetOut(outputBuf) @@ -71,7 +71,7 @@ func TestGendocsCmd_IfInvalidDemandValue_ReturnError(t *testing.T) { name: String }`) - genDocsCmd := MakeGenDocCommand(getTestConfig(t)) + genDocsCmd := MakeGenDocCommand() genDocsCmd.SetArgs([]string{ "--demand", `{"User": invalid}`, "--url", strings.TrimPrefix(defra.server.URL, "http://"), @@ -90,7 +90,7 @@ func TestGendocsCmd_IfInvalidConfig_ReturnError(t *testing.T) { name: String }`) - genDocsCmd := MakeGenDocCommand(getTestConfig(t)) + genDocsCmd := MakeGenDocCommand() genDocsCmd.SetArgs([]string{ "--demand", `{"Unknown": 3}`, "--url", strings.TrimPrefix(defra.server.URL, "http://"), diff --git a/tests/gen/cli/util_test.go b/tests/gen/cli/util_test.go index 81d713955c..10bd98ca99 100644 --- a/tests/gen/cli/util_test.go +++ b/tests/gen/cli/util_test.go @@ -19,7 +19,6 @@ import ( "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" badgerds "github.com/sourcenetwork/defradb/datastore/badger/v4" "github.com/sourcenetwork/defradb/db" "github.com/sourcenetwork/defradb/errors" @@ -67,17 +66,6 @@ func start(ctx context.Context) (*defraInstance, error) { }, nil } -func getTestConfig(t *testing.T) *config.Config { - cfg := config.DefaultConfig() - cfg.Datastore.Store = "memory" - cfg.Net.P2PDisabled = true - cfg.Rootdir = t.TempDir() - cfg.Net.P2PAddresses = []string{"/ip4/127.0.0.1/tcp/0"} - cfg.API.Address = "127.0.0.1:0" - cfg.Persist() - return cfg -} - func startTestNode(t *testing.T) (*defraInstance, func()) { ctx := context.Background() di, err := start(ctx) diff --git a/tests/integration/net/order/tcp_test.go b/tests/integration/net/order/tcp_test.go index 8a419360d5..f80701c64c 100644 --- a/tests/integration/net/order/tcp_test.go +++ b/tests/integration/net/order/tcp_test.go @@ -16,16 +16,16 @@ import ( "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" + "github.com/sourcenetwork/defradb/net" testutils "github.com/sourcenetwork/defradb/tests/integration" ) // TestP2PWithSingleDocumentUpdatePerNode tests document syncing between two nodes with a single update per node func TestP2PWithSingleDocumentUpdatePerNode(t *testing.T) { test := P2PTestCase{ - NodeConfig: []*config.Config{ - randomNetworkingConfig(), - randomNetworkingConfig(), + NodeConfig: [][]net.NodeOpt{ + testutils.RandomNetworkingConfig()(), + testutils.RandomNetworkingConfig()(), }, NodePeers: map[int][]int{ 1: { @@ -74,9 +74,9 @@ func TestP2PWithSingleDocumentUpdatePerNode(t *testing.T) { // TestP2PWithMultipleDocumentUpdatesPerNode tests document syncing between two nodes with multiple updates per node. func TestP2PWithMultipleDocumentUpdatesPerNode(t *testing.T) { test := P2PTestCase{ - NodeConfig: []*config.Config{ - randomNetworkingConfig(), - randomNetworkingConfig(), + NodeConfig: [][]net.NodeOpt{ + testutils.RandomNetworkingConfig()(), + testutils.RandomNetworkingConfig()(), }, NodePeers: map[int][]int{ 1: { @@ -145,9 +145,9 @@ func TestP2FullPReplicator(t *testing.T) { require.NoError(t, err) test := P2PTestCase{ - NodeConfig: []*config.Config{ - randomNetworkingConfig(), - randomNetworkingConfig(), + NodeConfig: [][]net.NodeOpt{ + testutils.RandomNetworkingConfig()(), + testutils.RandomNetworkingConfig()(), }, NodeReplicators: map[int][]int{ 0: { diff --git a/tests/integration/net/order/utils.go b/tests/integration/net/order/utils.go index 178272b155..3ba5fc7f26 100644 --- a/tests/integration/net/order/utils.go +++ b/tests/integration/net/order/utils.go @@ -13,14 +13,12 @@ package order import ( "context" "fmt" - "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" coreDB "github.com/sourcenetwork/defradb/db" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/logging" @@ -50,7 +48,7 @@ const ( type P2PTestCase struct { Query string // Configuration parameters for each peer - NodeConfig []*config.Config + NodeConfig [][]net.NodeOpt // List of peers for each net. // Only peers with lower index than the node can be used in the list of peers. @@ -69,7 +67,12 @@ type P2PTestCase struct { ReplicatorResult map[int]map[string]map[string]any } -func setupDefraNode(t *testing.T, cfg *config.Config, seeds []string) (*net.Node, []client.DocID, error) { +func setupDefraNode( + t *testing.T, + opts []net.NodeOpt, + peers []string, + seeds []string, +) (*net.Node, []client.DocID, error) { ctx := context.Background() log.Info(ctx, "Building new memory store") @@ -92,40 +95,28 @@ func setupDefraNode(t *testing.T, cfg *config.Config, seeds []string) (*net.Node // init the P2P node var n *net.Node - log.Info(ctx, "Starting P2P node", logging.NewKV("P2P addresses", cfg.Net.P2PAddresses)) - n, err = net.NewNode( - ctx, - db, - net.WithListenAddresses(cfg.Net.P2PAddresses...), - net.WithEnablePubSub(cfg.Net.PubSubEnabled), - net.WithEnableRelay(cfg.Net.RelayEnabled), - ) + n, err = net.NewNode(ctx, db, opts...) if err != nil { return nil, nil, errors.Wrap("failed to start P2P node", err) } // parse peers and bootstrap - if len(cfg.Net.Peers) != 0 { - log.Info(ctx, "Parsing bootstrap peers", logging.NewKV("Peers", cfg.Net.Peers)) - addrs, err := netutils.ParsePeers(strings.Split(cfg.Net.Peers, ",")) + if len(peers) != 0 { + log.Info(ctx, "Parsing bootstrap peers", logging.NewKV("Peers", peers)) + addrs, err := netutils.ParsePeers(peers) if err != nil { - return nil, nil, errors.Wrap(fmt.Sprintf("failed to parse bootstrap peers %v", cfg.Net.Peers), err) + return nil, nil, errors.Wrap(fmt.Sprintf("failed to parse bootstrap peers %v", peers), err) } log.Info(ctx, "Bootstrapping with peers", logging.NewKV("Addresses", addrs)) n.Bootstrap(addrs) } + log.Info(ctx, "Starting P2P node", logging.NewKV("P2P addresses", n.PeerInfo().Addrs)) if err := n.Start(); err != nil { n.Close() return nil, nil, errors.Wrap("unable to start P2P listeners", err) } - var addresses []string - for _, addr := range n.ListenAddrs() { - addresses = append(addresses, addr.String()) - } - cfg.Net.P2PAddresses = addresses - return n, docIDs, nil } @@ -201,9 +192,8 @@ func executeTestCase(t *testing.T, test P2PTestCase) { for i, cfg := range test.NodeConfig { log.Info(ctx, fmt.Sprintf("Setting up node %d", i)) - cfg.Datastore.Badger.Path = t.TempDir() + var peerAddresses []string if peers, ok := test.NodePeers[i]; ok { - peerAddresses := []string{} for _, p := range peers { if p >= len(nodes) { log.Info(ctx, "cannot set a peer that hasn't been started. Skipping to next peer") @@ -215,9 +205,8 @@ func executeTestCase(t *testing.T, test P2PTestCase) { fmt.Sprintf("%s/p2p/%s", peerInfo.Addrs[0], peerInfo.ID), ) } - cfg.Net.Peers = strings.Join(peerAddresses, ",") } - n, d, err := setupDefraNode(t, cfg, test.SeedDocuments) + n, d, err := setupDefraNode(t, cfg, peerAddresses, test.SeedDocuments) require.NoError(t, err) if i == 0 { @@ -348,10 +337,3 @@ func executeTestCase(t *testing.T, test P2PTestCase) { n.DB.Close() } } - -func randomNetworkingConfig() *config.Config { - cfg := config.DefaultConfig() - cfg.Net.P2PAddresses = []string{"/ip4/127.0.0.1/tcp/0"} - cfg.Net.RelayEnabled = false - return cfg -} diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index 463d1bd653..4d48cb033b 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -14,8 +14,8 @@ import ( "time" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/logging" + "github.com/sourcenetwork/defradb/net" "github.com/sourcenetwork/defradb/tests/clients" "github.com/libp2p/go-libp2p/core/peer" @@ -523,10 +523,10 @@ func waitForSync( } func RandomNetworkingConfig() ConfigureNode { - return func() config.Config { - cfg := config.DefaultConfig() - cfg.Net.P2PAddresses = []string{"/ip4/127.0.0.1/tcp/0"} - cfg.Net.RelayEnabled = false - return *cfg + return func() []net.NodeOpt { + return []net.NodeOpt{ + net.WithListenAddresses("/ip4/127.0.0.1/tcp/0"), + net.WithEnableRelay(false), + } } } diff --git a/tests/integration/state.go b/tests/integration/state.go index ca795a2492..25a248413b 100644 --- a/tests/integration/state.go +++ b/tests/integration/state.go @@ -14,12 +14,11 @@ import ( "context" "testing" - "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/datastore" + "github.com/sourcenetwork/defradb/net" "github.com/sourcenetwork/defradb/tests/clients" ) @@ -53,14 +52,11 @@ type state struct { // These synchronisation channels allow async actions to track their completion. syncChans []chan struct{} - // The private keys for any nodes. - nodePrivateKeys []crypto.PrivKey - // The addresses of any nodes configured. nodeAddresses []peer.AddrInfo // The configurations for any nodes - nodeConfigs []config.Config + nodeConfigs [][]net.NodeOpt // The nodes active in this test. nodes []clients.Client @@ -108,9 +104,8 @@ func newState( allActionsDone: make(chan struct{}), subscriptionResultsChans: []chan func(){}, syncChans: []chan struct{}{}, - nodePrivateKeys: []crypto.PrivKey{}, nodeAddresses: []peer.AddrInfo{}, - nodeConfigs: []config.Config{}, + nodeConfigs: [][]net.NodeOpt{}, nodes: []clients.Client{}, dbPaths: []string{}, collections: [][]client.Collection{}, diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index 2c0e095fe8..942f0fc932 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -16,7 +16,7 @@ import ( "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" + "github.com/sourcenetwork/defradb/net" "github.com/sourcenetwork/defradb/tests/gen" "github.com/sourcenetwork/defradb/tests/predefined" ) @@ -54,7 +54,7 @@ type SetupComplete struct{} // Nodes may be explicitly referenced by index by other actions using `NodeID` properties. // If the action has a `NodeID` property and it is not specified, the action will be // effected on all nodes. -type ConfigureNode func() config.Config +type ConfigureNode func() []net.NodeOpt // Restart is an action that will close and then start all nodes. type Restart struct{} diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index afbd57bea9..fe1900b0a2 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -663,25 +663,18 @@ func restartNodes( continue } - key := s.nodePrivateKeys[i] - cfg := s.nodeConfigs[i] // We need to make sure the node is configured with its old address, otherwise // a new one may be selected and reconnnection to it will fail. var addresses []string for _, addr := range s.nodeAddresses[i].Addrs { addresses = append(addresses, addr.String()) } - cfg.Net.P2PAddresses = addresses + + nodeOpts := s.nodeConfigs[i] + nodeOpts = append(nodeOpts, net.WithListenAddresses(addresses...)) var n *net.Node - n, err = net.NewNode( - s.ctx, - db, - net.WithListenAddresses(cfg.Net.P2PAddresses...), - net.WithEnablePubSub(cfg.Net.PubSubEnabled), - net.WithEnableRelay(cfg.Net.RelayEnabled), - net.WithPrivateKey(key), - ) + n, err = net.NewNode(s.ctx, db, nodeOpts...) require.NoError(s.t, err) if err := n.Start(); err != nil { @@ -770,23 +763,17 @@ func configureNode( return } - cfg := action() db, path, err := setupDatabase(s) //disable change dector, or allow it? require.NoError(s.t, err) privateKey, _, err := crypto.GenerateKeyPair(crypto.Ed25519, 0) require.NoError(s.t, err) + nodeOpts := action() + nodeOpts = append(nodeOpts, net.WithPrivateKey(privateKey)) + var n *net.Node - log.Info(s.ctx, "Starting P2P node", logging.NewKV("P2P address", cfg.Net.P2PAddresses)) - n, err = net.NewNode( - s.ctx, - db, - net.WithListenAddresses(cfg.Net.P2PAddresses...), - net.WithEnablePubSub(cfg.Net.PubSubEnabled), - net.WithEnableRelay(cfg.Net.RelayEnabled), - net.WithPrivateKey(privateKey), - ) + n, err = net.NewNode(s.ctx, db, nodeOpts...) require.NoError(s.t, err) log.Info(s.ctx, "Starting P2P node", logging.NewKV("P2P address", n.PeerInfo())) @@ -796,8 +783,7 @@ func configureNode( } s.nodeAddresses = append(s.nodeAddresses, n.PeerInfo()) - s.nodeConfigs = append(s.nodeConfigs, cfg) - s.nodePrivateKeys = append(s.nodePrivateKeys, privateKey) + s.nodeConfigs = append(s.nodeConfigs, nodeOpts) c, err := setupClient(s, n) require.NoError(s.t, err)