From daf61f649f1b6658bee3227b9e2954dbde0a8b06 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 31 Jan 2024 09:51:16 -0800 Subject: [PATCH 1/8] refactor config --- config/config.go | 725 +++------------------------------- config/config.yaml | 79 ++++ config/config_test.go | 452 ++------------------- config/config_utils.go | 131 ------ config/config_utils_test.go | 108 ----- config/configfile.go | 89 ----- config/configfile_test.go | 155 -------- config/configfile_yaml.gotmpl | 65 --- config/errors.go | 219 ---------- 9 files changed, 179 insertions(+), 1844 deletions(-) create mode 100644 config/config.yaml delete mode 100644 config/config_utils.go delete mode 100644 config/config_utils_test.go delete mode 100644 config/configfile.go delete mode 100644 config/configfile_test.go delete mode 100644 config/configfile_yaml.gotmpl delete mode 100644 config/errors.go diff --git a/config/config.go b/config/config.go index ef3aeaa8cc..332b5f4549 100644 --- a/config/config.go +++ b/config/config.go @@ -1,703 +1,100 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -/* -Package config provides the central point for DefraDB's configuration and related facilities. - -[Config] embeds component-specific config structs. Each config struct can have a function providing -default options, a method providing test configurations, a method for validation, a method handling -deprecated fields (e.g. with warnings). This is extensible. - -The 'root directory' is where the configuration file and data of a DefraDB instance exists. -It is specified as a global flag `defradb --rootdir path/to/somewhere. - -Some packages of DefraDB provide their own configuration approach (logging, node). -For each, a way to go from top-level configuration to package-specific configuration is provided. - -Parameters are determined by, in order of least importance: defaults, configuration file, env. variables, and then CLI -flags. That is, CLI flags can override everything else. - -For example `DEFRA_DATASTORE_BADGER_PATH` matches [Config.Datastore.Badger.Path] and in the config file: - - datastore: - badger: - path: /tmp/badger - -This implementation does not support online modification of configuration. - -How to use, e.g. without using a rootdir: - - cfg := config.DefaultConfig() - cfg.NetConfig.P2PDisabled = true // as example - err := cfg.LoadWithRootdir(false) - if err != nil { - ... -*/ package config import ( "bytes" - "encoding/json" - "fmt" - "net" + _ "embed" + "os" "path/filepath" - "strconv" "strings" - "text/template" - "github.com/mitchellh/mapstructure" - ma "github.com/multiformats/go-multiaddr" - "github.com/spf13/pflag" - "github.com/spf13/viper" - "golang.org/x/net/idna" - - badgerds "github.com/sourcenetwork/defradb/datastore/badger/v4" "github.com/sourcenetwork/defradb/logging" + "github.com/spf13/viper" ) -var log = logging.MustNewLogger("config") - -const ( - DefaultAPIEmail = "example@example.com" - RootdirKey = "rootdircli" - defraEnvPrefix = "DEFRA" - logLevelDebug = "debug" - logLevelInfo = "info" - logLevelError = "error" - logLevelFatal = "fatal" -) - -// Config is DefraDB's main configuration struct, embedding component-specific config structs. -type Config struct { - Datastore *DatastoreConfig - API *APIConfig - Net *NetConfig - Log *LoggingConfig - Rootdir string - v *viper.Viper -} - -// DefaultConfig returns the default configuration (or panics). -func DefaultConfig() *Config { - cfg := &Config{ - Datastore: defaultDatastoreConfig(), - API: defaultAPIConfig(), - Net: defaultNetConfig(), - Log: defaultLogConfig(), - Rootdir: "", - v: viper.New(), - } - - cfg.v.SetEnvPrefix(defraEnvPrefix) - cfg.v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) - - cfg.v.SetConfigName(DefaultConfigFileName) - cfg.v.SetConfigType(configType) - - cfg.Persist() - - return cfg -} - -// Persist persists manually set config parameters to the viper config. -func (cfg *Config) Persist() { - // Load new values in viper. - b, err := cfg.toBytes() - if err != nil { - panic(err) - } - if err = cfg.v.ReadConfig(bytes.NewReader(b)); err != nil { - panic(NewErrReadingConfigFile(err)) - } -} - -// LoadWithRootdir loads a Config with parameters from defaults, config file, environment variables, and CLI flags. -// It loads from config file when `fromFile` is true, otherwise it loads directly from a default configuration. -// Use on a Config struct already loaded with default values from DefaultConfig(). -// To be executed once at the beginning of the program. -func (cfg *Config) LoadWithRootdir(withRootdir bool) error { - // Use default logging configuration here, so that - // we can log errors in a consistent way even in the case of early failure. - defaultLogCfg := defaultLogConfig() - if err := defaultLogCfg.load(); err != nil { - return err - } - - if err := cfg.LoadRootDirFromFlagOrDefault(); err != nil { - return err - } - - if withRootdir { - if err := cfg.v.ReadInConfig(); err != nil { - return NewErrReadingConfigFile(err) - } - } - - cfg.v.AutomaticEnv() - - if err := cfg.paramsPreprocessing(); err != nil { - return err - } - // We load the viper configuration in the Config struct. - if err := cfg.v.Unmarshal(cfg, viper.DecodeHook(mapstructure.TextUnmarshallerHookFunc())); err != nil { - return NewErrLoadingConfig(err) - } - if err := cfg.validate(); err != nil { - return err - } - if err := cfg.load(); err != nil { - return err - } - - return nil -} - -func (cfg *Config) LoadRootDirFromFlagOrDefault() error { - if cfg.Rootdir == "" { - rootdir := cfg.v.GetString(RootdirKey) - if rootdir != "" { - return cfg.setRootdir(rootdir) - } - - return cfg.setRootdir(DefaultRootDir()) - } - - return nil -} - -func (cfg *Config) setRootdir(rootdir string) error { - var err error - if rootdir == "" { - return NewErrInvalidRootDir(rootdir) - } - // using absolute rootdir for robustness. - cfg.Rootdir, err = filepath.Abs(rootdir) - if err != nil { - return err - } - cfg.v.AddConfigPath(cfg.Rootdir) - return nil -} - -func (cfg *Config) validate() error { - if err := cfg.Datastore.validate(); err != nil { - return NewErrFailedToValidateConfig(err) - } - if err := cfg.API.validate(); err != nil { - return NewErrFailedToValidateConfig(err) - } - if err := cfg.Net.validate(); err != nil { - return NewErrFailedToValidateConfig(err) - } - if err := cfg.Log.validate(); err != nil { - return NewErrFailedToValidateConfig(err) - } - return nil -} - -func (cfg *Config) paramsPreprocessing() error { - // We prefer using absolute paths, relative to the rootdir. - if !filepath.IsAbs(cfg.v.GetString("datastore.badger.path")) { - cfg.v.Set("datastore.badger.path", filepath.Join(cfg.Rootdir, cfg.v.GetString("datastore.badger.path"))) - } - if !filepath.IsAbs(cfg.v.GetString("api.privkeypath")) { - cfg.v.Set("api.privkeypath", filepath.Join(cfg.Rootdir, cfg.v.GetString("api.privkeypath"))) - } - if !filepath.IsAbs(cfg.v.GetString("api.pubkeypath")) { - cfg.v.Set("api.pubkeypath", filepath.Join(cfg.Rootdir, cfg.v.GetString("api.pubkeypath"))) - } - - // log.logger configuration as a string - logloggerAsStringSlice := cfg.v.GetStringSlice("log.logger") - if logloggerAsStringSlice != nil { - cfg.v.Set("log.logger", strings.Join(logloggerAsStringSlice, ";")) - } - - // Expand the passed in `~` if it wasn't expanded properly by the shell. - // That can happen when the parameters are passed from outside of a shell. - if err := expandHomeDir(&cfg.API.PrivKeyPath); err != nil { - return err - } - if err := expandHomeDir(&cfg.API.PubKeyPath); err != nil { - return err - } +// configName is the config file name +const configName = "config.yaml" - var bs ByteSize - if err := bs.Set(cfg.v.GetString("datastore.badger.valuelogfilesize")); err != nil { - return err - } - cfg.Datastore.Badger.ValueLogFileSize = bs +//go:embed config.yaml +var defaultConfig []byte - return nil +// relativePathKeys are config keys that will be made relative to the rootdir +var relativePathKeys = []string{ + "datastore.badger.path", + "api.pubkeypath", + "api.privkeypath", } -func (cfg *Config) load() error { - if err := cfg.Log.load(); err != nil { +// WriteDefaultConfig writes the default config file to the given rootdir. +func WriteDefaultConfig(rootdir string) error { + if err := os.MkdirAll(rootdir, 0755); err != nil { return err } - return nil -} - -// DatastoreConfig configures datastores. -type DatastoreConfig struct { - Store string - Memory MemoryConfig - Badger BadgerConfig - MaxTxnRetries int -} - -// BadgerConfig configures Badger's on-disk / filesystem mode. -type BadgerConfig struct { - Path string - ValueLogFileSize ByteSize - *badgerds.Options -} - -// MemoryConfig configures of Badger's memory mode. -type MemoryConfig struct { - Size uint64 -} - -func defaultDatastoreConfig() *DatastoreConfig { - // create a copy of the default badger options - opts := badgerds.DefaultOptions - return &DatastoreConfig{ - Store: "badger", - Badger: BadgerConfig{ - Path: "data", - ValueLogFileSize: 1 * GiB, - Options: &opts, - }, - MaxTxnRetries: 5, - } -} - -func (dbcfg DatastoreConfig) validate() error { - switch dbcfg.Store { - case "badger", "memory": - default: - return NewErrInvalidDatastoreType(dbcfg.Store) - } - return nil -} - -// APIConfig configures the API endpoints. -type APIConfig struct { - Address string - TLS bool - AllowedOrigins []string `mapstructure:"allowed-origins"` - PubKeyPath string - PrivKeyPath string - Email string + return os.WriteFile(filepath.Join(rootdir, configName), defaultConfig, 0666) } -func defaultAPIConfig() *APIConfig { - return &APIConfig{ - Address: "localhost:9181", - TLS: false, - AllowedOrigins: []string{}, - PubKeyPath: "certs/server.key", - PrivKeyPath: "certs/server.crt", - Email: DefaultAPIEmail, - } -} +// LoadConfig returns a new config with values from the config in the given rootdir. +func LoadConfig(rootdir string) (*viper.Viper, error) { + cfg := viper.New() -func (apicfg *APIConfig) validate() error { - if apicfg.Address == "" { - return ErrInvalidDatabaseURL - } + cfg.SetEnvPrefix("DEFRA") + cfg.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_")) - if apicfg.Address == "localhost" || net.ParseIP(apicfg.Address) != nil { //nolint:goconst - return ErrMissingPortNumber - } + cfg.SetConfigName(configName) + cfg.SetConfigType("yaml") - if isValidDomainName(apicfg.Address) { - return nil - } + cfg.AddConfigPath(rootdir) + cfg.AutomaticEnv() - host, _, err := net.SplitHostPort(apicfg.Address) + // load defaults first then merge persisted config + err := cfg.MergeConfig(bytes.NewBuffer(defaultConfig)) if err != nil { - return NewErrInvalidDatabaseURL(err) - } - if host == "localhost" { - return nil - } - if net.ParseIP(host) == nil { - return ErrNoPortWithDomain - } - - return nil -} - -func isValidDomainName(domain string) bool { - asciiDomain, err := idna.Registration.ToASCII(domain) - if err != nil { - return false - } - return asciiDomain == domain -} - -// AddressToURL provides the API address as URL. -func (apicfg *APIConfig) AddressToURL() string { - if apicfg.TLS { - return fmt.Sprintf("https://%s", apicfg.Address) - } - return fmt.Sprintf("http://%s", apicfg.Address) -} - -// NetConfig configures aspects of network and peer-to-peer. -type NetConfig struct { - P2PAddresses []string - P2PDisabled bool - Peers string - PubSubEnabled bool `mapstructure:"pubsub"` - RelayEnabled bool `mapstructure:"relay"` -} - -func defaultNetConfig() *NetConfig { - return &NetConfig{ - P2PAddresses: []string{"/ip4/127.0.0.1/tcp/9171"}, - P2PDisabled: false, - Peers: "", - PubSubEnabled: true, - RelayEnabled: false, - } -} - -func (netcfg *NetConfig) validate() error { - for _, addr := range netcfg.P2PAddresses { - _, err := ma.NewMultiaddr(addr) - if err != nil { - return NewErrInvalidP2PAddress(err, addr) - } - } - if len(netcfg.Peers) > 0 { - peers := strings.Split(netcfg.Peers, ",") - maddrs := make([]ma.Multiaddr, len(peers)) - for i, addr := range peers { - addr, err := ma.NewMultiaddr(addr) - if err != nil { - return NewErrInvalidBootstrapPeers(err, netcfg.Peers) - } - maddrs[i] = addr - } - } - return nil -} - -// LogConfig configures output and logger. -type LoggingConfig struct { - Level string - Stacktrace bool - Format string - Output string // logging actually supports multiple output paths, but here only one is supported - Caller bool - NoColor bool - Logger string - NamedOverrides map[string]*NamedLoggingConfig -} - -// NamedLoggingConfig is a named logging config, used for named overrides of the default config. -type NamedLoggingConfig struct { - Name string - LoggingConfig -} - -func defaultLogConfig() *LoggingConfig { - return &LoggingConfig{ - Level: logLevelInfo, - Stacktrace: false, - Format: "csv", - Output: "stderr", - Caller: false, - NoColor: false, - Logger: "", - NamedOverrides: make(map[string]*NamedLoggingConfig), - } -} - -// validate ensures that the logging config is valid. -func (logcfg *LoggingConfig) validate() error { - /* - `loglevel` is either a single value, or a single value with comma-separated list of key=value pairs, for which - the key is the name of the logger and the value is the log level, each logger name is unique, and value is valid. - - `--loglevels ,=,...` - */ - kvs := []map[string]string{} - validLevel := func(level string) bool { - for _, l := range []string{ - logLevelDebug, - logLevelInfo, - logLevelError, - logLevelFatal, - } { - if l == level { - return true - } - } - return false - } - ensureUniqueKeys := func(kvs []map[string]string) error { - keys := make(map[string]bool) - for _, kv := range kvs { - for k := range kv { - if keys[k] { - return NewErrDuplicateLoggerName(k) - } - keys[k] = true - } - } - return nil - } - - parts := strings.Split(logcfg.Level, ",") - if len(parts) > 0 { - if !validLevel(parts[0]) { - return NewErrInvalidLogLevel(parts[0]) - } - for _, kv := range parts[1:] { - parsedKV, err := parseKV(kv) - if err != nil { - return err - } - // ensure each value is a valid loglevel validLevel - if !validLevel(parsedKV[1]) { - return NewErrInvalidLogLevel(parsedKV[1]) - } - kvs = append(kvs, map[string]string{parsedKV[0]: parsedKV[1]}) - } - if err := ensureUniqueKeys(kvs); err != nil { - return err - } - } - - // logger: expect format like: `net,nocolor=true,level=debug;config,output=stdout,level=info` - if len(logcfg.Logger) != 0 { - namedconfigs := strings.Split(logcfg.Logger, ";") - for _, c := range namedconfigs { - parts := strings.Split(c, ",") - if len(parts) < 2 { - return NewErrLoggerConfig("unexpected format (expected: `module,key=value;module,key=value;...`") - } - if parts[0] == "" { - return ErrLoggerNameEmpty - } - for _, pair := range parts[1:] { - parsedKV, err := parseKV(pair) - if err != nil { - return err - } - if !isLowercaseAlpha(parsedKV[0]) { - return NewErrInvalidLoggerName(parsedKV[0]) - } - switch parsedKV[0] { - case "format", "output", "nocolor", "stacktrace", "caller": //nolint:goconst - // valid logger parameters - case "level": //nolint:goconst - // ensure each value is a valid loglevel validLevel - if !validLevel(parsedKV[1]) { - return NewErrInvalidLogLevel(parsedKV[1]) - } - default: - return NewErrUnknownLoggerParameter(parsedKV[0]) - } - } - } + return nil, err } - - return nil -} - -func (logcfg *LoggingConfig) load() error { - // load loglevel - parts := strings.Split(logcfg.Level, ",") - if len(parts) > 0 { - logcfg.Level = parts[0] - } - if len(parts) > 1 { - for _, kv := range parts[1:] { - parsedKV := strings.Split(kv, "=") - if len(parsedKV) != 2 { - return NewErrInvalidLogLevel(kv) - } - c, err := logcfg.GetOrCreateNamedLogger(parsedKV[0]) - if err != nil { - return NewErrCouldNotObtainLoggerConfig(err, parsedKV[0]) - } - c.Level = parsedKV[1] - } + err = cfg.MergeInConfig() + if _, ok := err.(viper.ConfigFileNotFoundError); !ok && err != nil { + return nil, err } - // load logger - // e.g. `net,nocolor=true,level=debug;config,output=stdout,level=info` - // logger has higher priority over loglevel whenever both touch the same parameters - if len(logcfg.Logger) != 0 { - s := strings.Split(logcfg.Logger, ";") - for _, v := range s { - vs := strings.Split(v, ",") - override, err := logcfg.GetOrCreateNamedLogger(vs[0]) - if err != nil { - return NewErrCouldNotObtainLoggerConfig(err, vs[0]) - } - override.Name = vs[0] - for _, v := range vs[1:] { - parsedKV := strings.Split(v, "=") - if len(parsedKV) != 2 { - return NewErrNotProvidedAsKV(v) - } - switch param := strings.ToLower(parsedKV[0]); param { - case "level": // string - override.Level = parsedKV[1] - case "format": // string - override.Format = parsedKV[1] - case "output": // string - override.Output = parsedKV[1] - case "stacktrace": // bool - if override.Stacktrace, err = strconv.ParseBool(parsedKV[1]); err != nil { - return NewErrCouldNotParseType(err, "bool") - } - case "nocolor": // bool - if override.NoColor, err = strconv.ParseBool(parsedKV[1]); err != nil { - return NewErrCouldNotParseType(err, "bool") - } - case "caller": // bool - if override.Caller, err = strconv.ParseBool(parsedKV[1]); err != nil { - return NewErrCouldNotParseType(err, "bool") - } - default: - return NewErrUnknownLoggerParameter(param) - } - } + // make paths relative to the rootdir + for _, key := range relativePathKeys { + path := cfg.GetString(key) + if !filepath.IsAbs(path) { + cfg.Set(key, filepath.Join(rootdir, path)) } } - c, err := logcfg.toLoggerConfig() - if err != nil { - return err - } - logging.SetConfig(c) - return nil -} - -func convertLoglevel(level string) (logging.LogLevel, error) { - switch level { - case logLevelDebug: - return logging.Debug, nil - case logLevelInfo: - return logging.Info, nil - case logLevelError: - return logging.Error, nil - case logLevelFatal: - return logging.Fatal, nil + var level int8 + switch value := cfg.GetString("log.level"); value { + case "debug": + level = logging.Debug + case "info": + level = logging.Info + case "error": + level = logging.Error + case "fatal": + level = logging.Fatal default: - return logging.LogLevel(0), NewErrInvalidLogLevel(level) + level = logging.Info } -} -// Exports the logging config to the logging library's config. -func (logcfg LoggingConfig) toLoggerConfig() (logging.Config, error) { - loglevel, err := convertLoglevel(logcfg.Level) - if err != nil { - return logging.Config{}, err - } - - var encfmt logging.EncoderFormat - switch logcfg.Format { + var format logging.EncoderFormat + switch value := cfg.GetString("log.format"); value { case "json": - encfmt = logging.JSON + format = logging.JSON case "csv": - encfmt = logging.CSV + format = logging.CSV default: - return logging.Config{}, NewErrInvalidLogFormat(logcfg.Format) - } - - // handle logger named overrides - overrides := make(map[string]logging.Config) - for name, cfg := range logcfg.NamedOverrides { - c, err := cfg.toLoggerConfig() - if err != nil { - return logging.Config{}, NewErrOverrideConfigConvertFailed(err, name) - } - overrides[name] = c + format = logging.CSV } - c := logging.Config{ - Level: logging.NewLogLevelOption(loglevel), - EnableStackTrace: logging.NewEnableStackTraceOption(logcfg.Stacktrace), - DisableColor: logging.NewDisableColorOption(logcfg.NoColor), - EncoderFormat: logging.NewEncoderFormatOption(encfmt), - OutputPaths: []string{logcfg.Output}, - EnableCaller: logging.NewEnableCallerOption(logcfg.Caller), - OverridesByLoggerName: overrides, - } - return c, nil -} - -// this is a copy that doesn't deep copy the NamedOverrides map -// copy is handled by runtime "pass-by-value" -func (logcfg LoggingConfig) copy() LoggingConfig { - logcfg.NamedOverrides = make(map[string]*NamedLoggingConfig) - return logcfg -} - -// GetOrCreateNamedLogger returns a named logger config, or creates a default one if it doesn't exist. -func (logcfg *LoggingConfig) GetOrCreateNamedLogger(name string) (*NamedLoggingConfig, error) { - if name == "" { - return nil, ErrLoggerNameEmpty - } - if namedCfg, exists := logcfg.NamedOverrides[name]; exists { - return namedCfg, nil - } - // create default and save to overrides - namedCfg := &NamedLoggingConfig{ - Name: name, - LoggingConfig: logcfg.copy(), - } - logcfg.NamedOverrides[name] = namedCfg - - return namedCfg, nil -} - -// BindFlag binds a CLI flag to a config key. -func (cfg *Config) BindFlag(key string, flag *pflag.Flag) error { - return cfg.v.BindPFlag(key, flag) -} + logging.SetConfig(logging.Config{ + Level: logging.NewLogLevelOption(level), + EnableStackTrace: logging.NewEnableStackTraceOption(cfg.GetBool("log.stacktrace")), + DisableColor: logging.NewDisableColorOption(cfg.GetBool("log.nocolor")), + EncoderFormat: logging.NewEncoderFormatOption(format), + OutputPaths: []string{cfg.GetString("log.output")}, + EnableCaller: logging.NewEnableCallerOption(cfg.GetBool("log.caller")), + }) -// ToJSON serializes the config to a JSON byte array. -func (c *Config) ToJSON() ([]byte, error) { - jsonbytes, err := json.Marshal(c) - if err != nil { - return []byte{}, NewErrConfigToJSONFailed(err) - } - return jsonbytes, nil -} - -// String serializes the config to a JSON string. -func (c *Config) String() string { - jsonbytes, err := c.ToJSON() - if err != nil { - return fmt.Sprintf("failed to convert config to string: %s", err) - } - return string(jsonbytes) -} - -func (c *Config) toBytes() ([]byte, error) { - tmpl := template.New("configTemplate") - configTemplate, err := tmpl.Parse(defaultConfigTemplate) - if err != nil { - return nil, NewErrConfigTemplateFailed(err) - } - var buffer bytes.Buffer - if err := configTemplate.Execute(&buffer, c); err != nil { - return nil, NewErrConfigTemplateFailed(err) - } - return buffer.Bytes(), nil + return cfg, nil } diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 0000000000..04cfa93684 --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,79 @@ +# DefraDB configuration (YAML) + +# The default DefraDB directory is "$HOME/.defradb". It can be changed via the --rootdir CLI flag. +# Relative paths are interpreted as being rooted in the DefraDB directory. + +datastore: + # Store can be badger | memory + # badger: fast pure Go key-value store optimized for SSDs (https://github.com/dgraph-io/badger) + # memory: in-memory version of badger + store: badger + + # Maximum number of times to retry failed transactions. + maxTxnRetries: 5 + + # badger specific settings + badger: + # The path to the database data file(s). + path: data + + # Maximum file size of the value log files. + # The in-memory file size will be 2*valuelogfilesize. + valueLogFileSize: 1073741824 + +api: + # Address of the HTTP API to listen on or connect to + address: localhost:9181 + + # The list of origins a cross-domain request can be executed from. + # allowed-origins: + # - localhost + + # Whether the API server should listen over HTTPS + tls: false + + # The path to the public key file. Ignored if domains is set. + pubKeyPath: cert.pub + + # The path to the private key file. Ignored if domains is set. + privKeyPath: cert.key + + # Email address to let the CA (Let's Encrypt) send notifications via email when there are issues (optional). + email: example@example.com + +net: + # Whether the P2P is disabled + p2pDisabled: false + + # Listening address of the P2P network + p2pAddresses: + - /ip4/127.0.0.1/tcp/9171 + + # Whether the node has pubsub enabled or not + pubSubEnabled: true + + # Enable libp2p's Circuit relay transport protocol https://docs.libp2p.io/concepts/circuit-relay/ + relay: false + + # List of peers to boostrap with, specified as multiaddresses (https://docs.libp2p.io/concepts/addressing/) + # peers: + # - /ip4/127.0.0.1/tcp/9171/QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N + +log: + # Log level. Options are debug, info, error, fatal + level: info + + # Include stack trace in error and fatal logs + stackTrace: true + + # Supported log formats are json, csv + format: csv + + # Where the log output is written to + output: stderr + + # Disable colored log output + noColor: false + + # Caller location in log output + caller: false diff --git a/config/config_test.go b/config/config_test.go index 859cdc5e04..d4d1df7ecd 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,435 +1,61 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - package config import ( - "encoding/json" "os" "path/filepath" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -var envVarsDifferent = map[string]string{ - "DEFRA_DATASTORE_STORE": "memory", - "DEFRA_DATASTORE_BADGER_PATH": "defra_data", - "DEFRA_API_ADDRESS": "localhost:9999", - "DEFRA_NET_P2PDISABLED": "true", - "DEFRA_NET_P2PADDRESSES": "/ip4/0.0.0.0/tcp/9876", - "DEFRA_NET_PUBSUB": "false", - "DEFRA_NET_RELAY": "false", - "DEFRA_LOG_LEVEL": "error", - "DEFRA_LOG_STACKTRACE": "true", - "DEFRA_LOG_FORMAT": "json", -} - -var envVarsInvalid = map[string]string{ - "DEFRA_DATASTORE_STORE": "^=+()&**()*(&))", - "DEFRA_DATASTORE_BADGER_PATH": "^=+()&**()*(&))", - "DEFRA_API_ADDRESS": "^=+()&**()*(&))", - "DEFRA_NET_P2PDISABLED": "^=+()&**()*(&))", - "DEFRA_NET_P2PADDRESSES": "^=+()&**()*(&))", - "DEFRA_NET_PUBSUB": "^=+()&**()*(&))", - "DEFRA_NET_RELAY": "^=+()&**()*(&))", - "DEFRA_LOG_LEVEL": "^=+()&**()*(&))", - "DEFRA_LOG_STACKTRACE": "^=+()&**()*(&))", - "DEFRA_LOG_FORMAT": "^=+()&**()*(&))", -} - -func FixtureEnvKeyValue(t *testing.T, key, value string) { - t.Helper() - os.Setenv(key, value) - t.Cleanup(func() { - os.Unsetenv(key) - }) -} - -func FixtureEnvVars(t *testing.T, envVars map[string]string) { - t.Helper() - for k, v := range envVars { - os.Setenv(k, v) - } - t.Cleanup(func() { - for k := range envVars { - os.Unsetenv(k) - } - }) -} - -func TestConfigValidateBasic(t *testing.T) { - cfg := DefaultConfig() - assert.NoError(t, cfg.validate()) - - err := cfg.validate() - assert.NoError(t, err) - // asserting equality of some unlikely-to-change default values - assert.Equal(t, "stderr", cfg.Log.Output) - assert.Equal(t, "csv", cfg.Log.Format) - assert.Equal(t, false, cfg.API.TLS) - assert.Equal(t, false, cfg.Net.RelayEnabled) -} - -func TestLoadIncorrectValuesFromConfigFile(t *testing.T) { - var cfg *Config - - testcases := []struct { - setter func() - err error - }{ - { - setter: func() { - cfg.Datastore.Store = "antibadger" - }, - err: ErrInvalidDatastoreType, - }, - { - setter: func() { - cfg.Log.Level = "antilevel" - }, - err: ErrInvalidLogLevel, - }, - { - setter: func() { - cfg.Log.Format = "antiformat" - }, - - err: ErrInvalidLogFormat, - }, - } - - for _, tc := range testcases { - cfg = DefaultConfig() - err := cfg.setRootdir(t.TempDir()) - assert.NoError(t, err) - tc.setter() - err = cfg.WriteConfigFile() - assert.NoError(t, err) - err = cfg.LoadWithRootdir(true) - assert.ErrorIs(t, err, tc.err) - } -} - -func TestJSONSerialization(t *testing.T) { - cfg := DefaultConfig() - var m map[string]any - - b, errSerialize := cfg.ToJSON() - errUnmarshal := json.Unmarshal(b, &m) - - assert.NoError(t, errUnmarshal) - assert.NoError(t, errSerialize) - for k, v := range m { - if k != "Rootdir" { // Rootdir is not serialized - assert.NotEmpty(t, v) - } - } -} - -func TestLoadValidationDefaultsConfigFileEnv(t *testing.T) { - tmpdir := t.TempDir() - cfg := DefaultConfig() - err := cfg.setRootdir(tmpdir) - assert.NoError(t, err) - FixtureEnvVars(t, envVarsDifferent) - errWriteConfig := cfg.WriteConfigFile() - - errLoad := cfg.LoadWithRootdir(true) - - assert.NoError(t, errWriteConfig) - assert.NoError(t, errLoad) - assert.Equal(t, "localhost:9999", cfg.API.Address) - assert.Equal(t, filepath.Join(tmpdir, "defra_data"), cfg.Datastore.Badger.Path) -} - -func TestLoadDefaultsEnv(t *testing.T) { - cfg := DefaultConfig() - FixtureEnvVars(t, envVarsDifferent) - - err := cfg.LoadWithRootdir(false) - - assert.NoError(t, err) - assert.Equal(t, "localhost:9999", cfg.API.Address) - assert.Equal(t, filepath.Join(cfg.Rootdir, "defra_data"), cfg.Datastore.Badger.Path) -} - -func TestEnvVariablesAllConsidered(t *testing.T) { - cfg := DefaultConfig() - FixtureEnvVars(t, envVarsDifferent) - - err := cfg.LoadWithRootdir(false) - - assert.NoError(t, err) - assert.Equal(t, "localhost:9999", cfg.API.Address) - assert.Equal(t, filepath.Join(cfg.Rootdir, "defra_data"), cfg.Datastore.Badger.Path) - assert.Equal(t, "memory", cfg.Datastore.Store) - assert.Equal(t, true, cfg.Net.P2PDisabled) - assert.Equal(t, []string{"/ip4/0.0.0.0/tcp/9876"}, cfg.Net.P2PAddresses) - assert.Equal(t, false, cfg.Net.PubSubEnabled) - assert.Equal(t, false, cfg.Net.RelayEnabled) - assert.Equal(t, "error", cfg.Log.Level) - assert.Equal(t, true, cfg.Log.Stacktrace) - assert.Equal(t, "json", cfg.Log.Format) -} - -func TestLoadNonExistingConfigFile(t *testing.T) { - cfg := DefaultConfig() - err := cfg.setRootdir(t.TempDir()) - assert.NoError(t, err) - err = cfg.LoadWithRootdir(true) - assert.ErrorIs(t, err, ErrReadingConfigFile) -} - -func TestLoadInvalidConfigFile(t *testing.T) { - cfg := DefaultConfig() - tmpdir := t.TempDir() - - errWrite := os.WriteFile( - filepath.Join(tmpdir, DefaultConfigFileName), - []byte("{"), - 0644, - ) - assert.NoError(t, errWrite) - - err := cfg.setRootdir(tmpdir) - assert.NoError(t, err) - errLoad := cfg.LoadWithRootdir(true) - assert.ErrorIs(t, errLoad, ErrReadingConfigFile) -} - -func TestInvalidEnvVars(t *testing.T) { - cfg := DefaultConfig() - FixtureEnvVars(t, envVarsInvalid) - - err := cfg.LoadWithRootdir(false) - - assert.ErrorIs(t, err, ErrLoadingConfig) -} +func TestLoadConfigNotExist(t *testing.T) { + rootdir := t.TempDir() + cfg, err := LoadConfig(rootdir) + require.NoError(t, err) -func TestCreateAndLoadCustomConfig(t *testing.T) { - testdir := t.TempDir() + assert.Equal(t, "badger", cfg.GetString("datastore.store")) + assert.Equal(t, 5, cfg.GetInt("datastore.maxtxnretries")) - cfg := DefaultConfig() - err := cfg.setRootdir(testdir) - assert.NoError(t, err) - // a few valid but non-default changes - cfg.Net.PubSubEnabled = false - cfg.Log.Level = "fatal" + assert.Equal(t, filepath.Join(rootdir, "data"), cfg.GetString("datastore.badger.path")) + assert.Equal(t, 1073741824, cfg.GetInt("datastore.badger.valuelogfilesize")) - err = cfg.CreateRootDirAndConfigFile() - assert.NoError(t, err) + assert.Equal(t, "localhost:9181", cfg.GetString("api.address")) + assert.Equal(t, false, cfg.GetBool("api.tls")) + assert.Equal(t, []string(nil), cfg.GetStringSlice("api.allowed-origins")) + assert.Equal(t, filepath.Join(rootdir, "cert.pub"), cfg.GetString("api.pubkeypath")) + assert.Equal(t, filepath.Join(rootdir, "cert.key"), cfg.GetString("api.privkeypath")) + assert.Equal(t, "example@example.com", cfg.GetString("api.email")) - assert.True(t, cfg.ConfigFileExists()) + assert.Equal(t, false, cfg.GetBool("net.p2pdisabled")) + assert.Equal(t, []string{"/ip4/127.0.0.1/tcp/9171"}, cfg.GetStringSlice("net.p2paddresses")) + assert.Equal(t, true, cfg.GetBool("net.pubsubenabled")) + assert.Equal(t, false, cfg.GetBool("net.relay")) + assert.Equal(t, []string(nil), cfg.GetStringSlice("net.peers")) - // check that the config file loads properly - cfg2 := DefaultConfig() - err = cfg2.setRootdir(testdir) - assert.NoError(t, err) - err = cfg2.LoadWithRootdir(true) - assert.NoError(t, err) - assert.Equal(t, cfg.Net.PubSubEnabled, cfg2.Net.PubSubEnabled) - assert.Equal(t, cfg.Log.Level, cfg2.Log.Level) + assert.Equal(t, "info", cfg.GetString("log.level")) + assert.Equal(t, true, cfg.GetBool("log.stacktrace")) + assert.Equal(t, "csv", cfg.GetString("log.format")) + assert.Equal(t, "stderr", cfg.GetString("log.output")) + assert.Equal(t, false, cfg.GetBool("log.nocolor")) + assert.Equal(t, false, cfg.GetBool("log.caller")) } -func TestLoadValidationEnvLoggingConfig(t *testing.T) { - FixtureEnvKeyValue(t, "DEFRA_LOG_LEVEL", "debug,net=info,log=error,cli=fatal") - cfg := DefaultConfig() - err := cfg.LoadWithRootdir(false) - assert.NoError(t, err) - assert.Equal(t, "debug", cfg.Log.Level) - for _, override := range cfg.Log.NamedOverrides { - switch override.Name { - case "net": - assert.Equal(t, "info", override.Level) - case "log": - assert.Equal(t, "error", override.Level) - case "cli": - assert.Equal(t, "fatal", override.Level) - default: - t.Fatal("unexpected named override") - } - } -} +func TestLoadConfigExists(t *testing.T) { + rootdir := t.TempDir() + err := WriteDefaultConfig(rootdir) + require.NoError(t, err) -func TestLoadValidationEnvLoggerConfig(t *testing.T) { - FixtureEnvKeyValue(t, "DEFRA_LOG_LOGGER", "net,nocolor=true,level=debug;config,output=stdout,level=info") - cfg := DefaultConfig() - err := cfg.LoadWithRootdir(false) - assert.NoError(t, err) - for _, override := range cfg.Log.NamedOverrides { - switch override.Name { - case "net": - assert.Equal(t, true, override.NoColor) - assert.Equal(t, "debug", override.Level) - case "config": - assert.Equal(t, "info", override.Level) - assert.Equal(t, "stdout", override.Output) - default: - t.Fatal("unexpected named override") - } - } + _, err = LoadConfig(rootdir) + require.NoError(t, err) } -func TestLoadValidationEnvLoggerConfigInvalid(t *testing.T) { - // logging config parameter not provided as = pair - FixtureEnvKeyValue(t, "DEFRA_LOG_LOGGER", "net,nocolor,true,level,debug;config,output,stdout,level,info") - cfg := DefaultConfig() - err := cfg.LoadWithRootdir(false) - assert.ErrorIs(t, err, ErrFailedToValidateConfig) - - // invalid logger names - FixtureEnvKeyValue(t, "DEFRA_LOG_LOGGER", "13;2134;™¡£¡™£∞¡™∞¡™£¢;1234;1") - cfg = DefaultConfig() - err = cfg.LoadWithRootdir(false) - assert.ErrorIs(t, err, ErrFailedToValidateConfig) -} - -func TestLoadValidationLoggerConfigFromEnvExhaustive(t *testing.T) { - FixtureEnvKeyValue(t, "DEFRA_LOG_LOGGER", "net,nocolor=true,level=debug;config,output=stdout,caller=false;logging,stacktrace=true,format=json") - cfg := DefaultConfig() - err := cfg.LoadWithRootdir(false) - assert.NoError(t, err) - for _, override := range cfg.Log.NamedOverrides { - switch override.Name { - case "net": - assert.Equal(t, true, override.NoColor) - assert.Equal(t, "debug", override.Level) - case "config": - assert.Equal(t, "stdout", override.Output) - assert.Equal(t, false, override.Caller) - case "logging": - assert.Equal(t, true, override.Stacktrace) - assert.Equal(t, "json", override.Format) - default: - t.Fatal("unexpected named override") - } - } -} - -func TestLoadValidationLoggerConfigFromEnvUnknownParam(t *testing.T) { - FixtureEnvKeyValue(t, "DEFRA_LOG_LOGGER", "net,unknown=true,level=debug") - cfg := DefaultConfig() - err := cfg.LoadWithRootdir(false) - assert.ErrorIs(t, err, ErrUnknownLoggerParameter) -} - -func TestLoadValidationInvalidDatastoreConfig(t *testing.T) { - FixtureEnvKeyValue(t, "DEFRA_DATASTORE_STORE", "antibadger") - cfg := DefaultConfig() - err := cfg.LoadWithRootdir(false) - assert.ErrorIs(t, err, ErrInvalidDatastoreType) -} - -func TestValidationLogger(t *testing.T) { - testCases := []struct { - input string - expectedErr error - }{ - {"node,level=debug,output=stdout", nil}, - {"node,level=fatal,format=csv", nil}, - {"node,level=warn", ErrInvalidLogLevel}, - {"node,level=debug;cli,", ErrNotProvidedAsKV}, - {"node,level", ErrNotProvidedAsKV}, - - {";", ErrInvalidLoggerConfig}, - {";;", ErrInvalidLoggerConfig}, - {",level=debug", ErrLoggerNameEmpty}, - {"node,bar=baz", ErrUnknownLoggerParameter}, // unknown parameter - {"m,level=debug,output-json", ErrNotProvidedAsKV}, // key-value pair with invalid separator - {"myModule,level=debug,extraPart", ErrNotProvidedAsKV}, // additional part after last key-value pair - {"myModule,=myValue", ErrNotProvidedAsKV}, // empty key - {",k=v", ErrLoggerNameEmpty}, // empty module - {";foo", ErrInvalidLoggerConfig}, // empty module name - {"k=v", ErrInvalidLoggerConfig}, // missing module - {"debug,net=,log=error,cli=fatal", ErrNotProvidedAsKV}, // empty value - - } - - for _, tc := range testCases { - cfg := DefaultConfig() - cfg.Log.Logger = tc.input - t.Log(tc.input) - err := cfg.validate() - assert.ErrorIs(t, err, tc.expectedErr) - } -} - -func TestValidationInvalidEmptyAPIAddress(t *testing.T) { - cfg := DefaultConfig() - cfg.API.Address = "" - err := cfg.validate() - assert.ErrorIs(t, err, ErrInvalidDatabaseURL) -} - -func TestValidationNetConfigPeers(t *testing.T) { - cfg := DefaultConfig() - cfg.Net.Peers = "/ip4/127.0.0.1/udp/1234,/ip4/7.7.7.7/tcp/4242/p2p/QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N" - err := cfg.validate() - assert.NoError(t, err) -} - -func TestValidationInvalidNetConfigPeers(t *testing.T) { - cfg := DefaultConfig() - cfg.Net.Peers = "&(*^(*&^(*&^(*&^))), mmmmh,123123" - err := cfg.validate() - assert.ErrorIs(t, err, ErrFailedToValidateConfig) -} - -func TestValidationInvalidLoggingConfig(t *testing.T) { - cfg := DefaultConfig() - cfg.Log.Level = "546578" - cfg.Log.Format = "*&)*&" - err := cfg.validate() - assert.ErrorIs(t, err, ErrInvalidLogLevel) -} - -func TestValidationAddressBasicIncomplete(t *testing.T) { - cfg := DefaultConfig() - cfg.API.Address = "localhost" - err := cfg.validate() - assert.ErrorIs(t, err, ErrFailedToValidateConfig) -} - -func TestValidationAddressLocalhostValid(t *testing.T) { - cfg := DefaultConfig() - cfg.API.Address = "localhost:9876" - err := cfg.validate() - assert.NoError(t, err) -} - -func TestValidationAddress0000Incomplete(t *testing.T) { - cfg := DefaultConfig() - cfg.API.Address = "0.0.0.0" - err := cfg.validate() - assert.ErrorIs(t, err, ErrFailedToValidateConfig) -} - -func TestValidationAddress0000Valid(t *testing.T) { - cfg := DefaultConfig() - cfg.API.Address = "0.0.0.0:9876" - err := cfg.validate() - assert.NoError(t, err) -} - -func TestValidationAddressDomainWithSubdomainValidWithTLSCorrectPortIsInvalid(t *testing.T) { - cfg := DefaultConfig() - cfg.API.Address = "sub.example.com:443" - cfg.API.TLS = true - err := cfg.validate() - assert.ErrorIs(t, err, ErrNoPortWithDomain) -} +func TestWriteDefaultConfig(t *testing.T) { + rootdir := t.TempDir() + err := WriteDefaultConfig(rootdir) + require.NoError(t, err) -func TestValidationAddressDomainWithSubdomainWrongPortIsInvalid(t *testing.T) { - cfg := DefaultConfig() - cfg.API.Address = "sub.example.com:9876" - err := cfg.validate() - assert.ErrorIs(t, err, ErrNoPortWithDomain) + data, err := os.ReadFile(filepath.Join(rootdir, configName)) + require.NoError(t, err) + assert.Equal(t, defaultConfig, data) } diff --git a/config/config_utils.go b/config/config_utils.go deleted file mode 100644 index 46e5762570..0000000000 --- a/config/config_utils.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package config - -import ( - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - "unicode" -) - -type ByteSize uint64 - -const ( - B ByteSize = 1 - KiB = B << 10 - MiB = KiB << 10 - GiB = MiB << 10 - TiB = GiB << 10 - PiB = TiB << 10 -) - -// UnmarshalText calls Set on ByteSize with the given text -func (bs *ByteSize) UnmarshalText(text []byte) error { - return bs.Set(string(text)) -} - -// String returns the string formatted output of ByteSize -func (bs *ByteSize) String() string { - const unit = 1024 - bsInt := int64(*bs) - if bsInt < unit { - return fmt.Sprintf("%d", bsInt) - } - div, exp := int64(unit), 0 - for n := bsInt / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%d%ciB", bsInt/div, "KMGTP"[exp]) -} - -// Type returns the type as a string. -func (bs *ByteSize) Type() string { - return "ByteSize" -} - -// Set parses a string into ByteSize -func (bs *ByteSize) Set(s string) error { - digitString := "" - unit := "" - for _, char := range s { - if unicode.IsDigit(char) { - digitString += string(char) - } else { - unit += string(char) - } - } - digits, err := strconv.Atoi(digitString) - if err != nil { - return NewErrUnableToParseByteSize(err) - } - - switch strings.ToUpper(strings.Trim(unit, " ")) { - case "B": - *bs = ByteSize(digits) * B - case "KB", "KIB": - *bs = ByteSize(digits) * KiB - case "MB", "MIB": - *bs = ByteSize(digits) * MiB - case "GB", "GIB": - *bs = ByteSize(digits) * GiB - case "TB", "TIB": - *bs = ByteSize(digits) * TiB - case "PB", "PIB": - *bs = ByteSize(digits) * PiB - default: - *bs = ByteSize(digits) - } - - return nil -} - -// expandHomeDir expands paths if they were passed in as `~` rather than `${HOME}` -// converts `~/.defradb/certs/server.crt` to `/home/username/.defradb/certs/server.crt`. -func expandHomeDir(path *string) error { - if *path == "~" { - return ErrPathCannotBeHomeDir - } else if strings.HasPrefix(*path, "~/") { - homeDir, err := os.UserHomeDir() - if err != nil { - return NewErrUnableToExpandHomeDir(err) - } - - // Use strings.HasPrefix so we don't match paths like "/x/~/x/" - *path = filepath.Join(homeDir, (*path)[2:]) - } - - return nil -} - -func isLowercaseAlpha(s string) bool { - for i := 0; i < len(s); i++ { - c := s[i] - if c < 'a' || c > 'z' { - return false - } - } - return true -} - -func parseKV(kv string) ([]string, error) { - parsedKV := strings.Split(kv, "=") - if len(parsedKV) != 2 { - return nil, NewErrNotProvidedAsKV(kv) - } - if parsedKV[0] == "" || parsedKV[1] == "" { - return nil, NewErrNotProvidedAsKV(kv) - } - return parsedKV, nil -} diff --git a/config/config_utils_test.go b/config/config_utils_test.go deleted file mode 100644 index f9eea47c16..0000000000 --- a/config/config_utils_test.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package config - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestUnmarshalByteSize(t *testing.T) { - var bs ByteSize - - b := []byte("10") - err := bs.UnmarshalText(b) - assert.NoError(t, err) - assert.Equal(t, 10*B, bs) - - b = []byte("10B") - err = bs.UnmarshalText(b) - assert.NoError(t, err) - assert.Equal(t, 10*B, bs) - - b = []byte("10 B") - err = bs.UnmarshalText(b) - assert.NoError(t, err) - assert.Equal(t, 10*B, bs) - - kb := []byte("10KB") - err = bs.UnmarshalText(kb) - assert.NoError(t, err) - assert.Equal(t, 10*KiB, bs) - - kb = []byte("10KiB") - err = bs.UnmarshalText(kb) - assert.NoError(t, err) - assert.Equal(t, 10*KiB, bs) - - kb = []byte("10 kb") - err = bs.UnmarshalText(kb) - assert.NoError(t, err) - assert.Equal(t, 10*KiB, bs) - - mb := []byte("10MB") - err = bs.UnmarshalText(mb) - assert.NoError(t, err) - assert.Equal(t, 10*MiB, bs) - - mb = []byte("10MiB") - err = bs.UnmarshalText(mb) - assert.NoError(t, err) - assert.Equal(t, 10*MiB, bs) - - gb := []byte("10GB") - err = bs.UnmarshalText(gb) - assert.NoError(t, err) - assert.Equal(t, 10*GiB, bs) - - gb = []byte("10GiB") - err = bs.UnmarshalText(gb) - assert.NoError(t, err) - assert.Equal(t, 10*GiB, bs) - - tb := []byte("10TB") - err = bs.UnmarshalText(tb) - assert.NoError(t, err) - assert.Equal(t, 10*TiB, bs) - - tb = []byte("10TiB") - err = bs.UnmarshalText(tb) - assert.NoError(t, err) - assert.Equal(t, 10*TiB, bs) - - pb := []byte("10PB") - err = bs.UnmarshalText(pb) - assert.NoError(t, err) - assert.Equal(t, 10*PiB, bs) - - pb = []byte("10PiB") - err = bs.UnmarshalText(pb) - assert.NoError(t, err) - assert.Equal(t, 10*PiB, bs) - - eb := []byte("१") - err = bs.UnmarshalText(eb) - assert.ErrorIs(t, err, ErrUnableToParseByteSize) -} - -func TestByteSizeType(t *testing.T) { - var bs ByteSize - assert.Equal(t, "ByteSize", bs.Type()) -} - -func TestByteSizeToString(t *testing.T) { - b := 999 * B - assert.Equal(t, "999", b.String()) - - mb := 10 * MiB - assert.Equal(t, "10MiB", mb.String()) -} diff --git a/config/configfile.go b/config/configfile.go deleted file mode 100644 index f13b6695e1..0000000000 --- a/config/configfile.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package config - -import ( - "context" - _ "embed" - "fmt" - "os" - "path/filepath" -) - -const ( - DefaultConfigFileName = "config.yaml" - configType = "yaml" - defaultDirPerm = 0o700 - defaultConfigFilePerm = 0o644 -) - -// defaultConfigTemplate must reflect the Config struct in content and configuration. -// All parameters must be represented here, to support Viper's automatic environment variable handling. -// We embed the default config template for clarity and to avoid autoformatters from breaking it. -// -//go:embed configfile_yaml.gotmpl -var defaultConfigTemplate string - -func (cfg *Config) ConfigFilePath() string { - return filepath.Join(cfg.Rootdir, DefaultConfigFileName) -} - -func (cfg *Config) WriteConfigFile() error { - path := cfg.ConfigFilePath() - buffer, err := cfg.toBytes() - if err != nil { - return err - } - if err := os.WriteFile(path, buffer, defaultConfigFilePerm); err != nil { - return NewErrFailedToWriteFile(err, path) - } - log.FeedbackInfo(context.Background(), fmt.Sprintf("Created config file at %v", path)) - return nil -} - -func (cfg *Config) DeleteConfigFile() error { - if err := os.Remove(cfg.ConfigFilePath()); err != nil { - return NewErrFailedToRemoveConfigFile(err) - } - log.FeedbackInfo(context.Background(), fmt.Sprintf("Deleted config file at %v", cfg.ConfigFilePath())) - return nil -} - -func (cfg *Config) CreateRootDirAndConfigFile() error { - if err := os.MkdirAll(cfg.Rootdir, defaultDirPerm); err != nil { - return err - } - log.FeedbackInfo(context.Background(), fmt.Sprintf("Created DefraDB root directory at %v", cfg.Rootdir)) - if err := cfg.WriteConfigFile(); err != nil { - return err - } - return nil -} - -func (cfg *Config) ConfigFileExists() bool { - statInfo, err := os.Stat(cfg.ConfigFilePath()) - existsAsFile := (err == nil && !statInfo.IsDir()) - return existsAsFile -} - -func DefaultRootDir() string { - home, err := os.UserHomeDir() - if err != nil { - log.FatalE(context.Background(), "error determining user directory", err) - } - return filepath.Join(home, ".defradb") -} - -func FolderExists(folderPath string) bool { - statInfo, err := os.Stat(folderPath) - existsAsFolder := (err == nil && statInfo.IsDir()) - return existsAsFolder -} diff --git a/config/configfile_test.go b/config/configfile_test.go deleted file mode 100644 index 5f7aed26aa..0000000000 --- a/config/configfile_test.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package config - -import ( - "bytes" - "os" - "path/filepath" - "runtime" - "testing" - "text/template" - - "github.com/stretchr/testify/assert" -) - -func TestConfigTemplateSerialize(t *testing.T) { - var buffer bytes.Buffer - cfg := DefaultConfig() - tmpl := template.New("configTemplate") - configTemplate, err := tmpl.Parse(defaultConfigTemplate) - assert.NoError(t, err) - err = configTemplate.Execute(&buffer, cfg) - assert.NoError(t, err) - _, err = cfg.ToJSON() - assert.NoError(t, err) -} - -func TestConfigTemplateExecutes(t *testing.T) { - cfg := DefaultConfig() - var buffer bytes.Buffer - tmpl := template.New("configTemplate") - configTemplate, err := tmpl.Parse(defaultConfigTemplate) - assert.NoError(t, err) - err = configTemplate.Execute(&buffer, cfg) - assert.NoError(t, err) -} - -func TestWritesConfigFile(t *testing.T) { - cfg := DefaultConfig() - tmpdir := t.TempDir() - cfg.Rootdir = tmpdir - err := cfg.WriteConfigFile() - assert.NoError(t, err) - path := filepath.Join(tmpdir, DefaultConfigFileName) - _, err = os.Stat(path) - assert.Nil(t, err) -} - -func TestWritesConfigFileErroneousPath(t *testing.T) { - cfg := DefaultConfig() - cfg.Rootdir = filepath.Join(t.TempDir(), "////*&^^(*8769876////bar") - err := cfg.WriteConfigFile() - assert.Error(t, err) -} - -func TestReadConfigFileForLogger(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skipf("Test is not supported on windows as it leaks resources, see https://github.com/sourcenetwork/defradb/issues/2057") - } - - cfg := DefaultConfig() - tmpdir := t.TempDir() - cfg.Rootdir = tmpdir - cfg.Log.Caller = true - cfg.Log.Format = "json" - cfg.Log.Level = logLevelDebug - cfg.Log.NoColor = true - cfg.Log.Output = filepath.Join(tmpdir, "log.txt") - cfg.Log.Stacktrace = true - - err := cfg.WriteConfigFile() - assert.NoError(t, err) - - assert.True(t, cfg.ConfigFileExists()) - - cfgFromFile := DefaultConfig() - err = cfgFromFile.setRootdir(tmpdir) - assert.NoError(t, err) - err = cfgFromFile.LoadWithRootdir(true) - assert.NoError(t, err) - - assert.Equal(t, cfg.Log.Caller, cfgFromFile.Log.Caller) - assert.Equal(t, cfg.Log.Format, cfgFromFile.Log.Format) - assert.Equal(t, cfg.Log.Level, cfgFromFile.Log.Level) - assert.Equal(t, cfg.Log.NoColor, cfgFromFile.Log.NoColor) - assert.Equal(t, cfg.Log.Output, cfgFromFile.Log.Output) - assert.Equal(t, cfg.Log.Stacktrace, cfgFromFile.Log.Stacktrace) -} - -func TestReadConfigFileForDatastore(t *testing.T) { - tmpdir := t.TempDir() - - cfg := DefaultConfig() - err := cfg.setRootdir(tmpdir) - assert.NoError(t, err) - cfg.Datastore.Store = "badger" - cfg.Datastore.Badger.Path = "dataPath" - cfg.Datastore.Badger.ValueLogFileSize = 512 * MiB - - err = cfg.WriteConfigFile() - assert.NoError(t, err) - - configPath := filepath.Join(tmpdir, DefaultConfigFileName) - _, err = os.Stat(configPath) - assert.NoError(t, err) - - cfgFromFile := DefaultConfig() - err = cfgFromFile.setRootdir(tmpdir) - assert.NoError(t, err) - err = cfgFromFile.LoadWithRootdir(true) - assert.NoError(t, err) - - assert.Equal(t, cfg.Datastore.Store, cfgFromFile.Datastore.Store) - assert.Equal(t, filepath.Join(tmpdir, cfg.Datastore.Badger.Path), cfgFromFile.Datastore.Badger.Path) - assert.Equal(t, cfg.Datastore.Badger.ValueLogFileSize, cfgFromFile.Datastore.Badger.ValueLogFileSize) -} - -func TestConfigFileExists(t *testing.T) { - cfg := DefaultConfig() - err := cfg.setRootdir(t.TempDir()) - assert.NoError(t, err) - assert.False(t, cfg.ConfigFileExists()) - - err = cfg.WriteConfigFile() - assert.NoError(t, err) - assert.True(t, cfg.ConfigFileExists()) -} - -func TestConfigFileExistsErroneousPath(t *testing.T) { - cfg := DefaultConfig() - cfg.Rootdir = filepath.Join(t.TempDir(), "////*&^^(*8769876////bar") - assert.False(t, cfg.ConfigFileExists()) -} - -func TestDeleteConfigFile(t *testing.T) { - cfg := DefaultConfig() - tmpdir := t.TempDir() - cfg.Rootdir = tmpdir - err := cfg.WriteConfigFile() - assert.NoError(t, err) - - assert.True(t, cfg.ConfigFileExists()) - - err = cfg.DeleteConfigFile() - assert.NoError(t, err) - assert.False(t, cfg.ConfigFileExists()) -} diff --git a/config/configfile_yaml.gotmpl b/config/configfile_yaml.gotmpl deleted file mode 100644 index 53d87c46d3..0000000000 --- a/config/configfile_yaml.gotmpl +++ /dev/null @@ -1,65 +0,0 @@ -# DefraDB configuration (YAML) - -# The default DefraDB directory is "$HOME/.defradb". It can be changed via the --rootdir CLI flag. -# Relative paths are interpreted as being rooted in the DefraDB directory. - -datastore: - # Store can be badger | memory - # badger: fast pure Go key-value store optimized for SSDs (https://github.com/dgraph-io/badger) - # memory: in-memory version of badger - store: {{ .Datastore.Store }} - badger: - # The path to the database data file(s). - path: {{ .Datastore.Badger.Path }} - # Maximum file size of the value log files. The in-memory file size will be 2*valuelogfilesize. - # Human friendly units can be used (ex: 500MB). - valuelogfilesize: {{ .Datastore.Badger.ValueLogFileSize }} - maxtxnretries: {{ .Datastore.MaxTxnRetries }} - # memory: - # size: {{ .Datastore.Memory.Size }} - -api: - # Address of the HTTP API to listen on or connect to - address: {{ .API.Address }} - # Whether the API server should listen over HTTPS - tls: {{ .API.TLS }} - # The list of origins a cross-domain request can be executed from. - # allowed-origins: {{ .API.AllowedOrigins }} - # The path to the public key file. Ignored if domains is set. - pubkeypath: {{ .API.PubKeyPath }} - # The path to the private key file. Ignored if domains is set. - privkeypath: {{ .API.PrivKeyPath }} - # Email address to let the CA (Let's Encrypt) send notifications via email when there are issues (optional). - # email: {{ .API.Email }} - -net: - # Whether the P2P is disabled - p2pdisabled: {{ .Net.P2PDisabled }} - # Listening address of the P2P network - p2paddresses: - {{ range $addr := .Net.P2PAddresses }} - - {{ . }} - {{ end }} - # Whether the node has pubsub enabled or not - pubsub: {{ .Net.PubSubEnabled }} - # Enable libp2p's Circuit relay transport protocol https://docs.libp2p.io/concepts/circuit-relay/ - relay: {{ .Net.RelayEnabled }} - # List of peers to boostrap with, specified as multiaddresses (https://docs.libp2p.io/concepts/addressing/) - peers: {{ .Net.Peers }} - -log: - # Log level. Options are debug, info, error, fatal - level: {{ .Log.Level }} - # Include stacktrace in error and fatal logs - stacktrace: {{ .Log.Stacktrace }} - # Supported log formats are json, csv - format: {{ .Log.Format }} - # Where the log output is written to - output: {{ .Log.Output }} - # Disable colored log output - nocolor: {{ .Log.NoColor }} - # Caller location in log output - caller: {{ .Log.Caller }} - # Provide specific named component logger configuration - # e.g. net,nocolor=true,level=debug;config,output=stdout,format=json - logger: {{ .Log.Logger }} \ No newline at end of file diff --git a/config/errors.go b/config/errors.go deleted file mode 100644 index 872b362b66..0000000000 --- a/config/errors.go +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package config - -import ( - "github.com/sourcenetwork/defradb/errors" -) - -const ( - errFailedToWriteFile string = "failed to write file" - errFailedToRemoveConfigFile string = "failed to remove config file" - errCannotBeHomeDir string = "path cannot be just ~ (home directory)" - errUnableToExpandHomeDir string = "unable to expand home directory" - errNoDatabaseURLProvided string = "no database URL provided" - errLoggingConfigNotObtained string = "could not get logging config" - errFailedToValidateConfig string = "failed to validate config" - errOverrideConfigConvertFailed string = "invalid override config" - errConfigToJSONFailed string = "failed to marshal Config to JSON" - errInvalidDatabaseURL string = "invalid database URL" - errInvalidRPCTimeout string = "invalid RPC timeout" - errInvalidRPCMaxConnectionIdle string = "invalid RPC MaxConnectionIdle" - errInvalidP2PAddress string = "invalid P2P address" - errInvalidRPCAddress string = "invalid RPC address" - errInvalidBootstrapPeers string = "invalid bootstrap peers" - errInvalidLogLevel string = "invalid log level" - errInvalidDatastoreType string = "invalid store type" - errInvalidLogFormat string = "invalid log format" - errInvalidNamedLoggerName string = "invalid named logger name" - errInvalidLoggerConfig string = "invalid logger config" - errConfigTemplateFailed string = "could not process config template" - errCouldNotObtainLoggerConfig string = "could not get named logger config" - errNotProvidedAsKV string = "logging config parameter was not provided as = pair" - errLoggerNameEmpty string = "logger name cannot be empty" - errCouldNotParseType string = "could not parse type" - errUnknownLoggerParameter string = "unknown logger parameter" - errInvalidLoggerName string = "invalid logger name" - errDuplicateLoggerName string = "duplicate logger name" - errReadingConfigFile string = "failed to read config" - errLoadingConfig string = "failed to load config" - errUnableToParseByteSize string = "unable to parse byte size" - errInvalidDatastorePath string = "invalid datastore path" - errMissingPortNumber string = "missing port number" - errNoPortWithDomain string = "cannot provide port with domain name" - errInvalidRootDir string = "invalid root directory" -) - -var ( - ErrFailedToWriteFile = errors.New(errFailedToWriteFile) - ErrFailedToRemoveConfigFile = errors.New(errFailedToRemoveConfigFile) - ErrPathCannotBeHomeDir = errors.New(errCannotBeHomeDir) - ErrUnableToExpandHomeDir = errors.New(errUnableToExpandHomeDir) - ErrNoDatabaseURLProvided = errors.New(errNoDatabaseURLProvided) - ErrInvalidDatabaseURL = errors.New(errInvalidDatabaseURL) - ErrLoggingConfigNotObtained = errors.New(errLoggingConfigNotObtained) - ErrFailedToValidateConfig = errors.New(errFailedToValidateConfig) - ErrInvalidRPCTimeout = errors.New(errInvalidRPCTimeout) - ErrInvalidRPCMaxConnectionIdle = errors.New(errInvalidRPCMaxConnectionIdle) - ErrInvalidP2PAddress = errors.New(errInvalidP2PAddress) - ErrInvalidRPCAddress = errors.New(errInvalidRPCAddress) - ErrInvalidBootstrapPeers = errors.New(errInvalidBootstrapPeers) - ErrInvalidLogLevel = errors.New(errInvalidLogLevel) - ErrInvalidDatastoreType = errors.New(errInvalidDatastoreType) - ErrOverrideConfigConvertFailed = errors.New(errOverrideConfigConvertFailed) - ErrInvalidLogFormat = errors.New(errInvalidLogFormat) - ErrConfigToJSONFailed = errors.New(errConfigToJSONFailed) - ErrInvalidNamedLoggerName = errors.New(errInvalidNamedLoggerName) - ErrConfigTemplateFailed = errors.New(errConfigTemplateFailed) - ErrCouldNotObtainLoggerConfig = errors.New(errCouldNotObtainLoggerConfig) - ErrNotProvidedAsKV = errors.New(errNotProvidedAsKV) - ErrLoggerNameEmpty = errors.New(errLoggerNameEmpty) - ErrCouldNotParseType = errors.New(errCouldNotParseType) - ErrUnknownLoggerParameter = errors.New(errUnknownLoggerParameter) - ErrInvalidLoggerName = errors.New(errInvalidLoggerName) - ErrDuplicateLoggerName = errors.New(errDuplicateLoggerName) - ErrReadingConfigFile = errors.New(errReadingConfigFile) - ErrLoadingConfig = errors.New(errLoadingConfig) - ErrUnableToParseByteSize = errors.New(errUnableToParseByteSize) - ErrInvalidLoggerConfig = errors.New(errInvalidLoggerConfig) - ErrorInvalidDatastorePath = errors.New(errInvalidDatastorePath) - ErrMissingPortNumber = errors.New(errMissingPortNumber) - ErrNoPortWithDomain = errors.New(errNoPortWithDomain) - ErrorInvalidRootDir = errors.New(errInvalidRootDir) -) - -func NewErrFailedToWriteFile(inner error, path string) error { - return errors.Wrap(errFailedToWriteFile, inner, errors.NewKV("path", path)) -} - -func NewErrFailedToRemoveConfigFile(inner error) error { - return errors.Wrap(errFailedToRemoveConfigFile, inner) -} - -func NewErrPathCannotBeHomeDir(inner error) error { - return errors.Wrap(errCannotBeHomeDir, inner) -} - -func NewErrUnableToExpandHomeDir(inner error) error { - return errors.Wrap(errUnableToExpandHomeDir, inner) -} - -func NewErrNoDatabaseURLProvided(inner error) error { - return errors.Wrap(errNoDatabaseURLProvided, inner) -} - -func NewErrInvalidDatabaseURL(inner error) error { - return errors.Wrap(errInvalidDatabaseURL, inner) -} - -func NewErrLoggingConfigNotObtained(inner error) error { - return errors.Wrap(errLoggingConfigNotObtained, inner) -} - -func NewErrFailedToValidateConfig(inner error) error { - return errors.Wrap(errFailedToValidateConfig, inner) -} - -func NewErrInvalidRPCTimeout(inner error, timeout string) error { - return errors.Wrap(errInvalidRPCTimeout, inner, errors.NewKV("timeout", timeout)) -} - -func NewErrInvalidRPCMaxConnectionIdle(inner error, timeout string) error { - return errors.Wrap(errInvalidRPCMaxConnectionIdle, inner, errors.NewKV("timeout", timeout)) -} - -func NewErrInvalidP2PAddress(inner error, address string) error { - return errors.Wrap(errInvalidP2PAddress, inner, errors.NewKV("address", address)) -} - -func NewErrInvalidRPCAddress(inner error, address string) error { - return errors.Wrap(errInvalidRPCAddress, inner, errors.NewKV("address", address)) -} - -func NewErrInvalidBootstrapPeers(inner error, peers string) error { - return errors.Wrap(errInvalidBootstrapPeers, inner, errors.NewKV("peers", peers)) -} - -func NewErrInvalidLogLevel(level string) error { - return errors.New(errInvalidLogLevel, errors.NewKV("level", level)) -} - -func NewErrInvalidDatastoreType(storeType string) error { - return errors.New(errInvalidDatastoreType, errors.NewKV("store_type", storeType)) -} - -func NewErrOverrideConfigConvertFailed(inner error, name string) error { - return errors.Wrap(errOverrideConfigConvertFailed, inner, errors.NewKV("name", name)) -} - -func NewErrInvalidLogFormat(format string) error { - return errors.New(errInvalidLogFormat, errors.NewKV("format", format)) -} - -func NewErrConfigToJSONFailed(inner error) error { - return errors.Wrap(errConfigToJSONFailed, inner) -} - -func NewErrInvalidNamedLoggerName(name string) error { - return errors.New(errInvalidNamedLoggerName, errors.NewKV("name", name)) -} - -func NewErrConfigTemplateFailed(inner error) error { - return errors.Wrap(errConfigTemplateFailed, inner) -} - -func NewErrCouldNotObtainLoggerConfig(inner error, name string) error { - return errors.Wrap(errCouldNotObtainLoggerConfig, inner, errors.NewKV("name", name)) -} - -func NewErrNotProvidedAsKV(kv string) error { - return errors.New(errNotProvidedAsKV, errors.NewKV("KV", kv)) -} - -func NewErrCouldNotParseType(inner error, name string) error { - return errors.Wrap(errCouldNotParseType, inner, errors.NewKV("name", name)) -} - -func NewErrUnknownLoggerParameter(name string) error { - return errors.New(errUnknownLoggerParameter, errors.NewKV("param", name)) -} - -func NewErrInvalidLoggerName(name string) error { - return errors.New(errInvalidLoggerName, errors.NewKV("name", name)) -} - -func NewErrDuplicateLoggerName(name string) error { - return errors.New(errDuplicateLoggerName, errors.NewKV("name", name)) -} - -func NewErrReadingConfigFile(inner error) error { - return errors.Wrap(errReadingConfigFile, inner) -} - -func NewErrLoadingConfig(inner error) error { - return errors.Wrap(errLoadingConfig, inner) -} - -func NewErrUnableToParseByteSize(err error) error { - return errors.Wrap(errUnableToParseByteSize, err) -} - -func NewErrLoggerConfig(s string) error { - return errors.New(errInvalidLoggerConfig, errors.NewKV("explanation", s)) -} - -func NewErrInvalidDatastorePath(path string) error { - return errors.New(errInvalidDatastorePath, errors.NewKV("path", path)) -} - -func NewErrInvalidRootDir(path string) error { - return errors.New(errInvalidRootDir, errors.NewKV("path", path)) -} From da779cd66fe4586c4fb84cb6047f5feecc4a6e1f Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 31 Jan 2024 11:12:21 -0800 Subject: [PATCH 2/8] update cli with refactored config --- cli/cli.go | 20 +++-- cli/client.go | 10 +-- cli/collection.go | 9 +-- cli/init.go | 83 ------------------- cli/root.go | 132 +++++++++++++++++++------------ cli/server_dump.go | 15 ++-- cli/start.go | 124 +++-------------------------- cli/tx_commit.go | 7 +- cli/tx_create.go | 3 +- cli/tx_discard.go | 7 +- cli/utils.go | 121 ++++++++++++++++++++++------ cmd/defradb/main.go | 3 +- cmd/genclidocs/main.go | 3 +- cmd/gendocs/main.go | 4 +- cmd/genmanpages/main.go | 3 +- config/config.go | 27 ++++--- http/server.go | 3 +- tests/clients/cli/wrapper_cli.go | 3 +- tests/gen/cli/gendocs.go | 2 +- 19 files changed, 249 insertions(+), 330 deletions(-) delete mode 100644 cli/init.go diff --git a/cli/cli.go b/cli/cli.go index 2ee882afce..279eb6c2da 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -16,14 +16,13 @@ package cli import ( "github.com/spf13/cobra" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/logging" ) var log = logging.MustNewLogger("cli") // NewDefraCommand returns the root command instanciated with its tree of subcommands. -func NewDefraCommand(cfg *config.Config) *cobra.Command { +func NewDefraCommand() *cobra.Command { p2p_collection := MakeP2PCollectionCommand() p2p_collection.AddCommand( MakeP2PCollectionAddCommand(), @@ -83,12 +82,12 @@ func NewDefraCommand(cfg *config.Config) *cobra.Command { tx := MakeTxCommand() tx.AddCommand( - MakeTxCreateCommand(cfg), - MakeTxCommitCommand(cfg), - MakeTxDiscardCommand(cfg), + MakeTxCreateCommand(), + MakeTxCommitCommand(), + MakeTxDiscardCommand(), ) - collection := MakeCollectionCommand(cfg) + collection := MakeCollectionCommand() collection.AddCommand( MakeCollectionGetCommand(), MakeCollectionListDocIDsCommand(), @@ -98,7 +97,7 @@ func NewDefraCommand(cfg *config.Config) *cobra.Command { MakeCollectionDescribeCommand(), ) - client := MakeClientCommand(cfg) + client := MakeClientCommand() client.AddCommand( MakeDumpCommand(), MakeRequestCommand(), @@ -111,13 +110,12 @@ func NewDefraCommand(cfg *config.Config) *cobra.Command { collection, ) - root := MakeRootCommand(cfg) + root := MakeRootCommand() root.AddCommand( client, - MakeStartCommand(cfg), - MakeServerDumpCmd(cfg), + MakeStartCommand(), + MakeServerDumpCmd(), MakeVersionCommand(), - MakeInitCommand(cfg), ) return root diff --git a/cli/client.go b/cli/client.go index 8866294f69..98f7b1ba49 100644 --- a/cli/client.go +++ b/cli/client.go @@ -12,11 +12,9 @@ package cli import ( "github.com/spf13/cobra" - - "github.com/sourcenetwork/defradb/config" ) -func MakeClientCommand(cfg *config.Config) *cobra.Command { +func MakeClientCommand() *cobra.Command { var txID uint64 var cmd = &cobra.Command{ Use: "client", @@ -24,13 +22,13 @@ func MakeClientCommand(cfg *config.Config) *cobra.Command { Long: `Interact with a DefraDB node. Execute queries, add schema types, obtain node info, etc.`, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := loadConfig(cfg); err != nil { + if err := setConfigContext(cmd); err != nil { return err } - if err := setTransactionContext(cmd, cfg, txID); err != nil { + if err := setTransactionContext(cmd, txID); err != nil { return err } - return setStoreContext(cmd, cfg) + return setStoreContext(cmd) }, } cmd.PersistentFlags().Uint64Var(&txID, "tx", 0, "Transaction ID") diff --git a/cli/collection.go b/cli/collection.go index 996af66f9a..5fdad1218a 100644 --- a/cli/collection.go +++ b/cli/collection.go @@ -16,11 +16,10 @@ import ( "github.com/spf13/cobra" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/datastore" ) -func MakeCollectionCommand(cfg *config.Config) *cobra.Command { +func MakeCollectionCommand() *cobra.Command { var txID uint64 var name string var schemaRoot string @@ -31,13 +30,13 @@ func MakeCollectionCommand(cfg *config.Config) *cobra.Command { Long: `Create, read, update, and delete documents within a collection.`, PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { // cobra does not chain pre run calls so we have to run them again here - if err := loadConfig(cfg); err != nil { + if err := setConfigContext(cmd); err != nil { return err } - if err := setTransactionContext(cmd, cfg, txID); err != nil { + if err := setTransactionContext(cmd, txID); err != nil { return err } - if err := setStoreContext(cmd, cfg); err != nil { + if err := setStoreContext(cmd); err != nil { return err } store := mustGetStoreContext(cmd) diff --git a/cli/init.go b/cli/init.go deleted file mode 100644 index f9af1850b7..0000000000 --- a/cli/init.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2022 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package cli - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/sourcenetwork/defradb/config" - "github.com/sourcenetwork/defradb/errors" -) - -/* -The `init` command initializes the configuration file and root directory. - -It covers three possible situations: -- root dir doesn't exist -- root dir exists and doesn't contain a config file -- root dir exists and contains a config file -*/ -func MakeInitCommand(cfg *config.Config) *cobra.Command { - var reinitialize bool - var cmd = &cobra.Command{ - Use: "init", - Short: "Initialize DefraDB's root directory and configuration file", - Long: `Initialize a directory for configuration and data at the given path. -Passed flags will be persisted in the stored configuration.`, - // Load a default configuration, considering env. variables and CLI flags. - PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { - if err := cfg.LoadWithRootdir(false); err != nil { - return errors.Wrap("failed to load configuration", err) - } - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - if config.FolderExists(cfg.Rootdir) { - if cfg.ConfigFileExists() { - if reinitialize { - if err := cfg.DeleteConfigFile(); err != nil { - return err - } - if err := cfg.WriteConfigFile(); err != nil { - return err - } - } else { - log.FeedbackError( - cmd.Context(), - fmt.Sprintf( - "Configuration file already exists at %v. Consider using --reinitialize", - cfg.ConfigFilePath(), - ), - ) - } - } else { - if err := cfg.WriteConfigFile(); err != nil { - return errors.Wrap("failed to create configuration file", err) - } - } - } else { - if err := cfg.CreateRootDirAndConfigFile(); err != nil { - return err - } - } - return nil - }, - } - - cmd.Flags().BoolVar( - &reinitialize, "reinitialize", false, - "Reinitialize the configuration file", - ) - - return cmd -} diff --git a/cli/root.go b/cli/root.go index 729b638f02..2278c1962e 100644 --- a/cli/root.go +++ b/cli/root.go @@ -11,14 +11,10 @@ package cli import ( - "context" - "github.com/spf13/cobra" - - "github.com/sourcenetwork/defradb/config" ) -func MakeRootCommand(cfg *config.Config) *cobra.Command { +func MakeRootCommand() *cobra.Command { var cmd = &cobra.Command{ SilenceUsage: true, Use: "defradb", @@ -28,81 +24,117 @@ func MakeRootCommand(cfg *config.Config) *cobra.Command { Start a DefraDB node, interact with a local or remote node, and much more. `, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return loadConfig(cfg) + return setConfigContext(cmd) }, } cmd.PersistentFlags().String( - "rootdir", "", + "rootdir", + "$HOME/.defradb", "Directory for data and configuration to use (default: $HOME/.defradb)", ) - err := cfg.BindFlag(config.RootdirKey, cmd.PersistentFlags().Lookup("rootdir")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind rootdir", err) - } cmd.PersistentFlags().String( - "loglevel", cfg.Log.Level, + "loglevel", + "info", "Log level to use. Options are debug, info, error, fatal", ) - err = cfg.BindFlag("log.level", cmd.PersistentFlags().Lookup("loglevel")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind log.loglevel", err) - } - - cmd.PersistentFlags().StringArray( - "logger", []string{}, - "Override logger parameters. Usage: --logger ,level=,output=,...", - ) - err = cfg.BindFlag("log.logger", cmd.PersistentFlags().Lookup("logger")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind log.logger", err) - } cmd.PersistentFlags().String( - "logoutput", cfg.Log.Output, + "logoutput", + "stderr", "Log output path", ) - err = cfg.BindFlag("log.output", cmd.PersistentFlags().Lookup("logoutput")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind log.output", err) - } cmd.PersistentFlags().String( - "logformat", cfg.Log.Format, + "logformat", + "csv", "Log format to use. Options are csv, json", ) - err = cfg.BindFlag("log.format", cmd.PersistentFlags().Lookup("logformat")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind log.format", err) - } cmd.PersistentFlags().Bool( - "logtrace", cfg.Log.Stacktrace, + "logtrace", + false, "Include stacktrace in error and fatal logs", ) - err = cfg.BindFlag("log.stacktrace", cmd.PersistentFlags().Lookup("logtrace")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind log.stacktrace", err) - } cmd.PersistentFlags().Bool( - "lognocolor", cfg.Log.NoColor, + "lognocolor", + false, "Disable colored log output", ) - err = cfg.BindFlag("log.nocolor", cmd.PersistentFlags().Lookup("lognocolor")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind log.nocolor", err) - } cmd.PersistentFlags().String( - "url", cfg.API.Address, + "url", + "localhost:9181", "URL of HTTP endpoint to listen on or connect to", ) - err = cfg.BindFlag("api.address", cmd.PersistentFlags().Lookup("url")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind api.address", err) - } + + cmd.PersistentFlags().StringArray( + "peers", + []string{}, + "List of peers to connect to", + ) + + cmd.PersistentFlags().Int( + "max-txn-retries", + 5, + "Specify the maximum number of retries per transaction", + ) + + cmd.PersistentFlags().String( + "store", + "badger", + "Specify the datastore to use (supported: badger, memory)", + ) + + cmd.PersistentFlags().Int( + "valuelogfilesize", + 1<<30, + "Specify the datastore value log file size (in bytes). In memory size will be 2*valuelogfilesize", + ) + + cmd.PersistentFlags().StringSlice( + "p2paddr", + []string{"/ip4/127.0.0.1/tcp/9171"}, + "Listen addresses for the p2p network (formatted as a libp2p MultiAddr)", + ) + + cmd.PersistentFlags().Bool( + "no-p2p", + false, + "Disable the peer-to-peer network synchronization system", + ) + + cmd.PersistentFlags().Bool( + "tls", + false, + "Enable serving the API over https", + ) + + cmd.PersistentFlags().StringArray( + "allowed-origins", + []string{}, + "List of origins to allow for CORS requests", + ) + + cmd.PersistentFlags().String( + "pubkeypath", + "cert.pub", + "Path to the public key for tls", + ) + + cmd.PersistentFlags().String( + "privkeypath", + "cert.key", + "Path to the private key for tls", + ) + + cmd.PersistentFlags().String( + "email", + "example@example.com", + "Email address used by the CA for notifications", + ) return cmd } diff --git a/cli/server_dump.go b/cli/server_dump.go index 0ba638d268..ccad7405d8 100644 --- a/cli/server_dump.go +++ b/cli/server_dump.go @@ -17,7 +17,6 @@ import ( "github.com/spf13/cobra" - "github.com/sourcenetwork/defradb/config" ds "github.com/sourcenetwork/defradb/datastore" badgerds "github.com/sourcenetwork/defradb/datastore/badger/v4" "github.com/sourcenetwork/defradb/db" @@ -25,13 +24,14 @@ import ( "github.com/sourcenetwork/defradb/logging" ) -func MakeServerDumpCmd(cfg *config.Config) *cobra.Command { +func MakeServerDumpCmd() *cobra.Command { var datastore string cmd := &cobra.Command{ Use: "server-dump", Short: "Dumps the state of the entire database", RunE: func(cmd *cobra.Command, _ []string) error { + cfg := mustGetConfigContext(cmd) log.FeedbackInfo(cmd.Context(), "Starting DefraDB process...") // setup signal handlers @@ -41,16 +41,17 @@ func MakeServerDumpCmd(cfg *config.Config) *cobra.Command { var rootstore ds.RootStore var err error if datastore == badgerDatastoreName { - info, err := os.Stat(cfg.Datastore.Badger.Path) + badgerPath := cfg.GetString("datastore.badger.path") + info, err := os.Stat(badgerPath) exists := (err == nil && info.IsDir()) if !exists { return errors.New(fmt.Sprintf( "badger store does not exist at %s. Try with an existing directory", - cfg.Datastore.Badger.Path, + badgerPath, )) } - log.FeedbackInfo(cmd.Context(), "Opening badger store", logging.NewKV("Path", cfg.Datastore.Badger.Path)) - rootstore, err = badgerds.NewDatastore(cfg.Datastore.Badger.Path, cfg.Datastore.Badger.Options) + log.FeedbackInfo(cmd.Context(), "Opening badger store", logging.NewKV("Path", badgerPath)) + rootstore, err = badgerds.NewDatastore(badgerPath, &badgerds.DefaultOptions) if err != nil { return errors.Wrap("could not open badger datastore", err) } @@ -68,7 +69,7 @@ func MakeServerDumpCmd(cfg *config.Config) *cobra.Command { }, } cmd.Flags().StringVar( - &datastore, "store", cfg.Datastore.Store, + &datastore, "store", "badger", "Datastore to use. Options are badger, memory", ) return cmd diff --git a/cli/start.go b/cli/start.go index 55b168205f..b7895c71cb 100644 --- a/cli/start.go +++ b/cli/start.go @@ -22,9 +22,9 @@ import ( badger "github.com/sourcenetwork/badger/v4" "github.com/spf13/cobra" + "github.com/spf13/viper" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" ds "github.com/sourcenetwork/defradb/datastore" badgerds "github.com/sourcenetwork/defradb/datastore/badger/v4" "github.com/sourcenetwork/defradb/db" @@ -37,22 +37,18 @@ import ( const badgerDatastoreName = "badger" -func MakeStartCommand(cfg *config.Config) *cobra.Command { +func MakeStartCommand() *cobra.Command { var cmd = &cobra.Command{ Use: "start", Short: "Start a DefraDB node", Long: "Start a DefraDB node.", // Load the root config if it exists, otherwise create it. PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { - if err := loadConfig(cfg); err != nil { - return err - } - if !cfg.ConfigFileExists() { - return createConfig(cfg) - } - return nil + return setConfigContext(cmd) }, RunE: func(cmd *cobra.Command, args []string) error { + cfg := mustGetConfigContext(cmd) + di, err := start(cmd.Context(), cfg) if err != nil { return err @@ -62,104 +58,6 @@ func MakeStartCommand(cfg *config.Config) *cobra.Command { }, } - cmd.Flags().String( - "peers", cfg.Net.Peers, - "List of peers to connect to", - ) - err := cfg.BindFlag("net.peers", cmd.Flags().Lookup("peers")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind net.peers", err) - } - - cmd.Flags().Int( - "max-txn-retries", cfg.Datastore.MaxTxnRetries, - "Specify the maximum number of retries per transaction", - ) - err = cfg.BindFlag("datastore.maxtxnretries", cmd.Flags().Lookup("max-txn-retries")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind datastore.maxtxnretries", err) - } - - cmd.Flags().String( - "store", cfg.Datastore.Store, - "Specify the datastore to use (supported: badger, memory)", - ) - err = cfg.BindFlag("datastore.store", cmd.Flags().Lookup("store")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind datastore.store", err) - } - - cmd.Flags().Var( - &cfg.Datastore.Badger.ValueLogFileSize, "valuelogfilesize", - "Specify the datastore value log file size (in bytes). In memory size will be 2*valuelogfilesize", - ) - err = cfg.BindFlag("datastore.badger.valuelogfilesize", cmd.Flags().Lookup("valuelogfilesize")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind datastore.badger.valuelogfilesize", err) - } - - cmd.Flags().StringSlice( - "p2paddr", cfg.Net.P2PAddresses, - "Listen addresses for the p2p network (formatted as a libp2p MultiAddr)", - ) - err = cfg.BindFlag("net.p2paddresses", cmd.Flags().Lookup("p2paddr")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind net.p2paddress", err) - } - - cmd.Flags().Bool( - "no-p2p", cfg.Net.P2PDisabled, - "Disable the peer-to-peer network synchronization system", - ) - err = cfg.BindFlag("net.p2pdisabled", cmd.Flags().Lookup("no-p2p")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind net.p2pdisabled", err) - } - - cmd.Flags().Bool( - "tls", cfg.API.TLS, - "Enable serving the API over https", - ) - err = cfg.BindFlag("api.tls", cmd.Flags().Lookup("tls")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind api.tls", err) - } - - cmd.Flags().StringArray( - "allowed-origins", cfg.API.AllowedOrigins, - "List of origins to allow for CORS requests", - ) - err = cfg.BindFlag("api.allowed-origins", cmd.Flags().Lookup("allowed-origins")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind api.allowed-origins", err) - } - - cmd.Flags().String( - "pubkeypath", cfg.API.PubKeyPath, - "Path to the public key for tls", - ) - err = cfg.BindFlag("api.pubkeypath", cmd.Flags().Lookup("pubkeypath")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind api.pubkeypath", err) - } - - cmd.Flags().String( - "privkeypath", cfg.API.PrivKeyPath, - "Path to the private key for tls", - ) - err = cfg.BindFlag("api.privkeypath", cmd.Flags().Lookup("privkeypath")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind api.privkeypath", err) - } - - cmd.Flags().String( - "email", cfg.API.Email, - "Email address used by the CA for notifications", - ) - err = cfg.BindFlag("api.email", cmd.Flags().Lookup("email")) - if err != nil { - log.FeedbackFatalE(context.Background(), "Could not bind api.email", err) - } return cmd } @@ -184,7 +82,7 @@ func (di *defraInstance) close(ctx context.Context) { } } -func start(ctx context.Context, cfg *config.Config) (*defraInstance, error) { +func start(ctx context.Context, cfg *viper.Viper) (*defraInstance, error) { log.FeedbackInfo(ctx, "Starting DefraDB service...") var rootstore ds.RootStore @@ -218,13 +116,13 @@ func start(ctx context.Context, cfg *config.Config) (*defraInstance, error) { // init the p2p node var node *net.Node - if !cfg.Net.P2PDisabled { + if !cfg.GetBool("net.p2pdisabled") { nodeOpts := []net.NodeOpt{ - net.WithListenAddresses(cfg.Net.P2PAddresses...), - net.WithEnablePubSub(cfg.Net.PubSubEnabled), - net.WithEnableRelay(cfg.Net.RelayEnabled), + net.WithListenAddresses(cfg.GetStringSlice("net.p2paddresses")...), + net.WithEnablePubSub(cfg.GetBool("net.pubsubenabled")), + net.WithEnableRelay(cfg.GetBool("net.relayenabled")), } - if cfg.Datastore.Store == badgerDatastoreName { + if cfg.GetString("datastore.store") == badgerDatastoreName { // It would be ideal to not have the key path tied to the datastore. // Running with memory store mode will always generate a random key. // Adding support for an ephemeral mode and moving the key to the diff --git a/cli/tx_commit.go b/cli/tx_commit.go index 260a274a08..d6fe6ea970 100644 --- a/cli/tx_commit.go +++ b/cli/tx_commit.go @@ -15,22 +15,23 @@ import ( "github.com/spf13/cobra" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/http" ) -func MakeTxCommitCommand(cfg *config.Config) *cobra.Command { +func MakeTxCommitCommand() *cobra.Command { var cmd = &cobra.Command{ Use: "commit [id]", Short: "Commit a DefraDB transaction.", Long: `Commit a DefraDB transaction.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) (err error) { + cfg := mustGetConfigContext(cmd) + id, err := strconv.ParseUint(args[0], 10, 64) if err != nil { return err } - tx, err := http.NewTransaction(cfg.API.Address, id) + tx, err := http.NewTransaction(cfg.GetString("api.address"), id) if err != nil { return err } diff --git a/cli/tx_create.go b/cli/tx_create.go index 987a784077..da239b6943 100644 --- a/cli/tx_create.go +++ b/cli/tx_create.go @@ -14,11 +14,10 @@ import ( "github.com/spf13/cobra" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/datastore" ) -func MakeTxCreateCommand(cfg *config.Config) *cobra.Command { +func MakeTxCreateCommand() *cobra.Command { var concurrent bool var readOnly bool var cmd = &cobra.Command{ diff --git a/cli/tx_discard.go b/cli/tx_discard.go index 351f919f53..259d183461 100644 --- a/cli/tx_discard.go +++ b/cli/tx_discard.go @@ -15,22 +15,23 @@ import ( "github.com/spf13/cobra" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/http" ) -func MakeTxDiscardCommand(cfg *config.Config) *cobra.Command { +func MakeTxDiscardCommand() *cobra.Command { var cmd = &cobra.Command{ Use: "discard [id]", Short: "Discard a DefraDB transaction.", Long: `Discard a DefraDB transaction.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) (err error) { + cfg := mustGetConfigContext(cmd) + id, err := strconv.ParseUint(args[0], 10, 64) if err != nil { return err } - tx, err := http.NewTransaction(cfg.API.Address, id) + tx, err := http.NewTransaction(cfg.GetString("api.address"), id) if err != nil { return err } diff --git a/cli/utils.go b/cli/utils.go index 8c1a40dc1f..79775c4650 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -17,6 +17,7 @@ import ( "github.com/libp2p/go-libp2p/core/crypto" "github.com/spf13/cobra" + "github.com/spf13/viper" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/config" @@ -27,6 +28,8 @@ import ( type contextKey string var ( + // cfgContextKey is the context key for the config. + cfgContextKey = contextKey("cfg") // txContextKey is the context key for the datastore.Txn // // This will only be set if a transaction id is specified. @@ -59,6 +62,13 @@ func mustGetP2PContext(cmd *cobra.Command) client.P2P { return cmd.Context().Value(dbContextKey).(client.P2P) } +// mustGetConfigContext returns the config for the current command context. +// +// If a config is not set in the current context this function panics. +func mustGetConfigContext(cmd *cobra.Command) *viper.Viper { + return cmd.Context().Value(cfgContextKey).(*viper.Viper) +} + // tryGetCollectionContext returns the collection for the current command context // and a boolean indicating if the collection was set. func tryGetCollectionContext(cmd *cobra.Command) (client.Collection, bool) { @@ -66,12 +76,96 @@ func tryGetCollectionContext(cmd *cobra.Command) (client.Collection, bool) { return col, ok } +// setConfigContext sets teh config for the current command context. +func setConfigContext(cmd *cobra.Command) error { + rootdir, err := cmd.PersistentFlags().GetString("rootdir") + if err != nil { + return err + } + cfg, err := config.LoadConfig(rootdir) + if err != nil { + return err + } + err = cfg.BindPFlag("log.level", cmd.PersistentFlags().Lookup("loglevel")) + if err != nil { + return err + } + err = cfg.BindPFlag("log.output", cmd.PersistentFlags().Lookup("logoutput")) + if err != nil { + return err + } + err = cfg.BindPFlag("log.format", cmd.PersistentFlags().Lookup("logformat")) + if err != nil { + return err + } + err = cfg.BindPFlag("log.stacktrace", cmd.PersistentFlags().Lookup("logtrace")) + if err != nil { + return err + } + err = cfg.BindPFlag("log.nocolor", cmd.PersistentFlags().Lookup("lognocolor")) + if err != nil { + return err + } + err = cfg.BindPFlag("api.address", cmd.PersistentFlags().Lookup("url")) + if err != nil { + return err + } + err = cfg.BindPFlag("net.peers", cmd.PersistentFlags().Lookup("peers")) + if err != nil { + return err + } + err = cfg.BindPFlag("datastore.maxtxnretries", cmd.PersistentFlags().Lookup("max-txn-retries")) + if err != nil { + return err + } + err = cfg.BindPFlag("datastore.store", cmd.PersistentFlags().Lookup("store")) + if err != nil { + return err + } + err = cfg.BindPFlag("datastore.badger.valuelogfilesize", cmd.PersistentFlags().Lookup("valuelogfilesize")) + if err != nil { + return err + } + err = cfg.BindPFlag("net.p2paddresses", cmd.PersistentFlags().Lookup("p2paddr")) + if err != nil { + return err + } + err = cfg.BindPFlag("net.p2pdisabled", cmd.PersistentFlags().Lookup("no-p2p")) + if err != nil { + return err + } + err = cfg.BindPFlag("api.tls", cmd.PersistentFlags().Lookup("tls")) + if err != nil { + return err + } + err = cfg.BindPFlag("api.allowed-origins", cmd.PersistentFlags().Lookup("allowed-origins")) + if err != nil { + return err + } + err = cfg.BindPFlag("api.pubkeypath", cmd.PersistentFlags().Lookup("pubkeypath")) + if err != nil { + return err + } + err = cfg.BindPFlag("api.privkeypath", cmd.PersistentFlags().Lookup("privkeypath")) + if err != nil { + return err + } + err = cfg.BindPFlag("api.email", cmd.PersistentFlags().Lookup("email")) + if err != nil { + return err + } + ctx := context.WithValue(cmd.Context(), cfgContextKey, cfg) + cmd.SetContext(ctx) + return nil +} + // setTransactionContext sets the transaction for the current command context. -func setTransactionContext(cmd *cobra.Command, cfg *config.Config, txId uint64) error { +func setTransactionContext(cmd *cobra.Command, txId uint64) error { if txId == 0 { return nil } - tx, err := http.NewTransaction(cfg.API.Address, txId) + cfg := mustGetConfigContext(cmd) + tx, err := http.NewTransaction(cfg.GetString("api.address"), txId) if err != nil { return err } @@ -81,8 +175,9 @@ func setTransactionContext(cmd *cobra.Command, cfg *config.Config, txId uint64) } // setStoreContext sets the store for the current command context. -func setStoreContext(cmd *cobra.Command, cfg *config.Config) error { - db, err := http.NewClient(cfg.API.Address) +func setStoreContext(cmd *cobra.Command) error { + cfg := mustGetConfigContext(cmd) + db, err := http.NewClient(cfg.GetString("api.address")) if err != nil { return err } @@ -96,24 +191,6 @@ func setStoreContext(cmd *cobra.Command, cfg *config.Config) error { return nil } -// loadConfig loads the rootDir containing the configuration file, -// otherwise warn about it and load a default configuration. -func loadConfig(cfg *config.Config) error { - if err := cfg.LoadRootDirFromFlagOrDefault(); err != nil { - return err - } - return cfg.LoadWithRootdir(cfg.ConfigFileExists()) -} - -// createConfig creates the config directories and writes -// the current config to a file. -func createConfig(cfg *config.Config) error { - if config.FolderExists(cfg.Rootdir) { - return cfg.WriteConfigFile() - } - return cfg.CreateRootDirAndConfigFile() -} - // loadOrGeneratePrivateKey loads the private key from the given path // or generates a new key and writes it to a file at the given path. func loadOrGeneratePrivateKey(path string) (crypto.PrivKey, error) { diff --git a/cmd/defradb/main.go b/cmd/defradb/main.go index 2406885a76..e827177132 100644 --- a/cmd/defradb/main.go +++ b/cmd/defradb/main.go @@ -15,12 +15,11 @@ import ( "os" "github.com/sourcenetwork/defradb/cli" - "github.com/sourcenetwork/defradb/config" ) // Execute adds all child commands to the root command and sets flags appropriately. func main() { - defraCmd := cli.NewDefraCommand(config.DefaultConfig()) + defraCmd := cli.NewDefraCommand() if err := defraCmd.Execute(); err != nil { // this error is okay to discard because cobra // logs any errors encountered during execution diff --git a/cmd/genclidocs/main.go b/cmd/genclidocs/main.go index f556c26d20..a9a6f198c4 100644 --- a/cmd/genclidocs/main.go +++ b/cmd/genclidocs/main.go @@ -21,7 +21,6 @@ import ( "github.com/spf13/cobra/doc" "github.com/sourcenetwork/defradb/cli" - "github.com/sourcenetwork/defradb/config" ) var path string @@ -33,7 +32,7 @@ func init() { func main() { flag.Parse() - defraCmd := cli.NewDefraCommand(config.DefaultConfig()) + defraCmd := cli.NewDefraCommand() defraCmd.DisableAutoGenTag = true if err := os.MkdirAll(path, os.ModePerm); err != nil { diff --git a/cmd/gendocs/main.go b/cmd/gendocs/main.go index 44901b0faf..4c1f7dcf9a 100644 --- a/cmd/gendocs/main.go +++ b/cmd/gendocs/main.go @@ -16,13 +16,11 @@ package main import ( "os" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/tests/gen/cli" ) func main() { - conf := config.DefaultConfig() - gendocsCmd := cli.MakeGenDocCommand(conf) + gendocsCmd := cli.MakeGenDocCommand() if err := gendocsCmd.Execute(); err != nil { // this error is okay to discard because cobra // logs any errors encountered during execution diff --git a/cmd/genmanpages/main.go b/cmd/genmanpages/main.go index 1a9b43df7c..ecdc78762d 100644 --- a/cmd/genmanpages/main.go +++ b/cmd/genmanpages/main.go @@ -22,7 +22,6 @@ import ( "github.com/spf13/cobra/doc" "github.com/sourcenetwork/defradb/cli" - "github.com/sourcenetwork/defradb/config" ) const defaultPerm os.FileMode = 0o777 @@ -41,7 +40,7 @@ func init() { func main() { flag.Parse() - defraCmd := cli.NewDefraCommand(config.DefaultConfig()) + defraCmd := cli.NewDefraCommand() if err := os.MkdirAll(dir, defaultPerm); err != nil { log.Fatal("Failed to create directory", err) diff --git a/config/config.go b/config/config.go index 332b5f4549..f8298f1270 100644 --- a/config/config.go +++ b/config/config.go @@ -11,18 +11,23 @@ import ( "github.com/spf13/viper" ) -// configName is the config file name -const configName = "config.yaml" - -//go:embed config.yaml -var defaultConfig []byte +const ( + // configName is the config file name + configName = "config.yaml" + // DefaultRoot is the name of the default root directory + DefaultRoot = "$HOME/.defradb" +) -// relativePathKeys are config keys that will be made relative to the rootdir -var relativePathKeys = []string{ - "datastore.badger.path", - "api.pubkeypath", - "api.privkeypath", -} +var ( + //go:embed config.yaml + defaultConfig []byte + // relativePathKeys are config keys that will be made relative to the rootdir + relativePathKeys = []string{ + "datastore.badger.path", + "api.pubkeypath", + "api.privkeypath", + } +) // WriteDefaultConfig writes the default config file to the given rootdir. func WriteDefaultConfig(rootdir string) error { diff --git a/http/server.go b/http/server.go index 768542c68d..5aea2a63b3 100644 --- a/http/server.go +++ b/http/server.go @@ -23,7 +23,6 @@ import ( "golang.org/x/crypto/acme/autocert" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/logging" ) @@ -237,7 +236,7 @@ func (s *Server) listenWithTLS(ctx context.Context) error { if s.options.Domain.HasValue() && s.options.Domain.Value() != "" { s.Addr = s.options.TLS.Value().Port - if s.options.TLS.Value().Email == "" || s.options.TLS.Value().Email == config.DefaultAPIEmail { + if s.options.TLS.Value().Email == "" || s.options.TLS.Value().Email == "example@example.com" { return ErrNoEmail } diff --git a/tests/clients/cli/wrapper_cli.go b/tests/clients/cli/wrapper_cli.go index 1f73b20e25..e8dc24270c 100644 --- a/tests/clients/cli/wrapper_cli.go +++ b/tests/clients/cli/wrapper_cli.go @@ -17,7 +17,6 @@ import ( "strings" "github.com/sourcenetwork/defradb/cli" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/datastore" ) @@ -67,7 +66,7 @@ func (w *cliWrapper) executeStream(ctx context.Context, args []string) (io.ReadC } args = append(args, "--url", w.address) - cmd := cli.NewDefraCommand(config.DefaultConfig()) + cmd := cli.NewDefraCommand() cmd.SetOut(stdOutWrite) cmd.SetErr(stdErrWrite) cmd.SetArgs(args) diff --git a/tests/gen/cli/gendocs.go b/tests/gen/cli/gendocs.go index 226d73bc97..c8c7259d02 100644 --- a/tests/gen/cli/gendocs.go +++ b/tests/gen/cli/gendocs.go @@ -27,7 +27,7 @@ import ( const defaultBatchSize = 1000 -func MakeGenDocCommand(cfg *config.Config) *cobra.Command { +func MakeGenDocCommand() *cobra.Command { var demandJSON string var cmd = &cobra.Command{ From 9739882c71faa0bfeaad1415884b73ad9c6f6e13 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 12 Feb 2024 16:30:08 -0800 Subject: [PATCH 3/8] move config into cli package --- cli/client.go | 2 +- cli/collection.go | 2 +- cli/config.go | 179 ++++++++++++++++++++++++ cli/config_test.go | 49 +++++++ cli/root.go | 30 ++-- cli/server_dump.go | 9 +- cli/start.go | 16 +-- cli/utils.go | 75 +--------- config/config.go | 105 -------------- config/config.yaml | 79 ----------- config/config_test.go | 61 -------- tests/gen/cli/gendocs.go | 26 +--- tests/gen/cli/gendocs_test.go | 6 +- tests/gen/cli/util_test.go | 12 -- tests/integration/net/order/tcp_test.go | 20 +-- tests/integration/net/order/utils.go | 48 ++----- tests/integration/p2p.go | 12 +- tests/integration/state.go | 11 +- tests/integration/test_case.go | 4 +- tests/integration/utils2.go | 32 ++--- 20 files changed, 305 insertions(+), 473 deletions(-) create mode 100644 cli/config.go create mode 100644 cli/config_test.go delete mode 100644 config/config.go delete mode 100644 config/config.yaml delete mode 100644 config/config_test.go diff --git a/cli/client.go b/cli/client.go index 98f7b1ba49..26dfe3524f 100644 --- a/cli/client.go +++ b/cli/client.go @@ -22,7 +22,7 @@ func MakeClientCommand() *cobra.Command { Long: `Interact with a DefraDB node. Execute queries, add schema types, obtain node info, etc.`, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := setConfigContext(cmd); err != nil { + if err := setConfigContext(cmd, false); err != nil { return err } if err := setTransactionContext(cmd, txID); err != nil { diff --git a/cli/collection.go b/cli/collection.go index 5fdad1218a..2e19c940b5 100644 --- a/cli/collection.go +++ b/cli/collection.go @@ -30,7 +30,7 @@ func MakeCollectionCommand() *cobra.Command { Long: `Create, read, update, and delete documents within a collection.`, PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { // cobra does not chain pre run calls so we have to run them again here - if err := setConfigContext(cmd); err != nil { + if err := setConfigContext(cmd, false); err != nil { return err } if err := setTransactionContext(cmd, txID); err != nil { diff --git a/cli/config.go b/cli/config.go new file mode 100644 index 0000000000..6ca3218c11 --- /dev/null +++ b/cli/config.go @@ -0,0 +1,179 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package cli + +import ( + _ "embed" + "errors" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/sourcenetwork/defradb/logging" +) + +// configPaths are config keys that will be made relative to the rootdir +var configPaths = []string{ + "datastore.badger.path", + "api.pubkeypath", + "api.privkeypath", +} + +// configFlags is a mapping of config keys to cli flags to bind to. +var configFlags = map[string]string{ + "log.level": "loglevel", + "log.output": "logoutput", + "log.format": "logformat", + "log.stacktrace": "logtrace", + "log.nocolor": "lognocolor", + "api.address": "url", + "datastore.maxtxnretries": "max-txn-retries", + "datastore.badger.inMemory": "in-memory", + "datastore.badger.valuelogfilesize": "valuelogfilesize", + "net.peers": "peers", + "net.p2paddresses": "p2paddr", + "net.p2pdisabled": "no-p2p", + "api.allowed-origins": "allowed-origins", + "api.pubkeypath": "pubkeypath", + "api.privkeypath": "privkeypath", +} + +// defaultConfig returns a new config with default values. +func defaultConfig() *viper.Viper { + cfg := viper.New() + + cfg.AutomaticEnv() + cfg.SetEnvPrefix("DEFRA") + cfg.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_")) + + cfg.SetConfigName("config") + cfg.SetConfigType("yaml") + + cfg.SetDefault("datastore.maxTxnRetries", 5) + + cfg.SetDefault("datastore.badger.path", "data") + cfg.SetDefault("datastore.badger.inMemory", false) + cfg.SetDefault("datastore.badger.valueLogFileSize", 1<<30) + + cfg.SetDefault("api.address", "127.0.0.1:9181") + cfg.SetDefault("api.pubKeyPath", "") + cfg.SetDefault("api.privKeyPath", "") + cfg.SetDefault("api.allowed-origins", []string{}) + + cfg.SetDefault("net.p2pDisabled", false) + cfg.SetDefault("net.p2pAddresses", []any{"/ip4/127.0.0.1/tcp/9171"}) + cfg.SetDefault("net.pubSubEnabled", true) + cfg.SetDefault("net.relay", false) + cfg.SetDefault("net.peers", []string{}) + + cfg.SetDefault("log.level", "info") + cfg.SetDefault("log.stackTrace", true) + cfg.SetDefault("log.format", "csv") + cfg.SetDefault("log.output", "stderr") + cfg.SetDefault("log.noColor", false) + cfg.SetDefault("log.caller", false) + + return cfg +} + +// loadConfig returns a new config with values from the config in the given rootdir. +func loadConfig(rootdir string, cmd *cobra.Command, create bool) (*viper.Viper, error) { + // rootdir defaults to $HOME/.defradb + home, err := os.UserHomeDir() + if err != nil { + return nil, err + } + if rootdir == "" { + rootdir = filepath.Join(home, ".defradb") + } + + cfg := defaultConfig() + cfg.AddConfigPath(rootdir) + + // make sure rootdir exists + if err := os.MkdirAll(rootdir, 0755); err != nil { + return nil, err + } + // attempt to read the existing config + err = cfg.ReadInConfig() + if create && errors.As(err, &viper.ConfigFileNotFoundError{}) { + err = cfg.SafeWriteConfig() + } + if err != nil && !errors.As(err, &viper.ConfigFileNotFoundError{}) { + return nil, err + } + + // bind cli flags to config keys + for key, flag := range configFlags { + err := cfg.BindPFlag(key, cmd.Root().PersistentFlags().Lookup(flag)) + if err != nil { + return nil, err + } + } + + // make paths relative to the rootdir + for _, key := range configPaths { + path := cfg.GetString(key) + if path != "" && !filepath.IsAbs(path) { + cfg.Set(key, filepath.Join(rootdir, path)) + } + } + + logCfg := loggingConfig(cfg.Sub("log")) + logCfg.OverridesByLoggerName = make(map[string]logging.Config) + + // apply named logging overrides + for key := range cfg.GetStringMap("log.overrides") { + logCfg.OverridesByLoggerName[key] = loggingConfig(cfg.Sub("log.overrides." + key)) + } + logging.SetConfig(logCfg) + + return cfg, nil +} + +// loggingConfig returns a new logging config from the given config. +func loggingConfig(cfg *viper.Viper) logging.Config { + var level int8 + switch value := cfg.GetString("level"); value { + case "debug": + level = logging.Debug + case "info": + level = logging.Info + case "error": + level = logging.Error + case "fatal": + level = logging.Fatal + default: + level = logging.Info + } + + var format logging.EncoderFormat + switch value := cfg.GetString("format"); value { + case "json": //nolint:goconst + format = logging.JSON + case "csv": + format = logging.CSV + default: + format = logging.CSV + } + + return logging.Config{ + Level: logging.NewLogLevelOption(level), + EnableStackTrace: logging.NewEnableStackTraceOption(cfg.GetBool("stacktrace")), + DisableColor: logging.NewDisableColorOption(cfg.GetBool("nocolor")), + EncoderFormat: logging.NewEncoderFormatOption(format), + OutputPaths: []string{cfg.GetString("output")}, + EnableCaller: logging.NewEnableCallerOption(cfg.GetBool("caller")), + } +} diff --git a/cli/config_test.go b/cli/config_test.go new file mode 100644 index 0000000000..d87350093c --- /dev/null +++ b/cli/config_test.go @@ -0,0 +1,49 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package cli + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadConfigAndCreate(t *testing.T) { + rootdir := t.TempDir() + cfg, err := loadConfig(rootdir, NewDefraCommand(), true) + require.NoError(t, err) + + assert.FileExists(t, filepath.Join(rootdir, "config.yaml")) + assert.Equal(t, 5, cfg.GetInt("datastore.maxtxnretries")) + + assert.Equal(t, filepath.Join(rootdir, "data"), cfg.GetString("datastore.badger.path")) + assert.Equal(t, 1<<30, cfg.GetInt("datastore.badger.valuelogfilesize")) + + assert.Equal(t, "127.0.0.1:9181", cfg.GetString("api.address")) + assert.Equal(t, []string{}, cfg.GetStringSlice("api.allowed-origins")) + assert.Equal(t, "", cfg.GetString("api.pubkeypath")) + assert.Equal(t, "", cfg.GetString("api.privkeypath")) + + assert.Equal(t, false, cfg.GetBool("net.p2pdisabled")) + assert.Equal(t, []string{"/ip4/127.0.0.1/tcp/9171"}, cfg.GetStringSlice("net.p2paddresses")) + assert.Equal(t, true, cfg.GetBool("net.pubsubenabled")) + assert.Equal(t, false, cfg.GetBool("net.relay")) + assert.Equal(t, []string{}, cfg.GetStringSlice("net.peers")) + + assert.Equal(t, "info", cfg.GetString("log.level")) + assert.Equal(t, true, cfg.GetBool("log.stacktrace")) + assert.Equal(t, "csv", cfg.GetString("log.format")) + assert.Equal(t, "stderr", cfg.GetString("log.output")) + assert.Equal(t, false, cfg.GetBool("log.nocolor")) + assert.Equal(t, false, cfg.GetBool("log.caller")) +} diff --git a/cli/root.go b/cli/root.go index 2278c1962e..1126f1ff58 100644 --- a/cli/root.go +++ b/cli/root.go @@ -24,13 +24,13 @@ func MakeRootCommand() *cobra.Command { Start a DefraDB node, interact with a local or remote node, and much more. `, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return setConfigContext(cmd) + return setConfigContext(cmd, false) }, } cmd.PersistentFlags().String( "rootdir", - "$HOME/.defradb", + "", "Directory for data and configuration to use (default: $HOME/.defradb)", ) @@ -66,7 +66,7 @@ Start a DefraDB node, interact with a local or remote node, and much more. cmd.PersistentFlags().String( "url", - "localhost:9181", + "127.0.0.1:9181", "URL of HTTP endpoint to listen on or connect to", ) @@ -82,10 +82,10 @@ Start a DefraDB node, interact with a local or remote node, and much more. "Specify the maximum number of retries per transaction", ) - cmd.PersistentFlags().String( - "store", - "badger", - "Specify the datastore to use (supported: badger, memory)", + cmd.PersistentFlags().Bool( + "in-memory", + false, + "Enables the badger in memory only datastore.", ) cmd.PersistentFlags().Int( @@ -106,12 +106,6 @@ Start a DefraDB node, interact with a local or remote node, and much more. "Disable the peer-to-peer network synchronization system", ) - cmd.PersistentFlags().Bool( - "tls", - false, - "Enable serving the API over https", - ) - cmd.PersistentFlags().StringArray( "allowed-origins", []string{}, @@ -120,21 +114,15 @@ Start a DefraDB node, interact with a local or remote node, and much more. cmd.PersistentFlags().String( "pubkeypath", - "cert.pub", + "", "Path to the public key for tls", ) cmd.PersistentFlags().String( "privkeypath", - "cert.key", + "", "Path to the private key for tls", ) - cmd.PersistentFlags().String( - "email", - "example@example.com", - "Email address used by the CA for notifications", - ) - return cmd } diff --git a/cli/server_dump.go b/cli/server_dump.go index 86ac665df1..ae03f45d51 100644 --- a/cli/server_dump.go +++ b/cli/server_dump.go @@ -12,27 +12,26 @@ package cli import ( "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/sourcenetwork/defradb/db" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/node" ) -func MakeServerDumpCmd(cfg *viper.Viper) *cobra.Command { +func MakeServerDumpCmd() *cobra.Command { cmd := &cobra.Command{ Use: "server-dump", Short: "Dumps the state of the entire database", RunE: func(cmd *cobra.Command, _ []string) error { + cfg := mustGetConfigContext(cmd) log.FeedbackInfo(cmd.Context(), "Dumping DB state...") - datastore := cfg.GetString("datastore.store") - if datastore == "memory" { + if cfg.GetBool("datastore.badger.inMemory") { return errors.New("server-side dump is only supported for the Badger datastore") } storeOpts := []node.StoreOpt{ node.WithPath(cfg.GetString("datastore.badger.path")), - node.WithInMemory(datastore == "memory"), + node.WithInMemory(cfg.GetBool("datastore.badger.inMemory")), } rootstore, err := node.NewStore(storeOpts...) if err != nil { diff --git a/cli/start.go b/cli/start.go index 7b9598c857..f6b3b70cb6 100644 --- a/cli/start.go +++ b/cli/start.go @@ -15,12 +15,10 @@ import ( "os" "os/signal" "path/filepath" - "strings" "syscall" "github.com/libp2p/go-libp2p/core/peer" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/sourcenetwork/defradb/db" "github.com/sourcenetwork/defradb/errors" @@ -30,16 +28,18 @@ import ( "github.com/sourcenetwork/defradb/node" ) -func MakeStartCommand(cfg *viper.Viper) *cobra.Command { +func MakeStartCommand() *cobra.Command { var cmd = &cobra.Command{ Use: "start", Short: "Start a DefraDB node", Long: "Start a DefraDB node.", // Load the root config if it exists, otherwise create it. PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { - return setConfigContext(cmd) + return setConfigContext(cmd, true) }, RunE: func(cmd *cobra.Command, args []string) error { + cfg := mustGetConfigContext(cmd) + dbOpts := []db.Option{ db.WithUpdateEvents(), db.WithMaxRetries(cfg.GetInt("datastore.MaxTxnRetries")), @@ -60,19 +60,19 @@ func MakeStartCommand(cfg *viper.Viper) *cobra.Command { storeOpts := []node.StoreOpt{ node.WithPath(cfg.GetString("datastore.badger.path")), - node.WithInMemory(cfg.GetString("datastore.store") == "memory"), + node.WithInMemory(cfg.GetBool("datastore.badger.inMemory")), } var peers []peer.AddrInfo - if val := cfg.GetString("net.peers"); val != "" { - addrs, err := netutils.ParsePeers(strings.Split(val, ",")) + if val := cfg.GetStringSlice("net.peers"); len(val) > 0 { + addrs, err := netutils.ParsePeers(val) if err != nil { return errors.Wrap(fmt.Sprintf("failed to parse bootstrap peers %s", val), err) } peers = addrs } - if cfg.GetString("datastore.store") == "badger" { + if cfg.GetBool("datastore.badger.inMemory") { // It would be ideal to not have the key path tied to the datastore. // Running with memory store mode will always generate a random key. // Adding support for an ephemeral mode and moving the key to the diff --git a/cli/utils.go b/cli/utils.go index fc8c724b1e..6ec7096479 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -21,7 +21,6 @@ import ( "github.com/spf13/viper" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/http" ) @@ -78,80 +77,12 @@ func tryGetCollectionContext(cmd *cobra.Command) (client.Collection, bool) { } // setConfigContext sets teh config for the current command context. -func setConfigContext(cmd *cobra.Command) error { - rootdir, err := cmd.PersistentFlags().GetString("rootdir") +func setConfigContext(cmd *cobra.Command, create bool) error { + rootdir, err := cmd.Root().PersistentFlags().GetString("rootdir") if err != nil { return err } - cfg, err := config.LoadConfig(rootdir) - if err != nil { - return err - } - err = cfg.BindPFlag("log.level", cmd.PersistentFlags().Lookup("loglevel")) - if err != nil { - return err - } - err = cfg.BindPFlag("log.output", cmd.PersistentFlags().Lookup("logoutput")) - if err != nil { - return err - } - err = cfg.BindPFlag("log.format", cmd.PersistentFlags().Lookup("logformat")) - if err != nil { - return err - } - err = cfg.BindPFlag("log.stacktrace", cmd.PersistentFlags().Lookup("logtrace")) - if err != nil { - return err - } - err = cfg.BindPFlag("log.nocolor", cmd.PersistentFlags().Lookup("lognocolor")) - if err != nil { - return err - } - err = cfg.BindPFlag("api.address", cmd.PersistentFlags().Lookup("url")) - if err != nil { - return err - } - err = cfg.BindPFlag("net.peers", cmd.PersistentFlags().Lookup("peers")) - if err != nil { - return err - } - err = cfg.BindPFlag("datastore.maxtxnretries", cmd.PersistentFlags().Lookup("max-txn-retries")) - if err != nil { - return err - } - err = cfg.BindPFlag("datastore.store", cmd.PersistentFlags().Lookup("store")) - if err != nil { - return err - } - err = cfg.BindPFlag("datastore.badger.valuelogfilesize", cmd.PersistentFlags().Lookup("valuelogfilesize")) - if err != nil { - return err - } - err = cfg.BindPFlag("net.p2paddresses", cmd.PersistentFlags().Lookup("p2paddr")) - if err != nil { - return err - } - err = cfg.BindPFlag("net.p2pdisabled", cmd.PersistentFlags().Lookup("no-p2p")) - if err != nil { - return err - } - err = cfg.BindPFlag("api.tls", cmd.PersistentFlags().Lookup("tls")) - if err != nil { - return err - } - err = cfg.BindPFlag("api.allowed-origins", cmd.PersistentFlags().Lookup("allowed-origins")) - if err != nil { - return err - } - err = cfg.BindPFlag("api.pubkeypath", cmd.PersistentFlags().Lookup("pubkeypath")) - if err != nil { - return err - } - err = cfg.BindPFlag("api.privkeypath", cmd.PersistentFlags().Lookup("privkeypath")) - if err != nil { - return err - } - err = cfg.BindPFlag("api.email", cmd.PersistentFlags().Lookup("email")) + cfg, err := loadConfig(rootdir, cmd, create) if err != nil { return err } diff --git a/config/config.go b/config/config.go deleted file mode 100644 index f8298f1270..0000000000 --- a/config/config.go +++ /dev/null @@ -1,105 +0,0 @@ -package config - -import ( - "bytes" - _ "embed" - "os" - "path/filepath" - "strings" - - "github.com/sourcenetwork/defradb/logging" - "github.com/spf13/viper" -) - -const ( - // configName is the config file name - configName = "config.yaml" - // DefaultRoot is the name of the default root directory - DefaultRoot = "$HOME/.defradb" -) - -var ( - //go:embed config.yaml - defaultConfig []byte - // relativePathKeys are config keys that will be made relative to the rootdir - relativePathKeys = []string{ - "datastore.badger.path", - "api.pubkeypath", - "api.privkeypath", - } -) - -// WriteDefaultConfig writes the default config file to the given rootdir. -func WriteDefaultConfig(rootdir string) error { - if err := os.MkdirAll(rootdir, 0755); err != nil { - return err - } - return os.WriteFile(filepath.Join(rootdir, configName), defaultConfig, 0666) -} - -// LoadConfig returns a new config with values from the config in the given rootdir. -func LoadConfig(rootdir string) (*viper.Viper, error) { - cfg := viper.New() - - cfg.SetEnvPrefix("DEFRA") - cfg.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_")) - - cfg.SetConfigName(configName) - cfg.SetConfigType("yaml") - - cfg.AddConfigPath(rootdir) - cfg.AutomaticEnv() - - // load defaults first then merge persisted config - err := cfg.MergeConfig(bytes.NewBuffer(defaultConfig)) - if err != nil { - return nil, err - } - err = cfg.MergeInConfig() - if _, ok := err.(viper.ConfigFileNotFoundError); !ok && err != nil { - return nil, err - } - - // make paths relative to the rootdir - for _, key := range relativePathKeys { - path := cfg.GetString(key) - if !filepath.IsAbs(path) { - cfg.Set(key, filepath.Join(rootdir, path)) - } - } - - var level int8 - switch value := cfg.GetString("log.level"); value { - case "debug": - level = logging.Debug - case "info": - level = logging.Info - case "error": - level = logging.Error - case "fatal": - level = logging.Fatal - default: - level = logging.Info - } - - var format logging.EncoderFormat - switch value := cfg.GetString("log.format"); value { - case "json": - format = logging.JSON - case "csv": - format = logging.CSV - default: - format = logging.CSV - } - - logging.SetConfig(logging.Config{ - Level: logging.NewLogLevelOption(level), - EnableStackTrace: logging.NewEnableStackTraceOption(cfg.GetBool("log.stacktrace")), - DisableColor: logging.NewDisableColorOption(cfg.GetBool("log.nocolor")), - EncoderFormat: logging.NewEncoderFormatOption(format), - OutputPaths: []string{cfg.GetString("log.output")}, - EnableCaller: logging.NewEnableCallerOption(cfg.GetBool("log.caller")), - }) - - return cfg, nil -} diff --git a/config/config.yaml b/config/config.yaml deleted file mode 100644 index 04cfa93684..0000000000 --- a/config/config.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# DefraDB configuration (YAML) - -# The default DefraDB directory is "$HOME/.defradb". It can be changed via the --rootdir CLI flag. -# Relative paths are interpreted as being rooted in the DefraDB directory. - -datastore: - # Store can be badger | memory - # badger: fast pure Go key-value store optimized for SSDs (https://github.com/dgraph-io/badger) - # memory: in-memory version of badger - store: badger - - # Maximum number of times to retry failed transactions. - maxTxnRetries: 5 - - # badger specific settings - badger: - # The path to the database data file(s). - path: data - - # Maximum file size of the value log files. - # The in-memory file size will be 2*valuelogfilesize. - valueLogFileSize: 1073741824 - -api: - # Address of the HTTP API to listen on or connect to - address: localhost:9181 - - # The list of origins a cross-domain request can be executed from. - # allowed-origins: - # - localhost - - # Whether the API server should listen over HTTPS - tls: false - - # The path to the public key file. Ignored if domains is set. - pubKeyPath: cert.pub - - # The path to the private key file. Ignored if domains is set. - privKeyPath: cert.key - - # Email address to let the CA (Let's Encrypt) send notifications via email when there are issues (optional). - email: example@example.com - -net: - # Whether the P2P is disabled - p2pDisabled: false - - # Listening address of the P2P network - p2pAddresses: - - /ip4/127.0.0.1/tcp/9171 - - # Whether the node has pubsub enabled or not - pubSubEnabled: true - - # Enable libp2p's Circuit relay transport protocol https://docs.libp2p.io/concepts/circuit-relay/ - relay: false - - # List of peers to boostrap with, specified as multiaddresses (https://docs.libp2p.io/concepts/addressing/) - # peers: - # - /ip4/127.0.0.1/tcp/9171/QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N - -log: - # Log level. Options are debug, info, error, fatal - level: info - - # Include stack trace in error and fatal logs - stackTrace: true - - # Supported log formats are json, csv - format: csv - - # Where the log output is written to - output: stderr - - # Disable colored log output - noColor: false - - # Caller location in log output - caller: false diff --git a/config/config_test.go b/config/config_test.go deleted file mode 100644 index d4d1df7ecd..0000000000 --- a/config/config_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package config - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestLoadConfigNotExist(t *testing.T) { - rootdir := t.TempDir() - cfg, err := LoadConfig(rootdir) - require.NoError(t, err) - - assert.Equal(t, "badger", cfg.GetString("datastore.store")) - assert.Equal(t, 5, cfg.GetInt("datastore.maxtxnretries")) - - assert.Equal(t, filepath.Join(rootdir, "data"), cfg.GetString("datastore.badger.path")) - assert.Equal(t, 1073741824, cfg.GetInt("datastore.badger.valuelogfilesize")) - - assert.Equal(t, "localhost:9181", cfg.GetString("api.address")) - assert.Equal(t, false, cfg.GetBool("api.tls")) - assert.Equal(t, []string(nil), cfg.GetStringSlice("api.allowed-origins")) - assert.Equal(t, filepath.Join(rootdir, "cert.pub"), cfg.GetString("api.pubkeypath")) - assert.Equal(t, filepath.Join(rootdir, "cert.key"), cfg.GetString("api.privkeypath")) - assert.Equal(t, "example@example.com", cfg.GetString("api.email")) - - assert.Equal(t, false, cfg.GetBool("net.p2pdisabled")) - assert.Equal(t, []string{"/ip4/127.0.0.1/tcp/9171"}, cfg.GetStringSlice("net.p2paddresses")) - assert.Equal(t, true, cfg.GetBool("net.pubsubenabled")) - assert.Equal(t, false, cfg.GetBool("net.relay")) - assert.Equal(t, []string(nil), cfg.GetStringSlice("net.peers")) - - assert.Equal(t, "info", cfg.GetString("log.level")) - assert.Equal(t, true, cfg.GetBool("log.stacktrace")) - assert.Equal(t, "csv", cfg.GetString("log.format")) - assert.Equal(t, "stderr", cfg.GetString("log.output")) - assert.Equal(t, false, cfg.GetBool("log.nocolor")) - assert.Equal(t, false, cfg.GetBool("log.caller")) -} - -func TestLoadConfigExists(t *testing.T) { - rootdir := t.TempDir() - err := WriteDefaultConfig(rootdir) - require.NoError(t, err) - - _, err = LoadConfig(rootdir) - require.NoError(t, err) -} - -func TestWriteDefaultConfig(t *testing.T) { - rootdir := t.TempDir() - err := WriteDefaultConfig(rootdir) - require.NoError(t, err) - - data, err := os.ReadFile(filepath.Join(rootdir, configName)) - require.NoError(t, err) - assert.Equal(t, defaultConfig, data) -} diff --git a/tests/gen/cli/gendocs.go b/tests/gen/cli/gendocs.go index bb7d9e35d3..aa0267fcce 100644 --- a/tests/gen/cli/gendocs.go +++ b/tests/gen/cli/gendocs.go @@ -20,7 +20,6 @@ import ( "github.com/spf13/cobra" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/http" "github.com/sourcenetwork/defradb/tests/gen" ) @@ -29,7 +28,7 @@ const defaultBatchSize = 1000 func MakeGenDocCommand() *cobra.Command { var demandJSON string - + var url string var cmd = &cobra.Command{ Use: "gendocs --demand ", Short: "Automatically generates documents for existing collections.", @@ -39,11 +38,7 @@ Example: The following command generates 100 User documents and 500 Device docum gendocs --demand '{"User": 100, "Device": 500 }'`, ValidArgs: []string{"demand"}, RunE: func(cmd *cobra.Command, args []string) error { - // cobra does not chain pre run calls so we have to run them again here - if err := loadConfig(cfg); err != nil { - return err - } - store, err := http.NewClient(cfg.API.Address) + store, err := http.NewClient(url) if err != nil { return err } @@ -101,15 +96,7 @@ Example: The following command generates 100 User documents and 500 Device docum }, } - cmd.PersistentFlags().String( - "url", cfg.API.Address, - "URL of HTTP endpoint to listen on or connect to", - ) - err := cfg.BindFlag("api.address", cmd.PersistentFlags().Lookup("url")) - if err != nil { - panic(err) - } - + cmd.Flags().StringVar(&url, "url", "localhost:9181", "URL of HTTP endpoint to listen on or connect to") cmd.Flags().StringVarP(&demandJSON, "demand", "d", "", "Documents' demand in JSON format") return cmd @@ -160,10 +147,3 @@ func colsToDefs(cols []client.Collection) []client.CollectionDefinition { } return colDefs } - -func loadConfig(cfg *config.Config) error { - if err := cfg.LoadRootDirFromFlagOrDefault(); err != nil { - return err - } - return cfg.LoadWithRootdir(cfg.ConfigFileExists()) -} diff --git a/tests/gen/cli/gendocs_test.go b/tests/gen/cli/gendocs_test.go index 1bd30297a6..35202ec9e0 100644 --- a/tests/gen/cli/gendocs_test.go +++ b/tests/gen/cli/gendocs_test.go @@ -37,7 +37,7 @@ func TestGendocsCmd_IfNoErrors_ReturnGenerationOutput(t *testing.T) { owner: User }`) - genDocsCmd := MakeGenDocCommand(getTestConfig(t)) + genDocsCmd := MakeGenDocCommand() outputBuf := bytes.NewBufferString("") genDocsCmd.SetOut(outputBuf) @@ -71,7 +71,7 @@ func TestGendocsCmd_IfInvalidDemandValue_ReturnError(t *testing.T) { name: String }`) - genDocsCmd := MakeGenDocCommand(getTestConfig(t)) + genDocsCmd := MakeGenDocCommand() genDocsCmd.SetArgs([]string{ "--demand", `{"User": invalid}`, "--url", strings.TrimPrefix(defra.server.URL, "http://"), @@ -90,7 +90,7 @@ func TestGendocsCmd_IfInvalidConfig_ReturnError(t *testing.T) { name: String }`) - genDocsCmd := MakeGenDocCommand(getTestConfig(t)) + genDocsCmd := MakeGenDocCommand() genDocsCmd.SetArgs([]string{ "--demand", `{"Unknown": 3}`, "--url", strings.TrimPrefix(defra.server.URL, "http://"), diff --git a/tests/gen/cli/util_test.go b/tests/gen/cli/util_test.go index 81d713955c..10bd98ca99 100644 --- a/tests/gen/cli/util_test.go +++ b/tests/gen/cli/util_test.go @@ -19,7 +19,6 @@ import ( "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" badgerds "github.com/sourcenetwork/defradb/datastore/badger/v4" "github.com/sourcenetwork/defradb/db" "github.com/sourcenetwork/defradb/errors" @@ -67,17 +66,6 @@ func start(ctx context.Context) (*defraInstance, error) { }, nil } -func getTestConfig(t *testing.T) *config.Config { - cfg := config.DefaultConfig() - cfg.Datastore.Store = "memory" - cfg.Net.P2PDisabled = true - cfg.Rootdir = t.TempDir() - cfg.Net.P2PAddresses = []string{"/ip4/127.0.0.1/tcp/0"} - cfg.API.Address = "127.0.0.1:0" - cfg.Persist() - return cfg -} - func startTestNode(t *testing.T) (*defraInstance, func()) { ctx := context.Background() di, err := start(ctx) diff --git a/tests/integration/net/order/tcp_test.go b/tests/integration/net/order/tcp_test.go index 8a419360d5..f80701c64c 100644 --- a/tests/integration/net/order/tcp_test.go +++ b/tests/integration/net/order/tcp_test.go @@ -16,16 +16,16 @@ import ( "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" + "github.com/sourcenetwork/defradb/net" testutils "github.com/sourcenetwork/defradb/tests/integration" ) // TestP2PWithSingleDocumentUpdatePerNode tests document syncing between two nodes with a single update per node func TestP2PWithSingleDocumentUpdatePerNode(t *testing.T) { test := P2PTestCase{ - NodeConfig: []*config.Config{ - randomNetworkingConfig(), - randomNetworkingConfig(), + NodeConfig: [][]net.NodeOpt{ + testutils.RandomNetworkingConfig()(), + testutils.RandomNetworkingConfig()(), }, NodePeers: map[int][]int{ 1: { @@ -74,9 +74,9 @@ func TestP2PWithSingleDocumentUpdatePerNode(t *testing.T) { // TestP2PWithMultipleDocumentUpdatesPerNode tests document syncing between two nodes with multiple updates per node. func TestP2PWithMultipleDocumentUpdatesPerNode(t *testing.T) { test := P2PTestCase{ - NodeConfig: []*config.Config{ - randomNetworkingConfig(), - randomNetworkingConfig(), + NodeConfig: [][]net.NodeOpt{ + testutils.RandomNetworkingConfig()(), + testutils.RandomNetworkingConfig()(), }, NodePeers: map[int][]int{ 1: { @@ -145,9 +145,9 @@ func TestP2FullPReplicator(t *testing.T) { require.NoError(t, err) test := P2PTestCase{ - NodeConfig: []*config.Config{ - randomNetworkingConfig(), - randomNetworkingConfig(), + NodeConfig: [][]net.NodeOpt{ + testutils.RandomNetworkingConfig()(), + testutils.RandomNetworkingConfig()(), }, NodeReplicators: map[int][]int{ 0: { diff --git a/tests/integration/net/order/utils.go b/tests/integration/net/order/utils.go index 178272b155..3ba5fc7f26 100644 --- a/tests/integration/net/order/utils.go +++ b/tests/integration/net/order/utils.go @@ -13,14 +13,12 @@ package order import ( "context" "fmt" - "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" coreDB "github.com/sourcenetwork/defradb/db" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/logging" @@ -50,7 +48,7 @@ const ( type P2PTestCase struct { Query string // Configuration parameters for each peer - NodeConfig []*config.Config + NodeConfig [][]net.NodeOpt // List of peers for each net. // Only peers with lower index than the node can be used in the list of peers. @@ -69,7 +67,12 @@ type P2PTestCase struct { ReplicatorResult map[int]map[string]map[string]any } -func setupDefraNode(t *testing.T, cfg *config.Config, seeds []string) (*net.Node, []client.DocID, error) { +func setupDefraNode( + t *testing.T, + opts []net.NodeOpt, + peers []string, + seeds []string, +) (*net.Node, []client.DocID, error) { ctx := context.Background() log.Info(ctx, "Building new memory store") @@ -92,40 +95,28 @@ func setupDefraNode(t *testing.T, cfg *config.Config, seeds []string) (*net.Node // init the P2P node var n *net.Node - log.Info(ctx, "Starting P2P node", logging.NewKV("P2P addresses", cfg.Net.P2PAddresses)) - n, err = net.NewNode( - ctx, - db, - net.WithListenAddresses(cfg.Net.P2PAddresses...), - net.WithEnablePubSub(cfg.Net.PubSubEnabled), - net.WithEnableRelay(cfg.Net.RelayEnabled), - ) + n, err = net.NewNode(ctx, db, opts...) if err != nil { return nil, nil, errors.Wrap("failed to start P2P node", err) } // parse peers and bootstrap - if len(cfg.Net.Peers) != 0 { - log.Info(ctx, "Parsing bootstrap peers", logging.NewKV("Peers", cfg.Net.Peers)) - addrs, err := netutils.ParsePeers(strings.Split(cfg.Net.Peers, ",")) + if len(peers) != 0 { + log.Info(ctx, "Parsing bootstrap peers", logging.NewKV("Peers", peers)) + addrs, err := netutils.ParsePeers(peers) if err != nil { - return nil, nil, errors.Wrap(fmt.Sprintf("failed to parse bootstrap peers %v", cfg.Net.Peers), err) + return nil, nil, errors.Wrap(fmt.Sprintf("failed to parse bootstrap peers %v", peers), err) } log.Info(ctx, "Bootstrapping with peers", logging.NewKV("Addresses", addrs)) n.Bootstrap(addrs) } + log.Info(ctx, "Starting P2P node", logging.NewKV("P2P addresses", n.PeerInfo().Addrs)) if err := n.Start(); err != nil { n.Close() return nil, nil, errors.Wrap("unable to start P2P listeners", err) } - var addresses []string - for _, addr := range n.ListenAddrs() { - addresses = append(addresses, addr.String()) - } - cfg.Net.P2PAddresses = addresses - return n, docIDs, nil } @@ -201,9 +192,8 @@ func executeTestCase(t *testing.T, test P2PTestCase) { for i, cfg := range test.NodeConfig { log.Info(ctx, fmt.Sprintf("Setting up node %d", i)) - cfg.Datastore.Badger.Path = t.TempDir() + var peerAddresses []string if peers, ok := test.NodePeers[i]; ok { - peerAddresses := []string{} for _, p := range peers { if p >= len(nodes) { log.Info(ctx, "cannot set a peer that hasn't been started. Skipping to next peer") @@ -215,9 +205,8 @@ func executeTestCase(t *testing.T, test P2PTestCase) { fmt.Sprintf("%s/p2p/%s", peerInfo.Addrs[0], peerInfo.ID), ) } - cfg.Net.Peers = strings.Join(peerAddresses, ",") } - n, d, err := setupDefraNode(t, cfg, test.SeedDocuments) + n, d, err := setupDefraNode(t, cfg, peerAddresses, test.SeedDocuments) require.NoError(t, err) if i == 0 { @@ -348,10 +337,3 @@ func executeTestCase(t *testing.T, test P2PTestCase) { n.DB.Close() } } - -func randomNetworkingConfig() *config.Config { - cfg := config.DefaultConfig() - cfg.Net.P2PAddresses = []string{"/ip4/127.0.0.1/tcp/0"} - cfg.Net.RelayEnabled = false - return cfg -} diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index 463d1bd653..4d48cb033b 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -14,8 +14,8 @@ import ( "time" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/logging" + "github.com/sourcenetwork/defradb/net" "github.com/sourcenetwork/defradb/tests/clients" "github.com/libp2p/go-libp2p/core/peer" @@ -523,10 +523,10 @@ func waitForSync( } func RandomNetworkingConfig() ConfigureNode { - return func() config.Config { - cfg := config.DefaultConfig() - cfg.Net.P2PAddresses = []string{"/ip4/127.0.0.1/tcp/0"} - cfg.Net.RelayEnabled = false - return *cfg + return func() []net.NodeOpt { + return []net.NodeOpt{ + net.WithListenAddresses("/ip4/127.0.0.1/tcp/0"), + net.WithEnableRelay(false), + } } } diff --git a/tests/integration/state.go b/tests/integration/state.go index ca795a2492..25a248413b 100644 --- a/tests/integration/state.go +++ b/tests/integration/state.go @@ -14,12 +14,11 @@ import ( "context" "testing" - "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" "github.com/sourcenetwork/defradb/datastore" + "github.com/sourcenetwork/defradb/net" "github.com/sourcenetwork/defradb/tests/clients" ) @@ -53,14 +52,11 @@ type state struct { // These synchronisation channels allow async actions to track their completion. syncChans []chan struct{} - // The private keys for any nodes. - nodePrivateKeys []crypto.PrivKey - // The addresses of any nodes configured. nodeAddresses []peer.AddrInfo // The configurations for any nodes - nodeConfigs []config.Config + nodeConfigs [][]net.NodeOpt // The nodes active in this test. nodes []clients.Client @@ -108,9 +104,8 @@ func newState( allActionsDone: make(chan struct{}), subscriptionResultsChans: []chan func(){}, syncChans: []chan struct{}{}, - nodePrivateKeys: []crypto.PrivKey{}, nodeAddresses: []peer.AddrInfo{}, - nodeConfigs: []config.Config{}, + nodeConfigs: [][]net.NodeOpt{}, nodes: []clients.Client{}, dbPaths: []string{}, collections: [][]client.Collection{}, diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index 2c0e095fe8..942f0fc932 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -16,7 +16,7 @@ import ( "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/config" + "github.com/sourcenetwork/defradb/net" "github.com/sourcenetwork/defradb/tests/gen" "github.com/sourcenetwork/defradb/tests/predefined" ) @@ -54,7 +54,7 @@ type SetupComplete struct{} // Nodes may be explicitly referenced by index by other actions using `NodeID` properties. // If the action has a `NodeID` property and it is not specified, the action will be // effected on all nodes. -type ConfigureNode func() config.Config +type ConfigureNode func() []net.NodeOpt // Restart is an action that will close and then start all nodes. type Restart struct{} diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index afbd57bea9..fe1900b0a2 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -663,25 +663,18 @@ func restartNodes( continue } - key := s.nodePrivateKeys[i] - cfg := s.nodeConfigs[i] // We need to make sure the node is configured with its old address, otherwise // a new one may be selected and reconnnection to it will fail. var addresses []string for _, addr := range s.nodeAddresses[i].Addrs { addresses = append(addresses, addr.String()) } - cfg.Net.P2PAddresses = addresses + + nodeOpts := s.nodeConfigs[i] + nodeOpts = append(nodeOpts, net.WithListenAddresses(addresses...)) var n *net.Node - n, err = net.NewNode( - s.ctx, - db, - net.WithListenAddresses(cfg.Net.P2PAddresses...), - net.WithEnablePubSub(cfg.Net.PubSubEnabled), - net.WithEnableRelay(cfg.Net.RelayEnabled), - net.WithPrivateKey(key), - ) + n, err = net.NewNode(s.ctx, db, nodeOpts...) require.NoError(s.t, err) if err := n.Start(); err != nil { @@ -770,23 +763,17 @@ func configureNode( return } - cfg := action() db, path, err := setupDatabase(s) //disable change dector, or allow it? require.NoError(s.t, err) privateKey, _, err := crypto.GenerateKeyPair(crypto.Ed25519, 0) require.NoError(s.t, err) + nodeOpts := action() + nodeOpts = append(nodeOpts, net.WithPrivateKey(privateKey)) + var n *net.Node - log.Info(s.ctx, "Starting P2P node", logging.NewKV("P2P address", cfg.Net.P2PAddresses)) - n, err = net.NewNode( - s.ctx, - db, - net.WithListenAddresses(cfg.Net.P2PAddresses...), - net.WithEnablePubSub(cfg.Net.PubSubEnabled), - net.WithEnableRelay(cfg.Net.RelayEnabled), - net.WithPrivateKey(privateKey), - ) + n, err = net.NewNode(s.ctx, db, nodeOpts...) require.NoError(s.t, err) log.Info(s.ctx, "Starting P2P node", logging.NewKV("P2P address", n.PeerInfo())) @@ -796,8 +783,7 @@ func configureNode( } s.nodeAddresses = append(s.nodeAddresses, n.PeerInfo()) - s.nodeConfigs = append(s.nodeConfigs, cfg) - s.nodePrivateKeys = append(s.nodePrivateKeys, privateKey) + s.nodeConfigs = append(s.nodeConfigs, nodeOpts) c, err := setupClient(s, n) require.NoError(s.t, err) From 6bd738a0f1b01a42e78b37b6eba34d569ef4ac15 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 12 Feb 2024 16:47:24 -0800 Subject: [PATCH 4/8] fix private key loading logic --- cli/start.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/start.go b/cli/start.go index f6b3b70cb6..81fff46193 100644 --- a/cli/start.go +++ b/cli/start.go @@ -72,12 +72,12 @@ func MakeStartCommand() *cobra.Command { peers = addrs } - if cfg.GetBool("datastore.badger.inMemory") { + if !cfg.GetBool("datastore.badger.inMemory") { // It would be ideal to not have the key path tied to the datastore. // Running with memory store mode will always generate a random key. // Adding support for an ephemeral mode and moving the key to the // config would solve both of these issues. - rootdir, err := cmd.PersistentFlags().GetString("rootdir") + rootdir, err := cmd.Root().PersistentFlags().GetString("rootdir") if err != nil { return err } From 4ee8d50801fae80de55c32af39055d2f0f29ccdf Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 12 Feb 2024 18:25:01 -0800 Subject: [PATCH 5/8] move rootdir to cli context --- cli/client.go | 5 ++++- cli/collection.go | 5 ++++- cli/config.go | 39 +++++++++++++++++++-------------------- cli/config_test.go | 16 ++++++++++++++-- cli/root.go | 7 +++++-- cli/start.go | 13 ++++++++----- cli/utils.go | 37 +++++++++++++++++++++++++++++++------ 7 files changed, 85 insertions(+), 37 deletions(-) diff --git a/cli/client.go b/cli/client.go index 26dfe3524f..f711c526d5 100644 --- a/cli/client.go +++ b/cli/client.go @@ -22,7 +22,10 @@ func MakeClientCommand() *cobra.Command { Long: `Interact with a DefraDB node. Execute queries, add schema types, obtain node info, etc.`, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := setConfigContext(cmd, false); err != nil { + if err := setRootDirContext(cmd); err != nil { + return err + } + if err := setConfigContext(cmd); err != nil { return err } if err := setTransactionContext(cmd, txID); err != nil { diff --git a/cli/collection.go b/cli/collection.go index 2e19c940b5..d1c1e6effe 100644 --- a/cli/collection.go +++ b/cli/collection.go @@ -30,7 +30,10 @@ func MakeCollectionCommand() *cobra.Command { Long: `Create, read, update, and delete documents within a collection.`, PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { // cobra does not chain pre run calls so we have to run them again here - if err := setConfigContext(cmd, false); err != nil { + if err := setRootDirContext(cmd); err != nil { + return err + } + if err := setConfigContext(cmd); err != nil { return err } if err := setTransactionContext(cmd, txID); err != nil { diff --git a/cli/config.go b/cli/config.go index 6ca3218c11..827a835c99 100644 --- a/cli/config.go +++ b/cli/config.go @@ -12,12 +12,11 @@ package cli import ( _ "embed" - "errors" "os" "path/filepath" "strings" - "github.com/spf13/cobra" + "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/sourcenetwork/defradb/logging" @@ -87,36 +86,36 @@ func defaultConfig() *viper.Viper { return cfg } -// loadConfig returns a new config with values from the config in the given rootdir. -func loadConfig(rootdir string, cmd *cobra.Command, create bool) (*viper.Viper, error) { - // rootdir defaults to $HOME/.defradb - home, err := os.UserHomeDir() - if err != nil { - return nil, err - } - if rootdir == "" { - rootdir = filepath.Join(home, ".defradb") - } - +// createConfig writes the default config file if one does not exist. +func createConfig(rootdir string) error { cfg := defaultConfig() cfg.AddConfigPath(rootdir) // make sure rootdir exists if err := os.MkdirAll(rootdir, 0755); err != nil { - return nil, err + return err } - // attempt to read the existing config - err = cfg.ReadInConfig() - if create && errors.As(err, &viper.ConfigFileNotFoundError{}) { - err = cfg.SafeWriteConfig() + err := cfg.SafeWriteConfig() + if _, ok := err.(viper.ConfigFileAlreadyExistsError); ok { //nolint:errorlint + return nil } - if err != nil && !errors.As(err, &viper.ConfigFileNotFoundError{}) { + return err +} + +// loadConfig returns a new config with values from the config in the given rootdir. +func loadConfig(rootdir string, flags *pflag.FlagSet) (*viper.Viper, error) { + cfg := defaultConfig() + cfg.AddConfigPath(rootdir) + + // attempt to read the existing config + err := cfg.ReadInConfig() + if _, ok := err.(viper.ConfigFileNotFoundError); err != nil && !ok { //nolint:errorlint return nil, err } // bind cli flags to config keys for key, flag := range configFlags { - err := cfg.BindPFlag(key, cmd.Root().PersistentFlags().Lookup(flag)) + err := cfg.BindPFlag(key, flags.Lookup(flag)) if err != nil { return nil, err } diff --git a/cli/config_test.go b/cli/config_test.go index d87350093c..fd63da0b86 100644 --- a/cli/config_test.go +++ b/cli/config_test.go @@ -18,16 +18,28 @@ import ( "github.com/stretchr/testify/require" ) -func TestLoadConfigAndCreate(t *testing.T) { +func TestCreateConfig(t *testing.T) { rootdir := t.TempDir() - cfg, err := loadConfig(rootdir, NewDefraCommand(), true) + err := createConfig(rootdir) + require.NoError(t, err) + + // ensure no errors when config already exists + err = createConfig(rootdir) require.NoError(t, err) assert.FileExists(t, filepath.Join(rootdir, "config.yaml")) +} + +func TestLoadConfigNotExist(t *testing.T) { + rootdir := t.TempDir() + cfg, err := loadConfig(rootdir, NewDefraCommand().PersistentFlags()) + require.NoError(t, err) + assert.Equal(t, 5, cfg.GetInt("datastore.maxtxnretries")) assert.Equal(t, filepath.Join(rootdir, "data"), cfg.GetString("datastore.badger.path")) assert.Equal(t, 1<<30, cfg.GetInt("datastore.badger.valuelogfilesize")) + assert.Equal(t, false, cfg.GetBool("datastore.badger.inmemory")) assert.Equal(t, "127.0.0.1:9181", cfg.GetString("api.address")) assert.Equal(t, []string{}, cfg.GetStringSlice("api.allowed-origins")) diff --git a/cli/root.go b/cli/root.go index 1126f1ff58..aa8fc116a3 100644 --- a/cli/root.go +++ b/cli/root.go @@ -24,14 +24,17 @@ func MakeRootCommand() *cobra.Command { Start a DefraDB node, interact with a local or remote node, and much more. `, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return setConfigContext(cmd, false) + if err := setRootDirContext(cmd); err != nil { + return err + } + return setConfigContext(cmd) }, } cmd.PersistentFlags().String( "rootdir", "", - "Directory for data and configuration to use (default: $HOME/.defradb)", + "Directory for persistent data (default: $HOME/.defradb)", ) cmd.PersistentFlags().String( diff --git a/cli/start.go b/cli/start.go index 81fff46193..49e8745e23 100644 --- a/cli/start.go +++ b/cli/start.go @@ -35,7 +35,13 @@ func MakeStartCommand() *cobra.Command { Long: "Start a DefraDB node.", // Load the root config if it exists, otherwise create it. PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { - return setConfigContext(cmd, true) + if err := setRootDirContext(cmd); err != nil { + return err + } + if err := createConfig(mustGetRootDir(cmd)); err != nil { + return err + } + return setConfigContext(cmd) }, RunE: func(cmd *cobra.Command, args []string) error { cfg := mustGetConfigContext(cmd) @@ -77,10 +83,7 @@ func MakeStartCommand() *cobra.Command { // Running with memory store mode will always generate a random key. // Adding support for an ephemeral mode and moving the key to the // config would solve both of these issues. - rootdir, err := cmd.Root().PersistentFlags().GetString("rootdir") - if err != nil { - return err - } + rootdir := mustGetRootDir(cmd) key, err := loadOrGeneratePrivateKey(filepath.Join(rootdir, "data", "key")) if err != nil { return err diff --git a/cli/utils.go b/cli/utils.go index 6ec7096479..e9cd305167 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -30,6 +30,8 @@ type contextKey string var ( // cfgContextKey is the context key for the config. cfgContextKey = contextKey("cfg") + // rootDirContextKey is the context key for the root directory. + rootDirContextKey = contextKey("rootDir") // txContextKey is the context key for the datastore.Txn // // This will only be set if a transaction id is specified. @@ -69,6 +71,13 @@ func mustGetConfigContext(cmd *cobra.Command) *viper.Viper { return cmd.Context().Value(cfgContextKey).(*viper.Viper) } +// mustGetRootDir returns the rootdir for the current command context. +// +// If a rootdir is not set in the current context this function panics. +func mustGetRootDir(cmd *cobra.Command) string { + return cmd.Context().Value(rootDirContextKey).(string) +} + // tryGetCollectionContext returns the collection for the current command context // and a boolean indicating if the collection was set. func tryGetCollectionContext(cmd *cobra.Command) (client.Collection, bool) { @@ -77,12 +86,10 @@ func tryGetCollectionContext(cmd *cobra.Command) (client.Collection, bool) { } // setConfigContext sets teh config for the current command context. -func setConfigContext(cmd *cobra.Command, create bool) error { - rootdir, err := cmd.Root().PersistentFlags().GetString("rootdir") - if err != nil { - return err - } - cfg, err := loadConfig(rootdir, cmd, create) +func setConfigContext(cmd *cobra.Command) error { + rootdir := mustGetRootDir(cmd) + flags := cmd.Root().PersistentFlags() + cfg, err := loadConfig(rootdir, flags) if err != nil { return err } @@ -123,6 +130,24 @@ func setStoreContext(cmd *cobra.Command) error { return nil } +// setRootDirContext sets the rootdir for the current command context. +func setRootDirContext(cmd *cobra.Command) error { + rootdir, err := cmd.Root().PersistentFlags().GetString("rootdir") + if err != nil { + return err + } + home, err := os.UserHomeDir() + if err != nil { + return err + } + if rootdir == "" { + rootdir = filepath.Join(home, ".defradb") + } + ctx := context.WithValue(cmd.Context(), rootDirContextKey, rootdir) + cmd.SetContext(ctx) + return nil +} + // loadOrGeneratePrivateKey loads the private key from the given path // or generates a new key and writes it to a file at the given path. func loadOrGeneratePrivateKey(path string) (crypto.PrivKey, error) { From de99a3617dbe4ba3e5ccd76693a00249e73f0aa4 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 13 Feb 2024 09:03:10 -0800 Subject: [PATCH 6/8] review suggestions --- cli/backup_export.go | 2 +- cli/backup_import.go | 2 +- cli/client.go | 8 +++--- cli/collection.go | 10 +++---- cli/collection_create.go | 2 +- cli/collection_delete.go | 2 +- cli/collection_describe.go | 4 +-- cli/collection_get.go | 2 +- cli/collection_list_doc_ids.go | 2 +- cli/collection_update.go | 2 +- cli/config.go | 51 +++++++++++++++------------------- cli/config_test.go | 8 +++--- cli/index_create.go | 2 +- cli/index_drop.go | 2 +- cli/index_list.go | 2 +- cli/p2p_collection_add.go | 2 +- cli/p2p_collection_getall.go | 2 +- cli/p2p_collection_remove.go | 2 +- cli/p2p_replicator_delete.go | 2 +- cli/p2p_replicator_getall.go | 2 +- cli/p2p_replicator_set.go | 2 +- cli/request.go | 2 +- cli/root.go | 12 ++++---- cli/schema_add.go | 2 +- cli/schema_describe.go | 2 +- cli/schema_migration_down.go | 2 +- cli/schema_migration_get.go | 2 +- cli/schema_migration_reload.go | 2 +- cli/schema_migration_set.go | 2 +- cli/schema_migration_up.go | 2 +- cli/schema_patch.go | 2 +- cli/schema_set_default.go | 2 +- cli/server_dump.go | 6 ++-- cli/start.go | 15 +++++----- cli/tx_commit.go | 2 +- cli/tx_discard.go | 2 +- cli/utils.go | 42 ++++++++++++++-------------- cli/view_add.go | 2 +- 38 files changed, 105 insertions(+), 109 deletions(-) diff --git a/cli/backup_export.go b/cli/backup_export.go index 9e8d1c056e..b905bdf9c7 100644 --- a/cli/backup_export.go +++ b/cli/backup_export.go @@ -38,7 +38,7 @@ Example: export data for the 'Users' collection: defradb client export --collection Users user_data.json`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) if !isValidExportFormat(format) { return ErrInvalidExportFormat diff --git a/cli/backup_import.go b/cli/backup_import.go index 35af345a0a..56f1907643 100644 --- a/cli/backup_import.go +++ b/cli/backup_import.go @@ -24,7 +24,7 @@ Example: import data to the database: defradb client import user_data.json`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) return store.BasicImport(cmd.Context(), args[0]) }, } diff --git a/cli/client.go b/cli/client.go index f711c526d5..532712e8f8 100644 --- a/cli/client.go +++ b/cli/client.go @@ -22,16 +22,16 @@ func MakeClientCommand() *cobra.Command { Long: `Interact with a DefraDB node. Execute queries, add schema types, obtain node info, etc.`, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := setRootDirContext(cmd); err != nil { + if err := setContextRootDir(cmd); err != nil { return err } - if err := setConfigContext(cmd); err != nil { + if err := setContextConfig(cmd); err != nil { return err } - if err := setTransactionContext(cmd, txID); err != nil { + if err := setContextTransaction(cmd, txID); err != nil { return err } - return setStoreContext(cmd) + return setContextStore(cmd) }, } cmd.PersistentFlags().Uint64Var(&txID, "tx", 0, "Transaction ID") diff --git a/cli/collection.go b/cli/collection.go index d1c1e6effe..b15a627cb4 100644 --- a/cli/collection.go +++ b/cli/collection.go @@ -30,19 +30,19 @@ func MakeCollectionCommand() *cobra.Command { Long: `Create, read, update, and delete documents within a collection.`, PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { // cobra does not chain pre run calls so we have to run them again here - if err := setRootDirContext(cmd); err != nil { + if err := setContextRootDir(cmd); err != nil { return err } - if err := setConfigContext(cmd); err != nil { + if err := setContextConfig(cmd); err != nil { return err } - if err := setTransactionContext(cmd, txID); err != nil { + if err := setContextTransaction(cmd, txID); err != nil { return err } - if err := setStoreContext(cmd); err != nil { + if err := setContextStore(cmd); err != nil { return err } - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var col client.Collection var cols []client.Collection diff --git a/cli/collection_create.go b/cli/collection_create.go index 82e1e5db09..efeee61494 100644 --- a/cli/collection_create.go +++ b/cli/collection_create.go @@ -40,7 +40,7 @@ Example: create from stdin `, Args: cobra.RangeArgs(0, 1), RunE: func(cmd *cobra.Command, args []string) error { - col, ok := tryGetCollectionContext(cmd) + col, ok := tryGetContextCollection(cmd) if !ok { return cmd.Usage() } diff --git a/cli/collection_delete.go b/cli/collection_delete.go index dcd7c9d872..d1f945d9ae 100644 --- a/cli/collection_delete.go +++ b/cli/collection_delete.go @@ -31,7 +31,7 @@ Example: delete by filter defradb client collection delete --name User --filter '{ "_gte": { "points": 100 } }' `, RunE: func(cmd *cobra.Command, args []string) error { - col, ok := tryGetCollectionContext(cmd) + col, ok := tryGetContextCollection(cmd) if !ok { return cmd.Usage() } diff --git a/cli/collection_describe.go b/cli/collection_describe.go index a21c4d0c10..d903e5c89e 100644 --- a/cli/collection_describe.go +++ b/cli/collection_describe.go @@ -35,9 +35,9 @@ Example: view collection by version id defradb client collection describe --version bae123 `, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) - col, ok := tryGetCollectionContext(cmd) + col, ok := tryGetContextCollection(cmd) if ok { return writeJSON(cmd, col.Definition()) } diff --git a/cli/collection_get.go b/cli/collection_get.go index d753e0a8db..55c84d6289 100644 --- a/cli/collection_get.go +++ b/cli/collection_get.go @@ -28,7 +28,7 @@ Example: `, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - col, ok := tryGetCollectionContext(cmd) + col, ok := tryGetContextCollection(cmd) if !ok { return cmd.Usage() } diff --git a/cli/collection_list_doc_ids.go b/cli/collection_list_doc_ids.go index d7009cb300..7112a88817 100644 --- a/cli/collection_list_doc_ids.go +++ b/cli/collection_list_doc_ids.go @@ -26,7 +26,7 @@ Example: defradb client collection docIDs --name User `, RunE: func(cmd *cobra.Command, args []string) error { - col, ok := tryGetCollectionContext(cmd) + col, ok := tryGetContextCollection(cmd) if !ok { return cmd.Usage() } diff --git a/cli/collection_update.go b/cli/collection_update.go index 9fd2deed3f..42354948a9 100644 --- a/cli/collection_update.go +++ b/cli/collection_update.go @@ -38,7 +38,7 @@ Example: update by docIDs `, Args: cobra.RangeArgs(0, 1), RunE: func(cmd *cobra.Command, args []string) error { - col, ok := tryGetCollectionContext(cmd) + col, ok := tryGetContextCollection(cmd) if !ok { return cmd.Usage() } diff --git a/cli/config.go b/cli/config.go index 827a835c99..ac2636c0cd 100644 --- a/cli/config.go +++ b/cli/config.go @@ -11,7 +11,6 @@ package cli import ( - _ "embed" "os" "path/filepath" "strings" @@ -22,6 +21,11 @@ import ( "github.com/sourcenetwork/defradb/logging" ) +const ( + configStoreBadger = "badger" + configStoreMemory = "memory" +) + // configPaths are config keys that will be made relative to the rootdir var configPaths = []string{ "datastore.badger.path", @@ -38,7 +42,7 @@ var configFlags = map[string]string{ "log.nocolor": "lognocolor", "api.address": "url", "datastore.maxtxnretries": "max-txn-retries", - "datastore.badger.inMemory": "in-memory", + "datastore.store": "store", "datastore.badger.valuelogfilesize": "valuelogfilesize", "net.peers": "peers", "net.p2paddresses": "p2paddr", @@ -48,6 +52,17 @@ var configFlags = map[string]string{ "api.privkeypath": "privkeypath", } +// bindConfigFlags binds the set of cli flags to config values. +func bindConfigFlags(cfg *viper.Viper, flags *pflag.FlagSet) error { + for key, flag := range configFlags { + err := cfg.BindPFlag(key, flags.Lookup(flag)) + if err != nil { + return err + } + } + return nil +} + // defaultConfig returns a new config with default values. func defaultConfig() *viper.Viper { cfg := viper.New() @@ -59,38 +74,22 @@ func defaultConfig() *viper.Viper { cfg.SetConfigName("config") cfg.SetConfigType("yaml") - cfg.SetDefault("datastore.maxTxnRetries", 5) - cfg.SetDefault("datastore.badger.path", "data") - cfg.SetDefault("datastore.badger.inMemory", false) - cfg.SetDefault("datastore.badger.valueLogFileSize", 1<<30) - - cfg.SetDefault("api.address", "127.0.0.1:9181") - cfg.SetDefault("api.pubKeyPath", "") - cfg.SetDefault("api.privKeyPath", "") - cfg.SetDefault("api.allowed-origins", []string{}) - - cfg.SetDefault("net.p2pDisabled", false) - cfg.SetDefault("net.p2pAddresses", []any{"/ip4/127.0.0.1/tcp/9171"}) cfg.SetDefault("net.pubSubEnabled", true) cfg.SetDefault("net.relay", false) - cfg.SetDefault("net.peers", []string{}) - - cfg.SetDefault("log.level", "info") - cfg.SetDefault("log.stackTrace", true) - cfg.SetDefault("log.format", "csv") - cfg.SetDefault("log.output", "stderr") - cfg.SetDefault("log.noColor", false) cfg.SetDefault("log.caller", false) return cfg } // createConfig writes the default config file if one does not exist. -func createConfig(rootdir string) error { +func createConfig(rootdir string, flags *pflag.FlagSet) error { cfg := defaultConfig() cfg.AddConfigPath(rootdir) + if err := bindConfigFlags(cfg, flags); err != nil { + return err + } // make sure rootdir exists if err := os.MkdirAll(rootdir, 0755); err != nil { return err @@ -112,13 +111,9 @@ func loadConfig(rootdir string, flags *pflag.FlagSet) (*viper.Viper, error) { if _, ok := err.(viper.ConfigFileNotFoundError); err != nil && !ok { //nolint:errorlint return nil, err } - // bind cli flags to config keys - for key, flag := range configFlags { - err := cfg.BindPFlag(key, flags.Lookup(flag)) - if err != nil { - return nil, err - } + if err := bindConfigFlags(cfg, flags); err != nil { + return nil, err } // make paths relative to the rootdir diff --git a/cli/config_test.go b/cli/config_test.go index fd63da0b86..210743477c 100644 --- a/cli/config_test.go +++ b/cli/config_test.go @@ -20,11 +20,11 @@ import ( func TestCreateConfig(t *testing.T) { rootdir := t.TempDir() - err := createConfig(rootdir) + err := createConfig(rootdir, NewDefraCommand().PersistentFlags()) require.NoError(t, err) // ensure no errors when config already exists - err = createConfig(rootdir) + err = createConfig(rootdir, NewDefraCommand().PersistentFlags()) require.NoError(t, err) assert.FileExists(t, filepath.Join(rootdir, "config.yaml")) @@ -39,7 +39,7 @@ func TestLoadConfigNotExist(t *testing.T) { assert.Equal(t, filepath.Join(rootdir, "data"), cfg.GetString("datastore.badger.path")) assert.Equal(t, 1<<30, cfg.GetInt("datastore.badger.valuelogfilesize")) - assert.Equal(t, false, cfg.GetBool("datastore.badger.inmemory")) + assert.Equal(t, "badger", cfg.GetString("datastore.store")) assert.Equal(t, "127.0.0.1:9181", cfg.GetString("api.address")) assert.Equal(t, []string{}, cfg.GetStringSlice("api.allowed-origins")) @@ -53,7 +53,7 @@ func TestLoadConfigNotExist(t *testing.T) { assert.Equal(t, []string{}, cfg.GetStringSlice("net.peers")) assert.Equal(t, "info", cfg.GetString("log.level")) - assert.Equal(t, true, cfg.GetBool("log.stacktrace")) + assert.Equal(t, false, cfg.GetBool("log.stacktrace")) assert.Equal(t, "csv", cfg.GetString("log.format")) assert.Equal(t, "stderr", cfg.GetString("log.output")) assert.Equal(t, false, cfg.GetBool("log.nocolor")) diff --git a/cli/index_create.go b/cli/index_create.go index 099eb7e7a6..bfe5ec64c2 100644 --- a/cli/index_create.go +++ b/cli/index_create.go @@ -37,7 +37,7 @@ Example: create a named index for 'Users' collection on 'name' field: defradb client index create --collection Users --fields name --name UsersByName`, ValidArgs: []string{"collection", "fields", "name"}, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var fields []client.IndexedFieldDescription for _, name := range fieldsArg { diff --git a/cli/index_drop.go b/cli/index_drop.go index 03639fb277..96f007268d 100644 --- a/cli/index_drop.go +++ b/cli/index_drop.go @@ -28,7 +28,7 @@ Example: drop the index 'UsersByName' for 'Users' collection: defradb client index create --collection Users --name UsersByName`, ValidArgs: []string{"collection", "name"}, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) col, err := store.GetCollectionByName(cmd.Context(), collectionArg) if err != nil { diff --git a/cli/index_list.go b/cli/index_list.go index 92ada3e007..bf1fd21251 100644 --- a/cli/index_list.go +++ b/cli/index_list.go @@ -30,7 +30,7 @@ Example: show all index for 'Users' collection: defradb client index list --collection Users`, ValidArgs: []string{"collection"}, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) switch { case collectionArg != "": diff --git a/cli/p2p_collection_add.go b/cli/p2p_collection_add.go index dedae0a358..8a867e6abb 100644 --- a/cli/p2p_collection_add.go +++ b/cli/p2p_collection_add.go @@ -31,7 +31,7 @@ Example: add multiple collections `, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - p2p := mustGetP2PContext(cmd) + p2p := mustGetContextP2P(cmd) var collectionIDs []string for _, id := range strings.Split(args[0], ",") { diff --git a/cli/p2p_collection_getall.go b/cli/p2p_collection_getall.go index 10d98582c6..8a005df801 100644 --- a/cli/p2p_collection_getall.go +++ b/cli/p2p_collection_getall.go @@ -22,7 +22,7 @@ func MakeP2PCollectionGetAllCommand() *cobra.Command { This is the list of collections of the node that are synchronized on the pubsub network.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - p2p := mustGetP2PContext(cmd) + p2p := mustGetContextP2P(cmd) cols, err := p2p.GetAllP2PCollections(cmd.Context()) if err != nil { diff --git a/cli/p2p_collection_remove.go b/cli/p2p_collection_remove.go index 8aa0b5b7df..7def06e779 100644 --- a/cli/p2p_collection_remove.go +++ b/cli/p2p_collection_remove.go @@ -31,7 +31,7 @@ Example: remove multiple collections `, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - p2p := mustGetP2PContext(cmd) + p2p := mustGetContextP2P(cmd) var collectionIDs []string for _, id := range strings.Split(args[0], ",") { diff --git a/cli/p2p_replicator_delete.go b/cli/p2p_replicator_delete.go index 6cc2ddf785..debd0ac280 100644 --- a/cli/p2p_replicator_delete.go +++ b/cli/p2p_replicator_delete.go @@ -32,7 +32,7 @@ Example: `, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - p2p := mustGetP2PContext(cmd) + p2p := mustGetContextP2P(cmd) var info peer.AddrInfo if err := json.Unmarshal([]byte(args[0]), &info); err != nil { diff --git a/cli/p2p_replicator_getall.go b/cli/p2p_replicator_getall.go index 4bdf6e8487..0e5549fea6 100644 --- a/cli/p2p_replicator_getall.go +++ b/cli/p2p_replicator_getall.go @@ -25,7 +25,7 @@ Example: defradb client p2p replicator getall `, RunE: func(cmd *cobra.Command, args []string) error { - p2p := mustGetP2PContext(cmd) + p2p := mustGetContextP2P(cmd) reps, err := p2p.GetAllReplicators(cmd.Context()) if err != nil { diff --git a/cli/p2p_replicator_set.go b/cli/p2p_replicator_set.go index 5d9c712a82..29109a920a 100644 --- a/cli/p2p_replicator_set.go +++ b/cli/p2p_replicator_set.go @@ -32,7 +32,7 @@ Example: `, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - p2p := mustGetP2PContext(cmd) + p2p := mustGetContextP2P(cmd) var info peer.AddrInfo if err := json.Unmarshal([]byte(args[0]), &info); err != nil { diff --git a/cli/request.go b/cli/request.go index 56e33d7c4a..d5e37e79a3 100644 --- a/cli/request.go +++ b/cli/request.go @@ -45,7 +45,7 @@ with the database more conveniently. To learn more about the DefraDB GraphQL Query Language, refer to https://docs.source.network.`, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var request string switch { diff --git a/cli/root.go b/cli/root.go index aa8fc116a3..e4ba349f76 100644 --- a/cli/root.go +++ b/cli/root.go @@ -24,10 +24,10 @@ func MakeRootCommand() *cobra.Command { Start a DefraDB node, interact with a local or remote node, and much more. `, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := setRootDirContext(cmd); err != nil { + if err := setContextRootDir(cmd); err != nil { return err } - return setConfigContext(cmd) + return setContextConfig(cmd) }, } @@ -85,10 +85,10 @@ Start a DefraDB node, interact with a local or remote node, and much more. "Specify the maximum number of retries per transaction", ) - cmd.PersistentFlags().Bool( - "in-memory", - false, - "Enables the badger in memory only datastore.", + cmd.PersistentFlags().String( + "store", + "badger", + "Specify the datastore to use (supported: badger, memory)", ) cmd.PersistentFlags().Int( diff --git a/cli/schema_add.go b/cli/schema_add.go index b93427a883..f987d062df 100644 --- a/cli/schema_add.go +++ b/cli/schema_add.go @@ -36,7 +36,7 @@ Example: add from stdin: Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var schema string switch { diff --git a/cli/schema_describe.go b/cli/schema_describe.go index 72d8eda474..7d8306e93b 100644 --- a/cli/schema_describe.go +++ b/cli/schema_describe.go @@ -39,7 +39,7 @@ Example: view a single schema by version id defradb client schema describe --version bae123 `, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var schemas []client.SchemaDescription switch { diff --git a/cli/schema_migration_down.go b/cli/schema_migration_down.go index 1dcb5e64da..659bc45df6 100644 --- a/cli/schema_migration_down.go +++ b/cli/schema_migration_down.go @@ -41,7 +41,7 @@ Example: migrate from stdin `, Args: cobra.RangeArgs(0, 1), RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var srcData []byte switch { diff --git a/cli/schema_migration_get.go b/cli/schema_migration_get.go index 43b66599b7..a21fc1dc5d 100644 --- a/cli/schema_migration_get.go +++ b/cli/schema_migration_get.go @@ -25,7 +25,7 @@ Example: Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) cfgs, err := store.LensRegistry().Config(cmd.Context()) if err != nil { diff --git a/cli/schema_migration_reload.go b/cli/schema_migration_reload.go index d04aebed65..4266b3ec3f 100644 --- a/cli/schema_migration_reload.go +++ b/cli/schema_migration_reload.go @@ -22,7 +22,7 @@ func MakeSchemaMigrationReloadCommand() *cobra.Command { Short: "Reload the schema migrations within DefraDB", Long: `Reload the schema migrations within DefraDB`, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) lens := store.LensRegistry() if tx, ok := cmd.Context().Value(txContextKey).(datastore.Txn); ok { diff --git a/cli/schema_migration_set.go b/cli/schema_migration_set.go index 280130b8db..2a9659d072 100644 --- a/cli/schema_migration_set.go +++ b/cli/schema_migration_set.go @@ -41,7 +41,7 @@ Example: add from stdin: Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`, Args: cobra.RangeArgs(2, 3), RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var lensCfgJson string switch { diff --git a/cli/schema_migration_up.go b/cli/schema_migration_up.go index 3b0b522349..4e3199f41e 100644 --- a/cli/schema_migration_up.go +++ b/cli/schema_migration_up.go @@ -41,7 +41,7 @@ Example: migrate from stdin `, Args: cobra.RangeArgs(0, 1), RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var srcData []byte switch { diff --git a/cli/schema_patch.go b/cli/schema_patch.go index 70f4283c85..e67a3aa086 100644 --- a/cli/schema_patch.go +++ b/cli/schema_patch.go @@ -39,7 +39,7 @@ Example: patch from stdin: To learn more about the DefraDB GraphQL Schema Language, refer to https://docs.source.network.`, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) var patch string switch { diff --git a/cli/schema_set_default.go b/cli/schema_set_default.go index cdb6bd8bd8..88343c1fd2 100644 --- a/cli/schema_set_default.go +++ b/cli/schema_set_default.go @@ -21,7 +21,7 @@ func MakeSchemaSetDefaultCommand() *cobra.Command { Long: `Set the default schema version`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) return store.SetDefaultSchemaVersion(cmd.Context(), args[0]) }, } diff --git a/cli/server_dump.go b/cli/server_dump.go index ae03f45d51..6d2b2cefc6 100644 --- a/cli/server_dump.go +++ b/cli/server_dump.go @@ -23,15 +23,15 @@ func MakeServerDumpCmd() *cobra.Command { Use: "server-dump", Short: "Dumps the state of the entire database", RunE: func(cmd *cobra.Command, _ []string) error { - cfg := mustGetConfigContext(cmd) + cfg := mustGetContextConfig(cmd) log.FeedbackInfo(cmd.Context(), "Dumping DB state...") - if cfg.GetBool("datastore.badger.inMemory") { + if cfg.GetString("datastore.store") != configStoreBadger { return errors.New("server-side dump is only supported for the Badger datastore") } storeOpts := []node.StoreOpt{ node.WithPath(cfg.GetString("datastore.badger.path")), - node.WithInMemory(cfg.GetBool("datastore.badger.inMemory")), + node.WithInMemory(cfg.GetString("datastore.store") != configStoreMemory), } rootstore, err := node.NewStore(storeOpts...) if err != nil { diff --git a/cli/start.go b/cli/start.go index 49e8745e23..d4e789cbc6 100644 --- a/cli/start.go +++ b/cli/start.go @@ -35,16 +35,17 @@ func MakeStartCommand() *cobra.Command { Long: "Start a DefraDB node.", // Load the root config if it exists, otherwise create it. PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { - if err := setRootDirContext(cmd); err != nil { + if err := setContextRootDir(cmd); err != nil { return err } - if err := createConfig(mustGetRootDir(cmd)); err != nil { + rootdir := mustGetContextRootDir(cmd) + if err := createConfig(rootdir, cmd.Root().PersistentFlags()); err != nil { return err } - return setConfigContext(cmd) + return setContextConfig(cmd) }, RunE: func(cmd *cobra.Command, args []string) error { - cfg := mustGetConfigContext(cmd) + cfg := mustGetContextConfig(cmd) dbOpts := []db.Option{ db.WithUpdateEvents(), @@ -66,7 +67,7 @@ func MakeStartCommand() *cobra.Command { storeOpts := []node.StoreOpt{ node.WithPath(cfg.GetString("datastore.badger.path")), - node.WithInMemory(cfg.GetBool("datastore.badger.inMemory")), + node.WithInMemory(cfg.GetString("datastore.store") == configStoreMemory), } var peers []peer.AddrInfo @@ -78,12 +79,12 @@ func MakeStartCommand() *cobra.Command { peers = addrs } - if !cfg.GetBool("datastore.badger.inMemory") { + if cfg.GetString("datastore.store") != configStoreMemory { // It would be ideal to not have the key path tied to the datastore. // Running with memory store mode will always generate a random key. // Adding support for an ephemeral mode and moving the key to the // config would solve both of these issues. - rootdir := mustGetRootDir(cmd) + rootdir := mustGetContextRootDir(cmd) key, err := loadOrGeneratePrivateKey(filepath.Join(rootdir, "data", "key")) if err != nil { return err diff --git a/cli/tx_commit.go b/cli/tx_commit.go index d6fe6ea970..f7ef112988 100644 --- a/cli/tx_commit.go +++ b/cli/tx_commit.go @@ -25,7 +25,7 @@ func MakeTxCommitCommand() *cobra.Command { Long: `Commit a DefraDB transaction.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) (err error) { - cfg := mustGetConfigContext(cmd) + cfg := mustGetContextConfig(cmd) id, err := strconv.ParseUint(args[0], 10, 64) if err != nil { diff --git a/cli/tx_discard.go b/cli/tx_discard.go index 259d183461..0a980a63f5 100644 --- a/cli/tx_discard.go +++ b/cli/tx_discard.go @@ -25,7 +25,7 @@ func MakeTxDiscardCommand() *cobra.Command { Long: `Discard a DefraDB transaction.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) (err error) { - cfg := mustGetConfigContext(cmd) + cfg := mustGetContextConfig(cmd) id, err := strconv.ParseUint(args[0], 10, 64) if err != nil { diff --git a/cli/utils.go b/cli/utils.go index e9cd305167..caeb282606 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -50,44 +50,44 @@ var ( colContextKey = contextKey("col") ) -// mustGetStoreContext returns the store for the current command context. +// mustGetContextStore returns the store for the current command context. // // If a store is not set in the current context this function panics. -func mustGetStoreContext(cmd *cobra.Command) client.Store { +func mustGetContextStore(cmd *cobra.Command) client.Store { return cmd.Context().Value(storeContextKey).(client.Store) } -// mustGetP2PContext returns the p2p implementation for the current command context. +// mustGetContextP2P returns the p2p implementation for the current command context. // // If a p2p implementation is not set in the current context this function panics. -func mustGetP2PContext(cmd *cobra.Command) client.P2P { +func mustGetContextP2P(cmd *cobra.Command) client.P2P { return cmd.Context().Value(dbContextKey).(client.P2P) } -// mustGetConfigContext returns the config for the current command context. +// mustGetContextConfig returns the config for the current command context. // // If a config is not set in the current context this function panics. -func mustGetConfigContext(cmd *cobra.Command) *viper.Viper { +func mustGetContextConfig(cmd *cobra.Command) *viper.Viper { return cmd.Context().Value(cfgContextKey).(*viper.Viper) } -// mustGetRootDir returns the rootdir for the current command context. +// mustGetContextRootDir returns the rootdir for the current command context. // // If a rootdir is not set in the current context this function panics. -func mustGetRootDir(cmd *cobra.Command) string { +func mustGetContextRootDir(cmd *cobra.Command) string { return cmd.Context().Value(rootDirContextKey).(string) } -// tryGetCollectionContext returns the collection for the current command context +// tryGetContextCollection returns the collection for the current command context // and a boolean indicating if the collection was set. -func tryGetCollectionContext(cmd *cobra.Command) (client.Collection, bool) { +func tryGetContextCollection(cmd *cobra.Command) (client.Collection, bool) { col, ok := cmd.Context().Value(colContextKey).(client.Collection) return col, ok } -// setConfigContext sets teh config for the current command context. -func setConfigContext(cmd *cobra.Command) error { - rootdir := mustGetRootDir(cmd) +// setContextConfig sets teh config for the current command context. +func setContextConfig(cmd *cobra.Command) error { + rootdir := mustGetContextRootDir(cmd) flags := cmd.Root().PersistentFlags() cfg, err := loadConfig(rootdir, flags) if err != nil { @@ -98,12 +98,12 @@ func setConfigContext(cmd *cobra.Command) error { return nil } -// setTransactionContext sets the transaction for the current command context. -func setTransactionContext(cmd *cobra.Command, txId uint64) error { +// setContextTransaction sets the transaction for the current command context. +func setContextTransaction(cmd *cobra.Command, txId uint64) error { if txId == 0 { return nil } - cfg := mustGetConfigContext(cmd) + cfg := mustGetContextConfig(cmd) tx, err := http.NewTransaction(cfg.GetString("api.address"), txId) if err != nil { return err @@ -113,9 +113,9 @@ func setTransactionContext(cmd *cobra.Command, txId uint64) error { return nil } -// setStoreContext sets the store for the current command context. -func setStoreContext(cmd *cobra.Command) error { - cfg := mustGetConfigContext(cmd) +// setContextStore sets the store for the current command context. +func setContextStore(cmd *cobra.Command) error { + cfg := mustGetContextConfig(cmd) db, err := http.NewClient(cfg.GetString("api.address")) if err != nil { return err @@ -130,8 +130,8 @@ func setStoreContext(cmd *cobra.Command) error { return nil } -// setRootDirContext sets the rootdir for the current command context. -func setRootDirContext(cmd *cobra.Command) error { +// setContextRootDir sets the rootdir for the current command context. +func setContextRootDir(cmd *cobra.Command) error { rootdir, err := cmd.Root().PersistentFlags().GetString("rootdir") if err != nil { return err diff --git a/cli/view_add.go b/cli/view_add.go index 46779fb784..6552100cf5 100644 --- a/cli/view_add.go +++ b/cli/view_add.go @@ -23,7 +23,7 @@ Example: add from an argument string: Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`, RunE: func(cmd *cobra.Command, args []string) error { - store := mustGetStoreContext(cmd) + store := mustGetContextStore(cmd) if len(args) != 2 { return ErrViewAddMissingArgs From 19e428bdc29fadada99694772065b18d8e8a9aca Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 13 Feb 2024 09:28:25 -0800 Subject: [PATCH 7/8] add config doc --- README.md | 2 ++ cli/config.go | 22 ++++++++--------- docs/config.md | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 11 deletions(-) create mode 100644 docs/config.md diff --git a/README.md b/README.md index f1ac6cb88f..a7156888b9 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,8 @@ In this document, we use the default configuration, which has the following beha The GraphQL endpoint can be used with a GraphQL client (e.g., Altair) to conveniently perform requests (`query`, `mutation`) and obtain schema introspection. +Read more about the configuration [here](./docs/config.md). + ## External port binding By default the HTTP API and P2P network will use localhost. If you want to expose the ports externally you need to specify the addresses in the config or command line parameters. diff --git a/cli/config.go b/cli/config.go index ac2636c0cd..51c59bb92a 100644 --- a/cli/config.go +++ b/cli/config.go @@ -52,17 +52,6 @@ var configFlags = map[string]string{ "api.privkeypath": "privkeypath", } -// bindConfigFlags binds the set of cli flags to config values. -func bindConfigFlags(cfg *viper.Viper, flags *pflag.FlagSet) error { - for key, flag := range configFlags { - err := cfg.BindPFlag(key, flags.Lookup(flag)) - if err != nil { - return err - } - } - return nil -} - // defaultConfig returns a new config with default values. func defaultConfig() *viper.Viper { cfg := viper.New() @@ -136,6 +125,17 @@ func loadConfig(rootdir string, flags *pflag.FlagSet) (*viper.Viper, error) { return cfg, nil } +// bindConfigFlags binds the set of cli flags to config values. +func bindConfigFlags(cfg *viper.Viper, flags *pflag.FlagSet) error { + for key, flag := range configFlags { + err := cfg.BindPFlag(key, flags.Lookup(flag)) + if err != nil { + return err + } + } + return nil +} + // loggingConfig returns a new logging config from the given config. func loggingConfig(cfg *viper.Viper) logging.Config { var level int8 diff --git a/docs/config.md b/docs/config.md new file mode 100644 index 0000000000..874a5da35a --- /dev/null +++ b/docs/config.md @@ -0,0 +1,64 @@ +# DefraDB configuration (YAML) + +The default DefraDB directory is `$HOME/.defradb`. It can be changed via the --rootdir CLI flag. + +Relative paths are interpreted as being rooted in the DefraDB directory. + +## `datastore.store` + +Store can be badger or memory. Defaults to `badger`. + +- badger: fast pure Go key-value store optimized for SSDs (https://github.com/dgraph-io/badger) +- memory: in-memory version of badger + +## `datastore.maxtxnretries` + +Maximum number of times to retry a failed transaction. Defaults to `5`. + +## `datastore.badger.path` + +The path to the database data file(s). Defaults to `data`. + +## `datastore.badger.valuelogfilesize` + +Maximum file size of the value log files. + +## `api.address` + +Address of the HTTP API to listen on or connect to. Defaults to `127.0.0.1:9181`. + +## `api.allowed-origins` + +The list of origins a cross-domain request can be executed from. + +## `api.pubkeypath` + +The path to the public key file for TLS / HTTPS. + +## `api.privkeypath` + +The path to the private key file for TLS / HTTPS. + +## `net.p2pdisabled` + +Whether P2P networking is disabled. Defaults to `false`. + +## `net.p2paddresses` + +List of addresses for the P2P network to listen on. Defaults to `/ip4/127.0.0.1/tcp/9171`. + +## `net.pubsubenabled` + +Whether PubSub is enabled. Defaults to `true`. + +## `net.peers` + +List of peers to boostrap with, specified as multiaddresses. + +https://docs.libp2p.io/concepts/addressing/ + +## `net.relay` + +Enable libp2p's Circuit relay transport protocol. Defaults to `false`. + +https://docs.libp2p.io/concepts/circuit-relay/ \ No newline at end of file From 9dfb1b66ba7c6faeab8c255c5aaa15ee51b9bea9 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 14 Feb 2024 09:07:07 -0800 Subject: [PATCH 8/8] review fixes --- cli/config.go | 32 ++++++++++++++++++++++---------- docs/config.md | 4 +++- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/cli/config.go b/cli/config.go index 51c59bb92a..bb57a8cb3d 100644 --- a/cli/config.go +++ b/cli/config.go @@ -22,8 +22,14 @@ import ( ) const ( - configStoreBadger = "badger" - configStoreMemory = "memory" + configStoreBadger = "badger" + configStoreMemory = "memory" + configLogFormatJSON = "json" + configLogFormatCSV = "csv" + configLogLevelInfo = "info" + configLogLevelDebug = "debug" + configLogLevelError = "error" + configLogLevelFatal = "fatal" ) // configPaths are config keys that will be made relative to the rootdir @@ -84,7 +90,10 @@ func createConfig(rootdir string, flags *pflag.FlagSet) error { return err } err := cfg.SafeWriteConfig() - if _, ok := err.(viper.ConfigFileAlreadyExistsError); ok { //nolint:errorlint + // error type is known and shouldn't be wrapped + // + //nolint:errorlint + if _, ok := err.(viper.ConfigFileAlreadyExistsError); ok { return nil } return err @@ -97,7 +106,10 @@ func loadConfig(rootdir string, flags *pflag.FlagSet) (*viper.Viper, error) { // attempt to read the existing config err := cfg.ReadInConfig() - if _, ok := err.(viper.ConfigFileNotFoundError); err != nil && !ok { //nolint:errorlint + // error type is known and shouldn't be wrapped + // + //nolint:errorlint + if _, ok := err.(viper.ConfigFileNotFoundError); err != nil && !ok { return nil, err } // bind cli flags to config keys @@ -140,13 +152,13 @@ func bindConfigFlags(cfg *viper.Viper, flags *pflag.FlagSet) error { func loggingConfig(cfg *viper.Viper) logging.Config { var level int8 switch value := cfg.GetString("level"); value { - case "debug": + case configLogLevelDebug: level = logging.Debug - case "info": + case configLogLevelInfo: level = logging.Info - case "error": + case configLogLevelError: level = logging.Error - case "fatal": + case configLogLevelFatal: level = logging.Fatal default: level = logging.Info @@ -154,9 +166,9 @@ func loggingConfig(cfg *viper.Viper) logging.Config { var format logging.EncoderFormat switch value := cfg.GetString("format"); value { - case "json": //nolint:goconst + case configLogFormatJSON: format = logging.JSON - case "csv": + case configLogFormatCSV: format = logging.CSV default: format = logging.CSV diff --git a/docs/config.md b/docs/config.md index 874a5da35a..5f8985f71c 100644 --- a/docs/config.md +++ b/docs/config.md @@ -13,7 +13,9 @@ Store can be badger or memory. Defaults to `badger`. ## `datastore.maxtxnretries` -Maximum number of times to retry a failed transaction. Defaults to `5`. +The number of retries to make in the event of a transaction conflict. Defaults to `5`. + +Currently this is only used within the P2P system and will not affect operations initiated by users. ## `datastore.badger.path`