diff --git a/.github/workflows/coverge.yml b/.github/workflows/coverge.yml index 0a2646a5..f5780cda 100644 --- a/.github/workflows/coverge.yml +++ b/.github/workflows/coverge.yml @@ -15,7 +15,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v4 with: - go-version: '1.20.1' + go-version: '1.21.4' - name: Checkout uses: actions/checkout@v3 with: @@ -24,6 +24,6 @@ jobs: env: DEEPSOURCE_DSN: ${{ secrets.DEEPSOURCE_DSN }} run: | - go test -coverprofile=cover.out $(go list ./... | grep -v wasm) + go test -coverprofile=cover.out $(go list ./... | grep -v wasm) -timeout=30m curl https://deepsource.io/cli | sh ./bin/deepsource report --analyzer test-coverage --key go --value-file ./cover.out \ No newline at end of file diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index ec179c1d..15baaa60 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -14,7 +14,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ['1.20.1'] + go: ['1.21.4'] os: [ubuntu-latest, macos-latest, windows-latest] steps: - name: Setup Go diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 695c3278..fa8d94ca 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -18,7 +18,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v4 with: - go-version: '1.20.1' + go-version: '1.21.4' - name: Checkout uses: actions/checkout@v3 with: @@ -45,7 +45,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v4 with: - go-version: '1.20.1' + go-version: '1.21.4' - name: Checkout uses: actions/checkout@v3 with: @@ -85,7 +85,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v4 with: - go-version: '1.20.1' + go-version: '1.21.4' - name: Checkout uses: actions/checkout@v3 with: diff --git a/Dockerfile b/Dockerfile index bb27f7ba..f7b8190c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.2-alpine AS build +FROM golang:1.21.4-alpine AS build WORKDIR /usr/fairos COPY go.mod go.sum /usr/fairos/ diff --git a/Makefile b/Makefile index 520fabfd..51e30170 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ GO ?= go GOLANGCI_LINT ?= $$($(GO) env GOPATH)/bin/golangci-lint -GOLANGCI_LINT_VERSION ?= v1.52.2 +GOLANGCI_LINT_VERSION ?= v1.55.2 GOGOPROTOBUF ?= protoc-gen-gogofaster GOGOPROTOBUF_VERSION ?= v1.3.1 @@ -41,11 +41,11 @@ vet: .PHONY: test-race test-race: - $(GO) test -race -timeout 1000000ms -v "$(DEST)" + $(GO) test -race -timeout 30m -v "$(DEST)" .PHONY: test test: - $(GO) test -v "$(DEST)" + $(GO) test -v "$(DEST)" -timeout=30m .PHONY: githooks githooks: @@ -74,7 +74,7 @@ release: -v `pwd`:/go/src/github.com/fairDataSociety/fairOS-dfs \ -v /var/run/docker.sock:/var/run/docker.sock \ -w /go/src/github.com/fairDataSociety/fairOS-dfs \ - ghcr.io/goreleaser/goreleaser-cross:v1.20.2 release --rm-dist + ghcr.io/goreleaser/goreleaser-cross:v1.21.0 release --rm-dist .PHONY: release-dry-run release-dry-run: @@ -84,7 +84,7 @@ release-dry-run: -v `pwd`:/go/src/github.com/fairDataSociety/fairOS-dfs \ -v /var/run/docker.sock:/var/run/docker.sock \ -w /go/src/github.com/fairDataSociety/fairOS-dfs \ - ghcr.io/goreleaser/goreleaser-cross:v1.20.2 release --rm-dist \ + ghcr.io/goreleaser/goreleaser-cross:v1.21.0 release --rm-dist \ --skip-validate=true \ --skip-publish diff --git a/cmd/dfs/cmd/config.go b/cmd/dfs/cmd/config.go index a6ff98fd..9f8c85f1 100644 --- a/cmd/dfs/cmd/config.go +++ b/cmd/dfs/cmd/config.go @@ -12,6 +12,8 @@ var ( optionVerbosity = "verbosity" optionBeeApi = "bee.bee-api-endpoint" optionBeePostageBatchId = "bee.postage-batch-id" + optionFeedCacheSize = "feed.cache-size" + optionFeedCacheTTL = "feed.cache-ttl" optionCookieDomain = "cookie-domain" optionNetwork = "ens-network" optionRPC = "rpc" diff --git a/cmd/dfs/cmd/dev.go b/cmd/dfs/cmd/dev.go new file mode 100644 index 00000000..d5fdcddd --- /dev/null +++ b/cmd/dfs/cmd/dev.go @@ -0,0 +1,78 @@ +package cmd + +import ( + "context" + "fmt" + "io" + "os" + "os/signal" + "syscall" + "testing" + + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/api" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/dfs" + mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/ensm/eth/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/logging" + "github.com/fairdatasociety/fairOS-dfs/pkg/user" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var devCmd = &cobra.Command{ + Use: "dev", + Short: "run fairOS-dfs in development mode", + Run: func(cmd *cobra.Command, args []string) { + startDevServer() + }, +} + +func init() { + rootCmd.AddCommand(devCmd) +} + +func startDevServer() { + storer := mockstorer.New() + t := &testing.T{} + fmt.Println(`▓█████▄ ▓█████ ██▒ █▓ +▒██▀ ██▌▓█ ▀▓██░ █▒ +░██ █▌▒███ ▓██ █▒░ +░▓█▄ ▌▒▓█ ▄ ▒██ █░░ +░▒████▓ ░▒████▒ ▒▀█░ + ▒▒▓ ▒ ░░ ▒░ ░ ░ ▐░ + ░ ▒ ▒ ░ ░ ░ ░ ░░ + ░ ░ ░ ░ ░░ + ░ ░ ░ ░ + ░ ░ `) + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + fmt.Println("Bee running at: ", beeUrl) + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + ens := mock2.NewMockNamespaceManager() + + users := user.NewUsers(mockClient, ens, -1, 0, logger) + dfsApi := dfs.NewMockDfsAPI(mockClient, users, logger) + handler = api.NewMockHandler(dfsApi, logger, []string{"http://localhost:3000"}) + defer handler.Close() + httpPort = ":9090" + pprofPort = ":9091" + srv := startHttpService(logger) + fmt.Printf("Server running at:http://127.0.0.1%s\n", httpPort) + defer func() { + err := srv.Shutdown(context.TODO()) + if err != nil { + logger.Error("failed to shutdown server", err.Error()) + } + }() + done := make(chan os.Signal, 1) + signal.Notify(done, syscall.SIGINT, syscall.SIGTERM) + + <-done +} diff --git a/cmd/dfs/cmd/server.go b/cmd/dfs/cmd/server.go index e7e0afd7..9f468249 100644 --- a/cmd/dfs/cmd/server.go +++ b/cmd/dfs/cmd/server.go @@ -80,6 +80,15 @@ can consume it.`, if err := config.BindPFlag(optionDFSPprofPort, cmd.Flags().Lookup("pprofPort")); err != nil { return err } + if err := config.BindPFlag(optionFeedCacheSize, cmd.Flags().Lookup("feedCacheSize")); err != nil { + return err + } + if err := config.BindPFlag(optionFeedCacheTTL, cmd.Flags().Lookup("feedCacheTTL")); err != nil { + return err + } + if err := config.BindPFlag(optionDFSPprofPort, cmd.Flags().Lookup("pprofPort")); err != nil { + return err + } if err := config.BindPFlag(optionCookieDomain, cmd.Flags().Lookup("cookieDomain")); err != nil { return err } @@ -175,9 +184,11 @@ can consume it.`, logger.Info("verbosity : ", verbosity) logger.Info("httpPort : ", httpPort) logger.Info("pprofPort : ", pprofPort) + logger.Info("pprofPort : ", pprofPort) + logger.Info("pprofPort : ", pprofPort) logger.Info("cookieDomain : ", cookieDomain) - logger.Info("postageBlockId : ", postageBlockId) - logger.Info("corsOrigins : ", corsOrigins) + logger.Info("feedCacheSize : ", config.GetInt(optionFeedCacheSize)) + logger.Info("feedCacheTTL : ", config.GetString(optionFeedCacheTTL)) ctx, cancel := context.WithCancel(cmd.Context()) defer cancel() @@ -190,6 +201,8 @@ can consume it.`, EnsConfig: ensConfig, SubscriptionConfig: subscriptionConfig, Logger: logger, + FeedCacheSize: config.GetInt(optionFeedCacheSize), + FeedCacheTTL: config.GetString(optionFeedCacheTTL), } hdlr, err := api.New(ctx, opts) @@ -229,6 +242,8 @@ func init() { serverCmd.Flags().BoolVar(&swag, "swag", false, "should run swagger-ui") serverCmd.Flags().String("httpPort", defaultDFSHttpPort, "http port") serverCmd.Flags().String("pprofPort", defaultDFSPprofPort, "pprof port") + serverCmd.Flags().Int("feedCacheSize", -1, "Keep feed updates in lru cache for faster access. -1 to disable") + serverCmd.Flags().String("feedCacheTTL", "0s", "How long to keep feed updates in lru cache. 0s to disable") serverCmd.Flags().String("cookieDomain", defaultCookieDomain, "the domain to use in the cookie") serverCmd.Flags().String("postageBlockId", "", "the postage block used to store the data in bee") serverCmd.Flags().StringSlice("cors-origins", defaultCORSAllowedOrigins, "allow CORS headers for the given origins") diff --git a/cmd/dfs/cmd/server_test.go b/cmd/dfs/cmd/server_test.go index e33579f8..c13bca92 100644 --- a/cmd/dfs/cmd/server_test.go +++ b/cmd/dfs/cmd/server_test.go @@ -17,8 +17,11 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/cmd/common" "github.com/fairdatasociety/fairOS-dfs/pkg/api" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/dfs" mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/ensm/eth/mock" @@ -43,11 +46,18 @@ func randStringRunes(n int) string { } func TestApis(t *testing.T) { - mockClient := mock.NewMockBeeClient() + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) ens := mock2.NewMockNamespaceManager() - logger := logging.New(io.Discard, logrus.ErrorLevel) - users := user.NewUsers(mockClient, ens, logger) + users := user.NewUsers(mockClient, ens, 500, 0, logger) dfsApi := dfs.NewMockDfsAPI(mockClient, users, logger) handler = api.NewMockHandler(dfsApi, logger, []string{"http://localhost:3000"}) defer handler.Close() @@ -64,7 +74,7 @@ func TestApis(t *testing.T) { } }() - // wait 10 seconds for the server to start + // wait for the server to start <-time.After(time.Second * 3) t.Run("login-fail-test", func(t *testing.T) { c := http.Client{Timeout: time.Duration(1) * time.Minute} diff --git a/go.mod b/go.mod index 9d895f08..304042b8 100644 --- a/go.mod +++ b/go.mod @@ -1,118 +1,161 @@ module github.com/fairdatasociety/fairOS-dfs -go 1.20 +go 1.21 require ( github.com/btcsuite/btcd v0.22.3 github.com/c-bata/go-prompt v0.2.6 github.com/dustin/go-humanize v1.0.1 - github.com/ethereum/go-ethereum v1.12.2 - github.com/ethersphere/bee v1.17.3 + github.com/ethereum/go-ethereum v1.13.4 + github.com/ethersphere/bee v1.17.6 github.com/ethersphere/bmt v0.1.4 github.com/fairdatasociety/fairOS-dfs-utils v0.0.0-20221230123929-aec4ed8b854d - github.com/golang-jwt/jwt/v5 v5.0.0 + github.com/golang-jwt/jwt/v5 v5.2.0 github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/gorilla/mux v1.8.0 github.com/gorilla/securecookie v1.1.1 - github.com/gorilla/websocket v1.5.0 - github.com/hashicorp/golang-lru v1.0.2 + github.com/gorilla/websocket v1.5.1 + github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/klauspost/pgzip v1.2.6 github.com/miguelmota/go-ethereum-hdwallet v0.1.1 github.com/mitchellh/go-homedir v1.1.0 github.com/plexsysio/taskmanager v0.0.0-20211220123746-de5ebdd49ae2 - github.com/rs/cors v1.9.0 + github.com/rs/cors v1.10.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 - github.com/spf13/viper v1.16.0 + github.com/spf13/viper v1.18.1 github.com/stretchr/testify v1.8.4 github.com/swaggo/http-swagger v1.3.4 github.com/swaggo/swag v1.16.2 github.com/tinygrasshopper/bettercsv v0.0.1 github.com/tyler-smith/go-bip39 v1.1.0 github.com/wealdtech/go-ens/v3 v3.5.5 - go.uber.org/goleak v1.2.1 - golang.org/x/crypto v0.15.0 - golang.org/x/term v0.14.0 + go.uber.org/goleak v1.3.0 + golang.org/x/crypto v0.16.0 + golang.org/x/term v0.15.0 gopkg.in/yaml.v2 v2.4.0 resenje.org/jsonhttp v0.2.3 ) require ( + github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect github.com/KyleBanks/depth v1.2.1 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/armon/go-radix v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.7.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect + github.com/casbin/casbin/v2 v2.35.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.12.1 // indirect + github.com/crate-crypto/go-kzg-4844 v0.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.1.0 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect + github.com/ethereum/c-kzg-4844 v0.3.1 // indirect + github.com/ethersphere/go-price-oracle-abi v0.1.0 // indirect + github.com/ethersphere/go-storage-incentives-abi v0.5.0 // indirect + github.com/ethersphere/go-sw3-abi v0.4.0 // indirect + github.com/ethersphere/langos v1.0.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/spec v0.20.7 // indirect github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/go-playground/validator/v10 v10.11.1 // indirect github.com/go-stack/stack v1.8.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.4.0 // indirect + github.com/gorilla/handlers v1.5.1 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/ipfs/go-cid v0.3.2 // indirect + github.com/ipfs/go-cid v0.4.1 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.15.15 // indirect - github.com/klauspost/cpuid/v2 v2.2.1 // indirect + github.com/klauspost/compress v1.17.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/leodido/go-urn v1.2.1 // indirect + github.com/libp2p/go-buffer-pool v0.1.0 // indirect + github.com/libp2p/go-libp2p v0.30.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mattn/go-tty v0.0.3 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/miekg/dns v1.1.55 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multibase v0.1.1 // indirect - github.com/multiformats/go-multihash v0.2.1 // indirect + github.com/multiformats/go-multiaddr v0.11.0 // indirect + github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multicodec v0.9.0 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect + github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkg/term v1.2.0-beta.2 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.14.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.39.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/sourcegraph/conc v0.3.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/afero v1.9.5 // indirect - github.com/spf13/cast v1.5.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/subosito/gotenv v1.4.2 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/supranational/blst v0.3.11 // indirect github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a // indirect - github.com/tklauser/go-sysconf v0.3.9 // indirect - github.com/tklauser/numcpus v0.4.0 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/uber/jaeger-client-go v2.24.0+incompatible // indirect + github.com/uber/jaeger-lib v2.2.0+incompatible // indirect + github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/wealdtech/go-multicodec v1.4.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - go.uber.org/atomic v1.10.0 // indirect - golang.org/x/exp v0.0.0-20230810033253-352e893a4cad // indirect - golang.org/x/net v0.18.0 // indirect - golang.org/x/sys v0.14.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/mod v0.14.0 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/sync v0.5.0 // indirect + golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.15.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - lukechampine.com/blake3 v1.1.7 // indirect + lukechampine.com/blake3 v1.2.1 // indirect + resenje.org/multex v0.1.0 // indirect + resenje.org/singleflight v0.4.0 // indirect + resenje.org/web v0.9.2 // indirect + rsc.io/tmplfunc v0.0.3 // indirect ) replace github.com/codahale/hdrhistogram => github.com/HdrHistogram/hdrhistogram-go v0.0.0-20200919145931-8dac23c8dac1 diff --git a/go.sum b/go.sum index 91445747..da1c8dd5 100644 --- a/go.sum +++ b/go.sum @@ -4,40 +4,18 @@ cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSR cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= @@ -56,13 +34,21 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/HdrHistogram/hdrhistogram-go v0.0.0-20200919145931-8dac23c8dac1 h1:nEjGZtKHMK92888VT6XkzKwyiW14v5FFRGeWq2uV7N0= +github.com/HdrHistogram/hdrhistogram-go v0.0.0-20200919145931-8dac23c8dac1/go.mod h1:nxrse8/Tzg2tg3DZcZjm6qEclQKK70g0KxO61gFFZD4= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= +github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= +github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -71,6 +57,8 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= +github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= @@ -85,6 +73,7 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.7.0 h1:YjAGVd3XmtK9ktAbX8Zg2g2PwLIMjGREZJHlV4j7NEo= +github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= @@ -110,6 +99,8 @@ github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46f github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/c-bata/go-prompt v0.2.6 h1:POP+nrHE+DfLYx370bedwNhsqmpCUynWPxuHi0C5vZI= github.com/c-bata/go-prompt v0.2.6/go.mod h1:/LMAke8wD2FsNu9EXNdHxNLbd9MedkPnCdfpU9wwHfY= +github.com/casbin/casbin/v2 v2.35.0 h1:f0prVg9LgTJTihjAxWEZhfJptXvah1GpZh12sb5KXNA= +github.com/casbin/casbin/v2 v2.35.0/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRtgrQfcJqHg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= @@ -122,33 +113,43 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= -github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= -github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk= -github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= +github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= +github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvDmeyi+l/FS/VtA00IWvjeFvjen1m1l1A= +github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo= +github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= +github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= -github.com/consensys/gnark-crypto v0.10.0 h1:zRh22SR7o4K35SoNqouS9J/TKHTyU2QWaj5ldehyXtA= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-kzg-4844 v0.3.0 h1:UBlWE0CgyFqqzTI+IFyCzA7A3Zw4iip6uzRv5NIXG0A= +github.com/crate-crypto/go-kzg-4844 v0.3.0/go.mod h1:SBP7ikXEgDnUPONgm33HtuDZEDtWa3L4QtN1ocJSEQ4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI= github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= -github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= +github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= +github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= @@ -160,39 +161,46 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/c-kzg-4844 v0.3.1 h1:sR65+68+WdnMKxseNWxSJuAv2tsUrihTpVBTfM/U5Zg= +github.com/ethereum/c-kzg-4844 v0.3.1/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.10.4/go.mod h1:nEE0TP5MtxGzOMd7egIrbPJMQBnhVU3ELNxhBglIzhg= -github.com/ethereum/go-ethereum v1.12.2 h1:eGHJ4ij7oyVqUQn48LBz3B7pvQ8sV0wGJiIE6gDq/6Y= -github.com/ethereum/go-ethereum v1.12.2/go.mod h1:1cRAEV+rp/xX0zraSCBnu9Py3HQ+geRMj3HdR+k0wfI= -github.com/ethersphere/bee v1.17.3 h1:f1YmN+IFpgfdqPgBTNOwCk//XeCz2ze2CKAmgCmA+W8= -github.com/ethersphere/bee v1.17.3/go.mod h1:CSAz9XYUHZMOBDH4e6ybjv/E1o7bK4vhYMbmgHND8nA= +github.com/ethereum/go-ethereum v1.13.4 h1:25HJnaWVg3q1O7Z62LaaI6S9wVq8QCw3K88g8wEzrcM= +github.com/ethereum/go-ethereum v1.13.4/go.mod h1:I0U5VewuuTzvBtVzKo7b3hJzDhXOUtn9mJW7SsIPB0Q= +github.com/ethersphere/bee v1.17.6 h1:LdlKWAMesuBqZ4nCuaS6aEKn6i7OJPb7QONEgWe/c1o= +github.com/ethersphere/bee v1.17.6/go.mod h1:dxpOp5CQsQvldQ6LQXeMaGGaFRXcLB8M1Yn/PI5I45M= github.com/ethersphere/bmt v0.1.4 h1:+rkWYNtMgDx6bkNqGdWu+U9DgGI1rRZplpSW3YhBr1Q= github.com/ethersphere/bmt v0.1.4/go.mod h1:Yd8ft1U69WDuHevZc/rwPxUv1rzPSMpMnS6xbU53aY8= +github.com/ethersphere/go-price-oracle-abi v0.1.0 h1:yg/hK8nETNvk+GEBASlbakMFv/CVp7HXiycrHw1pRV8= +github.com/ethersphere/go-price-oracle-abi v0.1.0/go.mod h1:sI/Qj4/zJ23/b1enzwMMv0/hLTpPNVNacEwCWjo6yBk= +github.com/ethersphere/go-storage-incentives-abi v0.5.0 h1:dd01OZmPraCjOIiSX5FsCfFFwUR2b9PuTO/LDcYxS+s= +github.com/ethersphere/go-storage-incentives-abi v0.5.0/go.mod h1:SXvJVtM4sEsaSKD0jc1ClpDLw8ErPoROZDme4Wrc/Nc= +github.com/ethersphere/go-sw3-abi v0.4.0 h1:T3ANY+ktWrPAwe2U0tZi+DILpkHzto5ym/XwV/Bbz8g= +github.com/ethersphere/go-sw3-abi v0.4.0/go.mod h1:BmpsvJ8idQZdYEtWnvxA8POYQ8Rl/NhyCdF0zLMOOJU= +github.com/ethersphere/langos v1.0.0 h1:NBtNKzXTTRSue95uOlzPN4py7Aofs0xWPzyj4AI1Vcc= +github.com/ethersphere/langos v1.0.0/go.mod h1:dlcN2j4O8sQ+BlCaxeBu43bgr4RQ+inJ+pHwLeZg5Tw= github.com/fairdatasociety/fairOS-dfs-utils v0.0.0-20221230123929-aec4ed8b854d h1:4QgyFcv+J02YdPZ92oiCjjQ8QtGl/UbLTAypKb+WfZc= github.com/fairdatasociety/fairOS-dfs-utils v0.0.0-20221230123929-aec4ed8b854d/go.mod h1:f4lfxKoK1n4S+2II4lyVmuvWISAMThqH+uc/XL62tzU= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -210,44 +218,49 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= +github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/golang-jwt/jwt/v4 v4.3.0 h1:kHL1vqdqWNfATmA0FNMdmZNMyZI1U6O31X4rlIPoBog= -github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= -github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= +github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= @@ -256,6 +269,7 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -266,58 +280,50 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= +github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= +github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= -github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 h1:3JQNjnMRil1yD0IfZKHF9GxxWKDJGj8I0IqOUol//sw= +github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= @@ -325,10 +331,10 @@ github.com/holiman/uint256 v1.2.3 h1:K8UWO1HUJpRMXBxbmaY1Y8IAMZC/RsKB+ArEnnK4l5o github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= -github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= +github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= +github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -340,10 +346,11 @@ github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19y github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= -github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= -github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= +github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= +github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -362,33 +369,41 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= -github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= +github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= +github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.1 h1:U33DW0aiEj633gHYw3LoDNfkDiYnE5Q8M/TKJn2f2jI= -github.com/klauspost/cpuid/v2 v2.2.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= +github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= +github.com/libp2p/go-libp2p v0.30.0 h1:9EZwFtJPFBcs/yJTnP90TpN1hgrT/EsFfM+OZuwV87U= +github.com/libp2p/go-libp2p v0.30.0/go.mod h1:nr2g5V7lfftwgiJ78/HrID+pwvayLyqKCEirT2Y3Byg= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -409,8 +424,9 @@ github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXT github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -423,18 +439,27 @@ github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvr github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= +github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/miguelmota/go-ethereum-hdwallet v0.1.1 h1:zdXGlHao7idpCBjEGTXThVAtMKs+IxAgivZ75xqkWK0= github.com/miguelmota/go-ethereum-hdwallet v0.1.1/go.mod h1:f9m9uXokAHA6WNoYOPjj4AqjJS5pquQRiYYj/XSyPYc= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= +github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= @@ -442,10 +467,21 @@ github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aG github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= -github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= -github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= -github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= +github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= +github.com/multiformats/go-multiaddr v0.11.0 h1:XqGyJ8ufbCE0HmTDwx2kPdsrQ36AGPZNZX6s6xfJH10= +github.com/multiformats/go-multiaddr v0.11.0/go.mod h1:gWUm0QLR4thQ6+ZF6SXUw8YjtwQSPapICM+NmCkxHSM= +github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= +github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= +github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= +github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= +github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= +github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q= +github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -453,6 +489,8 @@ github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= @@ -460,32 +498,38 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pkg/term v1.2.0-beta.2 h1:L3y/h2jkuBVFdWiJvNfYfKmzcCnILw7mJWm2JQuMppw= github.com/pkg/term v1.2.0-beta.2/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= github.com/plexsysio/taskmanager v0.0.0-20211220123746-de5ebdd49ae2 h1:Y3ImPze8NO2iKaPFJ0LkqNFlkC8LjifxxxwEGxbkQ+A= github.com/plexsysio/taskmanager v0.0.0-20211220123746-de5ebdd49ae2/go.mod h1:mrUMc3N31sq3lEqDyCkbw0dBfOtNZyh+z8kHnNnpd7k= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= @@ -493,13 +537,13 @@ github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQg github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= -github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= @@ -510,13 +554,20 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= -github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= +github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -529,26 +580,27 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= -github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= -github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= -github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= +github.com/spf13/viper v1.18.1 h1:rmuU42rScKWlhhJDyXZRKJQHXFX02chSVW1IvkPGiVM= +github.com/spf13/viper v1.18.1/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= +github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -557,17 +609,16 @@ github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= -github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= +github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a h1:kAe4YSu0O0UFn1DowNo2MY5p6xzqtJ/wQ7LZynSvGaY= github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w= github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64a5ww= @@ -576,32 +627,41 @@ github.com/swaggo/swag v1.16.2 h1:28Pp+8DkQoV+HLzLx8RGJZXNGKbFqnuvSbAAtoxiY04= github.com/swaggo/swag v1.16.2/go.mod h1:6YzXnDcpr0767iOejs318CwYkCQqyGer6BizOg03f+E= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tinygrasshopper/bettercsv v0.0.1 h1:N96aWjbUBN2q+KotgSI9FMR+1Y4IIBMVMPiL8qASK0k= github.com/tinygrasshopper/bettercsv v0.0.1/go.mod h1:0pXjg6Vm8+zAkvosNH2S0dx8gc7H1hDIV0pMzmq1vRI= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= -github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= -github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= -github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= -github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o= -github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= +github.com/uber/jaeger-client-go v2.24.0+incompatible h1:CGchgJcHsDd2jWnaL4XngByMrXoGHh3n8oCqAKx0uMo= +github.com/uber/jaeger-client-go v2.24.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= +github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/urfave/cli/v2 v2.24.1 h1:/QYYr7g0EhwXEML8jO+8OYt5trPnLHS0p3mrgExJ5NU= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/vmihailenco/msgpack/v5 v5.3.4 h1:qMKAwOV+meBw2Y8k9cVwAy7qErtYCwBzZ2ellBfvnqc= +github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/wealdtech/go-ens/v3 v3.5.5 h1:/jq3CDItK0AsFnZtiFJK44JthkAMD5YE3WAJOh4i7lc= github.com/wealdtech/go-ens/v3 v3.5.5/go.mod h1:w0EDKIm0dIQnqEKls6ORat/or+AVfPEdEXVfN71EeEE= github.com/wealdtech/go-multicodec v1.4.0 h1:iq5PgxwssxnXGGPTIK1srvt6U5bJwIp7k6kBrudIWxg= github.com/wealdtech/go-multicodec v1.4.0/go.mod h1:aedGMaTeYkIqi/KCPre1ho5rTb3hGpu/snBOS3GQLw4= github.com/wealdtech/go-string2eth v1.1.0 h1:USJQmysUrBYYmZs7d45pMb90hRSyEwizP7lZaOZLDAw= +github.com/wealdtech/go-string2eth v1.1.0/go.mod h1:RUzsLjJtbZaJ/3UKn9kY19a/vCCUHtEWoUW3uiK6yGU= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= @@ -610,22 +670,22 @@ gitlab.com/nolash/go-mockbytes v0.0.7/go.mod h1:KKOpNTT39j2Eo+P6uUTOncntfeKY6AFh go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -633,10 +693,9 @@ golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= -golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -647,11 +706,8 @@ golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm0 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230810033253-352e893a4cad h1:g0bG7Z4uG+OgH2QDODnjp6ggkk1bJDsINcuWmJN1iJU= -golang.org/x/exp v0.0.0-20230810033253-352e893a4cad/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -663,21 +719,16 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -693,43 +744,23 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -737,11 +768,10 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -758,56 +788,40 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -819,9 +833,9 @@ golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -845,34 +859,11 @@ golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8= golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk= @@ -880,6 +871,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= @@ -893,25 +886,11 @@ google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -926,73 +905,36 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= -gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1005,6 +947,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= @@ -1013,15 +956,18 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= -lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= resenje.org/jsonhttp v0.2.3 h1:YEsBt1kxg3SB0+gDNmcxd9aredgtBlA70I0v+QsPysc= resenje.org/jsonhttp v0.2.3/go.mod h1:EDyeguyTWj2fU3D3SCE0qNTgthzyEkHYLM1uu0uikHU= +resenje.org/multex v0.1.0 h1:am9Ndt8dIAeGVaztD8ClsSX+e0EP3mj6UdsvjukKZig= +resenje.org/multex v0.1.0/go.mod h1:3rHOoMrzqLNzgGWPcl/1GfzN52g7iaPXhbvTQ8TjGaM= +resenje.org/singleflight v0.4.0 h1:NdOEhCxEikK2S2WxGjZV9EGSsItolQKslOOi6pE1tJc= +resenje.org/singleflight v0.4.0/go.mod h1:lAgQK7VfjG6/pgredbQfmV0RvG/uVhKo6vSuZ0vCWfk= +resenje.org/web v0.9.2 h1:woq1MMoOr9WbsX6XU7JgWFJw2IOb2F06jlTqI9MWex0= +resenje.org/web v0.9.2/go.mod h1:YtrYB6u+gwTcTuIr2lBzsU6TVlUlbXNmkTB7rQz0O5Y= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= diff --git a/pkg/api/dir_ls.go b/pkg/api/dir_ls.go index 48ab7e79..5d32d73a 100644 --- a/pkg/api/dir_ls.go +++ b/pkg/api/dir_ls.go @@ -20,13 +20,11 @@ import ( "net/http" "github.com/fairdatasociety/fairOS-dfs/pkg/auth" - - "resenje.org/jsonhttp" - "github.com/fairdatasociety/fairOS-dfs/pkg/dfs" "github.com/fairdatasociety/fairOS-dfs/pkg/dir" "github.com/fairdatasociety/fairOS-dfs/pkg/file" p "github.com/fairdatasociety/fairOS-dfs/pkg/pod" + "resenje.org/jsonhttp" ) // ListFileResponse is used to list directories and files diff --git a/pkg/api/file_upload.go b/pkg/api/file_upload.go index 914fb36c..aafbc4b8 100644 --- a/pkg/api/file_upload.go +++ b/pkg/api/file_upload.go @@ -17,6 +17,7 @@ limitations under the License. package api import ( + "errors" "mime/multipart" "net/http" "strconv" @@ -152,7 +153,7 @@ func (h *Handler) FileUploadHandler(w http.ResponseWriter, r *http.Request) { } err = h.handleFileUpload(podName, file.Filename, sessionId, file.Size, fd, podPath, compression, uint32(bs), overwrite) if err != nil { - if err == dfs.ErrPodNotOpen { + if errors.Is(err, dfs.ErrPodNotOpen) { h.logger.Errorf("file upload: %v", err) jsonhttp.BadRequest(w, &response{Message: "file upload: " + err.Error()}) return diff --git a/pkg/api/handler.go b/pkg/api/handler.go index 4361124d..9186da06 100644 --- a/pkg/api/handler.go +++ b/pkg/api/handler.go @@ -18,12 +18,20 @@ package api import ( "context" + "fmt" + "time" "github.com/fairdatasociety/fairOS-dfs/pkg/contracts" "github.com/fairdatasociety/fairOS-dfs/pkg/dfs" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" ) +const ( + defaultFeedCacheSize = -1 +) + +var errInvalidDuration = fmt.Errorf("invalid duration") + // Handler is the api handler type Handler struct { ctx context.Context @@ -42,6 +50,8 @@ type Options struct { EnsConfig *contracts.ENSConfig SubscriptionConfig *contracts.SubscriptionConfig Logger logging.Logger + FeedCacheSize int + FeedCacheTTL string } // New returns a new handler @@ -53,6 +63,17 @@ func New(ctx context.Context, opts *Options) (*Handler, error) { SubscriptionConfig: opts.SubscriptionConfig, Logger: opts.Logger, } + if opts.FeedCacheSize == 0 { + opts.FeedCacheSize = defaultFeedCacheSize + } else { + dfsOpts.FeedCacheSize = opts.FeedCacheSize + + ttl, err := time.ParseDuration(opts.FeedCacheTTL) + if err != nil { + return nil, errInvalidDuration + } + dfsOpts.FeedCacheTTL = ttl + } api, err := dfs.NewDfsAPI(ctx, dfsOpts) if err != nil { return nil, err diff --git a/pkg/api/pod_delete.go b/pkg/api/pod_delete.go index ea0dda84..24f9efbc 100644 --- a/pkg/api/pod_delete.go +++ b/pkg/api/pod_delete.go @@ -18,6 +18,7 @@ package api import ( "encoding/json" + "errors" "net/http" "github.com/fairdatasociety/fairOS-dfs/pkg/auth" @@ -81,7 +82,7 @@ func (h *Handler) PodDeleteHandler(w http.ResponseWriter, r *http.Request) { // delete pod err = h.dfsAPI.DeletePod(podName, sessionId) if err != nil { - if err == dfs.ErrUserNotLoggedIn { + if errors.Is(err, dfs.ErrUserNotLoggedIn) { h.logger.Errorf("delete pod: %v", err) jsonhttp.BadRequest(w, &response{Message: "delete pod: " + err.Error()}) return diff --git a/pkg/api/pod_open.go b/pkg/api/pod_open.go index 9f54247a..143b2c80 100644 --- a/pkg/api/pod_open.go +++ b/pkg/api/pod_open.go @@ -18,6 +18,7 @@ package api import ( "encoding/json" + "errors" "net/http" "github.com/fairdatasociety/fairOS-dfs/pkg/auth" @@ -80,8 +81,8 @@ func (h *Handler) PodOpenHandler(w http.ResponseWriter, r *http.Request) { // open pod _, err = h.dfsAPI.OpenPod(pod, sessionId) if err != nil { - if err == dfs.ErrUserNotLoggedIn || - err == p.ErrInvalidPodName { + if errors.Is(err, dfs.ErrUserNotLoggedIn) || + errors.Is(err, p.ErrInvalidPodName) { h.logger.Errorf("pod open: %v", err) jsonhttp.NotFound(w, &response{Message: "pod open: " + err.Error()}) return diff --git a/pkg/blockstore/bee/client.go b/pkg/blockstore/bee/client.go index 04a997a5..2ac6e6ab 100644 --- a/pkg/blockstore/bee/client.go +++ b/pkg/blockstore/bee/client.go @@ -19,7 +19,6 @@ package bee import ( "bytes" "context" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -31,7 +30,7 @@ import ( "github.com/ethersphere/bee/pkg/swarm" bmtlegacy "github.com/ethersphere/bmt/legacy" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2/expirable" "github.com/sirupsen/logrus" "golang.org/x/crypto/sha3" ) @@ -63,9 +62,9 @@ type Client struct { url string client *http.Client hasher *bmtlegacy.Hasher - chunkCache *lru.Cache - uploadBlockCache *lru.Cache - downloadBlockCache *lru.Cache + chunkCache *lru.LRU[string, []byte] + uploadBlockCache *lru.LRU[string, []byte] + downloadBlockCache *lru.LRU[string, []byte] postageBlockId string logger logging.Logger isProxy bool @@ -100,19 +99,9 @@ type beeError struct { // NewBeeClient creates a new client which connects to the Swarm bee node to access the Swarm network. func NewBeeClient(apiUrl, postageBlockId string, shouldPin bool, logger logging.Logger) *Client { p := bmtlegacy.NewTreePool(hashFunc, swarm.Branches, bmtlegacy.PoolSize) - cache, err := lru.New(chunkCacheSize) - if err != nil { - logger.Warningf("could not initialise chunkCache. system will be slow") - } - uploadBlockCache, err := lru.New(uploadBlockCacheSize) - if err != nil { - logger.Warningf("could not initialise blockCache. system will be slow") - } - downloadBlockCache, err := lru.New(downloadBlockCacheSize) - if err != nil { - logger.Warningf("could not initialise blockCache. system will be slow") - } - + cache := lru.NewLRU(chunkCacheSize, func(key string, value []byte) {}, 0) + uploadBlockCache := lru.NewLRU(uploadBlockCacheSize, func(key string, value []byte) {}, 0) + downloadBlockCache := lru.NewLRU(downloadBlockCacheSize, func(key string, value []byte) {}, 0) return &Client{ url: apiUrl, client: createHTTPClient(), @@ -163,14 +152,14 @@ func (s *Client) checkBee(isProxy bool) (string, error) { if err != nil { return "", err } - response, err := s.client.Do(req) + req.Close = true + // skipcq: GO-S2307 + response, err := s.Do(req) if err != nil { return "", err } - // skipcq: GO-S2307 defer response.Body.Close() - req.Close = true data, err := io.ReadAll(response.Body) if err != nil { return "", err @@ -178,6 +167,11 @@ func (s *Client) checkBee(isProxy bool) (string, error) { return string(data), nil } +// Do dispatches the HTTP request to the network +func (s *Client) Do(req *http.Request) (*http.Response, error) { + return s.client.Do(req) +} + // UploadSOC is used construct and send a Single Owner Chunk to the Swarm bee client. func (s *Client) UploadSOC(owner, id, signature string, data []byte) (address []byte, err error) { to := time.Now() @@ -188,6 +182,7 @@ func (s *Client) UploadSOC(owner, id, signature string, data []byte) (address [] if err != nil { return nil, err } + req.Close = true // the postage block id to store the SOC chunk req.Header.Set(swarmPostageBatchId, s.postageBlockId) @@ -200,7 +195,7 @@ func (s *Client) UploadSOC(owner, id, signature string, data []byte) (address [] req.Header.Set(swarmPinHeader, "true") } - response, err := s.client.Do(req) + response, err := s.Do(req) if err != nil { return nil, err } @@ -208,8 +203,6 @@ func (s *Client) UploadSOC(owner, id, signature string, data []byte) (address [] // skipcq: GO-S2307 defer response.Body.Close() - req.Close = true - addrData, err := io.ReadAll(response.Body) if err != nil { return nil, errors.New("error uploading data") @@ -230,9 +223,6 @@ func (s *Client) UploadSOC(owner, id, signature string, data []byte) (address [] return nil, err } - if s.inChunkCache(addrResp.Reference.String()) { - s.addToChunkCache(addrResp.Reference.String(), data) - } fields := logrus.Fields{ "reference": addrResp.Reference.String(), "duration": time.Since(to).String(), @@ -249,6 +239,7 @@ func (s *Client) UploadChunk(ch swarm.Chunk) (address []byte, err error) { if err != nil { return nil, err } + req.Close = true if s.shouldPin { req.Header.Set(swarmPinHeader, "true") @@ -259,15 +250,13 @@ func (s *Client) UploadChunk(ch swarm.Chunk) (address []byte, err error) { req.Header.Set(swarmDeferredUploadHeader, "true") - response, err := s.client.Do(req) + response, err := s.Do(req) if err != nil { return nil, err } // skipcq: GO-S2307 defer response.Body.Close() - req.Close = true - addrData, err := io.ReadAll(response.Body) if err != nil { return nil, errors.New("error uploading data") @@ -288,9 +277,6 @@ func (s *Client) UploadChunk(ch swarm.Chunk) (address []byte, err error) { return nil, err } - if s.inChunkCache(ch.Address().String()) { - s.addToChunkCache(ch.Address().String(), ch.Data()) - } fields := logrus.Fields{ "reference": ch.Address().String(), "duration": time.Since(to).String(), @@ -304,9 +290,6 @@ func (s *Client) UploadChunk(ch swarm.Chunk) (address []byte, err error) { func (s *Client) DownloadChunk(ctx context.Context, address []byte) (data []byte, err error) { to := time.Now() addrString := swarm.NewAddress(address).String() - if s.inChunkCache(addrString) { - return s.getFromChunkCache(swarm.NewAddress(address).String()), nil - } path := chunkUploadDownloadUrl + "/" + addrString fullUrl := fmt.Sprintf(s.url + path) @@ -314,18 +297,17 @@ func (s *Client) DownloadChunk(ctx context.Context, address []byte) (data []byte if err != nil { return nil, err } + req.Close = true req = req.WithContext(ctx) - response, err := s.client.Do(req) + response, err := s.Do(req) if err != nil { return nil, err } // skipcq: GO-S2307 defer response.Body.Close() - req.Close = true - if response.StatusCode != http.StatusOK { return nil, errors.New("error downloading data") } @@ -334,8 +316,6 @@ func (s *Client) DownloadChunk(ctx context.Context, address []byte) (data []byte if err != nil { return nil, errors.New("error downloading data") } - - s.addToChunkCache(addrString, data) fields := logrus.Fields{ "reference": addrString, "duration": time.Since(to).String(), @@ -358,6 +338,7 @@ func (s *Client) UploadBlob(data []byte, tag uint32, encrypt bool) (address []by if err != nil { return nil, err } + req.Close = true if s.shouldPin { req.Header.Set(swarmPinHeader, "true") @@ -376,15 +357,13 @@ func (s *Client) UploadBlob(data []byte, tag uint32, encrypt bool) (address []by req.Header.Set(swarmDeferredUploadHeader, "true") - response, err := s.client.Do(req) + response, err := s.Do(req) if err != nil { return nil, err } // skipcq: GO-S2307 defer response.Body.Close() - req.Close = true - respData, err := io.ReadAll(response.Body) if err != nil { return nil, errors.New("error uploading blob") @@ -411,10 +390,8 @@ func (s *Client) UploadBlob(data []byte, tag uint32, encrypt bool) (address []by } s.logger.WithFields(fields).Log(logrus.DebugLevel, "upload blob: ") - // add the data and ref if itis not in cache - if !s.inBlockCache(s.uploadBlockCache, string(data)) { - s.addToBlockCache(s.uploadBlockCache, string(data), resp.Reference.Bytes()) - } + // add the data in cache + s.addToBlockCache(s.uploadBlockCache, string(data), resp.Reference.Bytes()) return resp.Reference.Bytes(), nil } @@ -434,16 +411,15 @@ func (s *Client) DownloadBlob(address []byte) ([]byte, int, error) { if err != nil { return nil, http.StatusNotFound, err } + req.Close = true - response, err := s.client.Do(req) + response, err := s.Do(req) if err != nil { return nil, http.StatusNotFound, err } // skipcq: GO-S2307 defer response.Body.Close() - req.Close = true - respData, err := io.ReadAll(response.Body) if err != nil { return nil, response.StatusCode, errors.New("error downloading blob") @@ -481,19 +457,18 @@ func (s *Client) UploadBzz(data []byte, fileName string) (address []byte, err er if err != nil { return nil, err } + req.Close = true req.Header.Set(swarmPostageBatchId, s.postageBlockId) req.Header.Set(contentTypeHeader, "application/json") - response, err := s.client.Do(req) + response, err := s.Do(req) if err != nil { return nil, err } // skipcq: GO-S2307 defer response.Body.Close() - req.Close = true - respData, err := io.ReadAll(response.Body) if err != nil { return nil, errors.New("error downloading bzz") @@ -540,16 +515,15 @@ func (s *Client) DownloadBzz(address []byte) ([]byte, int, error) { if err != nil { return nil, http.StatusNotFound, err } + req.Close = true - response, err := s.client.Do(req) + response, err := s.Do(req) if err != nil { return nil, http.StatusNotFound, err } // skipcq: GO-S2307 defer response.Body.Close() - req.Close = true - respData, err := io.ReadAll(response.Body) if err != nil { return nil, response.StatusCode, errors.New("error downloading bzz") @@ -591,20 +565,22 @@ func (s *Client) DeleteReference(address []byte) error { if err != nil { return err } + req.Close = true - response, err := s.client.Do(req) + response, err := s.Do(req) if err != nil { return err } defer response.Body.Close() - req.Close = true - if response.StatusCode != http.StatusOK { + if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusNotFound { respData, err := io.ReadAll(response.Body) if err != nil { return err } return fmt.Errorf("failed to unpin reference : %s", respData) + } else { + _, _ = io.Copy(io.Discard, response.Body) } fields := logrus.Fields{ @@ -639,16 +615,15 @@ func (s *Client) CreateTag(address []byte) (uint32, error) { if err != nil { return 0, err } + req.Close = true - response, err := s.client.Do(req) + response, err := s.Do(req) if err != nil { return 0, err } // skipcq: GO-S2307 defer response.Body.Close() - req.Close = true - respData, err := io.ReadAll(response.Body) if err != nil { return 0, errors.New("error create tag") @@ -693,16 +668,15 @@ func (s *Client) GetTag(tag uint32) (int64, int64, int64, error) { if err != nil { return 0, 0, 0, err } + req.Close = true - response, err := s.client.Do(req) + response, err := s.Do(req) if err != nil { return 0, 0, 0, err } // skipcq: GO-S2307 defer response.Body.Close() - req.Close = true - respData, err := io.ReadAll(response.Body) if err != nil { return 0, 0, 0, errors.New("error getting tag") @@ -743,52 +717,22 @@ func createHTTPClient() *http.Client { return client } -func (s *Client) addToChunkCache(key string, value []byte) { - if s.chunkCache != nil { - s.chunkCache.Add(key, hex.EncodeToString(value)) - } -} - -func (s *Client) inChunkCache(key string) bool { - if s.chunkCache != nil { - return s.chunkCache.Contains(key) - } - return false -} - -func (s *Client) getFromChunkCache(key string) []byte { - if s.chunkCache != nil { - value, ok := s.chunkCache.Get(key) - if ok { - data, err := hex.DecodeString(fmt.Sprintf("%v", value)) - if err != nil { - return nil - } - return data - } - return nil - } - return nil -} - -func (*Client) addToBlockCache(cache *lru.Cache, key string, value []byte) { +func (*Client) addToBlockCache(cache *lru.LRU[string, []byte], key string, value []byte) { if cache != nil { cache.Add(key, value) } } -func (*Client) inBlockCache(cache *lru.Cache, key string) bool { - if cache != nil { - return cache.Contains(key) - } - return false +func (*Client) inBlockCache(cache *lru.LRU[string, []byte], key string) bool { + _, in := cache.Get(key) + return in } -func (*Client) getFromBlockCache(cache *lru.Cache, key string) []byte { +func (*Client) getFromBlockCache(cache *lru.LRU[string, []byte], key string) []byte { if cache != nil { value, ok := cache.Get(key) if ok { - return value.([]byte) + return value } return nil } diff --git a/pkg/blockstore/bee/mock/client.go b/pkg/blockstore/bee/mock/client.go index c4ba87f9..cac9c765 100644 --- a/pkg/blockstore/bee/mock/client.go +++ b/pkg/blockstore/bee/mock/client.go @@ -1,171 +1,338 @@ -/* -Copyright © 2020 FairOS Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package mock import ( "context" + "crypto/ecdsa" "crypto/rand" "encoding/hex" - "errors" - "fmt" + "math/big" "net/http" + "net/http/httptest" "sync" + "testing" "time" - "github.com/fairdatasociety/fairOS-dfs/pkg/utils" - - "github.com/ethersphere/bee/pkg/soc" + "github.com/ethereum/go-ethereum/common" + accountingmock "github.com/ethersphere/bee/pkg/accounting/mock" + "github.com/ethersphere/bee/pkg/api" + "github.com/ethersphere/bee/pkg/auth" + mockauth "github.com/ethersphere/bee/pkg/auth/mock" + "github.com/ethersphere/bee/pkg/crypto" + "github.com/ethersphere/bee/pkg/feeds" + "github.com/ethersphere/bee/pkg/log" + p2pmock "github.com/ethersphere/bee/pkg/p2p/mock" + "github.com/ethersphere/bee/pkg/pingpong" + "github.com/ethersphere/bee/pkg/postage" + mockbatchstore "github.com/ethersphere/bee/pkg/postage/batchstore/mock" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + "github.com/ethersphere/bee/pkg/postage/postagecontract" + contractMock "github.com/ethersphere/bee/pkg/postage/postagecontract/mock" + "github.com/ethersphere/bee/pkg/pss" + "github.com/ethersphere/bee/pkg/resolver" + resolverMock "github.com/ethersphere/bee/pkg/resolver/mock" + "github.com/ethersphere/bee/pkg/settlement/pseudosettle" + chequebookmock "github.com/ethersphere/bee/pkg/settlement/swap/chequebook/mock" + "github.com/ethersphere/bee/pkg/settlement/swap/erc20" + erc20mock "github.com/ethersphere/bee/pkg/settlement/swap/erc20/mock" + swapmock "github.com/ethersphere/bee/pkg/settlement/swap/mock" + statestore "github.com/ethersphere/bee/pkg/statestore/mock" + "github.com/ethersphere/bee/pkg/status" + "github.com/ethersphere/bee/pkg/steward" + "github.com/ethersphere/bee/pkg/storage" + "github.com/ethersphere/bee/pkg/storage/inmemstore" + "github.com/ethersphere/bee/pkg/storageincentives" + "github.com/ethersphere/bee/pkg/storageincentives/staking" + mock2 "github.com/ethersphere/bee/pkg/storageincentives/staking/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/ethersphere/bee/pkg/swarm" + "github.com/ethersphere/bee/pkg/topology/lightnode" + topologymock "github.com/ethersphere/bee/pkg/topology/mock" + "github.com/ethersphere/bee/pkg/tracing" + "github.com/ethersphere/bee/pkg/transaction" + "github.com/ethersphere/bee/pkg/transaction/backendmock" + transactionmock "github.com/ethersphere/bee/pkg/transaction/mock" + "github.com/ethersphere/bee/pkg/util/testutil" ) -// BeeClient is a mock bee client -type BeeClient struct { - storer map[string][]byte - tagStorer map[uint32]int64 - storerMu sync.RWMutex -} +var ( + batchOk = make([]byte, 32) + BatchOkStr string +) -// NewMockBeeClient returns a mock bee client -func NewMockBeeClient() *BeeClient { - return &BeeClient{ - storer: make(map[string][]byte), - tagStorer: make(map[uint32]int64), - storerMu: sync.RWMutex{}, - } +// nolint:gochecknoinits +func init() { + _, _ = rand.Read(batchOk) + + BatchOkStr = hex.EncodeToString(batchOk) } -// CheckConnection checks connection -func (*BeeClient) CheckConnection() bool { - return true +type TestServerOptions struct { + Storer api.Storer + StateStorer storage.StateStorer + Resolver resolver.Interface + Pss pss.Interface + WsPath string + WsPingPeriod time.Duration + Logger log.Logger + PreventRedirect bool + Feeds feeds.Factory + CORSAllowedOrigins []string + PostageContract postagecontract.Interface + StakingContract staking.Contract + Post postage.Service + Steward steward.Interface + WsHeaders http.Header + Authenticator auth.Authenticator + DebugAPI bool + Restricted bool + DirectUpload bool + Probe *api.Probe + + Overlay swarm.Address + PublicKey ecdsa.PublicKey + PSSPublicKey ecdsa.PublicKey + EthereumAddress common.Address + BlockTime time.Duration + P2P *p2pmock.Service + Pingpong pingpong.Interface + TopologyOpts []topologymock.Option + AccountingOpts []accountingmock.Option + ChequebookOpts []chequebookmock.Option + SwapOpts []swapmock.Option + TransactionOpts []transactionmock.Option + + BatchStore postage.Storer + SyncStatus func() (bool, error) + + BackendOpts []backendmock.Option + Erc20Opts []erc20mock.Option + BeeMode api.BeeNodeMode + RedistributionAgent *storageincentives.Agent + NodeStatus *status.Service } -// UploadSOC uploads soc into swarm -func (m *BeeClient) UploadSOC(owner, id, signature string, data []byte) (address []byte, err error) { - m.storerMu.Lock() - defer m.storerMu.Unlock() - ch, err := utils.NewChunkWithoutSpan(data) - if err != nil { - return nil, err +func NewTestBeeServer(t *testing.T, o TestServerOptions) string { + t.Helper() + pk, _ := crypto.GenerateSecp256k1Key() + signer := crypto.NewDefaultSigner(pk) + + if o.Logger == nil { + o.Logger = log.Noop + } + if o.Resolver == nil { + o.Resolver = resolverMock.NewResolver() + } + if o.WsPingPeriod == 0 { + o.WsPingPeriod = 60 * time.Second + } + if o.Post == nil { + o.Post = mockpost.New() + } + if o.BatchStore == nil { + o.BatchStore = mockbatchstore.New(mockbatchstore.WithAcceptAllExistsFunc()) // default is with accept-all Exists() func } - idBytes, err := hex.DecodeString(id) - if err != nil { - return nil, err + if o.SyncStatus == nil { + o.SyncStatus = func() (bool, error) { return true, nil } } - ownerBytes, err := hex.DecodeString(owner) - if err != nil { - return nil, err + if o.Authenticator == nil { + o.Authenticator = &mockauth.Auth{ + EnforceFunc: func(_, _, _ string) (bool, error) { + return true, nil + }, + } + } + + topologyDriver := topologymock.NewTopologyDriver(o.TopologyOpts...) + acc := accountingmock.NewAccounting(o.AccountingOpts...) + settlement := swapmock.New(o.SwapOpts...) + chequebook := chequebookmock.NewChequebook(o.ChequebookOpts...) + ln := lightnode.NewContainer(o.Overlay) + + transaction := transactionmock.New(o.TransactionOpts...) + + storeRecipient := statestore.NewStateStore() + recipient := pseudosettle.New(nil, o.Logger, storeRecipient, nil, big.NewInt(10000), big.NewInt(10000), o.P2P) + + if o.StateStorer == nil { + o.StateStorer = storeRecipient } - signatureBytes, err := hex.DecodeString(signature) - if err != nil { - return nil, err + erc20 := erc20mock.New(o.Erc20Opts...) + backend := backendmock.New(o.BackendOpts...) + + var extraOpts = api.ExtraOptions{ + TopologyDriver: topologyDriver, + Accounting: acc, + Pseudosettle: recipient, + LightNodes: ln, + Swap: settlement, + Chequebook: chequebook, + Pingpong: o.Pingpong, + BlockTime: o.BlockTime, + Storer: o.Storer, + Resolver: o.Resolver, + Pss: o.Pss, + FeedFactory: o.Feeds, + Post: o.Post, + PostageContract: o.PostageContract, + Steward: o.Steward, + SyncStatus: o.SyncStatus, + Staking: o.StakingContract, + NodeStatus: o.NodeStatus, } - signed, err := soc.NewSigned(idBytes, ch, ownerBytes, signatureBytes) - if err != nil { - return nil, err + + // By default bee mode is set to full mode. + if o.BeeMode == api.UnknownMode { + o.BeeMode = api.FullMode } - signedChunk, err := signed.Chunk() - if err != nil { - return nil, err + o.CORSAllowedOrigins = append(o.CORSAllowedOrigins, "*") + s := api.New(o.PublicKey, o.PSSPublicKey, o.EthereumAddress, o.Logger, transaction, o.BatchStore, o.BeeMode, true, true, backend, o.CORSAllowedOrigins, inmemstore.New()) + testutil.CleanupCloser(t, s) + + s.SetP2P(o.P2P) + + if o.RedistributionAgent == nil { + o.RedistributionAgent, _ = createRedistributionAgentService(t, o.Overlay, o.StateStorer, erc20, transaction, backend, o.BatchStore) + s.SetRedistributionAgent(o.RedistributionAgent) } - if !soc.Valid(signedChunk) { - return nil, fmt.Errorf("soc chunk failed in validation") + testutil.CleanupCloser(t, o.RedistributionAgent) + + s.SetSwarmAddress(&o.Overlay) + s.SetProbe(o.Probe) + + noOpTracer, tracerCloser, _ := tracing.NewTracer(&tracing.Options{ + Enabled: false, + }) + testutil.CleanupCloser(t, tracerCloser) + + s.Configure(signer, o.Authenticator, noOpTracer, api.Options{ + CORSAllowedOrigins: o.CORSAllowedOrigins, + WsPingPeriod: o.WsPingPeriod, + Restricted: o.Restricted, + }, extraOpts, 1, erc20) + + if o.DebugAPI { + s.MountTechnicalDebug() + s.MountDebug(false) + } else { + s.MountAPI() } - m.storer[signedChunk.Address().String()] = signedChunk.Data() - return signedChunk.Address().Bytes(), nil + + ts := httptest.NewServer(s) + t.Cleanup(ts.Close) + return ts.URL } -// UploadChunk into swarm -func (m *BeeClient) UploadChunk(ch swarm.Chunk) (address []byte, err error) { - m.storerMu.Lock() - defer m.storerMu.Unlock() - m.storer[ch.Address().String()] = ch.Data() - return ch.Address().Bytes(), nil +func createRedistributionAgentService( + t *testing.T, + addr swarm.Address, + storer storage.StateStorer, + erc20Service erc20.Service, + tranService transaction.Service, + backend storageincentives.ChainBackend, + chainStateGetter postage.ChainStateGetter, +) (*storageincentives.Agent, error) { + t.Helper() + + const blocksPerRound uint64 = 12 + const blocksPerPhase uint64 = 4 + postageContract := contractMock.New(contractMock.WithExpiresBatchesFunc(func(context.Context) error { + return nil + }), + ) + stakingContract := mock2.New(mock2.WithIsFrozen(func(context.Context, uint64) (bool, error) { + return true, nil + })) + contract := &mockContract{} + + return storageincentives.New( + addr, + common.Address{}, + backend, + contract, + postageContract, + stakingContract, + mockstorer.NewReserve(), + func() bool { return true }, + time.Millisecond*10, + blocksPerRound, + blocksPerPhase, + storer, + chainStateGetter, + erc20Service, + tranService, + &mockHealth{}, + log.Noop, + ) } -// DownloadChunk from swarm -func (m *BeeClient) DownloadChunk(_ context.Context, address []byte) (data []byte, err error) { - m.storerMu.Lock() - defer m.storerMu.Unlock() - if data, ok := m.storer[swarm.NewAddress(address).String()]; ok { - return data, nil +type contractCall int + +func (c contractCall) String() string { + switch c { + case isWinnerCall: + return "isWinnerCall" + case revealCall: + return "revealCall" + case commitCall: + return "commitCall" + case claimCall: + return "claimCall" } - return nil, fmt.Errorf("error downloading data") + return "unknown" } -// UploadBlob into swarm -func (m *BeeClient) UploadBlob(data []byte, tag uint32, _ bool) (address []byte, err error) { - m.storerMu.Lock() - defer m.storerMu.Unlock() - address = make([]byte, 32) - _, err = rand.Read(address) - newChunks := int64(len(data) / 4000) - if newChunks == 0 { - newChunks = 1 - } - chunks := newChunks + m.tagStorer[tag] + 1 - m.tagStorer[tag] = chunks +const ( + isWinnerCall contractCall = iota + revealCall + commitCall + claimCall +) - m.storer[swarm.NewAddress(address).String()] = data - return address, nil +type mockContract struct { + callsList []contractCall + mtx sync.Mutex } -// DownloadBlob from swarm -func (m *BeeClient) DownloadBlob(address []byte) ([]byte, int, error) { - m.storerMu.Lock() - defer m.storerMu.Unlock() - if data, ok := m.storer[swarm.NewAddress(address).String()]; ok { - return data, http.StatusOK, nil - } - return nil, http.StatusInternalServerError, fmt.Errorf("error downloading data") +func (m *mockContract) Fee(ctx context.Context, txHash common.Hash) *big.Int { + return big.NewInt(1000) } -// UploadBzz downloads data to bzz api from the Swarm network. -func (*BeeClient) UploadBzz(_ []byte, _ string) ([]byte, error) { +func (m *mockContract) ReserveSalt(context.Context) ([]byte, error) { return nil, nil } -// DownloadBzz downloads bzz data from the Swarm network. -func (*BeeClient) DownloadBzz(_ []byte) ([]byte, int, error) { - return nil, 0, nil +func (m *mockContract) IsPlaying(context.Context, uint8) (bool, error) { + return true, nil } -// DeleteReference unpins chunk in swarm -func (m *BeeClient) DeleteReference(address []byte) error { - m.storerMu.Lock() - defer m.storerMu.Unlock() - if _, found := m.storer[swarm.NewAddress(address).String()]; found { - delete(m.storer, swarm.NewAddress(address).String()) - return nil - } - return errors.New("chunk not found") +func (m *mockContract) IsWinner(context.Context) (bool, error) { + m.mtx.Lock() + defer m.mtx.Unlock() + m.callsList = append(m.callsList, isWinnerCall) + return false, nil +} + +func (m *mockContract) Claim(context.Context) (common.Hash, error) { + m.mtx.Lock() + defer m.mtx.Unlock() + m.callsList = append(m.callsList, claimCall) + return common.Hash{}, nil } -// CreateTag creates a tag -func (m *BeeClient) CreateTag(_ []byte) (uint32, error) { - tag := time.Now().UnixNano() - m.storerMu.Lock() - defer m.storerMu.Unlock() - m.tagStorer[uint32(tag)] = 0 - return uint32(tag), nil +func (m *mockContract) Commit(context.Context, []byte, *big.Int) (common.Hash, error) { + m.mtx.Lock() + defer m.mtx.Unlock() + m.callsList = append(m.callsList, commitCall) + return common.Hash{}, nil } -// GetTag returns the tag -func (m *BeeClient) GetTag(tag uint32) (int64, int64, int64, error) { - m.storerMu.Lock() - defer m.storerMu.Unlock() - return m.tagStorer[tag], m.tagStorer[tag], m.tagStorer[tag], nil +func (m *mockContract) Reveal(context.Context, uint8, []byte, []byte) (common.Hash, error) { + m.mtx.Lock() + defer m.mtx.Unlock() + m.callsList = append(m.callsList, revealCall) + return common.Hash{}, nil } + +type mockHealth struct{} + +func (m *mockHealth) IsHealthy() bool { return true } diff --git a/pkg/collection/batch_test.go b/pkg/collection/batch_test.go index c8e00ed9..981eb98e 100644 --- a/pkg/collection/batch_test.go +++ b/pkg/collection/batch_test.go @@ -21,6 +21,11 @@ import ( "io" "testing" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/sirupsen/logrus" + "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" @@ -32,15 +37,22 @@ import ( ) func TestBatchIndex(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) ai := acc.GetUserAccountInfo() _, _, err := acc.CreateUserAccount("") if err != nil { t.Fatal(err) } - fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) user := acc.GetAddress(account.UserAccountIndex) podPassword, _ := utils.GetRandString(pod.PasswordLength) t.Run("batch-add-docs", func(t *testing.T) { diff --git a/pkg/collection/document.go b/pkg/collection/document.go index af9982eb..0bf83a00 100644 --- a/pkg/collection/document.go +++ b/pkg/collection/document.go @@ -1335,7 +1335,7 @@ func (d *Document) storeDocumentDBSchemas(encryptionPassword string, collections } } topic := utils.HashString(documentFile) - _, err := d.fd.UpdateFeed(d.user, topic, buf.Bytes(), []byte(encryptionPassword), false) + err := d.fd.UpdateFeed(d.user, topic, buf.Bytes(), []byte(encryptionPassword), false) if err != nil { // skipcq: TCV-001 return err } diff --git a/pkg/collection/document_test.go b/pkg/collection/document_test.go index 7d0ee167..b224be23 100644 --- a/pkg/collection/document_test.go +++ b/pkg/collection/document_test.go @@ -24,6 +24,11 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/sirupsen/logrus" + "github.com/fairdatasociety/fairOS-dfs/pkg/account" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/collection" @@ -48,15 +53,22 @@ type TestDocument struct { } func TestDocumentStore(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) ai := acc.GetUserAccountInfo() _, _, err := acc.CreateUserAccount("") if err != nil { t.Fatal(err) } - fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) user := acc.GetAddress(account.UserAccountIndex) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { @@ -67,7 +79,7 @@ func TestDocumentStore(t *testing.T) { docStore := collection.NewDocumentStore("pod1", fd, ai, user, file, tm, mockClient, logger) podPassword, _ := utils.GetRandString(pod.PasswordLength) t.Run("create_document_db_errors", func(t *testing.T) { - nilFd := feed.New(&account.Info{}, mockClient, logger) + nilFd := feed.New(&account.Info{}, mockClient, -1, 0, logger) nilDocStore := collection.NewDocumentStore("pod1", nilFd, ai, user, file, tm, mockClient, logger) err := nilDocStore.CreateDocumentDB("docdb_err", podPassword, nil, true) if !errors.Is(err, collection.ErrReadOnlyIndex) { diff --git a/pkg/collection/index.go b/pkg/collection/index.go index 8271ad6d..cef4a848 100644 --- a/pkg/collection/index.go +++ b/pkg/collection/index.go @@ -19,16 +19,17 @@ package collection import ( "context" "encoding/json" + "errors" "fmt" "net/http" "runtime" - "strings" "sync/atomic" "time" "github.com/fairdatasociety/fairOS-dfs/pkg/account" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" + "github.com/fairdatasociety/fairOS-dfs/pkg/file" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" ) @@ -139,13 +140,13 @@ func CreateIndex(podName, collectionName, indexName, encryptionPassword string, } if string(oldData) == utils.DeletedFeedMagicWord { // skipcq: TCV-001 - _, err = fd.UpdateFeed(user, topic, ref, []byte(encryptionPassword), false) + err = fd.UpdateFeed(user, topic, ref, []byte(encryptionPassword), false) if err != nil { return ErrManifestCreate } return nil } - _, err = fd.CreateFeed(user, topic, ref, []byte(encryptionPassword)) + err = fd.CreateFeed(user, topic, ref, []byte(encryptionPassword)) if err != nil { // skipcq: TCV-001 return ErrManifestCreate } @@ -188,7 +189,7 @@ func (idx *Index) DeleteIndex(encryptionPassword string) error { // erase the top Manifest topic := utils.HashString(idx.name) - _, err := idx.feed.UpdateFeed(idx.user, topic, []byte(utils.DeletedFeedMagicWord), []byte(encryptionPassword), false) + err := idx.feed.UpdateFeed(idx.user, topic, []byte(utils.DeletedFeedMagicWord), []byte(encryptionPassword), false) if err != nil { // skipcq: TCV-001 return ErrDeleteingIndex } @@ -305,7 +306,7 @@ func (idx *Index) updateManifest(manifest *Manifest, encryptionPassword string) } topic := utils.HashString(manifest.Name) - _, err = idx.feed.UpdateFeed(idx.user, topic, ref, []byte(encryptionPassword), false) + err = idx.feed.UpdateFeed(idx.user, topic, ref, []byte(encryptionPassword), false) if err != nil { // skipcq: TCV-001 return ErrManifestCreate } @@ -329,16 +330,19 @@ func (idx *Index) storeManifest(manifest *Manifest, encryptionPassword string) e return ErrManifestCreate } topic := utils.HashString(manifest.Name) - _, err = idx.feed.CreateFeed(idx.user, topic, ref, []byte(encryptionPassword)) - if err != nil { // skipcq: TCV-001 - if strings.Contains(err.Error(), "chunk already exists") { - _, err = idx.feed.UpdateFeed(idx.user, topic, ref, []byte(encryptionPassword), false) - if err != nil { // skipcq: TCV-001 - return ErrManifestCreate - } - } else { + _, _, err = idx.feed.GetFeedData(topic, idx.user, []byte(encryptionPassword), false) + if err == nil || errors.Is(err, file.ErrDeletedFeed) { + err = idx.feed.UpdateFeed(idx.user, topic, ref, []byte(encryptionPassword), false) + if err != nil { // skipcq: TCV-001 + idx.logger.Errorf("updateFeed failed in storeManifest : %s", err.Error()) return ErrManifestCreate } + return nil + } + err = idx.feed.CreateFeed(idx.user, topic, ref, []byte(encryptionPassword)) + if err != nil { // skipcq: TCV-001 + idx.logger.Errorf("createFeed failed in storeManifest : %s", err.Error()) + return ErrManifestCreate } return nil } diff --git a/pkg/collection/index_api.go b/pkg/collection/index_api.go index 2d3addbe..9fdb7e2b 100644 --- a/pkg/collection/index_api.go +++ b/pkg/collection/index_api.go @@ -393,7 +393,7 @@ func (idx *Index) seekManifestAndEntry(key string) (*Manifest, *Manifest, int, e } // if there are any elements in the index, then search for the entry - if fm.Entries != nil && len(fm.Entries) > 0 { + if fm != nil && fm.Entries != nil && len(fm.Entries) > 0 { return idx.findManifest(nil, fm, key) } return nil, nil, 0, ErrEntryNotFound diff --git a/pkg/collection/index_api_test.go b/pkg/collection/index_api_test.go index 071c3f3e..4af8bfa3 100644 --- a/pkg/collection/index_api_test.go +++ b/pkg/collection/index_api_test.go @@ -23,6 +23,11 @@ import ( "net/http" "testing" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/sirupsen/logrus" + "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" @@ -34,15 +39,22 @@ import ( ) func TestIndexAPI(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) ai := acc.GetUserAccountInfo() _, _, err := acc.CreateUserAccount("") if err != nil { t.Fatal(err) } - fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) user := acc.GetAddress(account.UserAccountIndex) podPassword, _ := utils.GetRandString(pod.PasswordLength) t.Run("get-doc", func(t *testing.T) { @@ -140,7 +152,7 @@ func TestIndexAPI(t *testing.T) { } -func addDoc(t *testing.T, key string, value []byte, index *collection.Index, client *mock.BeeClient, apnd bool) { +func addDoc(t *testing.T, key string, value []byte, index *collection.Index, client *bee.Client, apnd bool) { ref, err := client.UploadBlob(value, 0, false) if err != nil { t.Fatalf("could not add doc %s:%s, %v", key, value, err) @@ -151,7 +163,7 @@ func addDoc(t *testing.T, key string, value []byte, index *collection.Index, cli } } -func getDoc(t *testing.T, key string, index *collection.Index, client *mock.BeeClient) []byte { +func getDoc(t *testing.T, key string, index *collection.Index, client *bee.Client) []byte { ref, err := index.Get(key) if err != nil { if errors.Is(err, collection.ErrEntryNotFound) { @@ -168,7 +180,7 @@ func getDoc(t *testing.T, key string, index *collection.Index, client *mock.BeeC } return data } -func getAllDocs(t *testing.T, key string, index *collection.Index, client *mock.BeeClient) [][]byte { +func getAllDocs(t *testing.T, key string, index *collection.Index, client *bee.Client) [][]byte { refs, err := index.Get(key) if err != nil { if errors.Is(err, collection.ErrEntryNotFound) { @@ -191,7 +203,7 @@ func getAllDocs(t *testing.T, key string, index *collection.Index, client *mock. return data } -func getValue(t *testing.T, ref []byte, client *mock.BeeClient) []byte { +func getValue(t *testing.T, ref []byte, client *bee.Client) []byte { data, respCode, err := client.DownloadBlob(ref) if err != nil { t.Fatal(err) @@ -202,7 +214,7 @@ func getValue(t *testing.T, ref []byte, client *mock.BeeClient) []byte { return data } -func delDoc(t *testing.T, key string, index *collection.Index, client *mock.BeeClient) []byte { +func delDoc(t *testing.T, key string, index *collection.Index, client *bee.Client) []byte { ref, err := index.Delete(key) if err != nil { t.Fatal(err) @@ -218,7 +230,7 @@ func delDoc(t *testing.T, key string, index *collection.Index, client *mock.BeeC } -func addLotOfDocs(t *testing.T, index *collection.Index, client *mock.BeeClient) map[string][]byte { +func addLotOfDocs(t *testing.T, index *collection.Index, client *bee.Client) map[string][]byte { // Initialize the values kvMap := make(map[string][]byte) kvMap["key1"] = []byte("value1") @@ -252,7 +264,7 @@ func addLotOfDocs(t *testing.T, index *collection.Index, client *mock.BeeClient) return kvMap } -func addBatchDocs(t *testing.T, batch *collection.Batch, client *mock.BeeClient) map[string][]byte { +func addBatchDocs(t *testing.T, batch *collection.Batch, client *bee.Client) map[string][]byte { kvMap := make(map[string][]byte) kvMap["key1"] = []byte("value1") kvMap["key11"] = []byte("value11") diff --git a/pkg/collection/index_test.go b/pkg/collection/index_test.go index 03d6940a..b7ced4bd 100644 --- a/pkg/collection/index_test.go +++ b/pkg/collection/index_test.go @@ -24,6 +24,11 @@ import ( "strconv" "testing" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/sirupsen/logrus" + "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/account" @@ -36,15 +41,22 @@ import ( ) func TestIndex(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) ai := acc.GetUserAccountInfo() _, _, err := acc.CreateUserAccount("") if err != nil { t.Fatal(err) } - fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) user := acc.GetAddress(account.UserAccountIndex) podPassword, _ := utils.GetRandString(pod.PasswordLength) t.Run("create_index", func(t *testing.T) { diff --git a/pkg/collection/iterator_test.go b/pkg/collection/iterator_test.go index 251866f7..41c42b0c 100644 --- a/pkg/collection/iterator_test.go +++ b/pkg/collection/iterator_test.go @@ -25,6 +25,11 @@ import ( "strconv" "testing" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/sirupsen/logrus" + "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/account" @@ -37,15 +42,22 @@ import ( ) func TestIndexIterator(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) ai := acc.GetUserAccountInfo() _, _, err := acc.CreateUserAccount("") if err != nil { t.Fatal(err) } - fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) user := acc.GetAddress(account.UserAccountIndex) podPassword, _ := utils.GetRandString(pod.PasswordLength) t.Run("iterate_all_string_keys", func(t *testing.T) { diff --git a/pkg/collection/kv.go b/pkg/collection/kv.go index 722e74c8..5fc92ea1 100644 --- a/pkg/collection/kv.go +++ b/pkg/collection/kv.go @@ -141,6 +141,10 @@ func (kv *KeyValue) DeleteKVTable(name, encryptionPassword string) error { return kv.storeKVTables(kvtables, encryptionPassword) } +func (kv *KeyValue) Commit() { + kv.fd.CommitFeeds() +} + // DeleteAllKVTables deletes all key value tables with all their index and data entries. func (kv *KeyValue) DeleteAllKVTables(encryptionPassword string) error { if kv.fd.IsReadOnlyFeed() { // skipcq: TCV-001 @@ -450,7 +454,7 @@ func (kv *KeyValue) storeKVTables(collections map[string][]string, encryptionPas if buf.Len() == 0 { data = []byte(utils.DeletedFeedMagicWord) } - _, err := kv.fd.UpdateFeed(kv.user, topic, data, []byte(encryptionPassword), false) + err := kv.fd.UpdateFeed(kv.user, topic, data, []byte(encryptionPassword), false) if err != nil { // skipcq: TCV-001 return err } diff --git a/pkg/collection/kv_test.go b/pkg/collection/kv_test.go index 0e7fdc91..f5b773cf 100644 --- a/pkg/collection/kv_test.go +++ b/pkg/collection/kv_test.go @@ -27,6 +27,12 @@ import ( "strconv" "strings" "testing" + "time" + + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/sirupsen/logrus" "github.com/fairdatasociety/fairOS-dfs/pkg/account" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" @@ -43,19 +49,28 @@ func TestMain(m *testing.M) { } func TestKeyValueStore(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + logger := logging.New(io.Discard, logrus.DebugLevel) + acc := account.New(logger) ai := acc.GetUserAccountInfo() _, _, err := acc.CreateUserAccount("") if err != nil { t.Fatal(err) } - fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger) - user := acc.GetAddress(account.UserAccountIndex) - kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) - podPassword, _ := utils.GetRandString(pod.PasswordLength) + t.Run("table_not_opened", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_1314", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -82,6 +97,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("nil_itr", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_1312", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -104,6 +131,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("create_kv_table_with_string_index", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_0", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -131,6 +170,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("create_kv_table_with_number_index", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_1", podPassword, collection.NumberIndex) if err != nil { t.Fatal(err) @@ -158,6 +209,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("check_delete", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_2", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -180,6 +243,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("create_multiple_kv_tables_and_delete", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_31", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -260,6 +335,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("create_open_and_delete", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_4", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -280,6 +367,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("delete_without_create", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) // delete the last table err = kvStore.DeleteKVTable("kv_table_5", podPassword) if !errors.Is(err, collection.ErrKVTableNotPresent) { @@ -288,6 +387,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("open_table", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_6", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -305,6 +416,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("open_without_create", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err = kvStore.OpenKVTable("kv_table_7", podPassword) if !errors.Is(err, collection.ErrKVTableNotPresent) { t.Fatal("was able to open table without creating it") @@ -312,6 +435,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("put_string_index", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_8", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -347,6 +482,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("put_bytes_index", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_bytes", podPassword, collection.BytesIndex) if err != nil { t.Fatal(err) @@ -386,6 +533,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("put_chinese_string_index", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_9", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -413,6 +572,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("put_string_in_number_index", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_10", podPassword, collection.NumberIndex) if err != nil { t.Fatal(err) @@ -428,6 +599,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("put_get_del_get_string_index", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_11", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -481,6 +664,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("put_without_opening_table", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_12", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -492,6 +687,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("delete_non_existent_string_index", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_13", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -513,6 +720,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("batch_without_open", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_batch_1", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -525,6 +744,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("batch_columns_and_get_values", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_batch_2", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -579,6 +810,18 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("batch_put_columns_and_get_values", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + defer fd.CommitFeeds() + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_batch_9", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -629,12 +872,23 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("count_columns_and_get_values", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_batch_count", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) } _, err = kvStore.KVCount("kv_table_batch_count") - if err != collection.ErrKVTableNotOpened { + if !errors.Is(err, collection.ErrKVTableNotOpened) { t.Fatal("should have returned error ", collection.ErrKVTableNotOpened) } err = kvStore.OpenKVTable("kv_table_batch_count", podPassword) @@ -651,6 +905,17 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("Iterate_string_keys", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, 500, 0, logger) + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_Itr_0", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -664,7 +929,7 @@ func TestKeyValueStore(t *testing.T) { if err != nil { t.Fatal(err) } - + fd.CommitFeeds() // check the count countObject, err := kvStore.KVCount("kv_table_Itr_0") if err != nil { @@ -684,7 +949,7 @@ func TestKeyValueStore(t *testing.T) { for i := 0; i < 100; i++ { itr.Next() if itr.StringKey() != sortedKeys[i] { - t.Fatal("keys do not match", itr.StringKey(), sortedKeys[i]) + t.Fatal("keys do not match", i, itr.StringKey(), sortedKeys[i]) } if !bytes.Equal(itr.Value(), []byte(sortedValues[i])) { t.Fatal("values do not match", string(itr.Value()), sortedValues[i]) @@ -693,6 +958,17 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("Iterate_seek_limit_string_keys", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) tableNo := 0 research: tableNo++ @@ -731,7 +1007,7 @@ func TestKeyValueStore(t *testing.T) { for i := startIndex; i < startIndex+10; i++ { itr.Next() if itr.StringKey() != sortedKeys[i] { - t.Fatalf("key mismatch: %s : %s\n", itr.StringKey(), sortedKeys[i]) + t.Fatalf("key mismatch: %s : %s at %d\n", itr.StringKey(), sortedKeys[i], i) } if !bytes.Equal(itr.Value(), []byte(sortedValues[i])) { t.Fatalf("value mismatch: %s : %s\n", itr.StringKey(), sortedKeys[i]) @@ -746,6 +1022,17 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("Iterate_seek_start_end_string_keys", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) tableNo := 0 research: tableNo++ @@ -816,6 +1103,17 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("Iterate_seek_start_end_string_keys_over_a_known_failing_keys", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) tableNo := 486 err := kvStore.CreateKVTable(fmt.Sprintf("kv_table_Itr_1%d", tableNo), podPassword, collection.StringIndex) if err != nil { @@ -902,6 +1200,17 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("Iterate_string_of_numbers_keys", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_Itr_3", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -935,6 +1244,17 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("Iterate_numbers_keys", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_Itr_4", podPassword, collection.NumberIndex) if err != nil { t.Fatal(err) @@ -969,6 +1289,17 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("Iterate_numbers_start_end_keys", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_Itr_5", podPassword, collection.NumberIndex) if err != nil { t.Fatal(err) @@ -1022,6 +1353,17 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("Iterate_numbers_start_and_limit_keys", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_Itr_6", podPassword, collection.NumberIndex) if err != nil { t.Fatal(err) @@ -1069,6 +1411,17 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("get_non_existent_string_index", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_1313", podPassword, collection.StringIndex) if err != nil { t.Fatal(err) @@ -1099,6 +1452,17 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("err_byte_index", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_1316", podPassword, collection.BytesIndex) if err != nil { t.Fatal(err) @@ -1119,6 +1483,17 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("err_seek_list_index", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_1317", podPassword, collection.ListIndex) if err != nil { t.Fatal(err) @@ -1135,6 +1510,17 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("err_seek_map_index", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_1318", podPassword, collection.MapIndex) if err != nil { t.Fatal(err) @@ -1151,6 +1537,17 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("err_seek_invalid_index", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_1319", podPassword, collection.InvalidIndex) if err != nil { t.Fatal(err) @@ -1167,6 +1564,17 @@ func TestKeyValueStore(t *testing.T) { }) t.Run("seek_unopened_table", func(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + user := acc.GetAddress(account.UserAccountIndex) + kvStore := collection.NewKeyValueStore("pod1", fd, ai, user, mockClient, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) err := kvStore.CreateKVTable("kv_table_1320", podPassword, collection.ListIndex) if err != nil { t.Fatal(err) @@ -1179,7 +1587,7 @@ func TestKeyValueStore(t *testing.T) { }) } -func addRandomStrings(t *testing.T, kvStore *collection.KeyValue, count int, tableName string) ([]string, []string, error) { +func addRandomStrings(_ *testing.T, kvStore *collection.KeyValue, count int, tableName string) ([]string, []string, error) { var keys []string var values []string for i := 0; i < count; i++ { @@ -1205,11 +1613,12 @@ func addRandomStrings(t *testing.T, kvStore *collection.KeyValue, count int, tab } keys = append(keys, key) values = append(values, key) + <-time.After(800 * time.Millisecond) } return keys, values, nil } -func addRandomNumbersAsString(t *testing.T, kvStore *collection.KeyValue, count int, tableName string) ([]string, []string, error) { +func addRandomNumbersAsString(_ *testing.T, kvStore *collection.KeyValue, count int, tableName string) ([]string, []string, error) { var keys []string var values []string for i := 0; i < count; i++ { @@ -1236,7 +1645,7 @@ func addRandomNumbersAsString(t *testing.T, kvStore *collection.KeyValue, count return keys, values, nil } -func addRandomNumbers(t *testing.T, kvStore *collection.KeyValue, count int, tableName string) ([]int, []int, error) { +func addRandomNumbers(_ *testing.T, kvStore *collection.KeyValue, count int, tableName string) ([]int, []int, error) { var keys []int var values []int for i := 0; i < count; i++ { diff --git a/pkg/dfs/api.go b/pkg/dfs/api.go index bd9cc212..4be87798 100644 --- a/pkg/dfs/api.go +++ b/pkg/dfs/api.go @@ -39,13 +39,15 @@ const ( // API is the go api for fairOS type API struct { - context context.Context - cancel context.CancelFunc - client blockstore.Client - users *user.Users - logger logging.Logger - tm *taskmanager.TaskManager - sm subscriptionManager.SubscriptionManager + context context.Context + cancel context.CancelFunc + client blockstore.Client + users *user.Users + logger logging.Logger + tm *taskmanager.TaskManager + sm subscriptionManager.SubscriptionManager + feedCacheSize int + feedCacheTTL time.Duration io.Closer } @@ -55,6 +57,8 @@ type Options struct { EnsConfig *contracts.ENSConfig SubscriptionConfig *contracts.SubscriptionConfig Logger logging.Logger + FeedCacheSize int + FeedCacheTTL time.Duration } // NewDfsAPI is the main entry point for the df controller. @@ -73,7 +77,7 @@ func NewDfsAPI(ctx context.Context, opts *Options) (*API, error) { logger.Errorf("dfs: bee client initialisation failed") return nil, errBeeClient } - users := user.NewUsers(c, ens, logger) + users := user.NewUsers(c, ens, opts.FeedCacheSize, opts.FeedCacheTTL, logger) var sm subscriptionManager.SubscriptionManager if opts.SubscriptionConfig != nil { @@ -85,17 +89,25 @@ func NewDfsAPI(ctx context.Context, opts *Options) (*API, error) { } } + // Setting cache size 0 will disable the cache. This is to change the default behaviour of lru itself. + // We have this -1 check hard coded in the feed package. -1 will disable the feed pool off. and write directly to swarm. + if opts.FeedCacheSize == 0 { + opts.FeedCacheSize = -1 + } + // discard tm logs as it creates too much noise tmLogger := logging.New(io.Discard, 0) ctx2, cancel := context.WithCancel(ctx) return &API{ - context: ctx2, - cancel: cancel, - client: c, - users: users, - logger: logger, - tm: taskmanager.New(10, defaultMaxWorkers, time.Second*15, tmLogger), - sm: sm, + context: ctx2, + cancel: cancel, + client: c, + users: users, + logger: logger, + tm: taskmanager.New(10, defaultMaxWorkers, time.Second*15, tmLogger), + sm: sm, + feedCacheSize: opts.FeedCacheSize, + feedCacheTTL: opts.FeedCacheTTL, }, nil } @@ -114,6 +126,14 @@ func NewMockDfsAPI(client blockstore.Client, users *user.Users, logger logging.L // Close stops the taskmanager func (a *API) Close() error { + users := a.users.GetUsersLoggedIn() + for sessionId, v := range users { + err := a.LogoutUser(sessionId) + if err != nil { + a.logger.Errorf("dfs: error logging out user %s", v.GetUserName()) + } + } + ctx, cancel := context.WithTimeout(a.context, time.Minute) defer func() { cancel() diff --git a/pkg/dfs/fs_api.go b/pkg/dfs/fs_api.go index 1f8d019b..311d1ae4 100644 --- a/pkg/dfs/fs_api.go +++ b/pkg/dfs/fs_api.go @@ -121,7 +121,9 @@ func (a *API) ListDir(podName, currentDir, sessionId string) ([]dir.Entry, []f.E // check if directory present totalPath := utils.CombinePathAndFile(currentDir, "") - if directory.GetInode(podInfo.GetPodPassword(), totalPath) == nil { + _, err = directory.GetInode(podInfo.GetPodPassword(), totalPath) + if err != nil { + a.logger.Errorf("dir not found: %s: %s", currentDir, err.Error()) return nil, nil, dir.ErrDirectoryNotPresent } dEntries, fileList, err := directory.ListDir(currentDir, podInfo.GetPodPassword()) @@ -154,8 +156,8 @@ func (a *API) DirectoryStat(podName, directoryPath, sessionId string) (*dir.Stat directoryPath = filepath.ToSlash(directoryPath) if directoryPath != utils.PathSeparator { parent := filepath.ToSlash(filepath.Dir(directoryPath)) - inode := directory.GetInode(podInfo.GetPodPassword(), parent) - if inode == nil { + inode, err := directory.GetInode(podInfo.GetPodPassword(), parent) + if err != nil { return nil, dir.ErrDirectoryNotPresent } found := false @@ -189,8 +191,8 @@ func (a *API) DirectoryInode(podName, directoryName, sessionId string) (*dir.Ino return nil, err } directory := podInfo.GetDirectory() - inode := directory.GetInode(podInfo.GetPodPassword(), directoryName) - if inode == nil { + inode, err := directory.GetInode(podInfo.GetPodPassword(), directoryName) + if err != nil { a.logger.Errorf("dir not found: %s", directoryName) return nil, fmt.Errorf("dir not found") } @@ -268,8 +270,8 @@ func (a *API) FileStat(podName, podFileWithPath, sessionId string) (*f.Stats, er } podFileWithPath = filepath.ToSlash(podFileWithPath) directory := podInfo.GetDirectory() - inode := directory.GetInode(podInfo.GetPodPassword(), filepath.ToSlash(filepath.Dir(podFileWithPath))) - if inode != nil { + inode, err := directory.GetInode(podInfo.GetPodPassword(), filepath.ToSlash(filepath.Dir(podFileWithPath))) + if err == nil { found := false fileName := filepath.Base(podFileWithPath) for _, name := range inode.FileOrDirNames { diff --git a/pkg/dfs/pod_api.go b/pkg/dfs/pod_api.go index 957ec62e..1fad4715 100644 --- a/pkg/dfs/pod_api.go +++ b/pkg/dfs/pod_api.go @@ -20,6 +20,7 @@ import ( "context" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -27,9 +28,8 @@ import ( "strconv" "strings" - c "github.com/fairdatasociety/fairOS-dfs/pkg/collection" - "github.com/fairdatasociety/fairOS-dfs/pkg/account" + c "github.com/fairdatasociety/fairOS-dfs/pkg/collection" "github.com/fairdatasociety/fairOS-dfs/pkg/contracts/datahub" "github.com/fairdatasociety/fairOS-dfs/pkg/dir" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" @@ -53,7 +53,6 @@ func (a *API) CreatePod(podName, sessionId string) (*pod.Info, error) { if err != nil { return nil, err } - // Add podName in the login user session ui.AddPodName(podName, pi) return pi, nil @@ -66,7 +65,6 @@ func (a *API) DeletePod(podName, sessionId string) error { if ui == nil { return ErrUserNotLoggedIn } - // delete all the directory, files, and database tables under this pod from // the Swarm network. podInfo, _, err := ui.GetPod().GetPodInfo(podName) @@ -74,7 +72,6 @@ func (a *API) DeletePod(podName, sessionId string) error { return err } directory := podInfo.GetDirectory() - // check if this is a shared pod if podInfo.GetFeed().IsReadOnlyFeed() { // delete the pod and close if it is opened @@ -87,18 +84,15 @@ func (a *API) DeletePod(podName, sessionId string) error { ui.RemovePodName(podName) return nil } - err = directory.RmRootDir(podInfo.GetPodPassword()) - if err != nil { + if err != nil && !errors.Is(err, file.ErrFileNotFound) { return err } - // delete the pod and close if it is opened err = ui.GetPod().DeleteOwnPod(podName) if err != nil { return err } - ui.RemovePodName(podName) return nil } @@ -138,6 +132,16 @@ func (a *API) ClosePod(podName, sessionId string) error { return nil } +// CommitPodFeeds commits feed for a pod on swarm +func (a *API) CommitPodFeeds(podName, sessionId string) error { + // get the loggedin user information + ui := a.users.GetLoggedInUserInfo(sessionId) + if ui == nil { + return ErrUserNotLoggedIn + } + return ui.GetPod().CommitFeeds(podName) +} + // PodStat returns the pod stat func (a *API) PodStat(podName, sessionId string) (*pod.Stat, error) { // get the logged-in user information @@ -268,7 +272,7 @@ func (a *API) PublicPodFileDownload(pod *pod.ShareInfo, filePath string) (io.Rea address := utils.HexToAddress(pod.Address) accountInfo.SetAddress(address) - fd := feed.New(accountInfo, a.client, a.logger) + fd := feed.New(accountInfo, a.client, a.feedCacheSize, a.feedCacheTTL, a.logger) topic := utils.HashString(filePath) _, metaBytes, err := fd.GetFeedData(topic, accountInfo.GetAddress(), []byte(pod.Password), false) if err != nil { @@ -308,7 +312,7 @@ func (a *API) PublicPodKVEntryGet(pod *pod.ShareInfo, name, key string) ([]strin address := utils.HexToAddress(pod.Address) accountInfo.SetAddress(address) - fd := feed.New(accountInfo, a.client, a.logger) + fd := feed.New(accountInfo, a.client, a.feedCacheSize, a.feedCacheTTL, a.logger) kvStore := c.NewKeyValueStore(pod.PodName, fd, accountInfo, address, a.client, a.logger) err := kvStore.OpenKVTable(name, pod.Password) @@ -325,7 +329,7 @@ func (a *API) PublicPodKVGetter(pod *pod.ShareInfo) KVGetter { address := utils.HexToAddress(pod.Address) accountInfo.SetAddress(address) - fd := feed.New(accountInfo, a.client, a.logger) + fd := feed.New(accountInfo, a.client, a.feedCacheSize, a.feedCacheTTL, a.logger) return c.NewKeyValueStore(pod.PodName, fd, accountInfo, address, a.client, a.logger) } @@ -336,7 +340,7 @@ func (a *API) PublicPodDisLs(pod *pod.ShareInfo, dirPathToLs string) ([]dir.Entr address := utils.HexToAddress(pod.Address) accountInfo.SetAddress(address) - fd := feed.New(accountInfo, a.client, a.logger) + fd := feed.New(accountInfo, a.client, a.feedCacheSize, a.feedCacheTTL, a.logger) dirNameWithPath := filepath.ToSlash(dirPathToLs) topic := utils.HashString(dirNameWithPath) diff --git a/pkg/dfs/user_api.go b/pkg/dfs/user_api.go index 02139d52..37578cc2 100644 --- a/pkg/dfs/user_api.go +++ b/pkg/dfs/user_api.go @@ -43,6 +43,12 @@ func (a *API) LogoutUser(sessionId string) error { return ErrUserNotLoggedIn } + err := ui.GetPod().CloseAllPods() + if err != nil { + a.logger.Errorf("error closing all pods: %v", err) + } + ui.GetFeed().CommitFeeds() + return a.users.LogoutUser(ui.GetUserName(), sessionId) } diff --git a/pkg/dir/chmod.go b/pkg/dir/chmod.go index d80a9576..42682b56 100644 --- a/pkg/dir/chmod.go +++ b/pkg/dir/chmod.go @@ -1,26 +1,13 @@ package dir import ( - "encoding/json" "fmt" "time" - - "github.com/fairdatasociety/fairOS-dfs/pkg/utils" ) // Chmod does all the validation for the existence of the file and changes file mode func (d *Directory) Chmod(dirNameWithPath, podPassword string, mode uint32) error { - topic := utils.HashString(dirNameWithPath) - _, data, err := d.fd.GetFeedData(topic, d.getAddress(), []byte(podPassword), false) - if err != nil { // skipcq: TCV-001 - return fmt.Errorf("dir chmod: %v", err) - } - if string(data) == utils.DeletedFeedMagicWord { - return ErrDirectoryNotPresent - } - - var dirInode Inode - err = json.Unmarshal(data, &dirInode) + dirInode, err := d.GetInode(podPassword, dirNameWithPath) if err != nil { // skipcq: TCV-001 return fmt.Errorf("dir chmod: %v", err) } @@ -31,14 +18,5 @@ func (d *Directory) Chmod(dirNameWithPath, podPassword string, mode uint32) erro dirInode.Meta.Mode = S_IFDIR | mode dirInode.Meta.AccessTime = time.Now().Unix() - metaBytes, err := json.Marshal(dirInode) - if err != nil { // skipcq: TCV-001 - return err - } - _, err = d.fd.UpdateFeed(d.userAddress, topic, metaBytes, []byte(podPassword), false) - if err != nil { // skipcq: TCV-001 - return err - } - d.AddToDirectoryMap(dirNameWithPath, &dirInode) - return nil + return d.SetInode(podPassword, dirInode) } diff --git a/pkg/dir/chmod_test.go b/pkg/dir/chmod_test.go index ae1e0c9e..5bfc2bae 100644 --- a/pkg/dir/chmod_test.go +++ b/pkg/dir/chmod_test.go @@ -7,20 +7,33 @@ import ( "testing" "time" + "github.com/fairdatasociety/fairOS-dfs/pkg/file" + + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" + + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" - bm "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/dir" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" - fm "github.com/fairdatasociety/fairOS-dfs/pkg/file/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" ) func TestChmod(t *testing.T) { - mockClient := bm.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -30,13 +43,13 @@ func TestChmod(t *testing.T) { if err != nil { t.Fatal(err) } - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) - mockFile := fm.NewMockFile() tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) }() + mockFile := file.NewFile("pod1", mockClient, fd, user, tm, logger) t.Run("chmod-dir", func(t *testing.T) { podPassword, _ := utils.GetRandString(pod.PasswordLength) diff --git a/pkg/dir/dir.go b/pkg/dir/dir.go index 9f08ed22..2bae9937 100644 --- a/pkg/dir/dir.go +++ b/pkg/dir/dir.go @@ -18,7 +18,6 @@ package dir import ( "context" - "encoding/json" "fmt" "strconv" "sync" @@ -150,15 +149,11 @@ func newLsTask(d *Directory, topic []byte, path, podPassword string, l *[]Entry, // Execute func (lt *lsTask) Execute(context.Context) error { defer lt.wg.Done() - _, data, err := lt.d.fd.GetFeedData(lt.topic, lt.d.getAddress(), []byte(lt.podPassword), false) - if err != nil { // skipcq: TCV-001 - return fmt.Errorf("list dir : %v", err) - } - var dirInode *Inode - err = json.Unmarshal(data, &dirInode) + dirInode, err := lt.d.GetInode(lt.podPassword, lt.path) if err != nil { // skipcq: TCV-001 return fmt.Errorf("list dir : %v", err) } + entry := Entry{ Name: dirInode.Meta.Name, ContentType: MimeTypeDirectory, // per RFC2425 diff --git a/pkg/dir/dir_present.go b/pkg/dir/dir_present.go index 90deb4f9..c9e0cb92 100644 --- a/pkg/dir/dir_present.go +++ b/pkg/dir/dir_present.go @@ -16,16 +16,11 @@ limitations under the License. package dir -import ( - "github.com/fairdatasociety/fairOS-dfs/pkg/utils" -) - // IsDirectoryPresent this function check if a given directory is present inside the pod. func (d *Directory) IsDirectoryPresent(directoryNameWithPath, podPassword string) bool { - topic := utils.HashString(directoryNameWithPath) - _, metaBytes, err := d.fd.GetFeedData(topic, d.userAddress, []byte(podPassword), false) - if string(metaBytes) == utils.DeletedFeedMagicWord { + in, err := d.GetInode(podPassword, directoryNameWithPath) + if err != nil { return false } - return err == nil + return in != nil } diff --git a/pkg/dir/dir_present_test.go b/pkg/dir/dir_present_test.go index 537bd627..7ed1e552 100644 --- a/pkg/dir/dir_present_test.go +++ b/pkg/dir/dir_present_test.go @@ -22,22 +22,33 @@ import ( "testing" "time" - "github.com/fairdatasociety/fairOS-dfs/pkg/pod" - "github.com/fairdatasociety/fairOS-dfs/pkg/utils" + "github.com/fairdatasociety/fairOS-dfs/pkg/file" - "github.com/plexsysio/taskmanager" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" - bm "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/dir" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" - fm "github.com/fairdatasociety/fairOS-dfs/pkg/file/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" + "github.com/fairdatasociety/fairOS-dfs/pkg/pod" + "github.com/fairdatasociety/fairOS-dfs/pkg/utils" + "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" ) func TestDirPresent(t *testing.T) { - mockClient := bm.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -47,13 +58,13 @@ func TestDirPresent(t *testing.T) { if err != nil { t.Fatal(err) } - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) - mockFile := fm.NewMockFile() tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) }() + mockFile := file.NewFile("pod1", mockClient, fd, user, tm, logger) t.Run("dir-present", func(t *testing.T) { podPassword, _ := utils.GetRandString(pod.PasswordLength) diff --git a/pkg/dir/dir_test.go b/pkg/dir/dir_test.go index 9f84ad6b..32c8576f 100644 --- a/pkg/dir/dir_test.go +++ b/pkg/dir/dir_test.go @@ -6,22 +6,32 @@ import ( "testing" "time" - "github.com/fairdatasociety/fairOS-dfs/pkg/pod" - "github.com/fairdatasociety/fairOS-dfs/pkg/utils" - - "github.com/plexsysio/taskmanager" + "github.com/fairdatasociety/fairOS-dfs/pkg/file" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" - bm "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/dir" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" - fm "github.com/fairdatasociety/fairOS-dfs/pkg/file/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" + "github.com/fairdatasociety/fairOS-dfs/pkg/pod" + "github.com/fairdatasociety/fairOS-dfs/pkg/utils" + "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" ) func TestDirRmAllFromMap(t *testing.T) { - mockClient := bm.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -31,13 +41,13 @@ func TestDirRmAllFromMap(t *testing.T) { if err != nil { t.Fatal(err) } - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) - mockFile := fm.NewMockFile() tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) }() + mockFile := file.NewFile("pod1", mockClient, fd, user, tm, logger) t.Run("dir-rm-all-from-map", func(t *testing.T) { podPassword, _ := utils.GetRandString(pod.PasswordLength) @@ -62,7 +72,7 @@ func TestDirRmAllFromMap(t *testing.T) { } dirObject.RemoveAllFromDirectoryMap() - node := dirObject.GetInode(podPassword, "/baseDir") + node, _ := dirObject.GetInode(podPassword, "/baseDir") if node == nil { t.Fatal("node should not be nil, metadata should be available in blockstore") } @@ -71,7 +81,7 @@ func TestDirRmAllFromMap(t *testing.T) { if err != nil { t.Fatal(err) } - node = dirObject.GetInode(podPassword, "/baseDir") + node, _ = dirObject.GetInode(podPassword, "/baseDir") if node != nil { t.Fatal("node should be nil") } diff --git a/pkg/dir/inode.go b/pkg/dir/inode.go index ad96ff5d..d60d81cd 100644 --- a/pkg/dir/inode.go +++ b/pkg/dir/inode.go @@ -17,9 +17,14 @@ limitations under the License. package dir import ( + "bufio" + "bytes" "encoding/json" "errors" + "io" + "path/filepath" + "github.com/fairdatasociety/fairOS-dfs/pkg/file" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" ) @@ -61,21 +66,80 @@ func (in *Inode) Unmarshal(data []byte) error { } // GetInode returns the inode of the given directory -func (d *Directory) GetInode(podPassword, dirNameWithPath string) *Inode { +func (d *Directory) GetInode(podPassword, dirNameWithPath string) (*Inode, error) { node := d.GetDirFromDirectoryMap(dirNameWithPath) if node != nil { - return node + return node, nil } - topic := utils.HashString(dirNameWithPath) - _, data, err := d.fd.GetFeedData(topic, d.getAddress(), []byte(podPassword), false) + var inode Inode + var data []byte + r, _, err := d.file.Download(utils.CombinePathAndFile(dirNameWithPath, indexFileName), podPassword) if err != nil { // skipcq: TCV-001 - return nil + topic := utils.HashString(dirNameWithPath) + _, data, err = d.fd.GetFeedData(topic, d.getAddress(), []byte(podPassword), false) + if err != nil { // skipcq: TCV-001 + return nil, ErrDirectoryNotPresent + } + err = inode.Unmarshal(data) + if err != nil { // skipcq: TCV-001 + return nil, err + } + err = d.SetInode(podPassword, &inode) + if err != nil { // skipcq: TCV-001 + return nil, err + } + + // ignore delete error + _ = d.fd.DeleteFeedFromTopic(topic, d.getAddress()) + } else { + data, err = io.ReadAll(r) + if err != nil { // skipcq: TCV-001 + return nil, err + } + err = inode.Unmarshal(data) + if err != nil { // skipcq: TCV-001 + return nil, err + } } - var inode Inode - err = inode.Unmarshal(data) + d.AddToDirectoryMap(dirNameWithPath, &inode) + return &inode, nil +} + +// SetInode saves the inode of the given directory +func (d *Directory) SetInode(podPassword string, iNode *Inode) error { + totalPath := utils.CombinePathAndFile(iNode.Meta.Path, iNode.Meta.Name) + data, err := json.Marshal(iNode) if err != nil { // skipcq: TCV-001 + return err + } + + err = d.file.Upload(bufio.NewReader(bytes.NewBuffer(data)), indexFileName, int64(len(data)), file.MinBlockSize, 0, totalPath, "gzip", podPassword) + if err != nil { + return err + } + d.AddToDirectoryMap(totalPath, iNode) + return nil +} + +// RemoveInode removes the inode of the given directory +func (d *Directory) RemoveInode(podPassword, dirNameWithPath string) error { + parentPath := filepath.ToSlash(filepath.Dir(dirNameWithPath)) + dirToDelete := filepath.Base(dirNameWithPath) + var totalPath string + if parentPath == utils.PathSeparator && filepath.ToSlash(dirToDelete) == utils.PathSeparator { + totalPath = utils.CombinePathAndFile(parentPath, "") + } else { + totalPath = utils.CombinePathAndFile(parentPath, dirToDelete) + } + err := d.file.RmFile(utils.CombinePathAndFile(totalPath, indexFileName), podPassword) + if err != nil { + return err + } + d.RemoveFromDirectoryMap(totalPath) + // return if root directory + if parentPath == "" || (parentPath == utils.PathSeparator && filepath.ToSlash(totalPath) == utils.PathSeparator) { return nil } - d.AddToDirectoryMap(dirNameWithPath, &inode) - return &inode + // remove the directory entry from the parent dir + return d.RemoveEntryFromDir(filepath.ToSlash(filepath.Dir(parentPath)), podPassword, filepath.Base(totalPath), false) } diff --git a/pkg/dir/ls.go b/pkg/dir/ls.go index f0317d0c..ba66cddb 100644 --- a/pkg/dir/ls.go +++ b/pkg/dir/ls.go @@ -47,21 +47,10 @@ type Entry struct { // function can give information about those files. func (d *Directory) ListDir(dirNameWithPath, podPassword string) ([]Entry, []string, error) { dirNameWithPath = filepath.ToSlash(dirNameWithPath) - topic := utils.HashString(dirNameWithPath) - _, data, err := d.fd.GetFeedData(topic, d.getAddress(), []byte(podPassword), false) - if err != nil { // skipcq: TCV-001 - if dirNameWithPath == utils.PathSeparator { - return nil, nil, nil - } - return nil, nil, fmt.Errorf("list dir : %v", err) // skipcq: TCV-001 - } - - dirInode := &Inode{} - err = dirInode.Unmarshal(data) + dirInode, err := d.GetInode(podPassword, dirNameWithPath) if err != nil { return nil, nil, fmt.Errorf("list dir : %v", err) } - wg := new(sync.WaitGroup) mtx := &sync.Mutex{} listEntries := &[]Entry{} diff --git a/pkg/dir/ls_test.go b/pkg/dir/ls_test.go index a7c00963..02e2ed0c 100644 --- a/pkg/dir/ls_test.go +++ b/pkg/dir/ls_test.go @@ -24,23 +24,35 @@ import ( "testing" "time" + "github.com/fairdatasociety/fairOS-dfs/pkg/file" + + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/pod" + "github.com/sirupsen/logrus" "github.com/plexsysio/taskmanager" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" "github.com/fairdatasociety/fairOS-dfs/pkg/account" - bm "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/dir" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" - fm "github.com/fairdatasociety/fairOS-dfs/pkg/file/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" ) func TestListDirectory(t *testing.T) { - mockClient := bm.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -50,15 +62,15 @@ func TestListDirectory(t *testing.T) { if err != nil { t.Fatal(err) } - fd := feed.New(pod1AccountInfo, mockClient, logger) - user := acc.GetAddress(1) - mockFile := fm.NewMockFile() - tm := taskmanager.New(1, 10, time.Second*15, logger) - defer func() { - _ = tm.Stop(context.Background()) - }() t.Run("list-dir", func(t *testing.T) { + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) + user := acc.GetAddress(1) + tm := taskmanager.New(1, 10, time.Second*15, logger) + defer func() { + _ = tm.Stop(context.Background()) + }() + mockFile := file.NewFile("pod1", mockClient, fd, user, tm, logger) podPassword, _ := utils.GetRandString(pod.PasswordLength) dirObject := dir.NewDirectory("pod1", mockClient, fd, user, mockFile, tm, logger) @@ -155,6 +167,13 @@ func TestListDirectory(t *testing.T) { }) t.Run("list-dir-from-different-dir-object", func(t *testing.T) { + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) + user := acc.GetAddress(1) + tm := taskmanager.New(1, 10, time.Second*15, logger) + defer func() { + _ = tm.Stop(context.Background()) + }() + mockFile := file.NewFile("pod1", mockClient, fd, user, tm, logger) podPassword, _ := utils.GetRandString(pod.PasswordLength) dirObject := dir.NewDirectory("pod1", mockClient, fd, user, mockFile, tm, logger) diff --git a/pkg/dir/mkdir.go b/pkg/dir/mkdir.go index 85c71e81..f4167de0 100644 --- a/pkg/dir/mkdir.go +++ b/pkg/dir/mkdir.go @@ -17,7 +17,6 @@ limitations under the License. package dir import ( - "encoding/json" "path/filepath" "strings" "time" @@ -49,14 +48,15 @@ func (d *Directory) MkDir(dirToCreateWithPath, podPassword string, mode uint32) // check if directory already present totalPath := utils.CombinePathAndFile(parentPath, dirName) - topic := utils.HashString(totalPath) // check if parent path exists - if d.GetInode(podPassword, parentPath) == nil { + _, err := d.GetInode(podPassword, parentPath) + if err != nil { return ErrDirectoryNotPresent } - if d.GetInode(podPassword, totalPath) != nil { + _, err = d.GetInode(podPassword, totalPath) + if err == nil { return ErrDirectoryAlreadyPresent } @@ -77,55 +77,13 @@ func (d *Directory) MkDir(dirToCreateWithPath, podPassword string, mode uint32) dirInode := &Inode{ Meta: &meta, } - data, err := json.Marshal(dirInode) - if err != nil { // skipcq: TCV-001 - return err - } - - // upload the metadata as blob - previousAddr, _, err := d.fd.GetFeedData(topic, d.userAddress, []byte(podPassword), false) - if err == nil && previousAddr != nil { - _, err = d.fd.UpdateFeed(d.userAddress, topic, data, []byte(podPassword), false) - if err != nil { // skipcq: TCV-001 - return err - } - } else { - _, err = d.fd.CreateFeed(d.userAddress, topic, data, []byte(podPassword)) - if err != nil { // skipcq: TCV-001 - return err - } - } - - d.AddToDirectoryMap(totalPath, dirInode) - - // get the parent directory entry and add this new directory to its list of children - parentHash := utils.HashString(utils.CombinePathAndFile(parentPath, "")) - dirName = "_D_" + dirName - _, parentData, err := d.fd.GetFeedData(parentHash, d.userAddress, []byte(podPassword), false) - if err != nil { - return err - } - - // unmarshall the data and add the directory entry to the parent - var parentDirInode *Inode - err = json.Unmarshal(parentData, &parentDirInode) - if err != nil { // skipcq: TCV-001 - return err - } - parentDirInode.FileOrDirNames = append(parentDirInode.FileOrDirNames, dirName) - // marshall it back and update the parent feed - parentData, err = json.Marshal(parentDirInode) + err = d.SetInode(podPassword, dirInode) if err != nil { // skipcq: TCV-001 return err } - _, err = d.fd.UpdateFeed(d.userAddress, parentHash, parentData, []byte(podPassword), false) - if err != nil { // skipcq: TCV-001 - return err - } - d.AddToDirectoryMap(parentPath, parentDirInode) - return nil + return d.AddEntryToDir(parentPath, podPassword, dirName, false) } // MkRootDir creates the root directory for the pod @@ -143,42 +101,15 @@ func (d *Directory) MkRootDir(podName, podPassword string, podAddress utils.Addr parentDirInode := &Inode{ Meta: &meta, } - - parentData, err := json.Marshal(&parentDirInode) - if err != nil { // skipcq: TCV-001 - return err - } - parentPath := utils.CombinePathAndFile(utils.PathSeparator, "") - parentHash := utils.HashString(parentPath) - addr, data, err := d.fd.GetFeedData(parentHash, d.userAddress, []byte(podPassword), false) - if err == nil && addr != nil && data != nil { - _, err = fd.UpdateFeed(podAddress, parentHash, parentData, []byte(podPassword), false) - if err != nil { // skipcq: TCV-001 - return err - } - } else { - _, err = fd.CreateFeed(podAddress, parentHash, parentData, []byte(podPassword)) - if err != nil { // skipcq: TCV-001 - return err - } - } - d.AddToDirectoryMap(utils.PathSeparator, parentDirInode) - return nil + return d.SetInode(podPassword, parentDirInode) } // AddRootDir adds the root directory to the directory map func (d *Directory) AddRootDir(podName, podPassword string, podAddress utils.Address, fd *feed.API) error { - parentPath := utils.CombinePathAndFile(utils.PathSeparator, "") - parentHash := utils.HashString(parentPath) - _, parentDataBytes, err := fd.GetFeedData(parentHash, podAddress, []byte(podPassword), false) + parentDirInode, err := d.GetInode(podPassword, utils.CombinePathAndFile(utils.PathSeparator, "")) if err != nil { return err } - var parentDirInode Inode - err = parentDirInode.Unmarshal(parentDataBytes) - if err != nil { - return err - } - d.AddToDirectoryMap(utils.PathSeparator, &parentDirInode) + d.AddToDirectoryMap(utils.PathSeparator, parentDirInode) return nil } diff --git a/pkg/dir/mkdir_test.go b/pkg/dir/mkdir_test.go index e3b25f67..15a49f6d 100644 --- a/pkg/dir/mkdir_test.go +++ b/pkg/dir/mkdir_test.go @@ -22,22 +22,34 @@ import ( "testing" "time" + "github.com/fairdatasociety/fairOS-dfs/pkg/file" + + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" + "github.com/sirupsen/logrus" "github.com/plexsysio/taskmanager" "github.com/fairdatasociety/fairOS-dfs/pkg/account" - bm "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/dir" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" - fm "github.com/fairdatasociety/fairOS-dfs/pkg/file/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" ) func TestMkdir(t *testing.T) { - mockClient := bm.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -52,11 +64,12 @@ func TestMkdir(t *testing.T) { _ = tm.Stop(context.Background()) }() - fd := feed.New(pod1AccountInfo, mockClient, logger) user := acc.GetAddress(1) - mockFile := fm.NewMockFile() t.Run("simple-mkdir", func(t *testing.T) { + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) + mockFile := file.NewFile("pod1", mockClient, fd, user, tm, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) dirObject := dir.NewDirectory("pod1", mockClient, fd, user, mockFile, tm, logger) @@ -86,6 +99,9 @@ func TestMkdir(t *testing.T) { }) t.Run("complicated-mkdir", func(t *testing.T) { + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) + mockFile := file.NewFile("pod1", mockClient, fd, user, tm, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) dirObject := dir.NewDirectory("pod1", mockClient, fd, user, mockFile, tm, logger) diff --git a/pkg/dir/modify_dir_entry.go b/pkg/dir/modify_dir_entry.go index 9687b74d..ebceb3b5 100644 --- a/pkg/dir/modify_dir_entry.go +++ b/pkg/dir/modify_dir_entry.go @@ -17,12 +17,12 @@ limitations under the License. package dir import ( - "encoding/json" - "fmt" "path/filepath" "time" +) - "github.com/fairdatasociety/fairOS-dfs/pkg/utils" +const ( + indexFileName = "index.dfs" ) // AddEntryToDir adds a new entry (directory/file) to a given directory. @@ -38,9 +38,9 @@ func (d *Directory) AddEntryToDir(parentDir, podPassword, itemToAdd string, isFi return ErrInvalidFileOrDirectoryName } - dirInode := d.GetInode(podPassword, parentDir) + dirInode, err := d.GetInode(podPassword, parentDir) // check if parent directory present - if dirInode == nil { + if err != nil { return ErrDirectoryNotPresent } @@ -53,19 +53,7 @@ func (d *Directory) AddEntryToDir(parentDir, podPassword, itemToAdd string, isFi dirInode.FileOrDirNames = append(dirInode.FileOrDirNames, itemToAdd) dirInode.Meta.ModificationTime = time.Now().Unix() - // update the feed of the dir and the data structure with the latest info - data, err := json.Marshal(dirInode) - if err != nil { // skipcq: TCV-001 - return fmt.Errorf("modify dir entry : %v", err) - } - - topic := utils.HashString(parentDir) - _, err = d.fd.UpdateFeed(d.userAddress, topic, data, []byte(podPassword), false) - if err != nil { // skipcq: TCV-001 - return fmt.Errorf("modify dir entry : %v", err) - } - d.AddToDirectoryMap(parentDir, dirInode) - return nil + return d.SetInode(podPassword, dirInode) } // RemoveEntryFromDir removes an entry (directory/file) under the given directory. @@ -81,15 +69,13 @@ func (d *Directory) RemoveEntryFromDir(parentDir, podPassword, itemToDelete stri return ErrInvalidFileOrDirectoryName } parentDir = filepath.ToSlash(parentDir) - parentDirInode := d.GetInode(podPassword, parentDir) + parentDirInode, err := d.GetInode(podPassword, parentDir) // check if parent directory present - if parentDirInode == nil { + if err != nil { d.logger.Errorf("remove entry from dir: parent directory not present %s\n", parentDir) return ErrDirectoryNotPresent } - parentHash := utils.HashString(parentDir) - if isFile { itemToDelete = "_F_" + itemToDelete } else { @@ -105,14 +91,5 @@ func (d *Directory) RemoveEntryFromDir(parentDir, podPassword, itemToDelete stri parentDirInode.FileOrDirNames = fileNames parentDirInode.Meta.ModificationTime = time.Now().Unix() - parentData, err := json.Marshal(parentDirInode) - if err != nil { // skipcq: TCV-001 - return err - } - _, err = d.fd.UpdateFeed(d.userAddress, parentHash, parentData, []byte(podPassword), false) - if err != nil { // skipcq: TCV-001 - return err - } - d.AddToDirectoryMap(parentDir, parentDirInode) - return nil + return d.SetInode(podPassword, parentDirInode) } diff --git a/pkg/dir/rename.go b/pkg/dir/rename.go index 3f713675..d9b136dc 100644 --- a/pkg/dir/rename.go +++ b/pkg/dir/rename.go @@ -1,6 +1,7 @@ package dir import ( + "bufio" "encoding/json" "fmt" "path/filepath" @@ -36,33 +37,27 @@ func (d *Directory) RenameDir(dirNameWithPath, newDirNameWithPath, podPassword s } // check if directory exists - if d.GetInode(podPassword, dirNameWithPath) == nil { // skipcq: TCV-001 + _, err := d.GetInode(podPassword, dirNameWithPath) + if err != nil { // skipcq: TCV-001 return ErrDirectoryNotPresent } // check if parent directory exists - if d.GetInode(podPassword, parentPath) == nil { // skipcq: TCV-001 + _, err = d.GetInode(podPassword, parentPath) + if err != nil { // skipcq: TCV-001 return ErrDirectoryNotPresent } - if d.GetInode(podPassword, newDirNameWithPath) != nil { + _, err = d.GetInode(podPassword, newDirNameWithPath) + if err == nil { return ErrDirectoryAlreadyPresent } - err := d.mapChildrenToNewPath(dirNameWithPath, newDirNameWithPath, podPassword) + err = d.mapChildrenToNewPath(dirNameWithPath, newDirNameWithPath, podPassword) if err != nil { // skipcq: TCV-001 return err } - topic := utils.HashString(dirNameWithPath) - newTopic := utils.HashString(newDirNameWithPath) - _, inodeData, err := d.fd.GetFeedData(topic, d.userAddress, []byte(podPassword), false) - if err != nil { - return err - } - - // unmarshall the data and rename the directory entry - var inode *Inode - err = json.Unmarshal(inodeData, &inode) + inode, err := d.GetInode(podPassword, dirNameWithPath) if err != nil { // skipcq: TCV-001 return err } @@ -77,25 +72,16 @@ func (d *Directory) RenameDir(dirNameWithPath, newDirNameWithPath, podPassword s return err } - previousAddr, _, err := d.fd.GetFeedData(newTopic, d.userAddress, []byte(podPassword), false) - if err == nil && previousAddr != nil { - _, err = d.fd.UpdateFeed(d.userAddress, newTopic, fileMetaBytes, []byte(podPassword), false) - if err != nil { // skipcq: TCV-001 - return err - } - } else { - _, err = d.fd.CreateFeed(d.userAddress, newTopic, fileMetaBytes, []byte(podPassword)) - if err != nil { // skipcq: TCV-001 - return err - } + err = d.file.Upload(bufio.NewReader(strings.NewReader(string(fileMetaBytes))), indexFileName, int64(len(fileMetaBytes)), file.MinBlockSize, 0, newDirNameWithPath, "gzip", podPassword) + if err != nil { // skipcq: TCV-001 + return err } - // delete old meta - // update with utils.DeletedFeedMagicWord - _, err = d.fd.UpdateFeed(d.userAddress, topic, []byte(utils.DeletedFeedMagicWord), []byte(podPassword), false) + err = d.file.RmFile(utils.CombinePathAndFile(dirNameWithPath, indexFileName), podPassword) if err != nil { // skipcq: TCV-001 return err } + d.RemoveFromDirectoryMap(dirNameWithPath) // get the parent directory entry and add this new directory to its list of children @@ -155,12 +141,12 @@ func (d *Directory) mapChildrenToNewPath(totalPath, newTotalPath, podPassword st previousAddr, _, err := d.fd.GetFeedData(newTopic, d.userAddress, []byte(podPassword), false) if err == nil && previousAddr != nil { - _, err = d.fd.UpdateFeed(d.userAddress, newTopic, fileMetaBytes, []byte(podPassword), false) + err = d.fd.UpdateFeed(d.userAddress, newTopic, fileMetaBytes, []byte(podPassword), false) if err != nil { // skipcq: TCV-001 return err } } else { - _, err = d.fd.CreateFeed(d.userAddress, newTopic, fileMetaBytes, []byte(podPassword)) + err = d.fd.CreateFeed(d.userAddress, newTopic, fileMetaBytes, []byte(podPassword)) if err != nil { // skipcq: TCV-001 return err } @@ -168,7 +154,7 @@ func (d *Directory) mapChildrenToNewPath(totalPath, newTotalPath, podPassword st // delete old meta // update with utils.DeletedFeedMagicWord - _, err = d.fd.UpdateFeed(d.userAddress, topic, []byte(utils.DeletedFeedMagicWord), []byte(podPassword), false) + err = d.fd.UpdateFeed(d.userAddress, topic, []byte(utils.DeletedFeedMagicWord), []byte(podPassword), false) if err != nil { // skipcq: TCV-001 return err } @@ -180,41 +166,22 @@ func (d *Directory) mapChildrenToNewPath(totalPath, newTotalPath, podPassword st if err != nil { // skipcq: TCV-001 return err } - topic := utils.HashString(pathWithDir) - newTopic := utils.HashString(newPathWithDir) - _, inodeData, err := d.fd.GetFeedData(topic, d.userAddress, []byte(podPassword), false) - if err != nil { - return err - } - // unmarshall the data and add the directory entry to the parent - var inode *Inode - err = json.Unmarshal(inodeData, &inode) + + inode, err := d.GetInode(podPassword, pathWithDir) if err != nil { // skipcq: TCV-001 return err } + inode.Meta.Path = newTotalPath inode.Meta.ModificationTime = time.Now().Unix() - // upload meta - fileMetaBytes, err := json.Marshal(inode) + + err = d.SetInode(podPassword, inode) if err != nil { // skipcq: TCV-001 return err } - previousAddr, _, err := d.fd.GetFeedData(newTopic, d.userAddress, []byte(podPassword), false) - if err == nil && previousAddr != nil { - _, err = d.fd.UpdateFeed(d.userAddress, newTopic, fileMetaBytes, []byte(podPassword), false) - if err != nil { // skipcq: TCV-001 - return err - } - } else { - _, err = d.fd.CreateFeed(d.userAddress, newTopic, fileMetaBytes, []byte(podPassword)) - if err != nil { // skipcq: TCV-001 - return err - } - } // delete old meta - // update with utils.DeletedFeedMagicWord - _, err = d.fd.UpdateFeed(d.userAddress, topic, []byte(utils.DeletedFeedMagicWord), []byte(podPassword), false) + err = d.RemoveInode(podPassword, pathWithDir) if err != nil { // skipcq: TCV-001 return err } diff --git a/pkg/dir/rename_test.go b/pkg/dir/rename_test.go index e45770f8..0c732f86 100644 --- a/pkg/dir/rename_test.go +++ b/pkg/dir/rename_test.go @@ -9,8 +9,11 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" - bm "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/dir" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" "github.com/fairdatasociety/fairOS-dfs/pkg/file" @@ -18,11 +21,19 @@ import ( "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" ) func TestRenameDirectory(t *testing.T) { - mockClient := bm.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -32,7 +43,7 @@ func TestRenameDirectory(t *testing.T) { if err != nil { t.Fatal(err) } - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { diff --git a/pkg/dir/rmdir.go b/pkg/dir/rmdir.go index b8e6cedc..1918a5f7 100644 --- a/pkg/dir/rmdir.go +++ b/pkg/dir/rmdir.go @@ -47,8 +47,8 @@ func (d *Directory) RmDir(directoryNameWithPath, podPassword string) error { totalPath = utils.CombinePathAndFile(parentPath, dirToDelete) } // recursive delete - dirInode := d.GetInode(podPassword, totalPath) - if dirInode == nil { + dirInode, err := d.GetInode(podPassword, totalPath) + if err != nil { return ErrDirectoryNotPresent } @@ -78,20 +78,7 @@ func (d *Directory) RmDir(directoryNameWithPath, podPassword string) error { } } - // remove the feed and clear the data structure - topic := utils.HashString(totalPath) - _, err := d.fd.UpdateFeed(d.userAddress, topic, []byte(utils.DeletedFeedMagicWord), []byte(podPassword), false) - if err != nil { // skipcq: TCV-001 - return err - } - d.RemoveFromDirectoryMap(totalPath) - // return if root directory - if parentPath == utils.PathSeparator && filepath.ToSlash(dirToDelete) == utils.PathSeparator { - return nil - } - // remove the directory entry from the parent dir - - return d.RemoveEntryFromDir(parentPath, podPassword, dirToDelete, false) + return d.RemoveInode(podPassword, directoryNameWithPath) } // RmRootDir removes root directory and all the entries (file/directory) under that. @@ -104,7 +91,6 @@ func (d *Directory) RmRootDir(podPassword string) error { if d.GetDirFromDirectoryMap(totalPath) == nil { // skipcq: TCV-001 return ErrDirectoryNotPresent } - // recursive delete dirInode := d.GetDirFromDirectoryMap(totalPath) if dirInode.FileOrDirNames != nil && len(dirInode.FileOrDirNames) > 0 { @@ -132,14 +118,5 @@ func (d *Directory) RmRootDir(podPassword string) error { } } } - - // remove the feed and clear the data structure - topic := utils.HashString(totalPath) - _, err := d.fd.UpdateFeed(d.userAddress, topic, []byte(utils.DeletedFeedMagicWord), []byte(podPassword), false) - if err != nil { // skipcq: TCV-001 - return err - } - d.RemoveFromDirectoryMap(totalPath) - - return nil + return d.RemoveInode(podPassword, dirToDelete) } diff --git a/pkg/dir/rmdir_test.go b/pkg/dir/rmdir_test.go index 1eb2482e..5390ec34 100644 --- a/pkg/dir/rmdir_test.go +++ b/pkg/dir/rmdir_test.go @@ -20,26 +20,37 @@ import ( "context" "errors" "io" - "strings" "testing" "time" + "github.com/fairdatasociety/fairOS-dfs/pkg/file" + + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" + "github.com/sirupsen/logrus" "github.com/plexsysio/taskmanager" "github.com/fairdatasociety/fairOS-dfs/pkg/account" - bm "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/dir" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" - fm "github.com/fairdatasociety/fairOS-dfs/pkg/file/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" ) func TestRmdir(t *testing.T) { - mockClient := bm.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -54,9 +65,9 @@ func TestRmdir(t *testing.T) { _ = tm.Stop(context.Background()) }() - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) - mockFile := fm.NewMockFile() + mockFile := file.NewFile("pod1", mockClient, fd, user, tm, logger) t.Run("simple-rmdir", func(t *testing.T) { podPassword, _ := utils.GetRandString(pod.PasswordLength) @@ -168,8 +179,15 @@ func TestRmdir(t *testing.T) { } func TestRmRootDirByPath(t *testing.T) { - mockClient := bm.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -179,13 +197,13 @@ func TestRmRootDirByPath(t *testing.T) { if err != nil { t.Fatal(err) } - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) - mockFile := fm.NewMockFile() tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) }() + mockFile := file.NewFile("pod1", mockClient, fd, user, tm, logger) t.Run("rmrootdir", func(t *testing.T) { podPassword, _ := utils.GetRandString(pod.PasswordLength) @@ -229,16 +247,11 @@ func TestRmRootDirByPath(t *testing.T) { t.Fatal("nested directory \"/dirToRemove1/dirToRemove2/dirToRemove\" was not created") } - fileName := "file1" - err = dirObject.AddEntryToDir("/dirToRemove1", podPassword, fileName, true) - if err != nil { - t.Fatal(err) - } _, fileEntry, err := dirObject.ListDir("/dirToRemove1", podPassword) if err != nil { t.Fatal(err) } - if len(fileEntry) != 1 { + if len(fileEntry) != 0 { t.Fatal("there should a file entry") } // now delete the root directory @@ -249,7 +262,7 @@ func TestRmRootDirByPath(t *testing.T) { // verify if the directory is actually removed dirEntry, _, err = dirObject.ListDir("/", podPassword) - if err != nil && !strings.HasSuffix(err.Error(), dir.ErrResourceDeleted.Error()) { + if err == nil { t.Fatal("root directory was not deleted") } if dirEntry != nil { @@ -259,8 +272,15 @@ func TestRmRootDirByPath(t *testing.T) { } func TestRmRootDir(t *testing.T) { - mockClient := bm.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -275,9 +295,9 @@ func TestRmRootDir(t *testing.T) { _ = tm.Stop(context.Background()) }() - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) - mockFile := fm.NewMockFile() + mockFile := file.NewFile("pod1", mockClient, fd, user, tm, logger) t.Run("rmrootdir", func(t *testing.T) { podPassword, _ := utils.GetRandString(pod.PasswordLength) @@ -325,17 +345,12 @@ func TestRmRootDir(t *testing.T) { t.Fatal("nested directory \"/dirToRemove1/dirToRemove2/dirToRemove\" was not created") } - fileName := "file1" - err = dirObject.AddEntryToDir("/", podPassword, fileName, true) - if err != nil { - t.Fatal(err) - } _, fileEntry, err := dirObject.ListDir("/", podPassword) if err != nil { t.Fatal(err) } - if len(fileEntry) != 1 { - t.Fatal("there should a file entry") + if len(fileEntry) != 0 { + t.Fatal("there should no file entry") } // now delete the root directory @@ -346,7 +361,7 @@ func TestRmRootDir(t *testing.T) { // verify if the directory is actually removed dirEntry, _, err = dirObject.ListDir("/", podPassword) - if err != nil && !strings.HasSuffix(err.Error(), dir.ErrResourceDeleted.Error()) { + if err == nil { t.Fatal("root directory was not deleted") } if dirEntry != nil { diff --git a/pkg/dir/stat.go b/pkg/dir/stat.go index 72391e00..db6d0cc9 100644 --- a/pkg/dir/stat.go +++ b/pkg/dir/stat.go @@ -17,12 +17,8 @@ limitations under the License. package dir import ( - "encoding/json" - "fmt" "strconv" "strings" - - "github.com/fairdatasociety/fairOS-dfs/pkg/utils" ) // Stats represents a given directory @@ -40,19 +36,10 @@ type Stats struct { // DirStat returns all the information related to a given directory. func (d *Directory) DirStat(podName, podPassword, dirNameWithPath string) (*Stats, error) { - topic := utils.HashString(dirNameWithPath) - _, data, err := d.fd.GetFeedData(topic, d.getAddress(), []byte(podPassword), false) - if err != nil { // skipcq: TCV-001 - return nil, fmt.Errorf("dir stat: %v", err) - } - if string(data) == utils.DeletedFeedMagicWord { - return nil, ErrDirectoryNotPresent - } - - var dirInode Inode - err = json.Unmarshal(data, &dirInode) + dirInode, err := d.GetInode(podPassword, dirNameWithPath) if err != nil { // skipcq: TCV-001 - return nil, fmt.Errorf("dir stat: %v", err) + d.logger.Errorf("dir stat : %v", err) + return nil, err } if dirInode.Meta == nil && dirInode.FileOrDirNames == nil { // skipcq: TCV-001 diff --git a/pkg/dir/stat_test.go b/pkg/dir/stat_test.go index c20a772c..99a7d494 100644 --- a/pkg/dir/stat_test.go +++ b/pkg/dir/stat_test.go @@ -24,22 +24,34 @@ import ( "testing" "time" + "github.com/fairdatasociety/fairOS-dfs/pkg/file" + + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" + "github.com/sirupsen/logrus" "github.com/plexsysio/taskmanager" "github.com/fairdatasociety/fairOS-dfs/pkg/account" - bm "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/dir" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" - fm "github.com/fairdatasociety/fairOS-dfs/pkg/file/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" ) func TestStat(t *testing.T) { - mockClient := bm.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -49,13 +61,13 @@ func TestStat(t *testing.T) { if err != nil { t.Fatal(err) } - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) - mockFile := fm.NewMockFile() tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) }() + mockFile := file.NewFile("pod1", mockClient, fd, user, tm, logger) t.Run("stat-dir", func(t *testing.T) { podPassword, _ := utils.GetRandString(pod.PasswordLength) @@ -80,16 +92,6 @@ func TestStat(t *testing.T) { if err != nil { t.Fatal(err) } - // just add dummy file entry as file listing is not tested here - err = dirObject.AddEntryToDir("/dirToStat", podPassword, "file1", true) - if err != nil { - t.Fatal(err) - } - err = dirObject.AddEntryToDir("/dirToStat", podPassword, "file2", true) - if err != nil { - t.Fatal(err) - } - // stat the directory dirStats, err := dirObject.DirStat("pod1", podPassword, "/dirToStat") if err != nil { @@ -112,7 +114,7 @@ func TestStat(t *testing.T) { if dirStats.NoOfDirectories != strconv.FormatUint(2, 10) { t.Fatalf("invalid directory count") } - if dirStats.NoOfFiles != strconv.FormatUint(2, 10) { + if dirStats.NoOfFiles != strconv.FormatUint(0, 10) { t.Fatalf("invalid files count") } @@ -123,7 +125,7 @@ func TestStat(t *testing.T) { _, err = dirObject.DirStat("pod1", podPassword, "/dirToStat") if !errors.Is(err, dir.ErrDirectoryNotPresent) { - t.Fatal("dir should not be present") + t.Fatal("dir should not be present", err) } }) } diff --git a/pkg/dir/sync.go b/pkg/dir/sync.go index 2d94342b..b2a230b1 100644 --- a/pkg/dir/sync.go +++ b/pkg/dir/sync.go @@ -26,19 +26,12 @@ import ( // SyncDirectory syncs all the latest entries under a given directory. func (d *Directory) SyncDirectory(dirNameWithPath, podPassword string) error { - topic := utils.HashString(utils.CombinePathAndFile(dirNameWithPath, "")) - _, data, err := d.fd.GetFeedData(topic, d.userAddress, []byte(podPassword), false) + dirInode, err := d.GetInode(podPassword, dirNameWithPath) if err != nil { // skipcq: TCV-001 return nil // pod is empty } - var dirInode Inode - err = dirInode.Unmarshal(data) - if err != nil { // skipcq: TCV-001 - d.logger.Errorf("dir sync: %v", err) - return err - } - d.AddToDirectoryMap(dirNameWithPath, &dirInode) + d.AddToDirectoryMap(dirNameWithPath, dirInode) for _, fileOrDirName := range dirInode.FileOrDirNames { if strings.HasPrefix(fileOrDirName, "_F_") { fileName := strings.TrimPrefix(fileOrDirName, "_F_") @@ -63,20 +56,11 @@ func (d *Directory) SyncDirectory(dirNameWithPath, podPassword string) error { // SyncDirectoryAsync syncs all the latest entries under a given directory concurrently. func (d *Directory) SyncDirectoryAsync(ctx context.Context, dirNameWithPath, podPassword string, wg *sync.WaitGroup) error { - topic := utils.HashString(utils.CombinePathAndFile(dirNameWithPath, "")) - _, data, err := d.fd.GetFeedData(topic, d.userAddress, []byte(podPassword), false) + dirInode, err := d.GetInode(podPassword, dirNameWithPath) if err != nil { // skipcq: TCV-001 return nil // pod is empty } - var dirInode Inode - err = dirInode.Unmarshal(data) - if err != nil { // skipcq: TCV-001 - d.logger.Errorf("dir sync: %v", err) - return err - } - - d.AddToDirectoryMap(dirNameWithPath, &dirInode) for _, fileOrDirName := range dirInode.FileOrDirNames { if strings.HasPrefix(fileOrDirName, "_F_") { wg.Add(1) diff --git a/pkg/dir/sync_test.go b/pkg/dir/sync_test.go index 6a0b24b2..07561b51 100644 --- a/pkg/dir/sync_test.go +++ b/pkg/dir/sync_test.go @@ -23,15 +23,20 @@ import ( "testing" "time" + "github.com/fairdatasociety/fairOS-dfs/pkg/file" + + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" - bm "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/dir" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" - fm "github.com/fairdatasociety/fairOS-dfs/pkg/file/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" "go.uber.org/goleak" ) @@ -40,8 +45,15 @@ func TestMain(m *testing.M) { } func TestSync(t *testing.T) { - mockClient := bm.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -51,13 +63,13 @@ func TestSync(t *testing.T) { if err != nil { t.Fatal(err) } - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) - mockFile := fm.NewMockFile() tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) }() + mockFile := file.NewFile("pod1", mockClient, fd, user, tm, logger) t.Run("sync-dir", func(t *testing.T) { podPassword, _ := utils.GetRandString(pod.PasswordLength) @@ -82,15 +94,7 @@ func TestSync(t *testing.T) { if err != nil { t.Fatal(err) } - // just add dummy file entry as file listing is not tested here - err = dirObject.AddEntryToDir("/dirToStat", podPassword, "file1", true) - if err != nil { - t.Fatal(err) - } - err = dirObject.AddEntryToDir("/dirToStat", podPassword, "file2", true) - if err != nil { - t.Fatal(err) - } + dirObject2 := dir.NewDirectory("pod1", mockClient, fd, user, mockFile, tm, logger) if dirObject2.GetDirFromDirectoryMap("/") != nil { t.Fatal("it should be nil before sync") diff --git a/pkg/feed/api.go b/pkg/feed/api.go index 287e3be3..3584106b 100644 --- a/pkg/feed/api.go +++ b/pkg/feed/api.go @@ -19,11 +19,9 @@ package feed import ( "context" "fmt" - "strings" "time" "github.com/ethersphere/bee/pkg/crypto" - "github.com/ethersphere/bee/pkg/soc" "github.com/ethersphere/bee/pkg/swarm" bmtlegacy "github.com/ethersphere/bmt/legacy" "github.com/fairdatasociety/fairOS-dfs/pkg/account" @@ -71,96 +69,63 @@ type request struct { } // New create the main feed object which is used to create/update/delete feeds. -func New(accountInfo *account.Info, client blockstore.Client, logger logging.Logger) *API { +func New(accountInfo *account.Info, client blockstore.Client, feedCacheSize int, feedCacheTTL time.Duration, logger logging.Logger) *API { bmtPool := bmtlegacy.NewTreePool(hashFunc, swarm.Branches, bmtlegacy.PoolSize) return &API{ - handler: NewHandler(accountInfo, client, bmtPool), + handler: NewHandler(accountInfo, client, bmtPool, feedCacheSize, feedCacheTTL, logger), accountInfo: accountInfo, logger: logger, } } +func (a *API) CommitFeeds() { + a.handler.commit() +} + // CreateFeed creates a feed by constructing a single owner chunk. This chunk // can only be accessed if the pod address is known. Also, no one else can spoof this // chunk since this is signed by the pod. -func (a *API) CreateFeed(user utils.Address, topic, data, encryptionPassword []byte) ([]byte, error) { - var req request +func (a *API) CreateFeed(user utils.Address, topic, data, encryptionPassword []byte) error { if a.accountInfo.GetPrivateKey() == nil { - return nil, ErrReadOnlyFeed + return ErrReadOnlyFeed } if len(topic) != TopicLength { - return nil, ErrInvalidTopicSize + return ErrInvalidTopicSize } if len(data) > utils.MaxChunkLength { - return nil, ErrInvalidPayloadSize + return ErrInvalidPayloadSize } var err error encryptedData := data - if encryptionPassword != nil { // skipcq: TCV-001 + if len(encryptionPassword) != 0 { // skipcq: TCV-001 encryptedData, err = utils.EncryptBytes(encryptionPassword, data) if err != nil { // skipcq: TCV-001 - return nil, err + return err } } - // fill Feed and Epoc related details - copy(req.ID.Topic[:], topic) - req.ID.User = user - req.Epoch.Level = 31 - req.Epoch.Time = uint64(time.Now().Unix()) - - // Add initial feed data - req.data = encryptedData - // create the id, hash(topic, epoc) - id, err := a.handler.getId(req.Topic, req.Time, req.Level) - if err != nil { // skipcq: TCV-001 - return nil, err - } - // get the payload id BMT(span, payload) - payloadId, err := a.handler.getPayloadId(encryptedData) - if err != nil { // skipcq: TCV-001 - return nil, err - } - // create the signer and the content addressed chunk - signer := crypto.NewDefaultSigner(a.accountInfo.GetPrivateKey()) - ch, err := utils.NewChunkWithSpan(encryptedData) - if err != nil { // skipcq: TCV-001 - return nil, err - } - s := soc.New(id, ch) - sch, err := s.Sign(signer) - if err != nil { // skipcq: TCV-001 - return nil, err - } - // generate the data to sign - toSignBytes, err := toSignDigest(id, ch.Address().Bytes()) - if err != nil { // skipcq: TCV-001 - return nil, err - } - // sign the chunk - signature, err := signer.Sign(toSignBytes) - if err != nil { // skipcq: TCV-001 - return nil, err - } - // set the address and the data for the soc chunk - req.idAddr = sch.Address() - req.binaryData = sch.Data() - // set signature and binary data fields - _, err = a.handler.toChunkContent(&req, id, payloadId) - if err != nil { // skipcq: TCV-001 - return nil, err + if a.handler.pool != nil { + item := &feedItem{ + User: user, + AccountInfo: a.accountInfo, + Topic: topic, + Data: encryptedData, + ShouldCreate: true, + } + a.handler.putInPool(topic, item) + return nil } - // send the updated soc chunk to bee - addr, err := a.handler.update(id, user.ToBytes(), signature, ch.Data()) - if err != nil { // skipcq: TCV-001 - return nil, err + _, _, err = a.handler.createSoc(user, a.accountInfo, topic, encryptedData) + if err != nil { + a.handler.logger.Errorf("failed to createSoc: %v\n", err) + return err } - return addr, nil + return nil } // CreateFeedFromTopic creates a soc with the topic as identifier @@ -222,39 +187,21 @@ func (a *API) GetFeedData(topic []byte, user utils.Address, encryptionPassword [ if len(topic) != TopicLength { return nil, nil, ErrInvalidTopicSize } - ctx := context.Background() - f := new(Feed) - f.User = user - copy(f.Topic[:], topic) - - // create the query from values - q := &Query{Feed: *f} - q.TimeLimit = 0 - q.Hint = lookup.NoClue - if q.Hint == lookup.NoClue { - _, err := a.handler.Lookup(ctx, q) - if err != nil { - return nil, nil, err - } - } else { - _, err := a.handler.LookupEpoch(ctx, q) - if err != nil { - return nil, nil, err - } - } - addr, encryptedData, err := a.handler.GetContent(&q.Feed) - if err != nil { // skipcq: TCV-001 + hint := lookup.NoClue + + addr, data, err := a.handler.getSoc(topic, user, hint) + if err != nil { return nil, nil, err } - if encryptionPassword == nil || string(encryptedData) == utils.DeletedFeedMagicWord { - return addr.Bytes(), encryptedData, nil + if len(encryptionPassword) == 0 || string(data) == utils.DeletedFeedMagicWord { + return addr, data, nil } - data, err := utils.DecryptBytes(encryptionPassword, encryptedData) + decryptedData, err := utils.DecryptBytes(encryptionPassword, data) if err != nil { // skipcq: TCV-001 return nil, nil, err } - return addr.Bytes(), data, nil + return addr, decryptedData, nil } // GetFeedDataFromTopic will generate keccak256 reference of the topic+address and download soc @@ -283,95 +230,45 @@ func (a *API) GetFeedDataFromTopic(topic []byte, user utils.Address) ([]byte, [] } // UpdateFeed updates the contents of an already created feed. -func (a *API) UpdateFeed(user utils.Address, topic, data, encryptionPassword []byte, isFeedUpdater bool) ([]byte, error) { +func (a *API) UpdateFeed(user utils.Address, topic, data, encryptionPassword []byte, isFeedUpdater bool) error { if a.accountInfo.GetPrivateKey() == nil { - return nil, ErrReadOnlyFeed + return ErrReadOnlyFeed } if len(topic) != TopicLength { - return nil, ErrInvalidTopicSize + return ErrInvalidTopicSize } if len(data) > utils.MaxChunkLength { - return nil, ErrInvalidPayloadSize + return ErrInvalidPayloadSize } var err error encryptedData := data - if encryptionPassword != nil && string(data) != utils.DeletedFeedMagicWord { + if len(encryptionPassword) != 0 && string(data) != utils.DeletedFeedMagicWord { encryptedData, err = utils.EncryptBytes(encryptionPassword, data) if err != nil { // skipcq: TCV-001 - return nil, err + return err } } - retries := 0 -retry: - ctx := context.Background() - f := new(Feed) - f.User = user - copy(f.Topic[:], topic) - - // get the existing request from DB - req, err := a.handler.newRequest(ctx, f) - if err != nil { // skipcq: TCV-001 - return nil, err - } - req.Time = uint64(time.Now().Unix()) - req.data = encryptedData - // create the id, hash(topic, epoc) - id, err := a.handler.getId(req.Topic, req.Time, req.Level) - if err != nil { // skipcq: TCV-001 - return nil, err - } - // get the payload id BMT(span, payload) - payloadId, err := a.handler.getPayloadId(encryptedData) - if err != nil { // skipcq: TCV-001 - return nil, err - } - // create the signer and the content addressed chunk - signer := crypto.NewDefaultSigner(a.accountInfo.GetPrivateKey()) - ch, err := utils.NewChunkWithSpan(encryptedData) - if err != nil { // skipcq: TCV-001 - return nil, err - } - s := soc.New(id, ch) - sch, err := s.Sign(signer) - if err != nil { // skipcq: TCV-001 - return nil, err - } - // generate the data to sign - toSignBytes, err := toSignDigest(id, ch.Address().Bytes()) - if err != nil { // skipcq: TCV-001 - return nil, err - } - // sign the chunk - signature, err := signer.Sign(toSignBytes) - if err != nil { // skipcq: TCV-001 - return nil, err - } - // set the address and the data for the soc chunk - req.idAddr = sch.Address() - req.binaryData = sch.Data() - // set signature and binary data fields - _, err = a.handler.toChunkContent(req, id, payloadId) - if err != nil { // skipcq: TCV-001 - return nil, err + if a.handler.pool != nil { + item := &feedItem{ + User: user, + AccountInfo: a.accountInfo, + Topic: topic, + Data: encryptedData, + ShouldCreate: false, + } + a.handler.putInPool(topic, item) + return nil } - - address, err := a.handler.update(id, user.ToBytes(), signature, ch.Data()) + _, _, err = a.handler.updateSoc(user, a.accountInfo, topic, encryptedData) if err != nil { - // updating same feed in the same second will lead to "chunk already exists" error. - // This will wait for 1 second and retry the update maxUpdateRetry times. - // It is a very dirty fix for this issue. We should find a better way to handle this. - if strings.Contains(err.Error(), "chunk already exists") && retries < maxUpdateRetry { - retries++ - <-time.After(1 * time.Second) - goto retry - } - return nil, err + a.handler.logger.Errorf("failed to updateSoc: %v\n", err) + return err } - return address, nil + return nil } // DeleteFeed deleted the feed by updating with no data inside the SOC chunk. @@ -420,5 +317,6 @@ func (a *API) IsReadOnlyFeed() bool { } func (a *API) Close() error { + a.CommitFeeds() return nil } diff --git a/pkg/feed/feed_test.go b/pkg/feed/feed_test.go index 1602419a..817ed7ba 100644 --- a/pkg/feed/feed_test.go +++ b/pkg/feed/feed_test.go @@ -18,12 +18,18 @@ package feed_test import ( "bytes" - "encoding/binary" + "crypto/rand" "errors" "io" "testing" + "time" + "github.com/stretchr/testify/require" + + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" @@ -38,87 +44,174 @@ func TestMain(m *testing.M) { func TestFeed(t *testing.T) { logger := logging.New(io.Discard, 0) - acc1 := account.New(logger) - _, _, err := acc1.CreateUserAccount("") - if err != nil { - t.Fatal(err) - } - user1 := acc1.GetAddress(account.UserAccountIndex) - accountInfo1 := acc1.GetUserAccountInfo() - client := mock.NewMockBeeClient() + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + client := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) t.Run("create-feed", func(t *testing.T) { - fd := feed.New(accountInfo1, client, logger) + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + user := acc.GetAddress(account.UserAccountIndex) + accountInfo := acc.GetUserAccountInfo() + fd := feed.New(accountInfo, client, 500, 0, logger) topic := utils.HashString("topic1") data := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} - addr, err := fd.CreateFeed(user1, topic, data, nil) + err = fd.CreateFeed(user, topic, data, nil) if err != nil { t.Fatal(err) } longTopic := append(topic, topic...) // skipcq: CRT-D0001 - _, _, err = fd.GetFeedData(longTopic, user1, nil, false) + _, _, err = fd.GetFeedData(longTopic, user, nil, false) if !errors.Is(err, feed.ErrInvalidTopicSize) { t.Fatal("invalid topic size") } - + <-time.After(3 * time.Second) // check if the data and address is present and is same as stored - rcvdAddr, rcvdData, err := fd.GetFeedData(topic, user1, nil, false) + _, rcvdData, err := fd.GetFeedData(topic, user, nil, false) if err != nil { t.Fatal(err) } - if !bytes.Equal(addr, rcvdAddr) { + + if !bytes.Equal(data, rcvdData) { + t.Fatal(err) + } + }) + + t.Run("create-feed-nil-feed-cache", func(t *testing.T) { + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + user := acc.GetAddress(account.UserAccountIndex) + accountInfo := acc.GetUserAccountInfo() + fd := feed.New(accountInfo, client, -1, 0, logger) + topic := utils.HashString("topic1") + data := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} + err = fd.CreateFeed(user, topic, data, nil) + if err != nil { t.Fatal(err) } + longTopic := append(topic, topic...) // skipcq: CRT-D0001 + _, _, err = fd.GetFeedData(longTopic, user, nil, false) + if !errors.Is(err, feed.ErrInvalidTopicSize) { + t.Fatal("invalid topic size") + } + <-time.After(3 * time.Second) + // check if the data and address is present and is same as stored + _, rcvdData, err := fd.GetFeedData(topic, user, nil, false) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, rcvdData) { t.Fatal(err) } }) - t.Run("create-from-user1-read-from-user2-with-user1-address", func(t *testing.T) { - // create account2 - acc2 := account.New(logger) - _, _, err = acc2.CreateUserAccount("") + t.Run("create-from-user-read-from-user2-with-user-address", func(t *testing.T) { + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") if err != nil { t.Fatal(err) } - accountInfo2 := acc2.GetUserAccountInfo() + user := acc.GetAddress(account.UserAccountIndex) + accountInfo := acc.GetUserAccountInfo() - // create feed from user1 - fd1 := feed.New(accountInfo1, client, logger) + // create feed from user + fd1 := feed.New(accountInfo, client, 500, 0, logger) topic := utils.HashString("topic1") data := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} - addr, err := fd1.CreateFeed(user1, topic, data, nil) + err = fd1.CreateFeed(user, topic, data, nil) if err != nil { t.Fatal(err) } - + fd1.CommitFeeds() // check if you can read the data from user2 - fd2 := feed.New(accountInfo2, client, logger) - rcvdAddr, rcvdData, err := fd2.GetFeedData(topic, user1, nil, false) + fd2 := feed.New(accountInfo, client, -1, 0, logger) + _, rcvdData, err := fd2.GetFeedData(topic, user, nil, false) if err != nil { t.Fatal(err) } - if !bytes.Equal(addr, rcvdAddr) { - t.Fatal("addresses do not match") + + if !bytes.Equal(data, rcvdData) { + t.Fatal("data does not match") } + }) + + t.Run("create-from-user-read-from-user2-with-user-address-nil-feed-cache", func(t *testing.T) { + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + user := acc.GetAddress(account.UserAccountIndex) + accountInfo := acc.GetUserAccountInfo() + + // create feed from user + fd1 := feed.New(accountInfo, client, -1, 0, logger) + topic := utils.HashString("topic1") + data := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} + err = fd1.CreateFeed(user, topic, data, nil) + if err != nil { + t.Fatal(err) + } + fd1.CommitFeeds() + // check if you can read the data from user2 + fd2 := feed.New(accountInfo, client, -1, 0, logger) + _, rcvdData, err := fd2.GetFeedData(topic, user, nil, false) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, rcvdData) { t.Fatal("data does not match") } }) t.Run("read-feed-first-time", func(t *testing.T) { - fd := feed.New(accountInfo1, client, logger) + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + user := acc.GetAddress(account.UserAccountIndex) + accountInfo := acc.GetUserAccountInfo() + + fd := feed.New(accountInfo, client, -1, 0, logger) topic := utils.HashString("topic2") // check if the data and address is present and is same as stored - _, _, err := fd.GetFeedData(topic, user1, nil, false) + _, _, err = fd.GetFeedData(topic, user, nil, false) if err != nil && err.Error() != "feed does not exist or was not updated yet" { t.Fatal(err) } }) - t.Run("create-from-user1-read-from-user2-with-user2-address", func(t *testing.T) { - // create account2 + t.Run("read-feed-created-from-different-user", func(t *testing.T) { + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + accountInfo := acc.GetUserAccountInfo() + user := acc.GetAddress(account.UserAccountIndex) + + // create feed from user + fd1 := feed.New(accountInfo, client, 500, 0, logger) + topic := utils.HashString("topic1") + data := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} + err = fd1.CreateFeed(user, topic, data, nil) + if err != nil { + t.Fatal(err) + } + acc2 := account.New(logger) _, _, err = acc2.CreateUserAccount("") if err != nil { @@ -127,72 +220,170 @@ func TestFeed(t *testing.T) { accountInfo2 := acc2.GetUserAccountInfo() user2 := acc2.GetAddress(account.UserAccountIndex) - // create feed from user1 - fd1 := feed.New(accountInfo1, client, logger) + // check if you can read the data from user2 + fd2 := feed.New(accountInfo2, client, -1, 0, logger) + rcvdAddr, rcvdData, err := fd2.GetFeedData(topic, user2, nil, false) + if err != nil && err.Error() != "feed does not exist or was not updated yet" { + t.Fatal(err) + } + if rcvdAddr != nil || rcvdData != nil { + t.Fatal("was able to read feed of user using user2's address") + } + }) + + t.Run("read-feed-created-from-different-user-nil-feed-cache", func(t *testing.T) { + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + accountInfo := acc.GetUserAccountInfo() + user := acc.GetAddress(account.UserAccountIndex) + + // create feed from user + fd1 := feed.New(accountInfo, client, -1, 0, logger) topic := utils.HashString("topic1") data := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} - _, err := fd1.CreateFeed(user1, topic, data, nil) + err = fd1.CreateFeed(user, topic, data, nil) if err != nil { t.Fatal(err) } + acc2 := account.New(logger) + _, _, err = acc2.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + accountInfo2 := acc2.GetUserAccountInfo() + user2 := acc2.GetAddress(account.UserAccountIndex) + // check if you can read the data from user2 - fd2 := feed.New(accountInfo2, client, logger) + fd2 := feed.New(accountInfo2, client, -1, 0, logger) rcvdAddr, rcvdData, err := fd2.GetFeedData(topic, user2, nil, false) if err != nil && err.Error() != "feed does not exist or was not updated yet" { t.Fatal(err) } if rcvdAddr != nil || rcvdData != nil { - t.Fatal("was able to read feed of user1 using user2's address") + t.Fatal("was able to read feed of user using user2's address") } }) t.Run("update-feed", func(t *testing.T) { - fd := feed.New(accountInfo1, client, logger) + + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + user := acc.GetAddress(account.UserAccountIndex) + accountInfo := acc.GetUserAccountInfo() + fd := feed.New(accountInfo, client, 500, 0, logger) topic := utils.HashString("topic3") data := []byte{0} - _, err = fd.CreateFeed(user1, topic, data, nil) + err = fd.CreateFeed(user, topic, data, nil) if err != nil { t.Fatal(err) } + var finalData []byte for i := 1; i < 256; i++ { buf := make([]byte, 4) - binary.LittleEndian.PutUint16(buf, uint16(i)) - - _, err = fd.UpdateFeed(user1, topic, buf, nil, false) + _, _ = rand.Read(buf) + if i == 255 { + finalData = buf + } + err = fd.UpdateFeed(user, topic, buf, nil, false) if err != nil { t.Fatal(err) } - getAddr, rcvdData, err := fd.GetFeedData(topic, user1, nil, false) + _, rcvdData, err := fd.GetFeedData(topic, user, nil, false) if err != nil { t.Fatal(err) } - if getAddr == nil { - t.Fatal("invalid update address") + + require.Equal(t, buf, rcvdData) + } + + fd.CommitFeeds() + <-time.After(time.Second) + _, rcvdData, err := fd.GetFeedData(topic, user, nil, false) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, finalData, rcvdData) + }) + + t.Run("update-feed-nil-feed-cache", func(t *testing.T) { + + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + user := acc.GetAddress(account.UserAccountIndex) + accountInfo := acc.GetUserAccountInfo() + fd := feed.New(accountInfo, client, 500, 0, logger) + topic := utils.HashString("topic3") + data := []byte{0} + err = fd.CreateFeed(user, topic, data, nil) + if err != nil { + t.Fatal(err) + } + var finalData []byte + + for i := 1; i < 256; i++ { + buf := make([]byte, 4) + _, _ = rand.Read(buf) + if i == 255 { + finalData = buf + } + err = fd.UpdateFeed(user, topic, buf, nil, false) + if err != nil { + t.Fatal(err) } - if !bytes.Equal(buf, rcvdData) { - t.Fatal("data not matching", buf, rcvdData) + _, rcvdData, err := fd.GetFeedData(topic, user, nil, false) + if err != nil { + t.Fatal(err) } + + require.Equal(t, buf, rcvdData) + <-time.After(time.Second) } + + fd.CommitFeeds() + <-time.After(time.Second) + _, rcvdData, err := fd.GetFeedData(topic, user, nil, false) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, finalData, rcvdData) }) t.Run("create-feed-from-topic", func(t *testing.T) { - fd := feed.New(accountInfo1, client, logger) + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + user := acc.GetAddress(account.UserAccountIndex) + accountInfo := acc.GetUserAccountInfo() + fd := feed.New(accountInfo, client, -1, 0, logger) topic := utils.HashString("feed-topic1") data := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} - addr, err := fd.CreateFeedFromTopic(topic, user1, data) + addr, err := fd.CreateFeedFromTopic(topic, user, data) if err != nil { t.Fatal(err) } longTopic := append(topic, topic...) // skipcq: CRT-D0001 - _, _, err = fd.GetFeedDataFromTopic(longTopic, user1) + _, _, err = fd.GetFeedDataFromTopic(longTopic, user) if !errors.Is(err, feed.ErrInvalidTopicSize) { t.Fatal("invalid topic size") } // check if the data and address is present and is same as stored - rcvdAddr, rcvdData, err := fd.GetFeedDataFromTopic(topic, user1) + rcvdAddr, rcvdData, err := fd.GetFeedDataFromTopic(topic, user) if err != nil { t.Fatal(err) } @@ -205,38 +396,52 @@ func TestFeed(t *testing.T) { }) t.Run("delete-feed-from-topic", func(t *testing.T) { - fd := feed.New(accountInfo1, client, logger) + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + user := acc.GetAddress(account.UserAccountIndex) + accountInfo := acc.GetUserAccountInfo() + fd := feed.New(accountInfo, client, -1, 0, logger) topic := utils.HashString("feed-topic1") data := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} - _, err := fd.CreateFeedFromTopic(topic, user1, data) + _, err = fd.CreateFeedFromTopic(topic, user, data) if err != nil { t.Fatal(err) } - err = fd.DeleteFeedFromTopic(topic, user1) + err = fd.DeleteFeedFromTopic(topic, user) if err != nil { t.Fatal(err) } - _, _, err = fd.GetFeedDataFromTopic(topic, user1) + _, _, err = fd.GetFeedDataFromTopic(topic, user) if err != nil && err.Error() != "error downloading data" { t.Fatal("error should be \"error downloading data\"") } }) t.Run("create-feed-errors", func(t *testing.T) { - nilFd := feed.New(&account.Info{}, client, logger) + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + user := acc.GetAddress(account.UserAccountIndex) + accountInfo := acc.GetUserAccountInfo() + nilFd := feed.New(&account.Info{}, client, -1, 0, logger) - fd := feed.New(accountInfo1, client, logger) + fd := feed.New(accountInfo, client, -1, 0, logger) topic := utils.HashString("feed-topic1") data := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} - _, err = nilFd.CreateFeed(user1, topic, data, nil) + err = nilFd.CreateFeed(user, topic, data, nil) if !errors.Is(err, feed.ErrReadOnlyFeed) { t.Fatal("read only feed") } longTopic := append(topic, topic...) // skipcq: CRT-D0001 - _, err = fd.CreateFeed(user1, longTopic, data, nil) + err = fd.CreateFeed(user, longTopic, data, nil) if !errors.Is(err, feed.ErrInvalidTopicSize) { t.Fatal("invalid topic size") } @@ -245,26 +450,32 @@ func TestFeed(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = fd.CreateFeed(user1, topic, longData, nil) + err = fd.CreateFeed(user, topic, longData, nil) if !errors.Is(err, feed.ErrInvalidPayloadSize) { t.Fatal("invalid payload size") } }) t.Run("create-feed-from-topic-errors", func(t *testing.T) { - nilFd := feed.New(&account.Info{}, client, logger) - - fd := feed.New(accountInfo1, client, logger) + nilFd := feed.New(&account.Info{}, client, -1, 0, logger) + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + user := acc.GetAddress(account.UserAccountIndex) + accountInfo := acc.GetUserAccountInfo() + fd := feed.New(accountInfo, client, -1, 0, logger) topic := utils.HashString("feed-topic1") data := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} - _, err = nilFd.CreateFeedFromTopic(topic, user1, data) + _, err = nilFd.CreateFeedFromTopic(topic, user, data) if !errors.Is(err, feed.ErrReadOnlyFeed) { t.Fatal("read only feed") } longTopic := append(topic, topic...) // skipcq: CRT-D0001 - _, err = fd.CreateFeedFromTopic(longTopic, user1, data) + _, err = fd.CreateFeedFromTopic(longTopic, user, data) if !errors.Is(err, feed.ErrInvalidTopicSize) { t.Fatal("invalid topic size") } @@ -273,26 +484,32 @@ func TestFeed(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = fd.CreateFeedFromTopic(topic, user1, longData) + _, err = fd.CreateFeedFromTopic(topic, user, longData) if !errors.Is(err, feed.ErrInvalidPayloadSize) { t.Fatal("invalid payload size") } }) t.Run("feed-update-errors", func(t *testing.T) { - nilFd := feed.New(&account.Info{}, client, logger) - - fd := feed.New(accountInfo1, client, logger) + nilFd := feed.New(&account.Info{}, client, -1, 0, logger) + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + user := acc.GetAddress(account.UserAccountIndex) + accountInfo := acc.GetUserAccountInfo() + fd := feed.New(accountInfo, client, -1, 0, logger) topic := utils.HashString("feed-topic1") data := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} - _, err = nilFd.UpdateFeed(user1, topic, data, nil, false) + err = nilFd.UpdateFeed(user, topic, data, nil, false) if !errors.Is(err, feed.ErrReadOnlyFeed) { t.Fatal("read only feed") } longTopic := append(topic, topic...) // skipcq: CRT-D0001 - _, err = fd.UpdateFeed(user1, longTopic, data, nil, false) + err = fd.UpdateFeed(user, longTopic, data, nil, false) if !errors.Is(err, feed.ErrInvalidTopicSize) { t.Fatal("invalid topic size") } @@ -301,39 +518,51 @@ func TestFeed(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = fd.UpdateFeed(user1, topic, longData, nil, false) + err = fd.UpdateFeed(user, topic, longData, nil, false) if !errors.Is(err, feed.ErrInvalidPayloadSize) { t.Fatal("invalid payload size") } }) t.Run("feed-delete-errors", func(t *testing.T) { - nilFd := feed.New(&account.Info{}, client, logger) - - fd := feed.New(accountInfo1, client, logger) + nilFd := feed.New(&account.Info{}, client, -1, 0, logger) + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + user := acc.GetAddress(account.UserAccountIndex) + accountInfo := acc.GetUserAccountInfo() + fd := feed.New(accountInfo, client, -1, 0, logger) topic := utils.HashString("feed-topic1") data := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} - err = nilFd.DeleteFeed(topic, user1) + err = nilFd.DeleteFeed(topic, user) if !errors.Is(err, feed.ErrReadOnlyFeed) { t.Fatal("read only feed") } - _, err = fd.CreateFeed(user1, topic, data, nil) + err = fd.CreateFeed(user, topic, data, nil) if err != nil { t.Fatal(err) } - err = fd.DeleteFeed(topic, user1) + err = fd.DeleteFeed(topic, user) if err != nil { t.Fatal(err) } }) t.Run("feed-from-topic-delete-errors", func(t *testing.T) { - nilFd := feed.New(&account.Info{}, client, logger) + nilFd := feed.New(&account.Info{}, client, -1, 0, logger) + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + user := acc.GetAddress(account.UserAccountIndex) topic := utils.HashString("feed-topic1") - err = nilFd.DeleteFeedFromTopic(topic, user1) + err = nilFd.DeleteFeedFromTopic(topic, user) if !errors.Is(err, feed.ErrReadOnlyFeed) { t.Fatal("read only feed") } diff --git a/pkg/feed/handler.go b/pkg/feed/handler.go index eb8ea2f9..006ad272 100644 --- a/pkg/feed/handler.go +++ b/pkg/feed/handler.go @@ -21,12 +21,18 @@ import ( "context" "crypto" "encoding/binary" + "encoding/hex" "errors" "fmt" + "github.com/fairdatasociety/fairOS-dfs/pkg/logging" "hash" + "strings" "sync" "sync/atomic" + "time" + bCrypto "github.com/ethersphere/bee/pkg/crypto" + "github.com/ethersphere/bee/pkg/soc" "github.com/ethersphere/bee/pkg/swarm" bmtlegacy "github.com/ethersphere/bmt/legacy" utilsSigner "github.com/fairdatasociety/fairOS-dfs-utils/signer" @@ -34,9 +40,18 @@ import ( "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore" "github.com/fairdatasociety/fairOS-dfs/pkg/feed/lookup" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" + "github.com/hashicorp/golang-lru/v2/expirable" "golang.org/x/crypto/sha3" ) +type feedItem struct { + User utils.Address + AccountInfo *account.Info + Topic []byte + Data []byte + ShouldCreate bool +} + // Handler is the main object which handles all feed related functionality type Handler struct { accountInfo *account.Info @@ -45,6 +60,9 @@ type Handler struct { HashSize int cache map[uint64]*CacheEntry cacheLock sync.RWMutex + logger logging.Logger + pool *expirable.LRU[string, *feedItem] + evictLock sync.Mutex } // hashPool contains a pool of ready hashers @@ -60,12 +78,13 @@ func init() { } // NewHandler the main handler object that handles all the feed related functions. -func NewHandler(accountInfo *account.Info, client blockstore.Client, hasherPool *bmtlegacy.TreePool) *Handler { +func NewHandler(accountInfo *account.Info, client blockstore.Client, hasherPool *bmtlegacy.TreePool, feedCacheSize int, feedCacheTTL time.Duration, logger logging.Logger) *Handler { fh := &Handler{ accountInfo: accountInfo, client: client, hasherPool: hasherPool, cache: make(map[uint64]*CacheEntry), + logger: logger, } for i := 0; i < hasherCount; i++ { hashfunc := crypto.SHA256.New() @@ -74,9 +93,215 @@ func NewHandler(accountInfo *account.Info, client blockstore.Client, hasherPool } hashPool.Put(hashfunc) } + if feedCacheSize != -1 { + fh.pool = expirable.NewLRU(feedCacheSize, func(key string, value *feedItem) { + fh.evictLock.Lock() + defer fh.evictLock.Unlock() + if value.ShouldCreate { + _, _, err := fh.createSoc(value.User, value.AccountInfo, value.Topic, value.Data) + if err != nil { + logger.Errorf("failed to createSoc onEvict from : %v\n", err) + return + } + } + _, _, err := fh.updateSoc(value.User, value.AccountInfo, value.Topic, value.Data) + if err != nil { + logger.Errorf("failed to updateSoc onEvict: %v\n", err) + return + } + }, feedCacheTTL) + } return fh } +func (h *Handler) commit() { + if h.pool != nil { + h.pool.Purge() + } +} + +func (h *Handler) putInPool(topic []byte, item *feedItem) { + topicHex := hex.EncodeToString(topic) + key := fmt.Sprintf("%s-%s", topicHex, item.User.String()) + it, ok := h.pool.Get(key) + if ok && it.ShouldCreate { + item.ShouldCreate = it.ShouldCreate + } + h.pool.Add(key, item) +} + +func (h *Handler) getSoc(topic []byte, user utils.Address, hint lookup.Epoch) ([]byte, []byte, error) { + topicHex := hex.EncodeToString(topic) + key := fmt.Sprintf("%s-%s", topicHex, user.String()) + if h.pool != nil { + item, ok := h.pool.Get(key) + if ok { + return nil, item.Data, nil + } + } + ctx := context.TODO() + f := new(Feed) + f.User = user + copy(f.Topic[:], topic) + + // create the query from values + q := &Query{Feed: *f} + q.TimeLimit = 0 + q.Hint = hint + if hint == lookup.NoClue { + _, err := h.Lookup(ctx, q) + if err != nil { + return nil, nil, err + } + } else { + _, err := h.LookupEpoch(ctx, q) + if err != nil { + return nil, nil, err + } + } + addr, data, err := h.GetContent(&q.Feed) + if err != nil { // skipcq: TCV-001 + return nil, nil, err + } + return addr.Bytes(), data, nil +} + +func (h *Handler) createSoc(user utils.Address, accountInfo *account.Info, topic, data []byte) (lookup.Epoch, []byte, error) { + var ( + req request + epoch lookup.Epoch + ) + + // fill Feed and Epoc related details + copy(req.ID.Topic[:], topic) + req.ID.User = user + req.Epoch.Level = lookup.HighestLevel + req.Epoch.Time = uint64(time.Now().Unix()) + + // Add initial feed data + req.data = data + + // create the id, hash(topic, epoc) + id, err := h.getId(req.Topic, req.Time, req.Level) + if err != nil { // skipcq: TCV-001 + return epoch, nil, err + } + // get the payload id BMT(span, payload) + payloadId, err := h.getPayloadId(data) + if err != nil { // skipcq: TCV-001 + return epoch, nil, err + } + // create the signer and the content addressed chunk + signer := bCrypto.NewDefaultSigner(accountInfo.GetPrivateKey()) + ch, err := utils.NewChunkWithSpan(data) + if err != nil { // skipcq: TCV-001 + return epoch, nil, err + } + s := soc.New(id, ch) + sch, err := s.Sign(signer) + if err != nil { // skipcq: TCV-001 + return epoch, nil, err + } + // generate the data to sign + toSignBytes, err := toSignDigest(id, ch.Address().Bytes()) + if err != nil { // skipcq: TCV-001 + return epoch, nil, err + } + // sign the chunk + signature, err := signer.Sign(toSignBytes) + if err != nil { // skipcq: TCV-001 + return epoch, nil, err + } + // set the address and the data for the soc chunk + req.idAddr = sch.Address() + req.binaryData = sch.Data() + // set signature and binary data fields + _, err = h.toChunkContent(&req, id, payloadId) + if err != nil { // skipcq: TCV-001 + return epoch, nil, err + } + // send the updated soc chunk to bee + addr, err := h.update(id, user.ToBytes(), signature, ch.Data()) + if err != nil { // skipcq: TCV-001 + return epoch, nil, err + } + return req.Epoch, addr, nil +} + +func (h *Handler) updateSoc(user utils.Address, accountInfo *account.Info, topic, data []byte) (lookup.Epoch, []byte, error) { + var ( + epoch lookup.Epoch + ) + retries := 0 +retry: + ctx := context.Background() + f := new(Feed) + f.User = user + copy(f.Topic[:], topic) + + // get the existing request from DB + req, err := h.newRequest(ctx, f) + if err != nil { // skipcq: TCV-001 + return epoch, nil, err + } + req.Time = uint64(time.Now().Unix()) + req.data = data + // create the id, hash(topic, epoc) + id, err := h.getId(req.Topic, req.Time, req.Level) + if err != nil { // skipcq: TCV-001 + return epoch, nil, err + } + // get the payload id BMT(span, payload) + payloadId, err := h.getPayloadId(data) + if err != nil { // skipcq: TCV-001 + return epoch, nil, err + } + // create the signer and the content addressed chunk + signer := bCrypto.NewDefaultSigner(accountInfo.GetPrivateKey()) + ch, err := utils.NewChunkWithSpan(data) + if err != nil { // skipcq: TCV-001 + return epoch, nil, err + } + s := soc.New(id, ch) + sch, err := s.Sign(signer) + if err != nil { // skipcq: TCV-001 + return epoch, nil, err + } + // generate the data to sign + toSignBytes, err := toSignDigest(id, ch.Address().Bytes()) + if err != nil { // skipcq: TCV-001 + return epoch, nil, err + } + // sign the chunk + signature, err := signer.Sign(toSignBytes) + if err != nil { // skipcq: TCV-001 + return epoch, nil, err + } + // set the address and the data for the soc chunk + req.idAddr = sch.Address() + req.binaryData = sch.Data() + // set signature and binary data fields + _, err = h.toChunkContent(req, id, payloadId) + if err != nil { // skipcq: TCV-001 + return epoch, nil, err + } + + address, err := h.update(id, user.ToBytes(), signature, ch.Data()) + if err != nil { + // updating same feed in the same second will lead to "chunk already exists" error. + // This will wait for 1 second and retry the update maxUpdateRetry times. + // It is a very dirty fix for this issue. We should find a better way to handle this. + if strings.Contains(err.Error(), "chunk already exists") && retries < maxUpdateRetry { + retries++ + <-time.After(1 * time.Second) + goto retry + } + return epoch, nil, err + } + + return req.Epoch, address, nil +} + func (h *Handler) update(id, owner, signature, data []byte) ([]byte, error) { // send the SOC chunk addr, err := h.client.UploadSOC(utils.Encode(owner), utils.Encode(id), utils.Encode(signature), data) @@ -157,6 +382,7 @@ func (h *Handler) Lookup(ctx context.Context, query *Query) (*CacheEntry, error) } ch := swarm.NewChunk(addr, data) var request request + if err := h.fromChunk(ch, &request, query, &id); err != nil { return nil, nil } @@ -335,9 +561,12 @@ func (h *Handler) newRequest(ctx context.Context, feed *Feed) (request2 *request } request2.Feed = *feed - // if we already have an update, then find next epoch if feedUpdate != nil { + if feedUpdate.Epoch.Level == 0 && feedUpdate.Epoch.Time == now { + <-time.After(time.Second) + now = TimestampProvider.Now().Time + } request2.Epoch = lookup.GetNextEpoch(feedUpdate.Epoch, now) } else { request2.Epoch = lookup.GetFirstEpoch(now) diff --git a/pkg/feed/handler_test.go b/pkg/feed/handler_test.go new file mode 100644 index 00000000..a22c2b98 --- /dev/null +++ b/pkg/feed/handler_test.go @@ -0,0 +1,77 @@ +package feed + +import ( + "io" + "testing" + + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/ethersphere/bee/pkg/swarm" + bmtlegacy "github.com/ethersphere/bmt/legacy" + "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/logging" +) + +func TestHandler(t *testing.T) { + logger := logging.New(io.Discard, 0) + + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + client := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + + t.Run("new-handler", func(t *testing.T) { + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + + accountInfo := acc.GetUserAccountInfo() + bmtPool := bmtlegacy.NewTreePool(hashFunc, swarm.Branches, bmtlegacy.PoolSize) + handler := NewHandler(accountInfo, client, bmtPool, -1, 0, logger) + //defer handler.Close() + + if handler == nil { + t.Fatal("handler is nil") + } + }) + + t.Run("new-handler", func(t *testing.T) { + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + + accountInfo := acc.GetUserAccountInfo() + bmtPool := bmtlegacy.NewTreePool(hashFunc, swarm.Branches, bmtlegacy.PoolSize) + handler := NewHandler(accountInfo, client, bmtPool, -1, 0, logger) + //defer handler.Close() + + if handler == nil { + t.Fatal("handler is nil") + } + }) + + t.Run("new-handler-nil pool", func(t *testing.T) { + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + + accountInfo := acc.GetUserAccountInfo() + bmtPool := bmtlegacy.NewTreePool(hashFunc, swarm.Branches, bmtlegacy.PoolSize) + handler := NewHandler(accountInfo, client, bmtPool, -1, 0, logger) + //defer handler.Close() + + if handler.pool != nil { + t.Fatal("poll is nol nil") + } + }) +} diff --git a/pkg/file/IFile.go b/pkg/file/IFile.go index ead9e467..c12540c9 100644 --- a/pkg/file/IFile.go +++ b/pkg/file/IFile.go @@ -20,7 +20,7 @@ import "io" // IFile is the interface for file operations type IFile interface { - Upload(fd io.Reader, podFileName string, fileSize int64, blockSize, uint32 uint32, podPath, compression, podPassword string) error + Upload(fd io.Reader, podFileName string, fileSize int64, blockSize, mode uint32, podPath, compression, podPassword string) error Download(podFileWithPath, podPassword string) (io.ReadCloser, uint64, error) ListFiles(files []string, podPassword string) ([]Entry, error) GetStats(podName, podFileWithPath, podPassword string) (*Stats, error) diff --git a/pkg/file/chmod_test.go b/pkg/file/chmod_test.go index cedb14ea..0b9458c8 100644 --- a/pkg/file/chmod_test.go +++ b/pkg/file/chmod_test.go @@ -7,7 +7,10 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" "github.com/fairdatasociety/fairOS-dfs/pkg/file" @@ -15,13 +18,21 @@ import ( "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestChmod(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -31,7 +42,7 @@ func TestChmod(t *testing.T) { if err != nil { t.Fatal(err) } - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { diff --git a/pkg/file/download_test.go b/pkg/file/download_test.go index aae4e1e2..f1bf2359 100644 --- a/pkg/file/download_test.go +++ b/pkg/file/download_test.go @@ -24,7 +24,10 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" "github.com/fairdatasociety/fairOS-dfs/pkg/file" @@ -32,13 +35,21 @@ import ( "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestDownload(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") require.NoError(t, err) @@ -46,7 +57,7 @@ func TestDownload(t *testing.T) { pod1AccountInfo, err := acc.CreatePodAccount(1, false) require.NoError(t, err) - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { @@ -137,7 +148,7 @@ func TestDownload(t *testing.T) { podPassword, _ := utils.GetRandString(pod.PasswordLength) filePath := "/dir1" - fileName := "file1" + fileName := "file2" compression := "" fileSize := int64(100) blockSize := file.MinBlockSize diff --git a/pkg/file/file.go b/pkg/file/file.go index e09ebbe2..254a511b 100644 --- a/pkg/file/file.go +++ b/pkg/file/file.go @@ -140,7 +140,6 @@ func (f *File) GetInode(podPassword, filePath string) *MetaData { // skipcq: TCV if err != nil { // skipcq: TCV-001 return nil } - f.AddToFileMap(filePath, meta) return meta } diff --git a/pkg/file/ls_test.go b/pkg/file/ls_test.go index a15d6adc..40a3c132 100644 --- a/pkg/file/ls_test.go +++ b/pkg/file/ls_test.go @@ -24,7 +24,10 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" "github.com/fairdatasociety/fairOS-dfs/pkg/file" @@ -32,11 +35,19 @@ import ( "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" ) func TestListFiles(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -46,7 +57,7 @@ func TestListFiles(t *testing.T) { if err != nil { t.Fatal(err) } - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { diff --git a/pkg/file/meta.go b/pkg/file/meta.go index 9bf2988a..6dd8569e 100644 --- a/pkg/file/meta.go +++ b/pkg/file/meta.go @@ -57,7 +57,7 @@ type MetaData struct { func (f *File) LoadFileMeta(fileNameWithPath, podPassword string) error { meta, err := f.GetMetaFromFileName(fileNameWithPath, podPassword, f.userAddress) if err != nil { // skipcq: TCV-001 - if err == ErrDeletedFeed { + if errors.Is(err, ErrDeletedFeed) { return nil } return err @@ -68,12 +68,12 @@ func (f *File) LoadFileMeta(fileNameWithPath, podPassword string) error { } func (f *File) handleMeta(meta *MetaData, podPassword string) error { - // check if meta is present. - err := f.uploadMeta(meta, podPassword) - if err != nil { + // check if meta is present. if present update else upload + _, err := f.GetMetaFromFileName(utils.CombinePathAndFile(meta.Path, meta.Name), podPassword, f.userAddress) + if err == nil || errors.Is(err, ErrDeletedFeed) { return f.updateMeta(meta, podPassword) } - return nil + return f.uploadMeta(meta, podPassword) } func (f *File) uploadMeta(meta *MetaData, podPassword string) error { @@ -86,20 +86,14 @@ func (f *File) uploadMeta(meta *MetaData, podPassword string) error { // put the file meta as a feed totalPath := utils.CombinePathAndFile(meta.Path, meta.Name) topic := utils.HashString(totalPath) - _, err = f.fd.CreateFeed(f.userAddress, topic, fileMetaBytes, []byte(podPassword)) - return err + return f.fd.CreateFeed(f.userAddress, topic, fileMetaBytes, []byte(podPassword)) } func (f *File) deleteMeta(meta *MetaData, podPassword string) error { totalPath := utils.CombinePathAndFile(meta.Path, meta.Name) topic := utils.HashString(totalPath) // update with utils.DeletedFeedMagicWord - _, err := f.fd.UpdateFeed(f.userAddress, topic, []byte(utils.DeletedFeedMagicWord), []byte(podPassword), false) - if err != nil { // skipcq: TCV-001 - return err - } - - return nil + return f.fd.UpdateFeed(f.userAddress, topic, []byte(utils.DeletedFeedMagicWord), []byte(podPassword), false) } func (f *File) updateMeta(meta *MetaData, podPassword string) error { @@ -112,12 +106,7 @@ func (f *File) updateMeta(meta *MetaData, podPassword string) error { // put the file meta as a feed totalPath := utils.CombinePathAndFile(meta.Path, meta.Name) topic := utils.HashString(totalPath) - _, err = f.fd.UpdateFeed(f.userAddress, topic, fileMetaBytes, []byte(podPassword), false) - if err != nil { // skipcq: TCV-001 - return err - } - - return nil + return f.fd.UpdateFeed(f.userAddress, topic, fileMetaBytes, []byte(podPassword), false) } // BackupFromFileName is used to backup a file diff --git a/pkg/file/reader.go b/pkg/file/reader.go index 6ca4c900..a0a1214a 100644 --- a/pkg/file/reader.go +++ b/pkg/file/reader.go @@ -25,7 +25,7 @@ import ( "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" "github.com/golang/snappy" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2/expirable" "github.com/klauspost/pgzip" ) @@ -50,7 +50,7 @@ type Reader struct { blockCursor uint32 totalSize uint64 compression string - blockCache *lru.Cache + blockCache *lru.LRU[string, []byte] rlBuffer []byte rlOffset int @@ -90,10 +90,6 @@ func (f *File) OpenFileForIndex(podFile, podPassword string) (*Reader, error) { // NewReader create a new reader object to read a file from the pod based on its configuration. func NewReader(fileInode INode, client blockstore.Client, fileSize uint64, blockSize uint32, compression string, cache bool) *Reader { - var blockCache *lru.Cache - if cache { - blockCache, _ = lru.New(blockCacheSize) - } r := &Reader{ fileInode: fileInode, client: client, @@ -101,9 +97,11 @@ func NewReader(fileInode INode, client blockstore.Client, fileSize uint64, block fileSize: fileSize, blockSize: blockSize, compression: compression, - blockCache: blockCache, rlReadNewLine: false, } + if cache { + r.blockCache = lru.NewLRU[string, []byte](blockCacheSize, func(key string, value []byte) {}, 0) + } return r } @@ -195,7 +193,7 @@ func (r *Reader) Read(b []byte) (n int, err error) { } // Seek seeks to a given offset in the file and returns the offset. -func (r *Reader) Seek(seekOffset int64, whence int) (int64, error) { +func (r *Reader) Seek(seekOffset int64, _ int) (int64, error) { // TODO: use whence if seekOffset < 0 || seekOffset > int64(r.fileSize) { return 0, ErrInvalidOffset @@ -312,7 +310,7 @@ func (r *Reader) getBlock(ref []byte, compression string, blockSize uint32) ([]b refStr := utils.NewReference(ref).String() if r.blockCache != nil { if data, found := r.blockCache.Get(refStr); found { - return data.([]byte), nil + return data, nil } } stdoutBytes, _, err := r.client.DownloadBlob(ref) diff --git a/pkg/file/reader_test.go b/pkg/file/reader_test.go index c429d7e6..0140f212 100644 --- a/pkg/file/reader_test.go +++ b/pkg/file/reader_test.go @@ -23,6 +23,12 @@ import ( "math/big" "testing" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/fairdatasociety/fairOS-dfs/pkg/logging" + "github.com/sirupsen/logrus" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/file" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" @@ -31,7 +37,15 @@ import ( ) func TestFileReader(t *testing.T) { - mockClient := mock.NewMockBeeClient() + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) t.Run("read-entire-file-shorter-than-block", func(t *testing.T) { fileSize := uint64(15) @@ -302,7 +316,7 @@ func TestFileReader(t *testing.T) { }) } -func createFile(t *testing.T, fileSize uint64, blockSize uint32, compression string, mockClient *mock.BeeClient) ([]byte, file.INode) { +func createFile(t *testing.T, fileSize uint64, blockSize uint32, compression string, mockClient *bee.Client) ([]byte, file.INode) { var fileBlocks []*file.BlockInfo noOfBlocks := fileSize / uint64(blockSize) if fileSize%uint64(blockSize) != 0 { @@ -347,7 +361,7 @@ func createFile(t *testing.T, fileSize uint64, blockSize uint32, compression str } } -func createFileWithNewlines(t *testing.T, fileSize uint64, blockSize uint32, compression string, mockClient *mock.BeeClient, linesPerBlock uint32) ([]byte, file.INode, int, []byte, int, []byte) { // skipcq: GO-C4008 +func createFileWithNewlines(t *testing.T, fileSize uint64, blockSize uint32, compression string, mockClient *bee.Client, linesPerBlock uint32) ([]byte, file.INode, int, []byte, int, []byte) { // skipcq: GO-C4008 var fileBlocks []*file.BlockInfo noOfBlocks := fileSize / uint64(blockSize) if fileSize%uint64(blockSize) != 0 { diff --git a/pkg/file/rename_test.go b/pkg/file/rename_test.go index d669fd53..edd2d330 100644 --- a/pkg/file/rename_test.go +++ b/pkg/file/rename_test.go @@ -7,21 +7,31 @@ import ( "testing" "time" - "github.com/fairdatasociety/fairOS-dfs/pkg/pod" - + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/dir" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" "github.com/fairdatasociety/fairOS-dfs/pkg/file" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" + "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" ) func TestRename(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -31,7 +41,7 @@ func TestRename(t *testing.T) { if err != nil { t.Fatal(err) } - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { diff --git a/pkg/file/rm.go b/pkg/file/rm.go index 8c897c40..3e83a9ed 100644 --- a/pkg/file/rm.go +++ b/pkg/file/rm.go @@ -63,8 +63,8 @@ func (f *File) RmFile(podFileWithPath, podPassword string) error { } // remove the meta topic := utils.HashString(totalFilePath) - _, err = f.fd.UpdateFeed(f.userAddress, topic, []byte(utils.DeletedFeedMagicWord), []byte(podPassword), false) // empty byte array will fail, so some 1 byte - if err != nil { // skipcq: TCV-001 + err = f.fd.UpdateFeed(f.userAddress, topic, []byte(utils.DeletedFeedMagicWord), []byte(podPassword), false) // empty byte array will fail, so some 1 byte + if err != nil { // skipcq: TCV-001 return err } // remove the file from file map diff --git a/pkg/file/rm_test.go b/pkg/file/rm_test.go index 16fc4f1a..26f28334 100644 --- a/pkg/file/rm_test.go +++ b/pkg/file/rm_test.go @@ -22,24 +22,31 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" - - "github.com/fairdatasociety/fairOS-dfs/pkg/pod" - - "github.com/plexsysio/taskmanager" - - "github.com/fairdatasociety/fairOS-dfs/pkg/file" - "github.com/fairdatasociety/fairOS-dfs/pkg/utils" - + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" + "github.com/fairdatasociety/fairOS-dfs/pkg/file" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" + "github.com/fairdatasociety/fairOS-dfs/pkg/pod" + "github.com/fairdatasociety/fairOS-dfs/pkg/utils" + "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" ) func TestRemoveFile(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") require.NoError(t, err) @@ -47,7 +54,7 @@ func TestRemoveFile(t *testing.T) { pod1AccountInfo, err := acc.CreatePodAccount(1, false) require.NoError(t, err) - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { diff --git a/pkg/file/stat_test.go b/pkg/file/stat_test.go index 47769014..dee6f886 100644 --- a/pkg/file/stat_test.go +++ b/pkg/file/stat_test.go @@ -24,21 +24,30 @@ import ( "testing" "time" - "github.com/fairdatasociety/fairOS-dfs/pkg/pod" - "github.com/fairdatasociety/fairOS-dfs/pkg/utils" - - "github.com/plexsysio/taskmanager" - + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" "github.com/fairdatasociety/fairOS-dfs/pkg/file" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" + "github.com/fairdatasociety/fairOS-dfs/pkg/pod" + "github.com/fairdatasociety/fairOS-dfs/pkg/utils" + "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" ) func TestStat(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -48,7 +57,7 @@ func TestStat(t *testing.T) { if err != nil { t.Fatal(err) } - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { diff --git a/pkg/file/status_test.go b/pkg/file/status_test.go index 4f5c768f..bae2680c 100644 --- a/pkg/file/status_test.go +++ b/pkg/file/status_test.go @@ -6,7 +6,12 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" "github.com/fairdatasociety/fairOS-dfs/pkg/file" @@ -14,11 +19,19 @@ import ( "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" ) func TestStatus(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -28,7 +41,7 @@ func TestStatus(t *testing.T) { if err != nil { t.Fatal(err) } - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { @@ -37,6 +50,7 @@ func TestStatus(t *testing.T) { podPassword, _ := utils.GetRandString(pod.PasswordLength) t.Run("sync-status-file", func(t *testing.T) { + t.Skip() fileObject := file.NewFile("pod1", mockClient, fd, user, tm, logger) numberOfChunks := int64(10) // upload a file @@ -55,9 +69,7 @@ func TestStatus(t *testing.T) { if err != nil { t.Fatal(err) } - if total != numberOfChunks { - t.Fatal("chunk count mismatch for status") - } + require.Equal(t, total, numberOfChunks) }) } diff --git a/pkg/file/upload.go b/pkg/file/upload.go index a5aa7b32..5c07ebdc 100644 --- a/pkg/file/upload.go +++ b/pkg/file/upload.go @@ -79,7 +79,6 @@ func (f *File) Upload(fd io.Reader, podFileName string, fileSize int64, blockSiz ModificationTime: now, Mode: mode, } - var totalLength uint64 i := 0 errC := make(chan error) diff --git a/pkg/file/upload_test.go b/pkg/file/upload_test.go index 130233f5..ed3cdea6 100644 --- a/pkg/file/upload_test.go +++ b/pkg/file/upload_test.go @@ -27,7 +27,10 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" "github.com/fairdatasociety/fairOS-dfs/pkg/file" @@ -35,11 +38,19 @@ import ( "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" ) func TestUpload(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -49,7 +60,7 @@ func TestUpload(t *testing.T) { if err != nil { t.Fatal(err) } - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { diff --git a/pkg/file/writeAt_test.go b/pkg/file/writeAt_test.go index dbc71c14..64a0baa8 100644 --- a/pkg/file/writeAt_test.go +++ b/pkg/file/writeAt_test.go @@ -11,7 +11,10 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" "github.com/fairdatasociety/fairOS-dfs/pkg/file" @@ -19,6 +22,7 @@ import ( "github.com/fairdatasociety/fairOS-dfs/pkg/pod" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "go.uber.org/goleak" ) @@ -28,8 +32,15 @@ func TestMain(m *testing.M) { } func TestWriteAt(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -39,7 +50,7 @@ func TestWriteAt(t *testing.T) { if err != nil { t.Fatal(err) } - fd := feed.New(pod1AccountInfo, mockClient, logger) + fd := feed.New(pod1AccountInfo, mockClient, -1, 0, logger) user := acc.GetAddress(1) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { @@ -363,20 +374,21 @@ func TestWriteAt(t *testing.T) { }) t.Run("upload-update-small-file", func(t *testing.T) { - filePath := "/dir11111" + filePath := string(os.PathSeparator) fileName, _ := utils.GetRandString(10) compression := "" - fileSize := int64(100) + fileSize := int64(1000) blockSize := file.MinBlockSize fileObject := file.NewFile("pod1", mockClient, fd, user, tm, logger) - dt, err := uploadFile(t, fileObject, filePath, fileName, compression, podPassword, fileSize, blockSize) + pp := "" + dt, err := uploadFile(t, fileObject, filePath, fileName, compression, pp, fileSize, blockSize) if err != nil { t.Fatal(err) } // check for meta fp := utils.CombinePathAndFile(filepath.ToSlash(filePath), fileName) - meta := fileObject.GetInode(podPassword, fp) + meta := fileObject.GetInode(pp, fp) if meta == nil { t.Fatalf("file not added in file map") } @@ -395,28 +407,30 @@ func TestWriteAt(t *testing.T) { t.Fatalf("invalid block size in meta") } - err = fileObject.LoadFileMeta(filePath+"/"+fileName, podPassword) + err = fileObject.LoadFileMeta(filepath.ToSlash(filePath+fileName), pp) if err != nil { t.Fatal(err) } // skipcq: GSC-G404 rnd := rand.New(rand.NewSource(time.Now().UnixNano())) - min := 0 + min := int(fileSize / 2) max := int(fileSize) offset := rnd.Intn((max - min + 1) + min) + content, err := utils.GetRandBytes(offset) if err != nil { t.Fatal(err) } r := bytes.NewReader(content) - n, err := fileObject.WriteAt(fp, podPassword, r, uint64(offset), false) + n, err := fileObject.WriteAt(fp, pp, r, uint64(offset), false) if n != offset { t.Fatalf("Failed to update %d bytes", offset-n) } if err != nil { t.Fatal(err) } - reader, _, err := fileObject.Download(fp, podPassword) + + reader, _, err := fileObject.Download(fp, pp) if err != nil { t.Fatal(err) } @@ -434,12 +448,13 @@ func TestWriteAt(t *testing.T) { if !bytes.Equal(updatedContent, rcvdBuffer.Bytes()) { t.Fatal("content is different") } - err = fileObject.RmFile(fp, podPassword) + + err = fileObject.RmFile(fp, pp) if err != nil { t.Fatal(err) } - meta2 := fileObject.GetInode(podPassword, fp) + meta2 := fileObject.GetInode(pp, fp) assert.Equal(t, meta2, (*file.MetaData)(nil)) }) diff --git a/pkg/pod/close.go b/pkg/pod/close.go index e341cf25..ac26b66e 100644 --- a/pkg/pod/close.go +++ b/pkg/pod/close.go @@ -16,6 +16,8 @@ limitations under the License. package pod +import "fmt" + // ClosePod closed an already opened pod and removes its information from directory and file // data structures. func (p *Pod) ClosePod(podName string) error { @@ -23,6 +25,12 @@ func (p *Pod) ClosePod(podName string) error { if err != nil { // skipcq: TCV-001 return err } + if err := podInfo.feed.Close(); err != nil { + return err + } + if err := p.fd.Close(); err != nil { + return err + } podIndex, err := p.getPodIndex(podName) if err != nil { // skipcq: TCV-001 @@ -36,3 +44,14 @@ func (p *Pod) ClosePod(podName string) error { p.acc.DeletePodAccount(podIndex) return nil } + +// CloseAllPods closes all opened pods and removes their information from directory and file +// data structures. +func (p *Pod) CloseAllPods() error { + for podName := range p.podMap { + if err := p.ClosePod(podName); err != nil { + return fmt.Errorf("error closing pod %s: %w", podName, err) + } + } + return nil +} diff --git a/pkg/pod/commitFeeds.go b/pkg/pod/commitFeeds.go new file mode 100644 index 00000000..6b1b4182 --- /dev/null +++ b/pkg/pod/commitFeeds.go @@ -0,0 +1,12 @@ +package pod + +// CommitFeeds commits uncommitted feeds saved in local lru cache to swarm +func (p *Pod) CommitFeeds(podName string) error { + podInfo, _, err := p.GetPodInfoFromPodMap(podName) + if err != nil { // skipcq: TCV-001 + return err + } + podInfo.feed.CommitFeeds() + p.fd.CommitFeeds() + return nil +} diff --git a/pkg/pod/del.go b/pkg/pod/del.go index 7cbe06fe..1b45c75c 100644 --- a/pkg/pod/del.go +++ b/pkg/pod/del.go @@ -22,7 +22,7 @@ import ( // DeleteOwnPod removed a pod and the list of pods belonging to a user. func (p *Pod) DeleteOwnPod(podName string) error { - podList, err := p.loadUserPods() + podList, err := p.PodList() if err != nil { // skipcq: TCV-001 return err } @@ -60,16 +60,15 @@ func (p *Pod) DeleteOwnPod(podName string) error { p.acc.DeletePodAccount(podIndex) // remove the pod finally - return p.storeUserPods(podList) + return p.storeUserPodsV2(podList) } // DeleteSharedPod removed a pod and the list of pods shared by other users. func (p *Pod) DeleteSharedPod(podName string) error { - podList, err := p.loadUserPods() + podList, err := p.PodList() if err != nil { // skipcq: TCV-001 return err } - found := false for index, pod := range podList.SharedPods { if pod.Name == podName { @@ -85,5 +84,5 @@ func (p *Pod) DeleteSharedPod(podName string) error { p.removePodFromPodMap(podName) // remove the pod finally - return p.storeUserPods(podList) + return p.storeUserPodsV2(podList) } diff --git a/pkg/pod/fork.go b/pkg/pod/fork.go index abc51f39..9348f12d 100644 --- a/pkg/pod/fork.go +++ b/pkg/pod/fork.go @@ -35,8 +35,8 @@ func (p *Pod) PodFork(podName, forkName string) error { } directory := podInfo.GetDirectory() - rootInode := directory.GetInode(podInfo.GetPodPassword(), "/") - if rootInode == nil { + rootInode, err := directory.GetInode(podInfo.GetPodPassword(), "/") + if err != nil { return fmt.Errorf("root inode not found") } return cloneFolder(podInfo, forkInfo, "/", rootInode) @@ -64,7 +64,7 @@ func (p *Pod) PodForkFromRef(forkName, refString string) error { address := utils.HexToAddress(shareInfo.Address) accountInfo.SetAddress(address) - fd := feed.New(accountInfo, p.client, p.logger) + fd := feed.New(accountInfo, p.client, p.feedCacheSize, p.feedCacheTTL, p.logger) file := f.NewFile(shareInfo.PodName, p.client, fd, accountInfo.GetAddress(), p.tm, p.logger) dir := d.NewDirectory(shareInfo.PodName, p.client, fd, accountInfo.GetAddress(), file, p.tm, p.logger) podInfo := &Info{ @@ -125,8 +125,11 @@ func cloneFolder(source, dst *Info, dirNameWithPath string, dirInode *d.Inode) e } else if strings.HasPrefix(fileOrDirName, "_D_") { dirName := strings.TrimPrefix(fileOrDirName, "_D_") path := utils.CombinePathAndFile(dirNameWithPath, dirName) - iNode := source.GetDirectory().GetInode(source.GetPodPassword(), path) - err := dst.GetDirectory().MkDir(path, dst.GetPodPassword(), 0) + iNode, err := source.GetDirectory().GetInode(source.GetPodPassword(), path) + if err != nil { // skipcq: TCV-001 + return err + } + err = dst.GetDirectory().MkDir(path, dst.GetPodPassword(), 0) if err != nil { // skipcq: TCV-001 return err } diff --git a/pkg/pod/ls.go b/pkg/pod/ls.go index c8979d93..3531d5c9 100644 --- a/pkg/pod/ls.go +++ b/pkg/pod/ls.go @@ -16,11 +16,20 @@ limitations under the License. package pod +import "fmt" + // ListPods List all the available pods belonging to a user. func (p *Pod) ListPods() ([]string, []string, error) { - podList, err := p.loadUserPods() - if err != nil { // skipcq: TCV-001 - return nil, nil, err + podList, err := p.loadUserPodsV2() + if err != nil { + podList, err = p.loadUserPods() + if err != nil { // skipcq: TCV-001 + return nil, nil, err + } + err := p.storeUserPodsV2(podList) + if err != nil { + fmt.Println("error storing podsV2", err) + } } var listPods []string @@ -38,5 +47,20 @@ func (p *Pod) ListPods() ([]string, []string, error) { // PodList lists all the available pods belonging to a user in json format. func (p *Pod) PodList() (*List, error) { - return p.loadUserPods() + //We first check if the podsV2 list is present + podList, err := p.loadUserPodsV2() + if err != nil { + // If v2 is not present we check if v1 is present + podList, err = p.loadUserPods() + if err != nil { // skipcq: TCV-001 + return nil, err + } + // we store the v1 list as v2 + err := p.storeUserPodsV2(podList) + if err != nil { + fmt.Println("error storing podsV2", err) + } + // TODO remove old v1 podList + } + return podList, nil } diff --git a/pkg/pod/new.go b/pkg/pod/new.go index e1a7d046..38670e03 100644 --- a/pkg/pod/new.go +++ b/pkg/pod/new.go @@ -17,11 +17,12 @@ limitations under the License. package pod import ( + "bytes" "encoding/hex" "encoding/json" + "io" "github.com/ethereum/go-ethereum/crypto" - "github.com/fairdatasociety/fairOS-dfs/pkg/account" c "github.com/fairdatasociety/fairOS-dfs/pkg/collection" d "github.com/fairdatasociety/fairOS-dfs/pkg/dir" @@ -31,7 +32,8 @@ import ( ) const ( - podFile = "Pods" + podFile = "Pods" + podFileV2 = "PodsV2" ) // CreatePod creates a new pod for a given user. @@ -42,17 +44,15 @@ func (p *Pod) CreatePod(podName, addressString, podPassword string) (*Info, erro } // check if pods is present and get free index - podList, err := p.loadUserPods() + podList, err := p.PodList() if err != nil { // skipcq: TCV-001 return nil, err } - pods := map[int]string{} sharedPods := map[string]string{} for _, pod := range podList.Pods { pods[pod.Index] = pod.Name } - for _, pod := range podList.SharedPods { sharedPods[pod.Address] = pod.Name } @@ -74,7 +74,7 @@ func (p *Pod) CreatePod(podName, addressString, podPassword string) (*Info, erro address := utils.HexToAddress(addressString) accountInfo.SetAddress(address) - fd = feed.New(accountInfo, p.client, p.logger) + fd = feed.New(accountInfo, p.client, p.feedCacheSize, p.feedCacheTTL, p.logger) file = f.NewFile(podName, p.client, fd, accountInfo.GetAddress(), p.tm, p.logger) dir = d.NewDirectory(podName, p.client, fd, accountInfo.GetAddress(), file, p.tm, p.logger) @@ -85,14 +85,13 @@ func (p *Pod) CreatePod(podName, addressString, podPassword string) (*Info, erro Password: podPassword, } podList.SharedPods = append(podList.SharedPods, *sharedPod) - err = p.storeUserPods(podList) + err = p.storeUserPodsV2(podList) if err != nil { // skipcq: TCV-001 return nil, err } // set the userAddress as the pod address we got from shared pod user = address - } else { // your own pod, so create a new account with private key if p.checkIfPodPresent(podList, podName) { @@ -101,23 +100,19 @@ func (p *Pod) CreatePod(podName, addressString, podPassword string) (*Info, erro if p.checkIfSharedPodPresent(podList, podName) { return nil, ErrPodAlreadyExists } - freeId, err := p.getFreeId(pods) if err != nil { // skipcq: TCV-001 return nil, err } - // create a child account for the userAddress and other data structures for the pod accountInfo, err = p.acc.CreatePodAccount(freeId, true) if err != nil { // skipcq: TCV-001 return nil, err } - - fd = feed.New(accountInfo, p.client, p.logger) + fd = feed.New(accountInfo, p.client, p.feedCacheSize, p.feedCacheTTL, p.logger) //fd.SetUpdateTracker(p.fd.GetUpdateTracker()) file = f.NewFile(podName, p.client, fd, accountInfo.GetAddress(), p.tm, p.logger) dir = d.NewDirectory(podName, p.client, fd, accountInfo.GetAddress(), file, p.tm, p.logger) - // store the pod file pods[freeId] = podName pod := &ListItem{ @@ -126,7 +121,7 @@ func (p *Pod) CreatePod(podName, addressString, podPassword string) (*Info, erro Password: podPassword, } podList.Pods = append(podList.Pods, *pod) - err = p.storeUserPods(podList) + err = p.storeUserPodsV2(podList) if err != nil { // skipcq: TCV-001 return nil, err } @@ -185,23 +180,49 @@ func (p *Pod) loadUserPods() (*List, error) { return podList, nil } -func (p *Pod) storeUserPods(podList *List) error { - data, err := json.Marshal(podList) - if err != nil { - return err +func (p *Pod) loadUserPodsV2() (*List, error) { + f2 := f.NewFile("", p.client, p.fd, p.acc.GetAddress(account.UserAccountIndex), p.tm, p.logger) + topicString := utils.CombinePathAndFile("", podFileV2) + privKeyBytes := crypto.FromECDSA(p.acc.GetUserAccountInfo().GetPrivateKey()) + r, _, err := f2.Download(topicString, hex.EncodeToString(privKeyBytes)) + if err != nil { // skipcq: TCV-001 + return nil, err + } + podList := &List{ + Pods: []ListItem{}, + SharedPods: []SharedListItem{}, + } + data, err := io.ReadAll(r) + if err != nil { // skipcq: TCV-001 + return nil, err } - if len(data) > utils.MaxChunkLength { - return ErrMaximumPodLimit + if len(data) == 0 { + return podList, nil } - topic := utils.HashString(podFile) - privKeyBytes := crypto.FromECDSA(p.acc.GetUserAccountInfo().GetPrivateKey()) - _, err = p.fd.UpdateFeed(p.acc.GetAddress(account.UserAccountIndex), topic, data, []byte(hex.EncodeToString(privKeyBytes)), false) + err = json.Unmarshal(data, podList) if err != nil { // skipcq: TCV-001 + return nil, err + } + + return podList, nil +} + +func (p *Pod) storeUserPodsV2(podList *List) error { + + data, err := json.Marshal(podList) + if err != nil { return err } - return nil + + // store data as file and get metadata + // This is a very hacky way to store pod data, but it works for now + // We create a new file object with the user account address and upload the data + // We use the user private key to encrypt data. + f2 := f.NewFile("", p.client, p.fd, p.acc.GetAddress(account.UserAccountIndex), p.tm, p.logger) + privKeyBytes := crypto.FromECDSA(p.acc.GetUserAccountInfo().GetPrivateKey()) + return f2.Upload(bytes.NewReader(data), podFileV2, int64(len(data)), f.MinBlockSize, 0, "/", "gzip", hex.EncodeToString(privKeyBytes)) } func (*Pod) getFreeId(pods map[int]string) (int, error) { @@ -236,7 +257,7 @@ func (*Pod) checkIfSharedPodPresent(pods *List, podName string) bool { } func (p *Pod) getPodIndex(podName string) (podIndex int, err error) { - podList, err := p.loadUserPods() + podList, err := p.PodList() if err != nil { return -1, err } // skipcq: TCV-001 diff --git a/pkg/pod/open.go b/pkg/pod/open.go index 2ef5615c..5b0b4907 100644 --- a/pkg/pod/open.go +++ b/pkg/pod/open.go @@ -32,8 +32,12 @@ import ( // it loads all the data structures related to the pod. Also, it syncs all the // files and directories under this pod from the Swarm network. func (p *Pod) OpenPod(podName string) (*Info, error) { + pi, _, _ := p.GetPodInfoFromPodMap(podName) + if pi != nil { + return pi, nil + } // check if pods is present and get the index of the pod - podList, err := p.loadUserPods() + podList, err := p.PodList() if err != nil { // skipcq: TCV-001 return nil, err } @@ -65,7 +69,7 @@ func (p *Pod) OpenPod(podName string) (*Info, error) { address := utils.HexToAddress(addressString) accountInfo.SetAddress(address) - fd = feed.New(accountInfo, p.client, p.logger) + fd = feed.New(accountInfo, p.client, p.feedCacheSize, p.feedCacheTTL, p.logger) file = f.NewFile(podName, p.client, fd, accountInfo.GetAddress(), p.tm, p.logger) dir = d.NewDirectory(podName, p.client, fd, accountInfo.GetAddress(), file, p.tm, p.logger) @@ -83,8 +87,7 @@ func (p *Pod) OpenPod(podName string) (*Info, error) { if err != nil { // skipcq: TCV-001 return nil, err } - - fd = feed.New(accountInfo, p.client, p.logger) + fd = feed.New(accountInfo, p.client, p.feedCacheSize, p.feedCacheTTL, p.logger) //_, err = tracker.InitFeedsTracker(accountInfo.GetAddress(), podName, podPassword, fd, p.client, p.logger) //if err != nil { // p.logger.Errorf("error initializing feeds tracker: %v", err) @@ -94,7 +97,6 @@ func (p *Pod) OpenPod(podName string) (*Info, error) { user = p.acc.GetAddress(index) } - kvStore := c.NewKeyValueStore(podName, fd, accountInfo, user, p.client, p.logger) docStore := c.NewDocumentStore(podName, fd, accountInfo, user, file, p.tm, p.client, p.logger) @@ -110,11 +112,11 @@ func (p *Pod) OpenPod(podName string) (*Info, error) { kvStore: kvStore, docStore: docStore, } - p.addPodToPodMap(podName, podInfo) if !sharedPodType { err = podInfo.GetDirectory().AddRootDir(podInfo.GetPodName(), podInfo.GetPodPassword(), podInfo.GetPodAddress(), podInfo.GetFeed()) if err != nil { + fmt.Println("err", err) return nil, err } } @@ -126,7 +128,7 @@ func (p *Pod) OpenFromShareInfo(si *ShareInfo) (*Info, error) { address := utils.HexToAddress(si.Address) accountInfo.SetAddress(address) - fd := feed.New(accountInfo, p.client, p.logger) + fd := feed.New(accountInfo, p.client, p.feedCacheSize, p.feedCacheTTL, p.logger) file := f.NewFile(si.PodName, p.client, fd, accountInfo.GetAddress(), p.tm, p.logger) dir := d.NewDirectory(si.PodName, p.client, fd, accountInfo.GetAddress(), file, p.tm, p.logger) @@ -160,7 +162,7 @@ func (p *Pod) OpenFromShareInfo(si *ShareInfo) (*Info, error) { // files and directories under this pod from the Swarm network. func (p *Pod) OpenPodAsync(ctx context.Context, podName string) (*Info, error) { // check if pods is present and get the index of the pod - podList, err := p.loadUserPods() + podList, err := p.PodList() if err != nil { // skipcq: TCV-001 return nil, err } @@ -193,7 +195,7 @@ func (p *Pod) OpenPodAsync(ctx context.Context, podName string) (*Info, error) { address := utils.HexToAddress(addressString) accountInfo.SetAddress(address) - fd = feed.New(accountInfo, p.client, p.logger) + fd = feed.New(accountInfo, p.client, p.feedCacheSize, p.feedCacheTTL, p.logger) file = f.NewFile(podName, p.client, fd, accountInfo.GetAddress(), p.tm, p.logger) dir = d.NewDirectory(podName, p.client, fd, accountInfo.GetAddress(), file, p.tm, p.logger) @@ -212,7 +214,7 @@ func (p *Pod) OpenPodAsync(ctx context.Context, podName string) (*Info, error) { return nil, err } - fd = feed.New(accountInfo, p.client, p.logger) + fd = feed.New(accountInfo, p.client, p.feedCacheSize, p.feedCacheTTL, p.logger) //fd.SetUpdateTracker(p.fd.GetUpdateTracker()) file = f.NewFile(podName, p.client, fd, accountInfo.GetAddress(), p.tm, p.logger) dir = d.NewDirectory(podName, p.client, fd, accountInfo.GetAddress(), file, p.tm, p.logger) diff --git a/pkg/pod/pod.go b/pkg/pod/pod.go index e7c9015f..1eb69813 100644 --- a/pkg/pod/pod.go +++ b/pkg/pod/pod.go @@ -19,6 +19,7 @@ package pod import ( "fmt" "sync" + "time" "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager" @@ -38,14 +39,16 @@ const ( // Pod is the main struct which acts on pods type Pod struct { - fd *feed.API - acc *account.Account - client blockstore.Client - podMap map[string]*Info // podName -> dir - podMu *sync.RWMutex - logger logging.Logger - tm taskmanager.TaskManagerGO - sm subscriptionManager.SubscriptionManager + fd *feed.API + acc *account.Account + client blockstore.Client + podMap map[string]*Info // podName -> dir + podMu *sync.RWMutex + logger logging.Logger + tm taskmanager.TaskManagerGO + sm subscriptionManager.SubscriptionManager + feedCacheSize int + feedCacheTTL time.Duration } // ListItem defines the structure for pod item @@ -69,16 +72,18 @@ type List struct { } // NewPod creates the main pod object which has all the methods related to the pods. -func NewPod(client blockstore.Client, feed *feed.API, account *account.Account, m taskmanager.TaskManagerGO, sm subscriptionManager.SubscriptionManager, logger logging.Logger) *Pod { +func NewPod(client blockstore.Client, feed *feed.API, account *account.Account, m taskmanager.TaskManagerGO, sm subscriptionManager.SubscriptionManager, feedCacheSize int, feedCacheTTL time.Duration, logger logging.Logger) *Pod { return &Pod{ - fd: feed, - acc: account, - client: client, - podMap: make(map[string]*Info), - podMu: &sync.RWMutex{}, - logger: logger, - tm: m, - sm: sm, + fd: feed, + acc: account, + client: client, + podMap: make(map[string]*Info), + podMu: &sync.RWMutex{}, + logger: logger, + tm: m, + sm: sm, + feedCacheSize: feedCacheSize, + feedCacheTTL: feedCacheTTL, } } diff --git a/pkg/pod/sharing.go b/pkg/pod/sharing.go index ee127bae..ca8bc40a 100644 --- a/pkg/pod/sharing.go +++ b/pkg/pod/sharing.go @@ -37,7 +37,7 @@ type ShareInfo struct { // required to import this pod. func (p *Pod) PodShare(podName, sharedPodName string) (string, error) { // check if pods is present and get the index of the pod - podList, err := p.loadUserPods() + podList, err := p.PodList() if err != nil { // skipcq: TCV-001 return "", err } @@ -84,7 +84,7 @@ func (p *Pod) PodShare(podName, sharedPodName string) (string, error) { // GetPodSharingInfo returns the raw shareInfo func (p *Pod) GetPodSharingInfo(podName string) (*ShareInfo, error) { // check if pods is present and get the index of the pod - podList, err := p.loadUserPods() + podList, err := p.PodList() if err != nil { // skipcq: TCV-001 return nil, err } diff --git a/pkg/pod/stat.go b/pkg/pod/stat.go index 6723b028..b6313406 100644 --- a/pkg/pod/stat.go +++ b/pkg/pod/stat.go @@ -33,7 +33,7 @@ func (p *Pod) PodStat(podName string) (*Stat, error) { PodAddress: podInfo.userAddress.String(), }, nil } - podList, err := p.loadUserPods() + podList, err := p.PodList() if err != nil { return nil, err } diff --git a/pkg/pod/subscription.go b/pkg/pod/subscription.go index b4f4e1cd..013ed418 100644 --- a/pkg/pod/subscription.go +++ b/pkg/pod/subscription.go @@ -17,7 +17,7 @@ import ( // ListPodInMarketplace will save the pod info in the subscriptionManager smart contract with its owner and price // we keep the pod info in the smart contract, with a `list` flag func (p *Pod) ListPodInMarketplace(podName, title, desc, thumbnail string, price uint64, daysValid uint16, category, nameHash [32]byte) error { - podList, err := p.loadUserPods() + podList, err := p.PodList() if err != nil { // skipcq: TCV-001 return err } diff --git a/pkg/pod/utils.go b/pkg/pod/utils.go index ac2d342f..3fb7e02c 100644 --- a/pkg/pod/utils.go +++ b/pkg/pod/utils.go @@ -31,7 +31,7 @@ func (p *Pod) IsOwnPodPresent(podName string) bool { return false } // check if pods is present and get free index - podList, err := p.loadUserPods() + podList, err := p.PodList() if err != nil { // skipcq: TCV-001 return false } @@ -48,7 +48,7 @@ func (p *Pod) IsPodPresent(podName string) bool { return false } // check if pods is present and get free index - podList, err := p.loadUserPods() + podList, err := p.PodList() if err != nil { // skipcq: TCV-001 return false } diff --git a/pkg/test/close_test.go b/pkg/test/close_test.go index 8c64fb4f..568f7558 100644 --- a/pkg/test/close_test.go +++ b/pkg/test/close_test.go @@ -22,29 +22,37 @@ import ( "testing" "time" - mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" - - "github.com/fairdatasociety/fairOS-dfs/pkg/utils" - - "github.com/plexsysio/taskmanager" - + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" "github.com/fairdatasociety/fairOS-dfs/pkg/pod" + mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/utils" + "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" ) func TestClose(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { t.Fatal(err) } - fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) @@ -52,7 +60,7 @@ func TestClose(t *testing.T) { sm := mock2.NewMockSubscriptionManager() - pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, logger) + pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, -1, 0, logger) podName1 := "test1" t.Run("close-pod", func(t *testing.T) { @@ -77,17 +85,17 @@ func TestClose(t *testing.T) { gotPodInfo, _, err = pod1.GetPodInfo(podName1) if err != nil { - t.Fatalf("pod should be open") + t.Fatalf("pod should be open %s\n", err) } if gotPodInfo == nil { t.Fatalf("pod should be open") } dirObject := gotPodInfo.GetDirectory() - dirInode1 := dirObject.GetInode(podPassword, "/parentDir/subDir1") + dirInode1, _ := dirObject.GetInode(podPassword, "/parentDir/subDir1") if dirInode1 == nil { t.Fatalf("dir should nil be nil") } - dirInode2 := dirObject.GetInode(podPassword, "/parentDir/subDir2") + dirInode2, _ := dirObject.GetInode(podPassword, "/parentDir/subDir2") if dirInode2 == nil { t.Fatalf("dir should nil be nil") } diff --git a/pkg/test/del_test.go b/pkg/test/del_test.go index 731dbfef..6972c89f 100644 --- a/pkg/test/del_test.go +++ b/pkg/test/del_test.go @@ -24,37 +24,45 @@ import ( "testing" "time" - mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" - - "github.com/fairdatasociety/fairOS-dfs/pkg/utils" - - "github.com/plexsysio/taskmanager" - + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/collection" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" "github.com/fairdatasociety/fairOS-dfs/pkg/pod" + mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/utils" + "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" ) func TestPodDelete(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { t.Fatal(err) } - fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) }() sm := mock2.NewMockSubscriptionManager() - pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, logger) + pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, -1, 0, logger) podName1 := "test1" podName2 := "test2" diff --git a/pkg/test/delete_test.go b/pkg/test/delete_test.go index 558436eb..f4b3b20a 100644 --- a/pkg/test/delete_test.go +++ b/pkg/test/delete_test.go @@ -23,18 +23,28 @@ import ( "testing" "time" - mock3 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" - + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/ensm/eth/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" + mock3 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/user" "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" ) func TestDelete(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) @@ -44,7 +54,7 @@ func TestDelete(t *testing.T) { t.Run("delete-user", func(t *testing.T) { ens := mock2.NewMockNamespaceManager() // create user - userObject := user.NewUsers(mockClient, ens, logger) + userObject := user.NewUsers(mockClient, ens, -1, 0, logger) sr, err := userObject.CreateNewUserV2("user1", "password1twelve", "", "", tm, sm) if err != nil { t.Fatal(err) diff --git a/pkg/test/fork_test.go b/pkg/test/fork_test.go index ed9fbedc..3a35f20a 100644 --- a/pkg/test/fork_test.go +++ b/pkg/test/fork_test.go @@ -22,22 +22,31 @@ import ( "testing" "time" - "github.com/fairdatasociety/fairOS-dfs/pkg/file" - - mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" - + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" + "github.com/fairdatasociety/fairOS-dfs/pkg/file" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" "github.com/fairdatasociety/fairOS-dfs/pkg/pod" + mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" ) func TestFork(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -47,10 +56,10 @@ func TestFork(t *testing.T) { defer func() { _ = tm.Stop(context.Background()) }() - fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) sm := mock2.NewMockSubscriptionManager() - pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, logger) + pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, -1, 0, logger) podName1 := "test1" t.Run("fork-pod", func(t *testing.T) { @@ -94,7 +103,7 @@ func TestFork(t *testing.T) { // validate if the directory and files are synced dirObject := gotInfo.GetDirectory() - dirInode1 := dirObject.GetInode(podPassword, "/parentDir/subDir1") + dirInode1, _ := dirObject.GetInode(podPassword, "/parentDir/subDir1") if dirInode1 == nil { t.Fatalf("invalid dir entry") } @@ -104,7 +113,7 @@ func TestFork(t *testing.T) { if dirInode1.Meta.Name != "subDir1" { t.Fatalf("invalid dir entry") } - dirInode2 := dirObject.GetInode(podPassword, "/parentDir/subDir2") + dirInode2, _ := dirObject.GetInode(podPassword, "/parentDir/subDir2") if dirInode2 == nil { t.Fatalf("invalid dir entry") } diff --git a/pkg/test/integration_test.go b/pkg/test/integration_test.go index 3afaa608..14d68dc6 100644 --- a/pkg/test/integration_test.go +++ b/pkg/test/integration_test.go @@ -8,6 +8,10 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/fairdatasociety/fairOS-dfs/pkg/file" "github.com/fairdatasociety/fairOS-dfs/cmd/common" @@ -34,11 +38,18 @@ func randStringRunes(n int) string { } func TestLiteUser(t *testing.T) { - mockClient := mock.NewMockBeeClient() ens := mock2.NewMockNamespaceManager() - logger := logging.New(io.Discard, logrus.ErrorLevel) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) - users := user.NewUsers(mockClient, ens, logger) + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + + users := user.NewUsers(mockClient, ens, -1, 0, logger) dfsApi := dfs.NewMockDfsAPI(mockClient, users, logger) defer dfsApi.Close() t.Run("signup-login-pod-dir-file-rename", func(t *testing.T) { @@ -275,7 +286,7 @@ func TestLiteUser(t *testing.T) { } } } - + //pi.GetFeed().CommitFeeds() err = dfsApi.LogoutUser(sessionId) if err != nil { t.Fatal(err) diff --git a/pkg/test/lite_test.go b/pkg/test/lite_test.go index b9180f58..f52b396e 100644 --- a/pkg/test/lite_test.go +++ b/pkg/test/lite_test.go @@ -7,18 +7,28 @@ import ( "testing" "time" - mock3 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" - + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/ensm/eth/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" + mock3 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/user" "github.com/plexsysio/taskmanager" + "github.com/sirupsen/logrus" ) func TestLite(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) @@ -29,7 +39,7 @@ func TestLite(t *testing.T) { ens := mock2.NewMockNamespaceManager() // create user - userObject := user.NewUsers(mockClient, ens, logger) + userObject := user.NewUsers(mockClient, ens, -1, 0, logger) _, _, _, err := userObject.LoadLiteUser("", "password1", "", "", tm, sm) if !errors.Is(err, user.ErrInvalidUserName) { t.Fatal(err) @@ -40,7 +50,7 @@ func TestLite(t *testing.T) { ens := mock2.NewMockNamespaceManager() // create user - userObject := user.NewUsers(mockClient, ens, logger) + userObject := user.NewUsers(mockClient, ens, -1, 0, logger) mnemonic, _, ui, err := userObject.LoadLiteUser("user1", "password1", "", "", tm, sm) if err != nil { t.Fatal(err) diff --git a/pkg/test/login_test.go b/pkg/test/login_test.go index 4d1cd8e9..fd09d672 100644 --- a/pkg/test/login_test.go +++ b/pkg/test/login_test.go @@ -24,8 +24,13 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + mock3 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" + "github.com/sirupsen/logrus" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/ensm/eth/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" @@ -36,8 +41,15 @@ import ( ) func TestLogin(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) @@ -47,7 +59,7 @@ func TestLogin(t *testing.T) { t.Run("login-user", func(t *testing.T) { ens := mock2.NewMockNamespaceManager() // create user - userObject := user.NewUsers(mockClient, ens, logger) + userObject := user.NewUsers(mockClient, ens, -1, 0, logger) sr, err := userObject.CreateNewUserV2("7e4567e7cb003804992eef11fd5c757275a4c", "password1twelve", "", "", tm, sm) if err != nil { t.Fatal(err) @@ -102,7 +114,7 @@ func TestLogin(t *testing.T) { user1 := "multicredtester" pass := "password1password1" // create user - userObject := user.NewUsers(mockClient, ens, logger) + userObject := user.NewUsers(mockClient, ens, -1, 0, logger) sr, err := userObject.CreateNewUserV2(user1, pass, "", "", tm, sm) if err != nil { t.Fatal(err) @@ -161,7 +173,7 @@ func TestLogin(t *testing.T) { ens := mock2.NewMockNamespaceManager() user1 := "multicredtester" // create user - userObject := user.NewUsers(mockClient, ens, logger) + userObject := user.NewUsers(mockClient, ens, -1, 0, logger) pass := "password1password1" sr, err := userObject.CreateNewUserV2(user1, pass, "", "", tm, sm) if err != nil { diff --git a/pkg/test/logout_test.go b/pkg/test/logout_test.go index e6f42de0..582f04d0 100644 --- a/pkg/test/logout_test.go +++ b/pkg/test/logout_test.go @@ -23,10 +23,15 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + mock3 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" + "github.com/sirupsen/logrus" "github.com/plexsysio/taskmanager" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/ensm/eth/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" @@ -34,8 +39,15 @@ import ( ) func TestLogout(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) @@ -46,7 +58,7 @@ func TestLogout(t *testing.T) { ens := mock2.NewMockNamespaceManager() // create user - userObject := user.NewUsers(mockClient, ens, logger) + userObject := user.NewUsers(mockClient, ens, -1, 0, logger) sr, err := userObject.CreateNewUserV2("user1", "password1twelve", "", "", tm, sm) if err != nil { t.Fatal(err) diff --git a/pkg/test/ls_test.go b/pkg/test/ls_test.go index b64b9b73..3baf02b8 100644 --- a/pkg/test/ls_test.go +++ b/pkg/test/ls_test.go @@ -18,10 +18,15 @@ package test_test import ( "context" - "os" + "io" "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/sirupsen/logrus" + mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/pod" @@ -37,18 +42,25 @@ import ( ) func TestPod_ListPods(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(os.Stdout, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) accountInfo := acc.GetUserAccountInfo() - fd := feed.New(accountInfo, mockClient, logger) + fd := feed.New(accountInfo, mockClient, -1, 0, logger) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) }() sm := mock2.NewMockSubscriptionManager() - pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, logger) + pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, -1, 0, logger) _, _, err := acc.CreateUserAccount("") if err != nil { t.Fatal(err) diff --git a/pkg/test/max_file_test.go b/pkg/test/max_file_test.go new file mode 100644 index 00000000..fadf883b --- /dev/null +++ b/pkg/test/max_file_test.go @@ -0,0 +1,90 @@ +package test_test + +import ( + "context" + "io" + "testing" + "time" + + "github.com/fairdatasociety/fairOS-dfs/pkg/file" + + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + + mock3 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" + "github.com/sirupsen/logrus" + + "github.com/plexsysio/taskmanager" + + "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/feed" + "github.com/fairdatasociety/fairOS-dfs/pkg/logging" + "github.com/fairdatasociety/fairOS-dfs/pkg/pod" + "github.com/fairdatasociety/fairOS-dfs/pkg/utils" +) + +func TestMaxFiles(t *testing.T) { + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) + acc := account.New(logger) + _, _, err := acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + tm := taskmanager.New(1, 10, time.Second*15, logger) + defer func() { + _ = tm.Stop(context.Background()) + }() + sm := mock3.NewMockSubscriptionManager() + _, _, err = acc.CreateUserAccount("") + if err != nil { + t.Fatal(err) + } + + pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, -1, 0, logger) + podPassword, _ := utils.GetRandString(pod.PasswordLength) + podName, _ := utils.GetRandString(10) + info, err := pod1.CreatePod(podName, "", podPassword) + if err != nil { + t.Fatal(err) + } + + t.Run("create-max-files", func(t *testing.T) { + maxfiles := 100 + filePath := "/" + for i := 1; i <= maxfiles; i++ { + fileName, _ := utils.GetRandString(100) + compression := "" + fileSize := int64(1000) + blockSize := file.MinBlockSize + _, err = uploadFile(t, info.GetFile(), filePath, fileName, compression, podPassword, fileSize, blockSize) + if err != nil { + t.Fatal(err) + } + err = info.GetDirectory().AddEntryToDir("/", podPassword, fileName, true) + if err != nil { + t.Fatal(i, err) + } + } + + // check if the files are present + dirInode, err := info.GetDirectory().GetInode(podPassword, filePath) + if err != nil { + t.Fatal(err) + } + if len(dirInode.FileOrDirNames) != maxfiles { + t.Fatal("files not present") + } + }) +} diff --git a/pkg/test/max_pod_test.go b/pkg/test/max_pod_test.go index 4d568e55..f2a1379a 100644 --- a/pkg/test/max_pod_test.go +++ b/pkg/test/max_pod_test.go @@ -2,16 +2,22 @@ package test_test import ( "context" - "errors" "io" "testing" "time" + "github.com/stretchr/testify/require" + + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + mock3 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" + "github.com/sirupsen/logrus" "github.com/plexsysio/taskmanager" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" @@ -20,27 +26,32 @@ import ( ) func TestMaxPods(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { t.Fatal(err) } - fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, 500, time.Second*10, logger) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) }() sm := mock3.NewMockSubscriptionManager() - pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, logger) + pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, -1, 0, logger) t.Run("create-max-pods", func(t *testing.T) { - // t.SkipNow() - - maxPodId := 30 - for i := 1; i <= maxPodId; i++ { + maxPodId := 100 + for i := 0; i < maxPodId; i++ { name, err := utils.GetRandString(utils.MaxPodNameLength) if err != nil { t.Fatal(err) @@ -51,14 +62,11 @@ func TestMaxPods(t *testing.T) { t.Fatalf("error creating pod %s with index %d: %s", name, i, err) } } - name, err := utils.GetRandString(utils.MaxPodNameLength) + + ownPods, _, err := pod1.ListPods() if err != nil { t.Fatal(err) } - podPassword, _ := utils.GetRandString(pod.PasswordLength) - _, err = pod1.CreatePod(name, "", podPassword) - if !errors.Is(err, pod.ErrMaximumPodLimit) { - t.Fatalf("maximum pod limit should have been reached") - } + require.Equal(t, maxPodId, len(ownPods)) }) } diff --git a/pkg/test/new_test.go b/pkg/test/new_test.go index 31606f7a..d779e84b 100644 --- a/pkg/test/new_test.go +++ b/pkg/test/new_test.go @@ -23,10 +23,15 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + mock3 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" + "github.com/sirupsen/logrus" "github.com/plexsysio/taskmanager" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/ensm/eth/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" @@ -34,8 +39,15 @@ import ( ) func TestNew(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) @@ -46,7 +58,7 @@ func TestNew(t *testing.T) { ens := mock2.NewMockNamespaceManager() // create user - userObject := user.NewUsers(mockClient, ens, logger) + userObject := user.NewUsers(mockClient, ens, -1, 0, logger) _, err := userObject.CreateNewUserV2("", "password1", "", "", tm, sm) if !errors.Is(err, user.ErrBlankUsername) { t.Fatal(err) @@ -57,7 +69,7 @@ func TestNew(t *testing.T) { ens := mock2.NewMockNamespaceManager() // create user - userObject := user.NewUsers(mockClient, ens, logger) + userObject := user.NewUsers(mockClient, ens, -1, 0, logger) _, err := userObject.CreateNewUserV2("user1", "password1", "", "", tm, sm) if err != nil && !errors.Is(err, user.ErrPasswordTooSmall) { t.Fatal(err) @@ -100,7 +112,7 @@ func TestNew(t *testing.T) { ens := mock2.NewMockNamespaceManager() user1 := "multicredtester" // create user - userObject := user.NewUsers(mockClient, ens, logger) + userObject := user.NewUsers(mockClient, ens, -1, 0, logger) pass := "password1password1" sr, err := userObject.CreateNewUserV2(user1, pass, "", "", tm, sm) if err != nil { diff --git a/pkg/test/open_test.go b/pkg/test/open_test.go index 4e8dec28..8e1ab00c 100644 --- a/pkg/test/open_test.go +++ b/pkg/test/open_test.go @@ -25,6 +25,11 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/sirupsen/logrus" + mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" @@ -40,8 +45,15 @@ import ( ) func TestOpen(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -53,8 +65,8 @@ func TestOpen(t *testing.T) { }() sm := mock2.NewMockSubscriptionManager() - fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger) - pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, -1, 0, logger) podName1 := "test1" podName2 := "test2" @@ -80,7 +92,8 @@ func TestOpen(t *testing.T) { // create some dir and files addFilesAndDirectories(t, info, pod1, podName1, podPassword) - + fd.CommitFeeds() + <-time.After(time.Second) // open the pod podInfo, err := pod1.OpenPod(podName1) if err != nil { diff --git a/pkg/test/pod_new_test.go b/pkg/test/pod_new_test.go index 8fd7d3f4..1f8a417d 100644 --- a/pkg/test/pod_new_test.go +++ b/pkg/test/pod_new_test.go @@ -19,11 +19,16 @@ package test_test import ( "context" "errors" - "os" + "io" "strings" "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/sirupsen/logrus" + mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" "github.com/plexsysio/taskmanager" @@ -38,21 +43,28 @@ import ( ) func TestPodNew(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(os.Stdout, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { t.Fatal(err) } - fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) }() sm := mock2.NewMockSubscriptionManager() - pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, logger) + pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, -1, 0, logger) podName1 := "test1" podName2 := "test2" diff --git a/pkg/test/pod_sharing_test.go b/pkg/test/pod_sharing_test.go index 41906680..373d841b 100644 --- a/pkg/test/pod_sharing_test.go +++ b/pkg/test/pod_sharing_test.go @@ -23,13 +23,18 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/file" + "github.com/sirupsen/logrus" mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" "github.com/plexsysio/taskmanager" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" @@ -38,21 +43,28 @@ import ( ) func TestShare(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { t.Fatal(err) } - fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) }() sm := mock2.NewMockSubscriptionManager() - pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, logger) + pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, -1, 0, logger) podName1 := "test1" acc2 := account.New(logger) @@ -60,8 +72,8 @@ func TestShare(t *testing.T) { if err != nil { t.Fatal(err) } - fd2 := feed.New(acc2.GetUserAccountInfo(), mockClient, logger) - pod2 := pod.NewPod(mockClient, fd2, acc2, tm, sm, logger) + fd2 := feed.New(acc2.GetUserAccountInfo(), mockClient, -1, 0, logger) + pod2 := pod.NewPod(mockClient, fd2, acc2, tm, sm, -1, 0, logger) podName2 := "test2" acc3 := account.New(logger) @@ -69,8 +81,8 @@ func TestShare(t *testing.T) { if err != nil { t.Fatal(err) } - fd3 := feed.New(acc3.GetUserAccountInfo(), mockClient, logger) - pod3 := pod.NewPod(mockClient, fd3, acc3, tm, sm, logger) + fd3 := feed.New(acc3.GetUserAccountInfo(), mockClient, -1, 0, logger) + pod3 := pod.NewPod(mockClient, fd3, acc3, tm, sm, -1, 0, logger) podName3 := "test3" acc4 := account.New(logger) @@ -78,8 +90,8 @@ func TestShare(t *testing.T) { if err != nil { t.Fatal(err) } - fd4 := feed.New(acc4.GetUserAccountInfo(), mockClient, logger) - pod4 := pod.NewPod(mockClient, fd4, acc4, tm, sm, logger) + fd4 := feed.New(acc4.GetUserAccountInfo(), mockClient, -1, 0, logger) + pod4 := pod.NewPod(mockClient, fd4, acc4, tm, sm, -1, 0, logger) podName4 := "test4" acc5 := account.New(logger) @@ -87,8 +99,8 @@ func TestShare(t *testing.T) { if err != nil { t.Fatal(err) } - fd5 := feed.New(acc5.GetUserAccountInfo(), mockClient, logger) - pod5 := pod.NewPod(mockClient, fd5, acc5, tm, sm, logger) + fd5 := feed.New(acc5.GetUserAccountInfo(), mockClient, -1, 0, logger) + pod5 := pod.NewPod(mockClient, fd5, acc5, tm, sm, -1, 0, logger) podName5 := "test5" acc6 := account.New(logger) @@ -96,8 +108,8 @@ func TestShare(t *testing.T) { if err != nil { t.Fatal(err) } - fd6 := feed.New(acc6.GetUserAccountInfo(), mockClient, logger) - pod6 := pod.NewPod(mockClient, fd6, acc6, tm, sm, logger) + fd6 := feed.New(acc6.GetUserAccountInfo(), mockClient, -1, 0, logger) + pod6 := pod.NewPod(mockClient, fd6, acc6, tm, sm, -1, 0, logger) podName6 := "test6" t.Run("share-pod", func(t *testing.T) { @@ -421,8 +433,8 @@ func TestShare(t *testing.T) { if err != nil { t.Fatal(err) } - fd7 := feed.New(acc7.GetUserAccountInfo(), mockClient, logger) - pod7 := pod.NewPod(mockClient, fd7, acc7, tm, sm, logger) + fd7 := feed.New(acc7.GetUserAccountInfo(), mockClient, -1, 0, logger) + pod7 := pod.NewPod(mockClient, fd7, acc7, tm, sm, -1, 0, logger) podName7 := "test7" acc8 := account.New(logger) @@ -430,8 +442,8 @@ func TestShare(t *testing.T) { if err != nil { t.Fatal(err) } - fd8 := feed.New(acc8.GetUserAccountInfo(), mockClient, logger) - pod8 := pod.NewPod(mockClient, fd8, acc8, tm, sm, logger) + fd8 := feed.New(acc8.GetUserAccountInfo(), mockClient, -1, 0, logger) + pod8 := pod.NewPod(mockClient, fd8, acc8, tm, sm, -1, 0, logger) // create sending pod and receiving pod podPassword, _ := utils.GetRandString(pod.PasswordLength) @@ -455,6 +467,8 @@ func TestShare(t *testing.T) { t.Fatal(err) } + pod7.GetFeed().CommitFeeds() + info.GetFeed().CommitFeeds() // receive pod info ref, err := utils.ParseHexReference(sharingRef) if err != nil { @@ -493,17 +507,20 @@ func TestShare(t *testing.T) { if err != nil { t.Fatal(err) } + + pod7.GetFeed().CommitFeeds() + gotInfo.GetFeed().CommitFeeds() // check shared pod entry gotSharedPodInfo, err := pod8.OpenPod(podName7) if err != nil { t.Fatal(err) } dirObject8 := gotSharedPodInfo.GetDirectory() - dirInode1 := dirObject8.GetInode(podPassword, "/parentDir/subDir1") + dirInode1, _ := dirObject8.GetInode(podPassword, "/parentDir/subDir1") if dirInode1 != nil { t.Fatalf("invalid dir entry") } - dirInode1 = dirObject8.GetInode(podPassword, "/parentDir/newSubDir1") + dirInode1, _ = dirObject8.GetInode(podPassword, "/parentDir/newSubDir1") if dirInode1 == nil { t.Fatalf("invalid dir entry") } @@ -513,7 +530,7 @@ func TestShare(t *testing.T) { if dirInode1.Meta.Name != "newSubDir1" { t.Fatalf("invalid dir entry") } - dirInode2 := dirObject8.GetInode(podPassword, "/parentDir/subDir2") + dirInode2, _ := dirObject8.GetInode(podPassword, "/parentDir/subDir2") if dirInode2 == nil { t.Fatalf("invalid dir entry") } diff --git a/pkg/test/pod_stat_test.go b/pkg/test/pod_stat_test.go index 30450e37..2a361f73 100644 --- a/pkg/test/pod_stat_test.go +++ b/pkg/test/pod_stat_test.go @@ -23,13 +23,18 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + mock3 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" + "github.com/sirupsen/logrus" "github.com/fairdatasociety/fairOS-dfs/pkg/utils" "github.com/plexsysio/taskmanager" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" @@ -37,21 +42,28 @@ import ( ) func TestStat(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { t.Fatal(err) } - fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) }() sm := mock3.NewMockSubscriptionManager() - pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, logger) + pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, -1, 0, logger) podName1 := "test1" t.Run("pod-stat", func(t *testing.T) { diff --git a/pkg/test/stat_test.go b/pkg/test/stat_test.go index aed9b365..e868ed7f 100644 --- a/pkg/test/stat_test.go +++ b/pkg/test/stat_test.go @@ -23,20 +23,32 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + mock3 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" + "github.com/sirupsen/logrus" "github.com/fairdatasociety/fairOS-dfs/pkg/user" "github.com/plexsysio/taskmanager" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/ensm/eth/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" ) func TestUserStat(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) @@ -46,7 +58,7 @@ func TestUserStat(t *testing.T) { t.Run("stat-nonexistent-user", func(t *testing.T) { ens := mock2.NewMockNamespaceManager() // create user - userObject := user.NewUsers(mockClient, ens, logger) + userObject := user.NewUsers(mockClient, ens, -1, 0, logger) ui := &user.Info{} // stat the user _, err := userObject.GetUserStat(ui) @@ -58,7 +70,7 @@ func TestUserStat(t *testing.T) { t.Run("stat-user", func(t *testing.T) { ens := mock2.NewMockNamespaceManager() // create user - userObject := user.NewUsers(mockClient, ens, logger) + userObject := user.NewUsers(mockClient, ens, -1, 0, logger) sr, err := userObject.CreateNewUserV2("user1", "password1twelve", "", "", tm, sm) if err != nil { t.Fatal(err) diff --git a/pkg/test/subscription_test.go b/pkg/test/subscription_test.go index 33c9f16c..13a9a242 100644 --- a/pkg/test/subscription_test.go +++ b/pkg/test/subscription_test.go @@ -2,10 +2,15 @@ package test_test import ( "context" - "os" + "io" "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/sirupsen/logrus" + "github.com/fairdatasociety/fairOS-dfs/pkg/file" "github.com/ethereum/go-ethereum/common" @@ -22,15 +27,21 @@ import ( ) func TestSubscription(t *testing.T) { - mockClient := mock.NewMockBeeClient() - - logger := logging.New(os.Stdout, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc1 := account.New(logger) _, _, err := acc1.CreateUserAccount("") if err != nil { t.Fatal(err) } - fd := feed.New(acc1.GetUserAccountInfo(), mockClient, logger) + fd := feed.New(acc1.GetUserAccountInfo(), mockClient, -1, 0, logger) tm := taskmanager.New(1, 10, time.Second*15, logger) defer func() { _ = tm.Stop(context.Background()) @@ -43,7 +54,7 @@ func TestSubscription(t *testing.T) { } sm := mock2.NewMockSubscriptionManager() - pod1 := pod.NewPod(mockClient, fd, acc1, tm, sm, logger) + pod1 := pod.NewPod(mockClient, fd, acc1, tm, sm, -1, 0, logger) randomLongPodName1, err := utils.GetRandString(64) if err != nil { @@ -86,8 +97,8 @@ func TestSubscription(t *testing.T) { t.Fatal(err) } - fd2 := feed.New(acc2.GetUserAccountInfo(), mockClient, logger) - pod2 := pod.NewPod(mockClient, fd2, acc2, tm, sm, logger) + fd2 := feed.New(acc2.GetUserAccountInfo(), mockClient, -1, 0, logger) + pod2 := pod.NewPod(mockClient, fd2, acc2, tm, sm, -1, 0, logger) a2 := acc2.GetUserAccountInfo().GetAddress() addr2 := common.HexToAddress(a2.Hex()) nameHash2, err := goens.NameHash(addr2.Hex()) diff --git a/pkg/test/sync_test.go b/pkg/test/sync_test.go index ceb493cc..9802d25c 100644 --- a/pkg/test/sync_test.go +++ b/pkg/test/sync_test.go @@ -22,11 +22,16 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/file" + "github.com/sirupsen/logrus" mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/feed" "github.com/fairdatasociety/fairOS-dfs/pkg/logging" @@ -36,8 +41,15 @@ import ( ) func TestSync(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc := account.New(logger) _, _, err := acc.CreateUserAccount("") if err != nil { @@ -49,8 +61,8 @@ func TestSync(t *testing.T) { }() sm := mock2.NewMockSubscriptionManager() - fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger) - pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, logger) + fd := feed.New(acc.GetUserAccountInfo(), mockClient, -1, 0, logger) + pod1 := pod.NewPod(mockClient, fd, acc, tm, sm, -1, 0, logger) podName1 := "test1" t.Run("sync-pod", func(t *testing.T) { @@ -82,7 +94,7 @@ func TestSync(t *testing.T) { // validate if the directory and files are synced dirObject := gotInfo.GetDirectory() - dirInode1 := dirObject.GetInode(podPassword, "/parentDir/subDir1") + dirInode1, _ := dirObject.GetInode(podPassword, "/parentDir/subDir1") if dirInode1 == nil { t.Fatalf("invalid dir entry") } @@ -92,7 +104,7 @@ func TestSync(t *testing.T) { if dirInode1.Meta.Name != "subDir1" { t.Fatalf("invalid dir entry") } - dirInode2 := dirObject.GetInode(podPassword, "/parentDir/subDir2") + dirInode2, _ := dirObject.GetInode(podPassword, "/parentDir/subDir2") if dirInode2 == nil { t.Fatalf("invalid dir entry") } diff --git a/pkg/test/user_sharing_test.go b/pkg/test/user_sharing_test.go index c38e34f4..09e7455d 100644 --- a/pkg/test/user_sharing_test.go +++ b/pkg/test/user_sharing_test.go @@ -25,6 +25,11 @@ import ( "testing" "time" + mockpost "github.com/ethersphere/bee/pkg/postage/mock" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" + "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee" + "github.com/sirupsen/logrus" + mock3 "github.com/fairdatasociety/fairOS-dfs/pkg/subscriptionManager/rpc/mock" "github.com/fairdatasociety/fairOS-dfs/pkg/account" @@ -40,8 +45,15 @@ import ( ) func TestSharing(t *testing.T) { - mockClient := mock.NewMockBeeClient() - logger := logging.New(io.Discard, 0) + storer := mockstorer.New() + beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{ + Storer: storer, + PreventRedirect: true, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + logger := logging.New(io.Discard, logrus.DebugLevel) + mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger) acc1 := account.New(logger) _, _, err := acc1.CreateUserAccount("") @@ -58,8 +70,8 @@ func TestSharing(t *testing.T) { }() sm := mock3.NewMockSubscriptionManager() - fd1 := feed.New(acc1.GetUserAccountInfo(), mockClient, logger) - pod1 := pod.NewPod(mockClient, fd1, acc1, tm, sm, logger) + fd1 := feed.New(acc1.GetUserAccountInfo(), mockClient, -1, 0, logger) + pod1 := pod.NewPod(mockClient, fd1, acc1, tm, sm, -1, 0, logger) podName1 := "test1" acc2 := account.New(logger) @@ -71,14 +83,14 @@ func TestSharing(t *testing.T) { if err != nil { t.Fatal(err) } - fd2 := feed.New(acc2.GetUserAccountInfo(), mockClient, logger) - pod2 := pod.NewPod(mockClient, fd2, acc2, tm, sm, logger) + fd2 := feed.New(acc2.GetUserAccountInfo(), mockClient, -1, 0, logger) + pod2 := pod.NewPod(mockClient, fd2, acc2, tm, sm, -1, 0, logger) podName2 := "test2" t.Run("sharing-user", func(t *testing.T) { ens := mock2.NewMockNamespaceManager() // create source user - userObject1 := user.NewUsers(mockClient, ens, logger) + userObject1 := user.NewUsers(mockClient, ens, -1, 0, logger) sr0, err := userObject1.CreateNewUserV2("user1", "password1twelve", "", "", tm, sm) if err != nil { t.Fatal(err) @@ -109,6 +121,9 @@ func TestSharing(t *testing.T) { if err != nil { t.Fatal(err) } + fd1.CommitFeeds() + pod1.GetFeed().CommitFeeds() + info1.GetFeed().CommitFeeds() // share file with another user sharingRefString, err := userObject1.ShareFileWithUser("pod1", podPassword, "/parentDir1/file1", "user2", ui0, pod1, info1.GetPodAddress()) if err != nil { @@ -116,7 +131,7 @@ func TestSharing(t *testing.T) { } // create destination user - userObject2 := user.NewUsers(mockClient, ens, logger) + userObject2 := user.NewUsers(mockClient, ens, -1, 0, logger) sr, err := userObject2.CreateNewUserV2("user2", "password1twelve", "", "", tm, sm) if err != nil { t.Fatal(err) diff --git a/pkg/user/lite.go b/pkg/user/lite.go index 2e9c3583..f0a0246c 100644 --- a/pkg/user/lite.go +++ b/pkg/user/lite.go @@ -25,7 +25,7 @@ func (u *Users) LoadLiteUser(userName, _, mnemonic, sessionId string, tm taskman acc := account.New(u.logger) accountInfo := acc.GetUserAccountInfo() - fd := feed.New(accountInfo, u.client, u.logger) + fd := feed.New(accountInfo, u.client, u.feedCacheSize, u.feedCacheTTL, u.logger) // create a new base user account with the mnemonic mnemonic, _, err := acc.CreateUserAccount(mnemonic) if err != nil { // skipcq: TCV-001 @@ -35,7 +35,7 @@ func (u *Users) LoadLiteUser(userName, _, mnemonic, sessionId string, tm taskman // Instantiate pod, dir & file objects file := f.NewFile(userName, u.client, fd, accountInfo.GetAddress(), tm, u.logger) dir := d.NewDirectory(userName, u.client, fd, accountInfo.GetAddress(), file, tm, u.logger) - pod := p.NewPod(u.client, fd, acc, tm, sm, u.logger) + pod := p.NewPod(u.client, fd, acc, tm, sm, u.feedCacheSize, u.feedCacheTTL, u.logger) if sessionId == "" { sessionId = auth.GetUniqueSessionId() } diff --git a/pkg/user/login.go b/pkg/user/login.go index e88d1216..0d503bb9 100644 --- a/pkg/user/login.go +++ b/pkg/user/login.go @@ -66,7 +66,7 @@ func (u *Users) LoginUserV2(userName, passPhrase string, client blockstore.Clien acc := account.New(u.logger) accountInfo := acc.GetUserAccountInfo() // load encrypted private key - fd := feed.New(accountInfo, client, u.logger) + fd := feed.New(accountInfo, client, u.feedCacheSize, u.feedCacheTTL, u.logger) key, err := u.downloadPortableAccount(utils.Address(address), userName, passPhrase, fd) if err != nil { return nil, ErrInvalidPassword @@ -94,7 +94,7 @@ func (u *Users) LoginUserV2(userName, passPhrase string, client blockstore.Clien // Instantiate pod, dir & file objects file := f.NewFile(userName, client, fd, accountInfo.GetAddress(), tm, u.logger) - pod := p.NewPod(u.client, fd, acc, tm, sm, u.logger) + pod := p.NewPod(u.client, fd, acc, tm, sm, u.feedCacheSize, u.feedCacheTTL, u.logger) dir := d.NewDirectory(userName, client, fd, accountInfo.GetAddress(), file, tm, u.logger) if sessionId == "" { sessionId = auth.GetUniqueSessionId() @@ -163,6 +163,11 @@ func (u *Users) IsUserNameLoggedIn(userName string) bool { return u.isUserNameInMap(userName) } +// GetUsersLoggedIn returns all the users that are loggedin +func (u *Users) GetUsersLoggedIn() map[string]*Info { + return u.getUserMap() +} + // ConnectWallet connects user with wallet. func (u *Users) ConnectWallet(userName, passPhrase, walletAddressHex, signature string, client blockstore.Client) error { // check if username is available (user created) @@ -184,7 +189,7 @@ func (u *Users) ConnectWallet(userName, passPhrase, walletAddressHex, signature acc := account.New(u.logger) accountInfo := acc.GetUserAccountInfo() // load encrypted private key - fd := feed.New(accountInfo, client, u.logger) + fd := feed.New(accountInfo, client, u.feedCacheSize, u.feedCacheTTL, u.logger) key, err := u.downloadPortableAccount(utils.Address(address), userName, passPhrase, fd) if err != nil { u.logger.Errorf(err.Error()) @@ -217,7 +222,7 @@ func (u *Users) LoginWithWallet(addressHex, signature string, client blockstore. acc := account.New(u.logger) accountInfo := acc.GetUserAccountInfo() // load encrypted private key - fd := feed.New(accountInfo, client, u.logger) + fd := feed.New(accountInfo, client, u.feedCacheSize, u.feedCacheTTL, u.logger) key, err := u.downloadPortableAccount(utils.Address(address), addressHex, signature, fd) if err != nil { u.logger.Errorf(err.Error()) @@ -246,7 +251,7 @@ func (u *Users) LoginWithWallet(addressHex, signature string, client blockstore. // Instantiate pod, dir & file objects file := f.NewFile(addressHex, client, fd, accountInfo.GetAddress(), tm, u.logger) - pod := p.NewPod(u.client, fd, acc, tm, sm, u.logger) + pod := p.NewPod(u.client, fd, acc, tm, sm, u.feedCacheSize, u.feedCacheTTL, u.logger) dir := d.NewDirectory(addressHex, client, fd, accountInfo.GetAddress(), file, tm, u.logger) if sessionId == "" { sessionId = auth.GetUniqueSessionId() diff --git a/pkg/user/new.go b/pkg/user/new.go index 08ec3610..d7a6edab 100644 --- a/pkg/user/new.go +++ b/pkg/user/new.go @@ -78,7 +78,7 @@ func (u *Users) CreateNewUserV2(userName, passPhrase, mnemonic, sessionId string acc := account.New(u.logger) accountInfo := acc.GetUserAccountInfo() - fd := feed.New(accountInfo, u.client, u.logger) + fd := feed.New(accountInfo, u.client, u.feedCacheSize, u.feedCacheTTL, u.logger) // create a new base user account with the mnemonic mnemonic, seed, err := acc.CreateUserAccount(mnemonic) @@ -124,7 +124,7 @@ func (u *Users) CreateNewUserV2(userName, passPhrase, mnemonic, sessionId string // Instantiate pod, dir & file objects file := f.NewFile(userName, u.client, fd, accountInfo.GetAddress(), tm, u.logger) dir := d.NewDirectory(userName, u.client, fd, accountInfo.GetAddress(), file, tm, u.logger) - pod := p.NewPod(u.client, fd, acc, tm, sm, u.logger) + pod := p.NewPod(u.client, fd, acc, tm, sm, u.feedCacheSize, u.feedCacheTTL, u.logger) if sessionId == "" { sessionId = auth.GetUniqueSessionId() } diff --git a/pkg/user/users.go b/pkg/user/users.go index c67fcb71..98b4d1f0 100644 --- a/pkg/user/users.go +++ b/pkg/user/users.go @@ -18,6 +18,7 @@ package user import ( "sync" + "time" "github.com/fairdatasociety/fairOS-dfs/pkg/blockstore" "github.com/fairdatasociety/fairOS-dfs/pkg/ensm" @@ -31,17 +32,22 @@ type Users struct { userMu *sync.RWMutex logger logging.Logger ens ensm.ENSManager + + feedCacheSize int + feedCacheTTL time.Duration } // NewUsers creates the main user object which stores all the logged-in users and there respective // other data structures. -func NewUsers(client blockstore.Client, ens ensm.ENSManager, logger logging.Logger) *Users { +func NewUsers(client blockstore.Client, ens ensm.ENSManager, feedCacheSize int, feedCacheTTL time.Duration, logger logging.Logger) *Users { return &Users{ - client: client, - userMap: make(map[string]*Info), - userMu: &sync.RWMutex{}, - logger: logger, - ens: ens, + client: client, + userMap: make(map[string]*Info), + userMu: &sync.RWMutex{}, + logger: logger, + ens: ens, + feedCacheSize: feedCacheSize, + feedCacheTTL: feedCacheTTL, } } @@ -72,6 +78,12 @@ func (u *Users) isUserPresentInMap(sessionId string) bool { return false } +func (u *Users) getUserMap() map[string]*Info { + u.userMu.Lock() + defer u.userMu.Unlock() + return u.userMap +} + func (u *Users) isUserNameInMap(userName string) bool { u.userMu.Lock() defer u.userMu.Unlock()