diff --git a/.github/workflows/go-check.yml b/.github/workflows/go-check.yml index 25e1afd..251f7fa 100644 --- a/.github/workflows/go-check.yml +++ b/.github/workflows/go-check.yml @@ -11,12 +11,12 @@ jobs: env: RUNGOGENERATE: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: recursive - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: - go-version: "1.18.x" + go-version: "1.19.x" - name: Run repo-specific setup uses: ./.github/actions/go-check-setup if: hashFiles('./.github/actions/go-check-setup') != '' @@ -27,7 +27,7 @@ jobs: echo "RUNGOGENERATE=true" >> $GITHUB_ENV fi - name: Install staticcheck - run: go install honnef.co/go/tools/cmd/staticcheck@d7e217c1ff411395475b2971c0824e1e7cc1af98 # 2022.1 (v0.3.0) + run: go install honnef.co/go/tools/cmd/staticcheck@376210a89477dedbe6fdc4484b233998650d7b3c # 2022.1.3 (v0.3.3) - name: Check that go.mod is tidy uses: protocol/multiple-go-modules@v1.2 with: diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index b86241a..8a1697b 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -10,16 +10,16 @@ jobs: fail-fast: false matrix: os: [ "ubuntu", "windows", "macos" ] - go: [ "1.17.x", "1.18.x" ] + go: [ "1.18.x", "1.19.x" ] env: COVERAGES: "" runs-on: ${{ format('{0}-latest', matrix.os) }} name: ${{ matrix.os }} (go ${{ matrix.go }}) steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: recursive - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: ${{ matrix.go }} - name: Go information @@ -43,7 +43,7 @@ jobs: # Use -coverpkg=./..., so that we include cross-package coverage. # If package ./A imports ./B, and ./A's tests also cover ./B, # this means ./B's coverage will be significantly higher than 0%. - run: go test -v -coverprofile=module-coverage.txt -coverpkg=./... ./... + run: go test -v -shuffle=on -coverprofile=module-coverage.txt -coverpkg=./... ./... - name: Run tests (32 bit) if: ${{ matrix.os != 'macos' }} # can't run 32 bit tests on OSX. uses: protocol/multiple-go-modules@v1.2 @@ -52,7 +52,7 @@ jobs: with: run: | export "PATH=${{ env.PATH_386 }}:$PATH" - go test -v ./... + go test -v -shuffle=on ./... - name: Run tests with race detector if: ${{ matrix.os == 'ubuntu' }} # speed things up. Windows and OSX VMs are slow uses: protocol/multiple-go-modules@v1.2 @@ -62,7 +62,7 @@ jobs: shell: bash run: echo "COVERAGES=$(find . -type f -name 'module-coverage.txt' | tr -s '\n' ',' | sed 's/,$//')" >> $GITHUB_ENV - name: Upload coverage to Codecov - uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b # v2.1.0 + uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 with: files: '${{ env.COVERAGES }}' env_vars: OS=${{ matrix.os }}, GO=${{ matrix.go }} diff --git a/basic_ds_test.go b/basic_ds_test.go index 7957283..8388347 100644 --- a/basic_ds_test.go +++ b/basic_ds_test.go @@ -1,7 +1,7 @@ package datastore_test import ( - "io/ioutil" + "io" "log" "testing" @@ -16,7 +16,7 @@ func TestMapDatastore(t *testing.T) { func TestLogDatastore(t *testing.T) { defer log.SetOutput(log.Writer()) - log.SetOutput(ioutil.Discard) + log.SetOutput(io.Discard) ds := datastore.NewLogDatastore(datastore.NewMapDatastore(), "") dstest.SubtestAll(t, ds) } diff --git a/datastore.go b/datastore.go index 8926bb4..f14f17a 100644 --- a/datastore.go +++ b/datastore.go @@ -201,9 +201,9 @@ var ErrNotFound error = &dsError{error: errors.New("datastore: key not found"), // GetBackedHas provides a default Datastore.Has implementation. // It exists so Datastore.Has implementations can use it, like so: // -// func (*d SomeDatastore) Has(key Key) (exists bool, err error) { -// return GetBackedHas(d, key) -// } +// func (*d SomeDatastore) Has(key Key) (exists bool, err error) { +// return GetBackedHas(d, key) +// } func GetBackedHas(ctx context.Context, ds Read, key Key) (bool, error) { _, err := ds.Get(ctx, key) switch err { @@ -219,9 +219,9 @@ func GetBackedHas(ctx context.Context, ds Read, key Key) (bool, error) { // GetBackedSize provides a default Datastore.GetSize implementation. // It exists so Datastore.GetSize implementations can use it, like so: // -// func (*d SomeDatastore) GetSize(key Key) (size int, err error) { -// return GetBackedSize(d, key) -// } +// func (*d SomeDatastore) GetSize(key Key) (size int, err error) { +// return GetBackedSize(d, key) +// } func GetBackedSize(ctx context.Context, ds Read, key Key) (int, error) { value, err := ds.Get(ctx, key) if err == nil { diff --git a/examples/fs.go b/examples/fs.go index 53f370a..bb62aa3 100644 --- a/examples/fs.go +++ b/examples/fs.go @@ -5,9 +5,9 @@ // This means key some segments will not work. For example, the // following keys will result in unwanted behavior: // -// - "/foo/./bar" -// - "/foo/../bar" -// - "/foo\x00bar" +// - "/foo/./bar" +// - "/foo/../bar" +// - "/foo\x00bar" // // Keys that only differ in case may be confused with each other on // case insensitive file systems, for example in OS X. @@ -20,7 +20,6 @@ package examples import ( "context" "fmt" - "io/ioutil" "log" "os" "path/filepath" @@ -65,7 +64,7 @@ func (d *Datastore) Put(ctx context.Context, key ds.Key, value []byte) (err erro return err } - return ioutil.WriteFile(fn, value, 0666) + return os.WriteFile(fn, value, 0666) } // Sync would ensure that any previous Puts under the prefix are written to disk. @@ -81,7 +80,7 @@ func (d *Datastore) Get(ctx context.Context, key ds.Key) (value []byte, err erro return nil, ds.ErrNotFound } - return ioutil.ReadFile(fn) + return os.ReadFile(fn) } // Has returns whether the datastore has a value for a given key diff --git a/failstore/failstore.go b/failstore/failstore.go index 906f337..348b25e 100644 --- a/failstore/failstore.go +++ b/failstore/failstore.go @@ -110,7 +110,7 @@ func (d *Failstore) DiskUsage(ctx context.Context) (uint64, error) { return ds.DiskUsage(ctx, d.child) } -// Close implements the Datastore interface +// Close implements the Datastore interface func (d *Failstore) Close() error { return d.child.Close() } diff --git a/fuzz/cmd/compare/main.go b/fuzz/cmd/compare/main.go index 77a2d8c..6fb917d 100644 --- a/fuzz/cmd/compare/main.go +++ b/fuzz/cmd/compare/main.go @@ -2,7 +2,7 @@ package main import ( "fmt" - "io/ioutil" + "io" "os" ds "github.com/ipfs/go-datastore" @@ -28,9 +28,9 @@ func main() { var dat []byte var err error if *input == "" { - dat, err = ioutil.ReadAll(os.Stdin) + dat, err = io.ReadAll(os.Stdin) } else { - dat, err = ioutil.ReadFile(*input) + dat, err = os.ReadFile(*input) } if err != nil { fmt.Fprintf(os.Stderr, "Could not read %s: %v\n", *input, err) diff --git a/fuzz/cmd/isprefix/main.go b/fuzz/cmd/isprefix/main.go index a6f7555..1f6ec6c 100644 --- a/fuzz/cmd/isprefix/main.go +++ b/fuzz/cmd/isprefix/main.go @@ -4,7 +4,7 @@ package main import ( "fmt" - "io/ioutil" + "io" "os" ds "github.com/ipfs/go-datastore" @@ -47,9 +47,9 @@ func main() { var dat []byte var err error if *input == "" { - dat, err = ioutil.ReadAll(os.Stdin) + dat, err = io.ReadAll(os.Stdin) } else { - dat, err = ioutil.ReadFile(*input) + dat, err = os.ReadFile(*input) } if err != nil { fmt.Fprintf(os.Stderr, "could not read %s: %v\n", *input, err) diff --git a/fuzz/cmd/run/main.go b/fuzz/cmd/run/main.go index 332a544..2c4d84a 100644 --- a/fuzz/cmd/run/main.go +++ b/fuzz/cmd/run/main.go @@ -3,7 +3,6 @@ package main import ( "bufio" "fmt" - "io/ioutil" "os" fuzzer "github.com/ipfs/go-datastore/fuzz" @@ -22,7 +21,7 @@ func main() { fuzzer.Threads = *threads if *input != "" { - dat, err := ioutil.ReadFile(*input) + dat, err := os.ReadFile(*input) if err != nil { fmt.Fprintf(os.Stderr, "could not read %s: %v\n", *input, err) os.Exit(1) diff --git a/fuzz/fuzzer.go b/fuzz/fuzzer.go index 5513aac..688295f 100644 --- a/fuzz/fuzzer.go +++ b/fuzz/fuzzer.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "os" "sync" "sync/atomic" @@ -134,7 +133,7 @@ func Fuzz(data []byte) int { impls = append(impls, impl) } - defaultLoc, _ := ioutil.TempDir("", "fuzz-*") + defaultLoc, _ := os.MkdirTemp("", "fuzz-*") if len(impls) == 0 { fmt.Fprintf(os.Stderr, "No datastores to fuzz.\n") diff --git a/fuzz/go.mod b/fuzz/go.mod index 8f247e8..2c50368 100644 --- a/fuzz/go.mod +++ b/fuzz/go.mod @@ -1,6 +1,6 @@ module github.com/ipfs/go-datastore/fuzz -go 1.17 +go 1.18 require ( github.com/ipfs/go-datastore v0.4.4 diff --git a/go.mod b/go.mod index e87c063..ff1a30a 100644 --- a/go.mod +++ b/go.mod @@ -16,4 +16,4 @@ require ( go.uber.org/atomic v1.6.0 // indirect ) -go 1.17 +go 1.18 diff --git a/key.go b/key.go index 42cea30..24bfccb 100644 --- a/key.go +++ b/key.go @@ -18,17 +18,16 @@ Keys are meant to be unique across a system. Keys are hierarchical, incorporating more and more specific namespaces. Thus keys can be deemed 'children' or 'ancestors' of other keys:: - Key("/Comedy") - Key("/Comedy/MontyPython") + Key("/Comedy") + Key("/Comedy/MontyPython") Also, every namespace can be parametrized to embed relevant object information. For example, the Key `name` (most specific namespace) could include the object type:: - Key("/Comedy/MontyPython/Actor:JohnCleese") - Key("/Comedy/MontyPython/Sketch:CheeseShop") - Key("/Comedy/MontyPython/Sketch:CheeseShop/Character:Mousebender") - + Key("/Comedy/MontyPython/Actor:JohnCleese") + Key("/Comedy/MontyPython/Sketch:CheeseShop") + Key("/Comedy/MontyPython/Sketch:CheeseShop/Character:Mousebender") */ type Key struct { string @@ -114,15 +113,17 @@ func (k Key) Less(k2 Key) bool { } // List returns the `list` representation of this Key. -// NewKey("/Comedy/MontyPython/Actor:JohnCleese").List() -// ["Comedy", "MontyPythong", "Actor:JohnCleese"] +// +// NewKey("/Comedy/MontyPython/Actor:JohnCleese").List() +// ["Comedy", "MontyPythong", "Actor:JohnCleese"] func (k Key) List() []string { return strings.Split(k.string, "/")[1:] } // Reverse returns the reverse of this Key. -// NewKey("/Comedy/MontyPython/Actor:JohnCleese").Reverse() -// NewKey("/Actor:JohnCleese/MontyPython/Comedy") +// +// NewKey("/Comedy/MontyPython/Actor:JohnCleese").Reverse() +// NewKey("/Actor:JohnCleese/MontyPython/Comedy") func (k Key) Reverse() Key { l := k.List() r := make([]string, len(l)) @@ -133,52 +134,59 @@ func (k Key) Reverse() Key { } // Namespaces returns the `namespaces` making up this Key. -// NewKey("/Comedy/MontyPython/Actor:JohnCleese").Namespaces() -// ["Comedy", "MontyPython", "Actor:JohnCleese"] +// +// NewKey("/Comedy/MontyPython/Actor:JohnCleese").Namespaces() +// ["Comedy", "MontyPython", "Actor:JohnCleese"] func (k Key) Namespaces() []string { return k.List() } // BaseNamespace returns the "base" namespace of this key (path.Base(filename)) -// NewKey("/Comedy/MontyPython/Actor:JohnCleese").BaseNamespace() -// "Actor:JohnCleese" +// +// NewKey("/Comedy/MontyPython/Actor:JohnCleese").BaseNamespace() +// "Actor:JohnCleese" func (k Key) BaseNamespace() string { n := k.Namespaces() return n[len(n)-1] } // Type returns the "type" of this key (value of last namespace). -// NewKey("/Comedy/MontyPython/Actor:JohnCleese").Type() -// "Actor" +// +// NewKey("/Comedy/MontyPython/Actor:JohnCleese").Type() +// "Actor" func (k Key) Type() string { return NamespaceType(k.BaseNamespace()) } // Name returns the "name" of this key (field of last namespace). -// NewKey("/Comedy/MontyPython/Actor:JohnCleese").Name() -// "JohnCleese" +// +// NewKey("/Comedy/MontyPython/Actor:JohnCleese").Name() +// "JohnCleese" func (k Key) Name() string { return NamespaceValue(k.BaseNamespace()) } // Instance returns an "instance" of this type key (appends value to namespace). -// NewKey("/Comedy/MontyPython/Actor").Instance("JohnClesse") -// NewKey("/Comedy/MontyPython/Actor:JohnCleese") +// +// NewKey("/Comedy/MontyPython/Actor").Instance("JohnClesse") +// NewKey("/Comedy/MontyPython/Actor:JohnCleese") func (k Key) Instance(s string) Key { return NewKey(k.string + ":" + s) } // Path returns the "path" of this key (parent + type). -// NewKey("/Comedy/MontyPython/Actor:JohnCleese").Path() -// NewKey("/Comedy/MontyPython/Actor") +// +// NewKey("/Comedy/MontyPython/Actor:JohnCleese").Path() +// NewKey("/Comedy/MontyPython/Actor") func (k Key) Path() Key { s := k.Parent().string + "/" + NamespaceType(k.BaseNamespace()) return NewKey(s) } // Parent returns the `parent` Key of this Key. -// NewKey("/Comedy/MontyPython/Actor:JohnCleese").Parent() -// NewKey("/Comedy/MontyPython") +// +// NewKey("/Comedy/MontyPython/Actor:JohnCleese").Parent() +// NewKey("/Comedy/MontyPython") func (k Key) Parent() Key { n := k.List() if len(n) == 1 { @@ -188,8 +196,9 @@ func (k Key) Parent() Key { } // Child returns the `child` Key of this Key. -// NewKey("/Comedy/MontyPython").Child(NewKey("Actor:JohnCleese")) -// NewKey("/Comedy/MontyPython/Actor:JohnCleese") +// +// NewKey("/Comedy/MontyPython").Child(NewKey("Actor:JohnCleese")) +// NewKey("/Comedy/MontyPython/Actor:JohnCleese") func (k Key) Child(k2 Key) Key { switch { case k.string == "/": @@ -202,15 +211,17 @@ func (k Key) Child(k2 Key) Key { } // ChildString returns the `child` Key of this Key -- string helper. -// NewKey("/Comedy/MontyPython").ChildString("Actor:JohnCleese") -// NewKey("/Comedy/MontyPython/Actor:JohnCleese") +// +// NewKey("/Comedy/MontyPython").ChildString("Actor:JohnCleese") +// NewKey("/Comedy/MontyPython/Actor:JohnCleese") func (k Key) ChildString(s string) Key { return NewKey(k.string + "/" + s) } // IsAncestorOf returns whether this key is a prefix of `other` -// NewKey("/Comedy").IsAncestorOf("/Comedy/MontyPython") -// true +// +// NewKey("/Comedy").IsAncestorOf("/Comedy/MontyPython") +// true func (k Key) IsAncestorOf(other Key) bool { // equivalent to HasPrefix(other, k.string + "/") @@ -229,8 +240,9 @@ func (k Key) IsAncestorOf(other Key) bool { } // IsDescendantOf returns whether this key contains another as a prefix. -// NewKey("/Comedy/MontyPython").IsDescendantOf("/Comedy") -// true +// +// NewKey("/Comedy/MontyPython").IsDescendantOf("/Comedy") +// true func (k Key) IsDescendantOf(other Key) bool { return other.IsAncestorOf(k) } @@ -258,8 +270,9 @@ func (k *Key) UnmarshalJSON(data []byte) error { } // RandomKey returns a randomly (uuid) generated key. -// RandomKey() -// NewKey("/f98719ea086343f7b71f32ea9d9d521d") +// +// RandomKey() +// NewKey("/f98719ea086343f7b71f32ea9d9d521d") func RandomKey() Key { return NewKey(strings.Replace(uuid.New().String(), "-", "", -1)) } diff --git a/keytransform/doc.go b/keytransform/doc.go index b389dcf..d0195c9 100644 --- a/keytransform/doc.go +++ b/keytransform/doc.go @@ -6,20 +6,19 @@ // A KeyTransform is simply an interface with two functions, a conversion and // its inverse. For example: // -// import ( -// ktds "github.com/ipfs/go-datastore/keytransform" -// ds "github.com/ipfs/go-datastore" -// ) +// import ( +// ktds "github.com/ipfs/go-datastore/keytransform" +// ds "github.com/ipfs/go-datastore" +// ) // -// func reverseKey(k ds.Key) ds.Key { -// return k.Reverse() -// } -// -// func invertKeys(d ds.Datastore) { -// return ktds.Wrap(d, &ktds.Pair{ -// Convert: reverseKey, -// Invert: reverseKey, // reverse is its own inverse. -// }) -// } +// func reverseKey(k ds.Key) ds.Key { +// return k.Reverse() +// } // +// func invertKeys(d ds.Datastore) { +// return ktds.Wrap(d, &ktds.Pair{ +// Convert: reverseKey, +// Invert: reverseKey, // reverse is its own inverse. +// }) +// } package keytransform diff --git a/mount/mount.go b/mount/mount.go index 8885358..027bdca 100644 --- a/mount/mount.go +++ b/mount/mount.go @@ -191,11 +191,11 @@ func (h *querySet) next() (query.Result, bool) { // // Specifically, this function will return three slices: // -// * The matching datastores. -// * The prefixes where each matching datastore has been mounted. -// * The prefix within these datastores at which descendants of the passed key -// live. If the mounted datastore is fully contained within the given key, -// this will be /. +// - The matching datastores. +// - The prefixes where each matching datastore has been mounted. +// - The prefix within these datastores at which descendants of the passed key +// live. If the mounted datastore is fully contained within the given key, +// this will be /. // // By example, given the datastores: // diff --git a/namespace/doc.go b/namespace/doc.go index 9ff9a8c..286915e 100644 --- a/namespace/doc.go +++ b/namespace/doc.go @@ -3,22 +3,22 @@ // // Use the Wrap function to wrap a datastore with any Key prefix. For example: // -// import ( -// "fmt" +// import ( +// "fmt" // -// ds "github.com/ipfs/go-datastore" -// nsds "github.com/ipfs/go-datastore/namespace" -// ) +// ds "github.com/ipfs/go-datastore" +// nsds "github.com/ipfs/go-datastore/namespace" +// ) // -// func main() { -// mp := ds.NewMapDatastore() -// ns := nsds.Wrap(mp, ds.NewKey("/foo/bar")) +// func main() { +// mp := ds.NewMapDatastore() +// ns := nsds.Wrap(mp, ds.NewKey("/foo/bar")) // -// // in the Namespace Datastore: -// ns.Put(ds.NewKey("/beep"), "boop") -// v2, _ := ns.Get(ds.NewKey("/beep")) // v2 == "boop" +// // in the Namespace Datastore: +// ns.Put(ds.NewKey("/beep"), "boop") +// v2, _ := ns.Get(ds.NewKey("/beep")) // v2 == "boop" // -// // and, in the underlying MapDatastore: -// v3, _ := mp.Get(ds.NewKey("/foo/bar/beep")) // v3 == "boop" -// } +// // and, in the underlying MapDatastore: +// v3, _ := mp.Get(ds.NewKey("/foo/bar/beep")) // v3 == "boop" +// } package namespace diff --git a/query/query.go b/query/query.go index a390e5b..0a8898e 100644 --- a/query/query.go +++ b/query/query.go @@ -12,9 +12,9 @@ Query represents storage for any key-value pair. tl;dr: - queries are supported across datastores. - Cheap on top of relational dbs, and expensive otherwise. - Pick the right tool for the job! + queries are supported across datastores. + Cheap on top of relational dbs, and expensive otherwise. + Pick the right tool for the job! In addition to the key-value store get and set semantics, datastore provides an interface to retrieve multiple records at a time through @@ -24,11 +24,11 @@ database research, let’s summarize the operations datastore supports. Query Operations, applied in-order: - * prefix - scope the query to a given path prefix - * filters - select a subset of values by applying constraints - * orders - sort the results by applying sort conditions, hierarchically. - * offset - skip a number of results (for efficient pagination) - * limit - impose a numeric limit on the number of results + - prefix - scope the query to a given path prefix + - filters - select a subset of values by applying constraints + - orders - sort the results by applying sort conditions, hierarchically. + - offset - skip a number of results (for efficient pagination) + - limit - impose a numeric limit on the number of results Datastore combines these operations into a simple Query class that allows applications to define their constraints in a simple, generic, way without @@ -41,13 +41,13 @@ backed datastore. Notes: - * Prefix: When a query filters by prefix, it selects keys that are strict + - Prefix: When a query filters by prefix, it selects keys that are strict children of the prefix. For example, a prefix "/foo" would select "/foo/bar" but not "/foobar" or "/foo", - * Orders: Orders are applied hierarchically. Results are sorted by the first + - Orders: Orders are applied hierarchically. Results are sorted by the first ordering, then entries equal under the first ordering are sorted with the second ordering, etc. - * Limits & Offset: Limits and offsets are applied after everything else. + - Limits & Offset: Limits and offsets are applied after everything else. */ type Query struct { Prefix string // namespaces the query to results whose keys have Prefix @@ -127,24 +127,23 @@ type Result struct { // Results is a set of Query results. This is the interface for clients. // Example: // -// qr, _ := myds.Query(q) -// for r := range qr.Next() { -// if r.Error != nil { -// // handle. -// break -// } +// qr, _ := myds.Query(q) +// for r := range qr.Next() { +// if r.Error != nil { +// // handle. +// break +// } // -// fmt.Println(r.Entry.Key, r.Entry.Value) -// } +// fmt.Println(r.Entry.Key, r.Entry.Value) +// } // // or, wait on all results at once: // -// qr, _ := myds.Query(q) -// es, _ := qr.Rest() -// for _, e := range es { -// fmt.Println(e.Key, e.Value) -// } -// +// qr, _ := myds.Query(q) +// es, _ := qr.Rest() +// for _, e := range es { +// fmt.Println(e.Key, e.Value) +// } type Results interface { Query() Query // the query these Results correspond to Next() <-chan Result // returns a channel to wait for the next result @@ -203,13 +202,12 @@ func (r *results) Query() Query { // Implementors of datastores and their clients must respect the // Process of the Request: // -// * clients must call r.Process().Close() on an early exit, so +// - clients must call r.Process().Close() on an early exit, so // implementations can reclaim resources. -// * if the Entries are read to completion (channel closed), Process +// - if the Entries are read to completion (channel closed), Process // should be closed automatically. -// * datastores must respect <-Process.Closing(), which intermediates +// - datastores must respect <-Process.Closing(), which intermediates // an early close signal from the client. -// type ResultBuilder struct { Query Query Process goprocess.Process