upgrade pebble (#139)

* upgrade pebble

* update minor differences from latest pebble

* update make bench

* ensure we are closing iterators in our tests

.github/workflows/go.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v3
       with:
-        go-version: 1.21.1
+        go-version: 1.23.1
 
     - name: Build Example
       run: go build -v ./_examples/simple/...

Makefile

@@ -39,7 +39,10 @@ test-all: test-clean
 test-with-reset: db-reset test-all
 
 test-clean:
-	GOGC=off go clean -testcache
+	GOGC=off go clean -testcache && rm -rf test_db tmp_db
+
+bench:
+	@cd _benchmarks && go test -timeout=25m -bench=.
 
 todo:
 	@git grep TODO -- './*' ':!./vendor/' ':!./Makefile' || :

backup_test.go

@@ -97,6 +97,9 @@ func TestBond_BackupRestore(t *testing.T) {
 	// make sure both db has same keys and values.
 	itr := db.Iter(&IterOptions{})
 	itr2 := db2.Iter(&IterOptions{})
+	defer itr.Close()
+	defer itr2.Close()
+
 	for _, _ = itr.First(), itr2.First(); itr.Valid(); _, _ = itr.Next(), itr2.Next() {
 		require.Equal(t, itr.Key(), itr2.Key())
 		require.Equal(t, itr.Value(), itr2.Value())

bond.go

@@ -389,6 +389,7 @@ func (db *_db) Dump(_ context.Context, path string, tables []TableID, withIndex
 	defer snapshot.Close()
 
 	grp := new(errgroup.Group)
+
 	// write all the table data to the sst file.
 	for _, tableID := range tables {
 		tablePath := filepath.Join(path, fmt.Sprintf("table_%d", tableID))
@@ -413,6 +414,7 @@ func (db *_db) Dump(_ context.Context, path string, tables []TableID, withIndex
 		if !withIndex {
 			continue
 		}
+
 		// write all the index data to sst file.
 		indexes := db.getIndexIDS(tableID)
 		for _, index := range indexes {
@@ -438,7 +440,7 @@ func (db *_db) Dump(_ context.Context, path string, tables []TableID, withIndex
 	return grp.Wait()
 }
 
-func (db *_db) Restore(_ context.Context, path string, tables []TableID, withIndex bool) error {
+func (db *_db) Restore(ctx context.Context, path string, tables []TableID, withIndex bool) error {
 	buf, err := os.ReadFile(filepath.Join(path, "VERSION"))
 	if err != nil {
 		return err
@@ -495,7 +497,7 @@ func (db *_db) Restore(_ context.Context, path string, tables []TableID, withInd
 	if err != nil {
 		return err
 	}
-	return db.pebble.Ingest(ssts)
+	return db.pebble.Ingest(ctx, ssts)
 }
 
 func (db *_db) getIndexIDS(tableID TableID) []IndexID {
@@ -518,15 +520,17 @@ func (db *_db) getIndexIDS(tableID TableID) []IndexID {
 		}
 		prefix[1] = indexID + 1
 	}
+	itr.Close()
 	return indexIDS
 }
 
 // write all the key/value of iterator to the SST file.
 func iteratorToSST(itr Iterator, path string) error {
 	defer itr.Close()
+
 	// sst reader
 	currentFileID := 1
-	file, err := vfs.Default.Create(filepath.Join(path, fmt.Sprintf("%d.sst", currentFileID)))
+	file, err := vfs.Default.Create(filepath.Join(path, fmt.Sprintf("%d.sst", currentFileID)), vfs.WriteCategoryUnspecified)
 	if err != nil {
 		return err
 	}
@@ -538,16 +542,17 @@ func iteratorToSST(itr Iterator, path string) error {
 
 	for itr.First(); itr.Valid(); itr.Next() {
 		if err := writer.Set(itr.Key(), itr.Value()); err != nil {
+			writer.Close()
 			return err
 		}
 
 		// Replace the old writer with new writer after the old writer reaches it's capacity.
-		if writer.EstimatedSize() > exportFileSize {
+		if writer.Raw().EstimatedSize() > exportFileSize {
 			if err := writer.Close(); err != nil {
 				return err
 			}
 			currentFileID++
-			file, err = vfs.Default.Create(filepath.Join(path, fmt.Sprintf("%d.sst", currentFileID)))
+			file, err = vfs.Default.Create(filepath.Join(path, fmt.Sprintf("%d.sst", currentFileID)), vfs.WriteCategoryUnspecified)
 			if err != nil {
 				return err
 			}

bond_test.go

@@ -22,7 +22,10 @@ func setupDB(name string, serializer ...Serializer[any]) DB {
 		options.Serializer = serializer[0]
 	}
 
-	db, _ := Open(name, options)
+	db, err := Open(name, options)
+	if err != nil {
+		panic(err)
+	}
 	return db
 }
 
@@ -31,8 +34,17 @@ func tearDownDatabase(db DB) {
 }
 
 func tearDownDB(name string, db DB) {
-	_ = db.Close()
-	_ = os.RemoveAll(name)
+	// NOTE: we intentionally panic on Close here, as it will catch any
+	// leaking iterators which would otherwise be difficult to debug,
+	// and are relevant to ensure we close.
+	err := db.Close()
+	if err != nil {
+		panic(err)
+	}
+	err = os.RemoveAll(name)
+	if err != nil {
+		panic(err)
+	}
 }
 
 func TestBond_Open(t *testing.T) {

go.mod

@@ -1,58 +1,63 @@
 module github.com/go-bond/bond
 
-go 1.21.1
+go 1.22.0
+
+toolchain go1.23.1
 
 require (
 	github.com/bits-and-blooms/bloom/v3 v3.7.0
-	github.com/cockroachdb/pebble v0.0.0-20240321023028-8a097e8ae86b
+	github.com/cockroachdb/pebble v0.0.0-20240925204210-c88c74713c2d
 	github.com/dustin/go-humanize v1.0.1
 	github.com/fatih/structs v1.1.0
-	github.com/fxamacker/cbor/v2 v2.6.0
-	github.com/go-resty/resty/v2 v2.12.0
+	github.com/fxamacker/cbor/v2 v2.7.0
+	github.com/go-resty/resty/v2 v2.15.2
 	github.com/google/uuid v1.6.0
-	github.com/klauspost/compress v1.17.7
+	github.com/klauspost/compress v1.17.10
 	github.com/lithammer/go-jump-consistent-hash v1.0.2
 	github.com/stretchr/testify v1.9.0
-	github.com/tinylib/msgp v1.1.9
-	github.com/urfave/cli/v2 v2.27.1
+	github.com/tinylib/msgp v1.2.1
+	github.com/urfave/cli/v2 v2.27.4
 	github.com/vmihailenco/msgpack/v5 v5.4.1
-	golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81
-	golang.org/x/sync v0.6.0
+	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0
+	golang.org/x/sync v0.8.0
 )
 
 require (
-	github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect
+	github.com/DataDog/zstd v1.5.6 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bits-and-blooms/bitset v1.13.0 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/cockroachdb/errors v1.11.1 // indirect
+	github.com/bits-and-blooms/bitset v1.14.3 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cockroachdb/crlib v0.0.0-20240904210011-3474b320e3bf // indirect
+	github.com/cockroachdb/errors v1.11.3 // indirect
+	github.com/cockroachdb/fifo v0.0.0-20240816210425-c5d0cb0b6fc0 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/redact v1.1.5 // indirect
-	github.com/cockroachdb/swiss v0.0.0-20240303172742-c161743eb608 // indirect
+	github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 // indirect
 	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/getsentry/sentry-go v0.27.0 // indirect
+	github.com/getsentry/sentry-go v0.29.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
+	github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/philhofer/fwd v1.1.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.19.0 // indirect
-	github.com/prometheus/client_model v0.6.0 // indirect
-	github.com/prometheus/common v0.51.0 // indirect
-	github.com/prometheus/procfs v0.13.0 // indirect
-	github.com/rogpeppe/go-internal v1.12.0 // indirect
+	github.com/prometheus/client_golang v1.20.4 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.59.1 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/rogpeppe/go-internal v1.13.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
-	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
-	golang.org/x/net v0.22.0 // indirect
-	golang.org/x/sys v0.18.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	golang.org/x/net v0.29.0 // indirect
+	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/text v0.18.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )