mdbx: v13 (#139)

.github/workflows/test.yml

@@ -12,9 +12,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.20'
+          go-version: '1.21'
 
-      - run: GODEBUG=cgocheck=2 make race
+      - run: make race
 
   win:
     strategy:
@@ -33,14 +33,13 @@ jobs:
 
       - name: Install dependencies
         run: |
-          choco upgrade mingw -y --no-progress --version 13.2.0
-          choco install cmake -y --no-progress --version 3.27.8
+          choco upgrade mingw -y --no-progress --version 13.2.0 --allow-downgrade
+          choco install cmake -y --no-progress --version 3.31.2 --allow-downgrade
 
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.20'
+          go-version: '1.21'
 
-      - run: go env
-      - run: go test -p 1 ./mdbx
-      - run: go test -p 1 ./exp/mdbxpool
+      - run: go test ./mdbx
+      - run: go test ./exp/mdbxpool

Makefile

@@ -4,7 +4,7 @@
 MASTER_COMMIT=`git rev-parse --short origin/master`
 
 deps: lintci-deps
-	go get -d ./...
+	go get ./...
 
 all: deps
 
@@ -20,7 +20,7 @@ lint:
 
 lintci-deps:
 	rm -f ./build/bin/golangci-lint
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./build/bin v1.59.1
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./build/bin v1.62.2
 
 clean:
 	cd mdbxdist && make clean
@@ -29,5 +29,5 @@ tools: clean
 	cd mdbxdist && MDBX_BUILD_TIMESTAMP=unknown CFLAGS="${CFLAGS} -Wno-unknown-warning-option -Wno-enum-int-mismatch -Wno-strict-prototypes -Wno-unused-but-set-variable" make tools
 
 cp:
-	cp mdbxdist/mdbx.h mdbx/
-	cp mdbxdist/mdbx.c mdbx/
+	#cd ../libmdbx && make dist
+	cp -R ./../libmdbx/dist/* ./mdbxdist/

README.md

@@ -147,6 +147,12 @@ On FreeBSD 10, you must explicitly set `CC` (otherwise it will fail with a crypt
 
 In `libmdbx` repo: `make dist && cp -R ./dist/* ./../mdbx-go/mdbxdist/`. Then in mdbx-go repo: `make cp`
 
+On mac: 
+```
+brew install --default-names gnu-sed
+PATH="/usr/local/opt/gnu-sed/libexec/gnubin:$PATH" make cp
+```
+
 ## Build binaries
 
 In mdbx-go repo: `MDBX_BUILD_TIMESTAMP=unknown make tools`

go.mod

@@ -2,4 +2,4 @@ module github.com/erigontech/mdbx-go
 
 go 1.20
 
-require github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d
+require github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e

go.sum

@@ -1,2 +1,2 @@
-github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d h1:Azx2B59D4+zpVVtuYb8Oe3uOLi/ift4xfwKdhBX0Cy0=
-github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
+github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e h1:8AnObPi8WmIgjwcidUxaREhXMSpyUJeeSrIkZTXdabw=
+github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=

mdbx/Notes

@@ -0,0 +1,3 @@
+This package contains only mdbx.h and mdbx.c - to minimize go build time/size.
+But full version of libmdbx (produced by it's `make dist` command) is in `./../mdbxdist/`. License is also there.
+

mdbx/cursor.go

@@ -36,6 +36,8 @@ const (
 	Set          = C.MDBX_SET            // The specified key.
 	SetKey       = C.MDBX_SET_KEY        // Get key and data at the specified key.
 	SetRange     = C.MDBX_SET_RANGE      // The first key no less than the specified key.
+
+	LesserThan = C.MDBX_TO_KEY_LESSER_THAN
 )
 
 // The MDB_MULTIPLE and MDB_RESERVE flags are special and do not fit the

mdbx/env.go

@@ -32,16 +32,17 @@ const (
 	EnvDefaults = C.MDBX_ENV_DEFAULTS
 	LifoReclaim = C.MDBX_LIFORECLAIM
 	//FixedMap    = C.MDBX_FIXEDMAP   // Danger zone. Map memory at a fixed address.
-	NoSubdir      = C.MDBX_NOSUBDIR // Argument to Open is a file, not a directory.
-	Accede        = C.MDBX_ACCEDE
-	Coalesce      = C.MDBX_COALESCE
-	Readonly      = C.MDBX_RDONLY     // Used in several functions to denote an object as readonly.
-	WriteMap      = C.MDBX_WRITEMAP   // Use a writable memory map.
-	NoMetaSync    = C.MDBX_NOMETASYNC // Don't fsync metapage after commit.
-	UtterlyNoSync = C.MDBX_UTTERLY_NOSYNC
-	SafeNoSync    = C.MDBX_SAFE_NOSYNC
-	Durable       = C.MDBX_SYNC_DURABLE
-	NoTLS         = C.MDBX_NOTLS // Danger zone. When unset reader locktable slots are tied to their thread.
+	NoSubdir        = C.MDBX_NOSUBDIR // Argument to Open is a file, not a directory.
+	Accede          = C.MDBX_ACCEDE
+	Coalesce        = C.MDBX_COALESCE
+	Readonly        = C.MDBX_RDONLY     // Used in several functions to denote an object as readonly.
+	WriteMap        = C.MDBX_WRITEMAP   // Use a writable memory map.
+	NoMetaSync      = C.MDBX_NOMETASYNC // Don't fsync metapage after commit.
+	UtterlyNoSync   = C.MDBX_UTTERLY_NOSYNC
+	SafeNoSync      = C.MDBX_SAFE_NOSYNC
+	Durable         = C.MDBX_SYNC_DURABLE
+	NoTLS           = C.MDBX_NOTLS           // Danger zone. When unset reader locktable slots are tied to their thread.
+	NoStickyThreads = C.MDBX_NOSTICKYTHREADS // Danger zone. Like MDBX_NOTLS. But also allow move RwTx between threads. Still require to call Begin/Rollback in stame thread.
 	//NoLock      = C.MDBX_NOLOCK     // Danger zone. MDBX does not use any locks.
 	NoReadahead = C.MDBX_NORDAHEAD // Disable readahead. Requires OS support.
 	NoMemInit   = C.MDBX_NOMEMINIT // Disable MDBX memory initialization.
@@ -106,12 +107,18 @@ const (
 	OptSpillMinDenominator          = C.MDBX_opt_spill_min_denominator
 	OptSpillParent4ChildDenominator = C.MDBX_opt_spill_parent4child_denominator
 	OptMergeThreshold16dot16Percent = C.MDBX_opt_merge_threshold_16dot16_percent
+	OptPreferWafInsteadofBalance    = C.MDBX_opt_prefer_waf_insteadof_balance
 )
 
 var (
 	LoggerDoNotChange = C.MDBX_LOGGER_DONTCHANGE
 )
 
+// Label - will be added to error messages. For better understanding - which DB has problem.
+type Label string
+
+const Default Label = "default"
+
 // DBI is a handle for a database in an Env.
 //
 // See MDBX_dbi
@@ -122,7 +129,8 @@ type DBI C.MDBX_dbi
 //
 // See MDBX_env.
 type Env struct {
-	_env *C.MDBX_env
+	_env  *C.MDBX_env
+	label Label
 
 	// closeLock is used to allow the Txn finalizer to check if the Env has
 	// been closed, so that it may know if it must abort.
@@ -132,8 +140,8 @@ type Env struct {
 // NewEnv allocates and initializes a new Env.
 //
 // See mdbx_env_create.
-func NewEnv() (*Env, error) {
-	env := new(Env)
+func NewEnv(label Label) (*Env, error) {
+	env := &Env{label: label}
 	ret := C.mdbx_env_create(&env._env)
 	if ret != success {
 		return nil, operrno("mdbx_env_create", ret)
@@ -151,6 +159,7 @@ func (env *Env) Open(path string, flags uint, mode os.FileMode) error {
 	ret := C.mdbx_env_open(env._env, cpath, C.MDBX_env_flags_t(NoTLS|flags), C.mdbx_mode_t(mode))
 	return operrno("mdbx_env_open", ret)
 }
+func (env *Env) Label() Label { return env.label }
 
 var errNotOpen = errors.New("enivornment is not open")
 
@@ -365,7 +374,24 @@ func (env *Env) Info(txn *Txn) (*EnvInfo, error) {
 	if ret != success {
 		return nil, operrno("mdbx_env_info", ret)
 	}
-	info := EnvInfo{
+	return castEnvInfo(_info), nil
+}
+
+func PreOpenSnapInfo(path string) (*EnvInfo, error) {
+	cpath := C.CString(path)
+	defer C.free(unsafe.Pointer(cpath))
+
+	var _info C.MDBX_envinfo
+	var bytes C.size_t = C.size_t(unsafe.Sizeof(_info))
+	ret := C.mdbx_preopen_snapinfo(cpath, &_info, bytes)
+	if ret != success {
+		return nil, operrno("mdbx_preopen_snapinfo", ret)
+	}
+	return castEnvInfo(_info), nil
+}
+
+func castEnvInfo(_info C.MDBX_envinfo) *EnvInfo {
+	return &EnvInfo{
 		MapSize: int64(_info.mi_mapsize),
 		Geo: EnvInfoGeo{
 			Lower:   uint64(_info.mi_geo.lower),
@@ -402,7 +428,6 @@ func (env *Env) Info(txn *Txn) (*EnvInfo, error) {
 		SinceReaderCheck:  toDuration(_info.mi_since_reader_check_seconds16dot16),
 		Flags:             uint(_info.mi_mode),
 	}
-	return &info, nil
 }
 
 // Sync flushes buffers to disk.  If force is true a synchronous flush occurs

mdbx/env_test.go

@@ -45,7 +45,7 @@ import (
 //}
 
 func TestEnv_Open_notExist(t *testing.T) {
-	env, err := NewEnv()
+	env, err := NewEnv(Default)
 	if err != nil {
 		t.Fatalf("create: %s", err)
 	}
@@ -59,7 +59,23 @@ func TestEnv_Open_notExist(t *testing.T) {
 }
 
 func TestEnv_Open(t *testing.T) {
-	env, err1 := NewEnv()
+	env, err1 := NewEnv(Default)
+	if err1 != nil {
+		t.Error(err1)
+		return
+	}
+	defer env.Close()
+
+	// open an environment at a temporary path.
+	path := t.TempDir()
+	err := env.Open(path, 0, 0664)
+	if err != nil {
+		t.Errorf("open: %s", err)
+	}
+}
+
+func TestEnv_PreOpen(t *testing.T) {
+	env, err1 := NewEnv(Default)
 	if err1 != nil {
 		t.Error(err1)
 		return
@@ -72,6 +88,15 @@ func TestEnv_Open(t *testing.T) {
 	if err != nil {
 		t.Errorf("open: %s", err)
 	}
+	env.Close()
+
+	_info, err := PreOpenSnapInfo(path)
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("%#v\n", _info)
+
 }
 
 /*
@@ -154,7 +179,7 @@ func TestEnv_Flags(t *testing.T) {
 func TestEnv_SetMaxReader(t *testing.T) {
 	dir := t.TempDir()
 
-	env, err := NewEnv()
+	env, err := NewEnv(Default)
 	if err != nil {
 		t.Error(err)
 	}
@@ -193,7 +218,7 @@ func TestEnv_SetMaxReader(t *testing.T) {
 }
 
 func TestEnv_SetDebug(t *testing.T) {
-	env, err := NewEnv()
+	env, err := NewEnv(Default)
 	if err != nil {
 		t.Error(err)
 	}
@@ -493,7 +518,7 @@ func setup(t testing.TB) (*Env, string) {
 }
 
 func setupFlags(t testing.TB, flags uint) (env *Env, path string) {
-	env, err := NewEnv()
+	env, err := NewEnv(Default)
 	if err != nil {
 		t.Fatalf("env: %s", err)
 	}

mdbx/error.go

@@ -87,10 +87,10 @@ var CorruptErrorMessage = CorruptErrorHardwareRecommendations + " " + CorruptErr
 
 func (e Errno) Error() string {
 	if e == Corrupted {
-		return "MDBX_FATAL: " + CorruptErrorMessage
+		return fmt.Sprintf("MDBX_FATAL(%d): ", int(e)) + CorruptErrorMessage
 	}
 	if e == Panic {
-		return "MDBX_PANIC: " + CorruptErrorMessage
+		return fmt.Sprintf("MDBX_PANIC(%d): ", int(e)) + CorruptErrorMessage
 	}
 	return C.GoString(C.mdbx_strerror(C.int(e)))
 }