V 0.9.3 (#3)

.github/workflows/test.yml

@@ -22,7 +22,8 @@ jobs:
   test:
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-10.15, windows-latest] # list of os: https://github.com/actions/virtual-environments
+        os: [ubuntu-20.04, macos-10.15] # list of os: https://github.com/actions/virtual-environments
+#        os: [ubuntu-20.04, macos-10.15, windows-latest] # list of os: https://github.com/actions/virtual-environments
         go: [ '1.15', '1.13' ]
     runs-on: ${{ matrix.os }}
 

Makefile

@@ -9,11 +9,6 @@ GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)"
 deps: lintci-deps
 	go get -d ./...
 
-bin:
-	mkdir -p bin
-	GOBIN=${PWD}/bin go install ./exp/cmd/...
-	GOBIN=${PWD}/bin go install ./cmd/...
-
 all: deps check race bin
 
 test: mdbx-build
@@ -40,4 +35,4 @@ clean:
 
 mdbx-build:
 	echo "Building mdbx"
-	cd mdbx/dist/ && make mdbx && cat config.h
+	cd mdbx/dist/ && make clean && make config.h && CFLAGS_EXTRA="-Wno-deprecated-declarations" make mdbx-static.o

README.md

@@ -139,10 +139,7 @@ questions of why to use one database or the other.
 
 ##Build
 
-There is no dependency on shared libraries.  So most users can simply install
-using `go get`.
-
-`go get github.com/torquem-ch/mdbx-go/mdbx`
+There is no dependency on shared libraries. `go get github.com/torquem-ch/mdbx-go/mdbx`. But it require to call `make mdbx-build` before running any go command.
 
 On FreeBSD 10, you must explicitly set `CC` (otherwise it will fail with a
 cryptic error), for example:

mdbx/.gitignore

@@ -2,6 +2,7 @@
 *.dylib
 *.o
 config.h
+config_darwin.h
 
 mdbx_chk
 mdbx_copy

mdbx/cursor.go

@@ -88,7 +88,7 @@ func (c *Cursor) Renew(txn *Txn) error {
 func (c *Cursor) close() bool {
 	if c._c != nil {
 		if c.txn._txn == nil && !c.txn.readonly {
-			// the cursor has already been released by MDBX.
+			// the cursor has already been released by LMDB.
 		} else {
 			C.mdbx_cursor_close(c._c)
 		}

mdbx/cursor_test.go

@@ -228,9 +228,9 @@ func TestCursor_Get_KV(t *testing.T) {
 				err = txn.Put(dbi, k, v, 0)
 			}
 		}
-		put([]byte("key"), []byte("1"))
-		put([]byte("key"), []byte("2"))
-		put([]byte("key"), []byte("3"))
+		put([]byte("k1"), []byte("v1"))
+		put([]byte("k1"), []byte("v2"))
+		put([]byte("k1"), []byte("v3"))
 		return err
 	})
 	if err != nil {
@@ -244,18 +244,23 @@ func TestCursor_Get_KV(t *testing.T) {
 		}
 		defer cur.Close()
 
-		k, v, err := cur.Get([]byte("key"), []byte("0"), GetBothRange)
+		k, v, err := cur.Get([]byte("k1"), []byte("v0"), GetBothRange)
 		if err != nil {
 			return err
 		}
-		if string(k) != "key" {
-			t.Errorf("unexpected key: %q (not %q)", k, "key")
+		if string(k) != "k1" {
+			t.Errorf("unexpected key: %q (not %q)", k, "k1")
 		}
-		if string(v) != "1" {
+		if string(v) != "v1" {
 			t.Errorf("unexpected value: %q (not %q)", k, "1")
 		}
 
-		_, _, err = cur.Get([]byte("key"), []byte("1"), GetBoth)
+		_, _, err = cur.Get([]byte("k0"), []byte("v0"), GetBothRange)
+		if !IsErrno(err, NotFound) {
+			t.Errorf("unexpected error: %s", err)
+		}
+
+		_, _, err = cur.Get([]byte("k1"), []byte("v1"), GetBoth)
 		return err
 	})
 	if err != nil {
@@ -302,16 +307,22 @@ func TestDupCmpExcludeSuffix32(t *testing.T) {
 	}
 
 	err = env.Update(func(txn *Txn) (err error) {
-		put := func(k, v []byte) {
-			if err == nil {
-				err = txn.Put(dbi, k, v, 0)
-			}
+		err = txn.Put(dbi, []byte{0}, hash32Bytes, Append|AppendDup)
+		if err != nil {
+			panic(err)
+		}
+		err = txn.Put(dbi, []byte{0}, append([]byte{0}, hash32Bytes...), AppendDup)
+		if err != nil {
+			panic(err)
+		}
+		err = txn.Put(dbi, []byte{0}, append([]byte{0, 0}, hash32Bytes...), AppendDup)
+		if err != nil {
+			panic(err)
+		}
+		err = txn.Put(dbi, []byte{1}, hash32Bytes, Append|AppendDup)
+		if err != nil {
+			panic(err)
 		}
-		put([]byte{1}, hash32Bytes)
-		put([]byte{0}, append([]byte{0, 0}, hash32Bytes...))
-		put([]byte{0}, append([]byte{0}, hash32Bytes...))
-		put([]byte{0}, hash32Bytes)
-
 		return err
 	})
 	if err != nil {
@@ -325,12 +336,23 @@ func TestDupCmpExcludeSuffix32(t *testing.T) {
 		}
 
 		defer cur.Close()
+		k, v, err := cur.Get(nil, nil, Last)
+		if err != nil {
+			return err
+		}
+		if !bytes.Equal(k, []byte{1}) {
+			t.Errorf("unexpected order: %x (not %x)", k, []byte{1})
+		}
+		if !bytes.Equal(v, hash32Bytes) {
+			t.Errorf("unexpected order: %x (not %x)", v, hash32Bytes)
+		}
+
 		_, _, err = cur.Get([]byte{0}, nil, First)
 		if err != nil {
 			return err
 		}
 
-		_, v, err := cur.Get(nil, nil, FirstDup)
+		_, v, err = cur.Get(nil, nil, FirstDup)
 		if err != nil {
 			return err
 		}
@@ -353,7 +375,7 @@ func TestDupCmpExcludeSuffix32(t *testing.T) {
 			t.Errorf("unexpected order: %x (not %x)", v, append([]byte{0, 0}, hash32Bytes...))
 		}
 
-		k, v, err := cur.Get(nil, nil, Next)
+		k, v, err = cur.Get(nil, nil, Next)
 		if err != nil {
 			return err
 		}

mdbx/dist/CMakeLists.txt

@@ -1,5 +1,5 @@
 ##
-##  Copyright 2020 Leonid Yuriev <[email protected]>
+##  Copyright 2020-2021 Leonid Yuriev <[email protected]>
 ##  and other libmdbx authors: please see AUTHORS file.
 ##  All rights reserved.
 ##
@@ -340,10 +340,14 @@ if(NOT DEFINED MDBX_CXX_STANDARD)
     set(MDBX_CXX_STANDARD 98)
   endif()
 endif()
-if(NOT HAS_C11 LESS 0)
-  set(MDBX_C_STANDARD 11)
-else()
-  set(MDBX_C_STANDARD 99)
+if(NOT DEFINED MDBX_C_STANDARD)
+  # MSVC >= 19.28 (Microsoft Visual Studio 16.8) is mad!
+  #               It unable process Windows SDK headers in the C11 mode!
+  if(HAS_C11 LESS 0 OR (MSVC AND MSVC_VERSION GREATER 1927))
+    set(MDBX_C_STANDARD 99)
+  else()
+    set(MDBX_C_STANDARD 11)
+  endif()
 endif()
 
 if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows" AND EXISTS "${MDBX_SOURCE_DIR}/ntdll.def")
@@ -434,7 +438,6 @@ add_mdbx_option(MDBX_BUILD_TOOLS "Build MDBX tools (mdbx_chk/stat/dump/load/copy
 CMAKE_DEPENDENT_OPTION(MDBX_INSTALL_MANPAGES "Install man-pages for MDBX tools (mdbx_chk/stat/dump/load/copy)" ON MDBX_BUILD_TOOLS OFF)
 add_mdbx_option(MDBX_TXN_CHECKOWNER "Checking transaction matches the calling thread inside libmdbx's API" ON)
 add_mdbx_option(MDBX_ENV_CHECKPID "Paranoid checking PID inside libmdbx's API" AUTO)
-add_mdbx_option(MDBX_HUGE_TRANSACTIONS "Support for huge write-transactions" OFF)
 mark_as_advanced(MDBX_ENV_CHECKPID)
 if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
   add_mdbx_option(MDBX_DISABLE_GNU_SOURCE "Don't use GNU/Linux libc extensions" OFF)
@@ -460,6 +463,7 @@ mark_as_advanced(MDBX_LOCKING)
 add_mdbx_option(MDBX_TRUST_RTC "Does a system have battery-backed Real-Time Clock or just a fake" AUTO)
 mark_as_advanced(MDBX_TRUST_RTC)
 option(MDBX_FORCE_ASSERTIONS "Force enable assertion checking" OFF)
+option(MDBX_DISABLE_PAGECHECKS "Disable some checks to reduce an overhead and detection probability of database corruption to a values closer to the LMDB" OFF)
 
 if(NOT MDBX_AMALGAMATED_SOURCE)
   add_mdbx_option(MDBX_ALLOY_BUILD "Build MDBX library through single/alloyed object file" ON)
@@ -756,14 +760,21 @@ if(NOT CMAKE_CONFIGURATION_TYPES)
   endif()
 endif()
 
+# choice target to fetch definitions and options
+if(MDBX_BUILD_SHARED_LIBRARY)
+  set(target4fetch mdbx)
+else()
+  set(target4fetch mdbx-static)
+endif()
+
 # get definitions
-get_target_property(defs_list mdbx-static COMPILE_DEFINITIONS)
+get_target_property(defs_list ${target4fetch} COMPILE_DEFINITIONS)
 if(defs_list)
   list(APPEND MDBX_BUILD_FLAGS ${defs_list})
 endif()
 
 # get target compile options
-get_target_property(options_list mdbx-static COMPILE_OPTIONS)
+get_target_property(options_list ${target4fetch} COMPILE_OPTIONS)
 if(options_list)
   list(APPEND MDBX_BUILD_FLAGS ${options_list})
 endif()

mdbx/dist/ChangeLog.md

@@ -1,18 +1,112 @@
 ChangeLog
 ---------
 
-## v0.9.2 (in development)
+## v0.9.4 (in development) scheduled at 2021-02-23
 
 TODO:
 
- - Rework/speedup the implementation of the dirty page list (lazy compactification, lazy sorting via merge).
- - Finalize C++ API (few typos and trivia bugs are likely for now).
- - Packages for ROSA Linux, ALT Linux, Fedora/RHEL, Debian/Ubuntu.
+ - Engage new terminology (https://github.com/erthink/libmdbx/issues/137).
+ - Resolve few TODOs (https://github.com/erthink/libmdbx/issues/124, https://github.com/erthink/libmdbx/issues/127,
+   https://github.com/erthink/libmdbx/issues/115).
+ - Finalize C++ API (few typos and trivia bugs are still likely for now).
+ - Packages for [ROSA Linux](https://www.rosalinux.ru/), [ALT Linux](https://www.altlinux.org/), Fedora/RHEL, Debian/Ubuntu.
+
+Acknowledgements:
+
+ - [Alex Sharov](https://github.com/AskAlexSharov) for [mdbx-go](https://github.com/torquem-ch/mdbx-go).
+
+New features:
+
+ - Added `MDBX_DISABLE_PAGECHECKS` build option to disable some checks to reduce an overhead
+   and detection probability of database corruption to a values closer to the LMDB.
+   The `MDBX_DISABLE_PAGECHECKS=1` provides a performance boost of about 10% in CRUD scenarios,
+   and conjointly with the `MDBX_ENV_CHECKPID=0` and `MDBX_TXN_CHECKOWNER=0` options can yield
+   up to 30% more performance compared to LMDB.
+
+Fixes:
+
+ - Fixed performance regression due non-optimal C11 atomics usage (https://github.com/erthink/libmdbx/issues/160).
+
+
+## v0.9.3 at 2021-02-02
+
+Acknowledgements:
+
+ - [Mahlon E. Smith](http://www.martini.nu/) for [FreeBSD port of libmdbx](https://svnweb.freebsd.org/ports/head/databases/mdbx/).
+ - [장세연](http://www.castis.com) for bug fixing and PR.
+ - [Clément Renault](https://github.com/Kerollmops/heed) for [Heed](https://github.com/Kerollmops/heed) fully typed Rust wrapper.
+ - [Alex Sharov](https://github.com/AskAlexSharov) for bug reporting.
+ - [Noel Kuntze](https://github.com/Thermi) for bug reporting.
+
+Removed options and features:
+
+ - Drop `MDBX_HUGE_TRANSACTIONS` build-option (now no longer required).
+
+New features:
+
+ - Package for FreeBSD is available now by Mahlon E. Smith.
+ - New API functions to get/set various options (https://github.com/erthink/libmdbx/issues/128):
+    - the maximum number of named databases for the environment;
+    - the maximum number of threads/reader slots;
+    - threshold (since the last unsteady commit) to force flush the data buffers to disk;
+    - relative period (since the last unsteady commit) to force flush the data buffers to disk;
+    - limit to grow a list of reclaimed/recycled page's numbers for finding a sequence of contiguous pages for large data items;
+    - limit to grow a cache of dirty pages for reuse in the current transaction;
+    - limit of a pre-allocated memory items for dirty pages;
+    - limit of dirty pages for a write transaction;
+    - initial allocation size for dirty pages list of a write transaction;
+    - maximal part of the dirty pages may be spilled when necessary;
+    - minimal part of the dirty pages should be spilled when necessary;
+    - how much of the parent transaction dirty pages will be spilled while start each child transaction;
+ - Unlimited/Dynamic size of retired and dirty page lists (https://github.com/erthink/libmdbx/issues/123).
+ - Added `-p` option (purge subDB before loading) to `mdbx_load` tool.
+ - Reworked spilling of large transaction and committing of nested transactions:
+    - page spilling code reworked to avoid the flaws and bugs inherited from LMDB;
+    - limit for number of dirty pages now is controllable at runtime;
+    - a spilled pages, including overflow/large pages, now can be reused and refunded/compactified in nested transactions;
+    - more effective refunding/compactification especially for the loosed page cache.
+ - Added `MDBX_ENABLE_REFUND` and `MDBX_PNL_ASCENDING` internal/advanced build options.
+ - Added `mdbx_default_pagesize()` function.
+ - Better support architectures with a weak/relaxed memory consistency model (ARM, AARCH64, PPC, MIPS, RISC-V, etc) by means [C11 atomics](https://en.cppreference.com/w/c/atomic).
+ - Speed up page number lists and dirty page lists (https://github.com/erthink/libmdbx/issues/132).
+ - Added `LIBMDBX_NO_EXPORTS_LEGACY_API` build option.
+
+Fixes:
+
+ - Fixed missing cleanup (null assigned) in the C++ commit/abort (https://github.com/erthink/libmdbx/pull/143).
+ - Fixed `mdbx_realloc()` for case of nullptr and `MDBX_AVOID_CRT=ON` for Windows.
+ - Fixed the possibility to use invalid and renewed (closed & re-opened, dropped & re-created) DBI-handles (https://github.com/erthink/libmdbx/issues/146).
+ - Fixed 4-byte aligned access to 64-bit integers, including access to the `bootid` meta-page's field (https://github.com/erthink/libmdbx/issues/153).
+ - Fixed minor/potential memory leak during page flushing and unspilling.
+ - Fixed handling states of cursors's and subDBs's for nested transactions.
+ - Fixed page leak in extra rare case the list of retired pages changed during update GC on transaction commit.
+ - Fixed assertions to avoid false-positive UB detection by CLANG/LLVM (https://github.com/erthink/libmdbx/issues/153).
+ - Fixed `MDBX_TXN_FULL` and regressive `MDBX_KEYEXIST` during large transaction commit with `MDBX_LIFORECLAIM` (https://github.com/erthink/libmdbx/issues/123).
+ - Fixed auto-recovery (`weak->steady` with the same boot-id) when Database size at last weak checkpoint is large than at last steady checkpoint.
+ - Fixed operation on systems with unusual small/large page size, including PowerPC (https://github.com/erthink/libmdbx/issues/157).
+
+
+## v0.9.2 at 2020-11-27
+
+Acknowledgements:
+
+ - Jens Alfke (Mobile Architect at [Couchbase](https://www.couchbase.com/)) for [NimDBX](https://github.com/snej/nimdbx).
+ - Clément Renault (CTO at [MeiliSearch](https://www.meilisearch.com/)) for [mdbx-rs](https://github.com/Kerollmops/mdbx-rs).
+ - Alex Sharov (Go-Lang Teach Lead at [TurboGeth/Ethereum](https://ethereum.org/)) for an extreme test cases and bug reporting.
+ - George Hazan (CTO at [Miranda NG](https://www.miranda-ng.org/)) for bug reporting.
+ - [Positive Technologies](https://www.ptsecurity.com/) for funding and [The Standoff](https://standoff365.com/).
 
 Added features:
+
  - Provided package for [buildroot](https://buildroot.org/).
+ - Binding for Nim is [available](https://github.com/snej/nimdbx) now by Jens Alfke.
  - Added `mdbx_env_delete()` for deletion an environment files in a proper and multiprocess-safe way.
  - Added `mdbx_txn_commit_ex()` with collecting latency information.
+ - Fast completion pure nested transactions.
+ - Added `LIBMDBX_INLINE_API` macro and inline versions of some API functions.
+ - Added `mdbx_cursor_copy()` function.
+ - Extended tests for checking cursor tracking.
+ - Added `MDBX_SET_LOWERBOUND` operation for `mdbx_cursor_get()`.
 
 Fixes:
 
@@ -27,6 +121,20 @@ Fixes:
  - Fixed copy&paste typos.
  - Fixed minor false-positive GCC warning.
  - Added workaround for broken `DEFINE_ENUM_FLAG_OPERATORS` from Windows SDK.
+ - Fixed cursor state after multimap/dupsort repeated deletes (https://github.com/erthink/libmdbx/issues/121).
+ - Added `SIGPIPE` suppression for internal thread during `mdbx_env_copy()`.
+ - Fixed extra-rare `MDBX_KEY_EXIST` error during `mdbx_commit()` (https://github.com/erthink/libmdbx/issues/131).
+ - Fixed spilled pages checking (https://github.com/erthink/libmdbx/issues/126).
+ - Fixed `mdbx_load` for 'plain text' and without `-s name` cases (https://github.com/erthink/libmdbx/issues/136).
+ - Fixed save/restore/commit of cursors for nested transactions.
+ - Fixed cursors state in rare/special cases (move next beyond end-of-data, after deletion and so on).
+ - Added workaround for MSVC 19.28 (Visual Studio 16.8) (but may still hang during compilation).
+ - Fixed paranoidal Clang C++ UB for bitwise operations with flags defined by enums.
+ - Fixed large pages checking (for compatibility and to avoid false-positive errors from `mdbx_chk`).
+ - Added workaround for Wine (https://github.com/miranda-ng/miranda-ng/issues/1209).
+ - Fixed `ERROR_NOT_SUPPORTED` while opening DB by UNC pathnames (https://github.com/miranda-ng/miranda-ng/issues/2627).
+ - Added handling `EXCEPTION_POSSIBLE_DEADLOCK` condition for Windows.
+
 
 ## v0.9.1 2020-09-30
 

mdbx/dist/GNUmakefile

@@ -72,7 +72,7 @@ strip: all
 	strip libmdbx.$(SO_SUFFIX) $(TOOLS)
 
 clean:
-	rm -rf $(TOOLS) mdbx_test @* *.[ao] *.[ls]o *~ tmp.db/* \
+	rm -rf $(TOOLS) mdbx_test @* *.[ao] *.[ls]o *.$(SO_SUFFIX) *.dSYM *~ tmp.db/* \
 		*.gcov *.log *.err src/*.o test/*.o mdbx_example dist \
 		config.h src/config.h src/version.c *.tar*