Merge pull request #5815 from knst/bp-multiprocess-p1

backport: bitcoin#18677, bitcoin#19685, bitcoin#20046, bitcoin#20549 (multiprocess related)

.gitignore

@@ -9,6 +9,8 @@ reset-files.bash
 src/dash
 src/dashd
 src/dash-cli
+src/dash-gui
+src/dash-node
 src/dash-tx
 src/dash-wallet
 src/test/fuzz/*

.gitlab-ci.yml

@@ -204,6 +204,17 @@ x86_64-pc-linux-gnu-nowallet:
     HOST: x86_64-pc-linux-gnu
     DEP_OPTS: "NO_WALLET=1"
 
+## TODO: # Skip on non-depends, non-multiprocess PRs
+## if: type != pull_request OR commit_message =~ /depends:|multiprocess:/
+#x86_64-pc-linux-gnu-multiprocess:
+#  extends:
+#    - .build-depends-template
+#    - .skip-in-fast-mode-template
+#  variables:
+#    HOST: x86_64-pc-linux-gnu
+#    DEP_OPTS: "MULTIPROCESS=1"
+#
+
 x86_64-apple-darwin:
   extends:
     - .build-depends-template
@@ -299,6 +310,15 @@ linux64_nowallet-build:
   variables:
     BUILD_TARGET: linux64_nowallet
 
+#linux64_multiprocess-build:
+#  extends:
+#    - .build-template
+#    - .skip-in-fast-mode-template
+#  needs:
+#    - x86_64-pc-linux-gnu-multiprocess
+#  variables:
+#    BUILD_TARGET: linux64_multiprocess
+
 #linux64_valgrind-build:
 #  extends:
 #    - .build-template
@@ -362,6 +382,17 @@ linux64_ubsan-test:
   variables:
     BUILD_TARGET: linux64_ubsan
 
+# TODO: enable multiprocess back in CI once it has any value
+# or in case if any new backports to test
+#linux64_multiprocess-test:
+#  extends:
+#    - .test-template
+#    - .skip-in-fast-mode-template
+#  needs:
+#    - linux64_multiprocess-build
+#  variables:
+#    BUILD_TARGET: linux64_multiprocess
+
 #linux64_valgrind-test:
 #  extends:
 #    - .test-template

.travis.yml

@@ -250,6 +250,11 @@ after_success:
       env: >-
         FILE_ENV="./ci/test/00_setup_env_native_fuzz_with_valgrind.sh"
 
+    - stage: test
+      name: 'x86_64 Linux [GOAL: install]  [focal]  [multiprocess]'
+      env: >-
+        FILE_ENV="./ci/test/00_setup_env_native_multiprocess.sh"
+
     - stage: test
       name: 'x86_64 Linux  [GOAL: install]  [focal]  [no wallet]'
       env: >-

Makefile.am

@@ -27,6 +27,8 @@ BITCOIN_QT_BIN=$(top_builddir)/src/qt/$(BITCOIN_GUI_NAME)$(EXEEXT)
 BITCOIN_CLI_BIN=$(top_builddir)/src/$(BITCOIN_CLI_NAME)$(EXEEXT)
 BITCOIN_TX_BIN=$(top_builddir)/src/$(BITCOIN_TX_NAME)$(EXEEXT)
 BITCOIN_WALLET_BIN=$(top_builddir)/src/$(BITCOIN_WALLET_TOOL_NAME)$(EXEEXT)
+BITCOIN_NODE_BIN=$(top_builddir)/src/$(BITCOIN_MP_NODE_NAME)$(EXEEXT)
+BITCOIN_GUI_BIN=$(top_builddir)/src/$(BITCOIN_MP_GUI_NAME)$(EXEEXT)
 BITCOIN_WIN_INSTALLER=$(PACKAGE)-$(PACKAGE_VERSION)-win64-setup$(EXEEXT)
 
 empty :=
@@ -155,6 +157,12 @@ $(BITCOIN_TX_BIN): FORCE
 $(BITCOIN_WALLET_BIN): FORCE
 	$(MAKE) -C src $(@F)
 
+$(BITCOIN_NODE_BIN): FORCE
+	$(MAKE) -C src $(@F)
+
+$(BITCOIN_GUI_BIN): FORCE
+	$(MAKE) -C src $(@F)
+
 if USE_LCOV
 LCOV_FILTER_PATTERN = \
 	-p "/usr/local/" \

ci/dash/test_integrationtests.sh

@@ -42,7 +42,7 @@ echo "Using socketevents mode: $SOCKETEVENTS"
 EXTRA_ARGS="--dashd-arg=-socketevents=$SOCKETEVENTS"
 
 set +e
-LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ./test/functional/test_runner.py --ci --attempts=3 --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --failfast --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS $EXTRA_ARGS
+LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} ./test/functional/test_runner.py --ci --attempts=3 --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --failfast --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS $EXTRA_ARGS
 RESULT=$?
 set -e
 

ci/test/00_setup_env.sh

@@ -39,6 +39,7 @@ export USE_BUSY_BOX=${USE_BUSY_BOX:-false}
 export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true}
 export RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:-true}
 export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false}
+export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-}
 export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false}
 export RUN_SYMBOL_TESTS=${RUN_SYMBOL_TESTS:-true}
 export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed}

ci/test/00_setup_env_native_multiprocess.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=ci_native_multiprocess
+export PACKAGES="cmake python3"
+export DEP_OPTS="MULTIPROCESS=1"
+export GOAL="install"
+export BITCOIN_CONFIG=""
+export TEST_RUNNER_ENV="BITCOIND=dash-node"

configure.ac

@@ -24,6 +24,9 @@ BITCOIN_GUI_NAME=dash-qt
 BITCOIN_CLI_NAME=dash-cli
 BITCOIN_TX_NAME=dash-tx
 BITCOIN_WALLET_TOOL_NAME=dash-wallet
+dnl Multi Process
+BITCOIN_MP_NODE_NAME=dash-node
+BITCOIN_MP_GUI_NAME=dash-gui
 
 dnl Unless the user specified ARFLAGS, force it to be cr
 dnl This is also the default as-of libtool 2.4.7
@@ -256,6 +259,24 @@ AC_ARG_ENABLE([zmq],
   [use_zmq=$enableval],
   [use_zmq=yes])
 
+AC_ARG_WITH([libmultiprocess],
+  [AS_HELP_STRING([--with-libmultiprocess=yes|no|auto],
+  [Build with libmultiprocess library. (default: auto, i.e. detect with pkg-config)])],
+  [with_libmultiprocess=$withval],
+  [with_libmultiprocess=auto])
+
+AC_ARG_WITH([mpgen],
+  [AS_HELP_STRING([--with-mpgen=yes|no|auto|PREFIX],
+  [Build with libmultiprocess codegen tool. Useful to specify different libmultiprocess host system library and build system codegen tool prefixes when cross-compiling (default is host system libmultiprocess prefix)])],
+  [with_mpgen=$withval],
+  [with_mpgen=auto])
+
+AC_ARG_ENABLE([multiprocess],
+  [AS_HELP_STRING([--enable-multiprocess],
+  [build multiprocess dash-node, dash-wallet, and dash-gui executables in addition to monolithic dashd and dash-qt executables. Requires libmultiprocess library. Experimental (default is no)])],
+  [enable_multiprocess=$enableval],
+  [enable_multiprocess=no])
+
 AC_ARG_ENABLE(man,
     [AS_HELP_STRING([--disable-man],
                     [do not install man pages (default is to install)])],,
@@ -1530,6 +1551,48 @@ dnl check if libgmp is present
 AC_CHECK_HEADER([gmp.h],, AC_MSG_ERROR(libgmp headers missing))
 AC_CHECK_LIB([gmp],  [__gmpz_init],GMP_LIBS=-lgmp, AC_MSG_ERROR(libgmp missing))
 
+dnl libmultiprocess library check
+
+libmultiprocess_found=no
+if test "x$with_libmultiprocess" = xyes || test "x$with_libmultiprocess" = xauto; then
+  m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [
+     libmultiprocess_found=yes;
+     libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`;
+  ], [true])])
+elif test "x$with_libmultiprocess" != xno; then
+  AC_MSG_ERROR([--with-libmultiprocess=$with_libmultiprocess value is not yes, auto, or no])
+fi
+AC_SUBST(LIBMULTIPROCESS_CFLAGS)
+AC_SUBST(LIBMULTIPROCESS_LIBS)
+
+dnl Enable multiprocess check
+
+if test "x$enable_multiprocess" = xyes; then
+  if test "x$libmultiprocess_found" != xyes; then
+    AC_MSG_ERROR([--enable-multiprocess=yes option specified but libmultiprocess library was not found. May need to install libmultiprocess library, or specify install path with PKG_CONFIG_PATH environment variable. Running 'pkg-config --debug libmultiprocess' may be helpful for debugging.])
+  fi
+  build_multiprocess=yes
+elif test "x$enable_multiprocess" = xauto; then
+  build_multiprocess=$libmultiprocess_found
+else
+  build_multiprocess=no
+fi
+
+AM_CONDITIONAL([BUILD_MULTIPROCESS],[test "x$build_multiprocess" = xyes])
+AM_CONDITIONAL([BUILD_BITCOIN_NODE], [test "x$build_multiprocess" = xyes])
+AM_CONDITIONAL([BUILD_BITCOIN_GUI], [test "x$build_multiprocess" = xyes])
+
+dnl codegen tools check
+
+if test x$build_multiprocess != xno; then
+  if test "x$with_mpgen" = xyes || test "x$with_mpgen" = xauto; then
+    MPGEN_PREFIX="$libmultiprocess_prefix"
+  elif test "x$with_mpgen" != xno; then
+    MPGEN_PREFIX="$with_mpgen";
+  fi
+  AC_SUBST(MPGEN_PREFIX)
+fi
+
 AC_MSG_CHECKING([whether to build dashd])
 AM_CONDITIONAL([BUILD_BITCOIND], [test x$build_bitcoind = xyes])
 AC_MSG_RESULT($build_bitcoind)
@@ -1760,6 +1823,8 @@ AC_SUBST(BITCOIN_GUI_NAME)
 AC_SUBST(BITCOIN_CLI_NAME)
 AC_SUBST(BITCOIN_TX_NAME)
 AC_SUBST(BITCOIN_WALLET_TOOL_NAME)
+AC_SUBST(BITCOIN_MP_NODE_NAME)
+AC_SUBST(BITCOIN_MP_GUI_NAME)
 
 AC_SUBST(RELDFLAGS)
 AC_SUBST(DEBUG_CPPFLAGS)
@@ -1853,6 +1918,7 @@ esac
 
 echo
 echo "Options used to compile and link:"
+echo "  multiprocess  = $build_multiprocess"
 echo "  with wallet         = $enable_wallet"
 echo "  with gui / qt       = $bitcoin_enable_qt"
 if test "x$enable_wallet" != "xno"; then

depends/Makefile

@@ -40,6 +40,7 @@ NO_WALLET ?=
 NO_ZMQ ?=
 NO_UPNP ?=
 NO_NATPMP ?=
+MULTIPROCESS ?=
 FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources
 
 BUILD = $(shell ./config.guess)
@@ -152,6 +153,7 @@ upnp_packages_$(NO_UPNP) = $(upnp_packages)
 natpmp_packages_$(NO_NATPMP) = $(natpmp_packages)
 
 zmq_packages_$(NO_ZMQ) = $(zmq_packages)
+multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages)
 
 packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_)
 native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages)
@@ -160,6 +162,11 @@ ifneq ($(zmq_packages_),)
 packages += $(zmq_packages)
 endif
 
+ifeq ($(multiprocess_packages_),)
+packages += $(multiprocess_packages)
+native_packages += $(multiprocess_native_packages)
+endif
+
 all_packages = $(packages) $(native_packages)
 
 meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk
@@ -227,6 +234,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
             -e 's|@no_sqlite@|$(NO_SQLITE)|' \
             -e 's|@no_upnp@|$(NO_UPNP)|' \
             -e 's|@no_natpmp@|$(NO_NATPMP)|' \
+            -e 's|@multiprocess@|$(MULTIPROCESS)|' \
             -e 's|@debug@|$(DEBUG)|' \
             $< > $@
 	touch $@

depends/README.md

@@ -107,6 +107,7 @@ The following can be set when running make: `make FOO=bar`
 - `NO_SQLITE`: Don't download/build/cache SQLite
 - `NO_UPNP`: Don't download/build/cache packages needed for enabling UPnP
 - `NO_NATPMP`: Don't download/build/cache packages needed for enabling NAT-PMP
+- `MULTIPROCESS`: build libmultiprocess (experimental, requires cmake)
 - `DEBUG`: Disable some optimizations and enable more runtime checking
 - `HOST_ID_SALT`: Optional salt to use when generating host package ids
 - `BUILD_ID_SALT`: Optional salt to use when generating build package ids

depends/config.site.in

@@ -26,6 +26,9 @@ fi
 if test -z "$with_qt_bindir" && test -z "@no_qt@"; then
   with_qt_bindir="${depends_prefix}/native/bin"
 fi
+if test -z "$with_mpgen" && test -n "@multiprocess@"; then
+  with_mpgen="${depends_prefix}/native"
+fi
 
 if test -z "$with_qrencode" && test -n "@no_qr@"; then
   with_qrencode=no
@@ -43,6 +46,10 @@ if test -z "$with_sqlite" && test -n "@no_sqlite@"; then
   with_sqlite=no
 fi
 
+if test -z "$enable_multiprocess" && test -n "@multiprocess@"; then
+  enable_multiprocess=yes
+fi
+
 if test -z "$with_miniupnpc" && test -n "@no_upnp@"; then
   with_miniupnpc=no
 fi

depends/funcs.mk

@@ -138,11 +138,11 @@ $(1)_config_env+=$($(1)_config_env_$(host_arch)_$(host_os)) $($(1)_config_env_$(
 
 $(1)_config_env+=PKG_CONFIG_LIBDIR=$($($(1)_type)_prefix)/lib/pkgconfig
 $(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig
+$(1)_config_env+=CMAKE_MODULE_PATH=$($($(1)_type)_prefix)/lib/cmake
 $(1)_config_env+=PATH=$(build_prefix)/bin:$(PATH)
 $(1)_build_env+=PATH=$(build_prefix)/bin:$(PATH)
 $(1)_stage_env+=PATH=$(build_prefix)/bin:$(PATH)
 $(1)_autoconf=./configure --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)"
-
 ifneq ($($(1)_nm),)
 $(1)_autoconf += NM="$$($(1)_nm)"
 endif
@@ -164,6 +164,22 @@ endif
 ifneq ($($(1)_ldflags),)
 $(1)_autoconf += LDFLAGS="$$($(1)_ldflags)"
 endif
+
+$(1)_cmake=env CC="$$($(1)_cc)" \
+               CFLAGS="$$($(1)_cppflags) $$($(1)_cflags)" \
+               CXX="$$($(1)_cxx)" \
+               CXXFLAGS="$$($(1)_cppflags) $$($(1)_cxxflags)" \
+               LDFLAGS="$$($(1)_ldflags)" \
+             cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)"
+ifeq ($($(1)_type),build)
+$(1)_cmake += -DCMAKE_INSTALL_RPATH:PATH="$$($($(1)_type)_prefix)/lib"
+else
+ifneq ($(host),$(build))
+$(1)_cmake += -DCMAKE_SYSTEM_NAME=$($(host_os)_cmake_system)
+$(1)_cmake += -DCMAKE_C_COMPILER_TARGET=$(host)
+$(1)_cmake += -DCMAKE_CXX_COMPILER_TARGET=$(host)
+endif
+endif
 endef
 
 define int_add_cmds

depends/hosts/android.mk

@@ -9,3 +9,4 @@ android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang++
 android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang
 android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ranlib
 endif
+android_cmake_system=Android

depends/hosts/darwin.mk

@@ -120,3 +120,5 @@ darwin_release_CXXFLAGS=$(darwin_release_CFLAGS)
 
 darwin_debug_CFLAGS=-O1
 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS)
+
+darwin_cmake_system=Darwin

depends/hosts/linux.mk

@@ -29,3 +29,4 @@ i686_linux_CXX=$(default_host_CXX) -m32
 x86_64_linux_CC=$(default_host_CC) -m64
 x86_64_linux_CXX=$(default_host_CXX) -m64
 endif
+linux_cmake_system=Linux

depends/hosts/mingw32.mk

@@ -8,3 +8,5 @@ mingw32_debug_CFLAGS=-O1
 mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS)
 
 mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC
+
+mingw_cmake_system=Windows

depends/packages/capnp.mk

@@ -0,0 +1,18 @@
+package=capnp
+$(package)_version=$(native_$(package)_version)
+$(package)_download_path=$(native_$(package)_download_path)
+$(package)_file_name=$(native_$(package)_file_name)
+$(package)_sha256_hash=$(native_$(package)_sha256_hash)
+$(package)_dependencies=native_$(package)
+
+define $(package)_config_cmds
+  $($(package)_autoconf) --with-external-capnp
+endef
+
+define $(package)_build_cmds
+  $(MAKE)
+endef
+
+define $(package)_stage_cmds
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install
+endef

depends/packages/libmultiprocess.mk

@@ -0,0 +1,18 @@
+package=libmultiprocess
+$(package)_version=$(native_$(package)_version)
+$(package)_download_path=$(native_$(package)_download_path)
+$(package)_file_name=$(native_$(package)_file_name)
+$(package)_sha256_hash=$(native_$(package)_sha256_hash)
+$(package)_dependencies=native_$(package) boost capnp
+
+define $(package)_config_cmds
+  $($(package)_cmake)
+endef
+
+define $(package)_build_cmds
+  $(MAKE)
+endef
+
+define $(package)_stage_cmds
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install
+endef