Merge pull request #431 from oxen-io/dev

Merge dev to stable for 2.1.1 release

.drone.jsonnet

@@ -1,7 +1,9 @@
 
 local default_deps_base='libsystemd-dev libboost-program-options-dev libboost-system-dev libboost-test-dev ' +
     'libsqlite3-dev libsodium-dev libssl-dev pkg-config';
-local default_deps='g++ ' + default_deps_base; // g++ sometimes needs replacement
+local default_deps_nocxx='libsodium-dev ' + default_deps_base; // libsodium-dev needs to be >= 1.0.18
+local default_deps='g++ ' + default_deps_nocxx; // g++ sometimes needs replacement
+local docker_base = 'registry.oxen.rocks/lokinet-ci-';
 
 local submodules_commands = ['git fetch --tags', 'git submodule update --init --recursive --depth=1'];
 local submodules = {
@@ -12,15 +14,15 @@ local submodules = {
 
 local apt_get_quiet = 'apt-get -o=Dpkg::Use-Pty=0 -q';
 
-
 // Regular build on a debian-like system:
 local debian_pipeline(name, image,
         arch='amd64',
         deps=default_deps,
         build_type='Release',
         lto=false,
         build_tests=true,
-        run_tests=false, # Runs full test suite
+        run_tests=true, # Runs full test suite
+        test_oxen_storage=true, # Makes sure oxen-storage --version runs
         cmake_extra='',
         extra_cmds=[],
         extra_steps=[],
@@ -53,6 +55,7 @@ local debian_pipeline(name, image,
                     cmake_extra,
                 'ninja -j' + jobs + ' -v',
             ] +
+            (if test_oxen_storage then ['./httpserver/oxen-storage --version'] else []) +
             (if run_tests then ['./unit_test/Test'] else []) +
             extra_cmds,
         }
@@ -64,7 +67,8 @@ local mac_builder(name,
         build_type='Release',
         lto=false,
         build_tests=true,
-        run_tests=false,
+        run_tests=true,
+        test_oxen_storage=true, # Makes sure oxen-storage --version runs
         cmake_extra='',
         extra_cmds=[],
         extra_steps=[],
@@ -91,6 +95,7 @@ local mac_builder(name,
                     cmake_extra,
                 'ninja -j' + jobs + ' -v'
             ] +
+            (if test_oxen_storage then ['./httpserver/oxen-storage --version'] else []) +
             (if run_tests then ['./unit_test/Test'] else []) +
             extra_cmds,
         }
@@ -109,25 +114,27 @@ local static_build_deps='autoconf automake make file libtool pkg-config patch op
 
 [
     // Various debian builds
-    debian_pipeline("Debian (w/ tests) (amd64)", "debian:sid", lto=true, run_tests=true),
+    debian_pipeline("Debian (amd64)", docker_base+"debian-sid", lto=true),
     debian_pipeline("Debian Debug (amd64)", "debian:sid", build_type='Debug'),
-    debian_pipeline("Debian clang-11 (amd64)", "debian:sid", deps='clang-11 '+default_deps_base,
+    debian_pipeline("Debian clang-11 (amd64)", docker_base+"debian-sid", deps='clang-11 '+default_deps_base,
                     cmake_extra='-DCMAKE_C_COMPILER=clang-11 -DCMAKE_CXX_COMPILER=clang++-11 ', lto=true),
-    debian_pipeline("Debian buster (i386)", "i386/debian:buster"),
-    debian_pipeline("Ubuntu focal (amd64)", "ubuntu:focal"),
+    debian_pipeline("Debian buster (i386)", "i386/debian:buster", deps=default_deps_base+' g++ make file',
+                    cmake_extra='-DDOWNLOAD_SODIUM=ON'),
+    debian_pipeline("Ubuntu focal (amd64)", docker_base+"ubuntu-focal"),
 
     // ARM builds (ARM64 and armhf)
-    debian_pipeline("Debian (ARM64)", "debian:sid", arch="arm64", build_tests=false),
-    debian_pipeline("Debian buster (armhf)", "arm32v7/debian:buster", arch="arm64", build_tests=false),
+    debian_pipeline("Debian (ARM64)", "debian:sid", arch="arm64"),
+    debian_pipeline("Debian buster (armhf)", "arm32v7/debian:buster", arch="arm64",
+                    cmake_extra='-DDOWNLOAD_SODIUM=ON', deps=default_deps_base+' g++ make file'),
 
     // Static build (on bionic) which gets uploaded to oxen.rocks:
-    debian_pipeline("Static (bionic amd64)", "ubuntu:bionic", deps='g++-8 '+static_build_deps,
+    debian_pipeline("Static (bionic amd64)", docker_base+"ubuntu-bionic", deps='g++-8 '+static_build_deps,
                     cmake_extra='-DBUILD_STATIC_DEPS=ON -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8',
-                    build_tests=false, lto=true, extra_cmds=static_check_and_upload),
+                    lto=true, extra_cmds=static_check_and_upload),
 
     // Macos builds:
     mac_builder('macOS (Static)', cmake_extra='-DBUILD_STATIC_DEPS=ON',
-                build_tests=false, lto=true, extra_cmds=static_check_and_upload),
-    mac_builder('macOS (Release)', run_tests=true),
-    mac_builder('macOS (Debug)', build_type='Debug', cmake_extra='-DBUILD_DEBUG_UTILS=ON'),
+                lto=true, extra_cmds=static_check_and_upload),
+    mac_builder('macOS (Release)'),
+    mac_builder('macOS (Debug)', build_type='Debug'),
 ]

CMakeLists.txt

@@ -13,7 +13,7 @@ endif()
 cmake_minimum_required(VERSION 3.10)
 
 project(storage_server
-    VERSION 2.1.0
+    VERSION 2.1.1
     LANGUAGES CXX C)
 
 option(INTEGRATION_TEST "build for integration test" OFF)
@@ -59,16 +59,32 @@ if(BUILD_STATIC_DEPS)
   include(StaticBuild)
 else()
   find_package(PkgConfig REQUIRED)
-  pkg_check_modules(SODIUM REQUIRED IMPORTED_TARGET libsodium>=1.0.17)
+  find_package(Boost REQUIRED system program_options)
+  find_package(OpenSSL REQUIRED)
+endif()
+
+if(NOT TARGET sodium)
+  # Allow -D DOWNLOAD_SODIUM=FORCE to download without even checking for a local libsodium
+  option(DOWNLOAD_SODIUM "Allow libsodium to be downloaded and built locally if not found on the system" OFF)
+  if(NOT DOWNLOAD_SODIUM STREQUAL "FORCE" AND NOT BUILD_STATIC_DEPS)
+    find_package(PkgConfig REQUIRED)
+    pkg_check_modules(SODIUM libsodium>=1.0.18 IMPORTED_TARGET)
+  endif()
+
   add_library(sodium INTERFACE)
-  target_link_libraries(sodium INTERFACE PkgConfig::SODIUM)
+  if(SODIUM_FOUND AND NOT DOWNLOAD_SODIUM STREQUAL "FORCE" AND NOT BUILD_STATIC_DEPS)
+    target_link_libraries(sodium INTERFACE PkgConfig::SODIUM)
+  else()
+    if(NOT DOWNLOAD_SODIUM AND NOT BUILD_STATIC_DEPS)
+      message(FATAL_ERROR "Could not find libsodium >= 1.0.18; either install it on your system or use -DDOWNLOAD_SODIUM=ON to download and build an internal copy")
+    endif()
+    message(STATUS "Sodium >= 1.0.18 not found, but DOWNLOAD_SODIUM specified, so downloading it")
+    include(DownloadLibSodium)
+    target_link_libraries(sodium INTERFACE sodium_vendor)
+  endif()
 
   # Need this target export so that loki-mq properly picks up sodium
   export(TARGETS sodium NAMESPACE sodium:: FILE sodium-exports.cmake)
-
-  find_package(Boost REQUIRED system program_options)
-
-  find_package(OpenSSL REQUIRED)
 endif()
 
 include(cmake/check_for_std_filesystem.cmake)
@@ -81,8 +97,25 @@ add_subdirectory(utils)
 add_subdirectory(crypto)
 add_subdirectory(storage)
 add_subdirectory(httpserver)
-set(BUILD_SHARED_LIBS OFF CACHE BOOL "disable shared libraries") # Tells loki-mq to do a static build
-add_subdirectory(vendors/loki-mq)
+
+
+
+option(FORCE_OXENMQ_SUBMODULE "force using oxenmq submodule" OFF)
+if(NOT FORCE_OXENMQ_SUBMODULE)
+  find_package(PkgConfig REQUIRED)
+  pkg_check_modules(OXENMQ liboxenmq>=1.2.5)
+endif()
+if(OXENMQ_FOUND)
+  add_library(oxenmq INTERFACE)
+  link_dep_libs(oxenmq INTERFACE "${OXENMQ_LIBRARY_DIRS}" ${OXENMQ_LIBRARIES})
+  target_include_directories(oxenmq INTERFACE ${OXENMQ_INCLUDE_DIRS})
+  add_library(oxenmq::oxenmq ALIAS oxenmq)
+  message(STATUS "Found system liboxenmq ${OXENMQ_VERSION}")
+else()
+  message(STATUS "using oxenmq submodule")
+  set(BUILD_SHARED_LIBS OFF CACHE BOOL "disable shared libraries") # Tells oxen-mq to do a static build
+  add_subdirectory(vendors/loki-mq)
+endif()
 
 if (BUILD_TESTS)
     add_subdirectory(unit_test)

cmake/DownloadLibSodium.cmake

@@ -0,0 +1,51 @@
+
+set(LIBSODIUM_PREFIX ${CMAKE_BINARY_DIR}/libsodium)
+set(LIBSODIUM_URL https://github.com/jedisct1/libsodium/releases/download/1.0.18-RELEASE/libsodium-1.0.18.tar.gz https://download.libsodium.org/libsodium/releases/libsodium-1.0.18.tar.gz)
+set(LIBSODIUM_HASH SHA512=17e8638e46d8f6f7d024fe5559eccf2b8baf23e143fadd472a7d29d228b186d86686a5e6920385fe2020729119a5f12f989c3a782afbd05a8db4819bb18666ef)
+
+if(SODIUM_TARBALL_URL)
+    # make a build time override of the tarball url so we can fetch it if the original link goes away
+    set(LIBSODIUM_URL ${SODIUM_TARBALL_URL})
+endif()
+
+
+file(MAKE_DIRECTORY ${LIBSODIUM_PREFIX}/include)
+
+include(ExternalProject)
+include(ProcessorCount)
+ProcessorCount(PROCESSOR_COUNT)
+if(PROCESSOR_COUNT EQUAL 0)
+    set(PROCESSOR_COUNT 1)
+endif()
+
+set(sodium_cc ${CMAKE_C_COMPILER})
+if(CCACHE_PROGRAM)
+  set(sodium_cc "${CCACHE_PROGRAM} ${sodium_cc}")
+endif()
+set(SODIUM_CONFIGURE ./configure --prefix=${LIBSODIUM_PREFIX} --enable-static --disable-shared --with-pic --quiet CC=${sodium_cc})
+if (CMAKE_C_COMPILER_ARG1)
+  set(SODIUM_CONFIGURE ${SODIUM_CONFIGURE} CPPFLAGS=${CMAKE_C_COMPILER_ARG1})
+endif()
+
+if (CROSS_TARGET)
+    set(SODIUM_CONFIGURE ${SODIUM_CONFIGURE} --target=${CROSS_TARGET} --host=${CROSS_TARGET})
+endif()
+
+
+ExternalProject_Add(libsodium_external
+    BUILD_IN_SOURCE ON
+    PREFIX ${LIBSODIUM_PREFIX}
+    URL ${LIBSODIUM_URL}
+    URL_HASH ${LIBSODIUM_HASH}
+    CONFIGURE_COMMAND ${SODIUM_CONFIGURE}
+    BUILD_COMMAND make -j${PROCESSOR_COUNT}
+    INSTALL_COMMAND ${MAKE}
+    BUILD_BYPRODUCTS ${LIBSODIUM_PREFIX}/lib/libsodium.a ${LIBSODIUM_PREFIX}/include
+    )
+
+add_library(sodium_vendor STATIC IMPORTED GLOBAL)
+add_dependencies(sodium_vendor libsodium_external)
+set_target_properties(sodium_vendor PROPERTIES
+    IMPORTED_LOCATION ${LIBSODIUM_PREFIX}/lib/libsodium.a
+    INTERFACE_INCLUDE_DIRECTORIES ${LIBSODIUM_PREFIX}/include
+    )

cmake/StaticBuild.cmake

@@ -229,7 +229,7 @@ set(boost_libs program_options system)
 set(boost_with_libs_extra)
 if(BUILD_TESTS)
     list(APPEND boost_libs unit_test_framework)
-    set(boost_with_libs_extra "${boost_with_libs_extra} --with-unit_test_framework")
+    list(APPEND boost_with_libs_extra --with-test)
 endif()
 string(REPLACE ";" "," boost_with_libraries "${boost_libs}")
 set(boost_static_libraries)

cmake/archive.cmake

@@ -6,7 +6,7 @@ find_package(Git)
 set(git_tag "-unknown")
 if(GIT_FOUND)
   execute_process(COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD RESULT_VARIABLE ret OUTPUT_VARIABLE curr_commit OUTPUT_STRIP_TRAILING_WHITESPACE)
-  execute_process(COMMAND "${GIT_EXECUTABLE}" rev-parse master RESULT_VARIABLE ret2 OUTPUT_VARIABLE stable_commit OUTPUT_STRIP_TRAILING_WHITESPACE)
+  execute_process(COMMAND "${GIT_EXECUTABLE}" rev-parse stable RESULT_VARIABLE ret2 OUTPUT_VARIABLE stable_commit OUTPUT_STRIP_TRAILING_WHITESPACE)
   if(NOT ret AND curr_commit STREQUAL "${stable_commit}")
     # Get the tag description; for a tagged release this will be just the tag (v1.2.3); for
     # something following a tag this will be something like "v1.2.3-2-abcdef" for something 2

crypto/include/oxend_key.h

@@ -3,6 +3,7 @@
 #include <array>
 #include <cstddef>
 #include <functional>
+#include <optional>
 #include <string>
 #include <string_view>
 #include <utility>