diff --git a/.gitignore b/.gitignore index ae9fcf8..051ba44 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ # Ignore build files build .nih_c +.vs diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29..0000000 diff --git a/.travis.yml b/.travis.yml index d8fba53..16de619 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,88 +1,421 @@ -sudo: false language: cpp -dist: xenial +dist: bionic + +services: + - docker + +stages: + - windep-ssl + - windep-grpc + - windep-libarchive + - windep-boost + - build + # - winbuild1 + # - winbuild2 + # - winbuild3 + # - winbuild4 env: global: - # Maintenance note: to move to a new version - # of boost, update both BOOST_ROOT and BOOST_URL. - # Note that for simplicity, BOOST_ROOT's final - # namepart must match the folder name internal - # to boost's .tar.gz. - - LCOV_ROOT=$HOME/lcov - - LCOV_EXCLUDE_FILES="*/src/test/*" - - BOOST_ROOT=$HOME/boost_1_67_0 - - BOOST_URL='http://sourceforge.net/projects/boost/files/boost/1.67.0/boost_1_67_0.tar.gz' + - DOCKER_IMAGE="rippleci/rippled-ci-builder:2020-01-08" + - CMAKE_EXTRA_ARGS="-Dwerr=ON -Dwextra=ON" + - NINJA_BUILD=true + # change this if we get more VM capacity + - MAX_TIME_MIN=80 + - CACHE_DIR=${TRAVIS_HOME}/_cache + - NIH_CACHE_ROOT=${CACHE_DIR}/nih_c + - PARALLEL_TESTS=true + # this is NOT used by linux container based builds (which already have boost installed) + - BOOST_URL='https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.bz2' + - BOOST_URL2='https://downloads.sourceforge.net/project/boost/boost/1.70.0/boost_1_70_0.tar.bz2?r=&ts=1594393912&use_mirror=newcontinuum' + # Travis downloader doesn't seem to have updated certs. Using this option + # introduces obvious security risks, but they're Travis's risks. + # Note that this option is only used if the "normal" build fails. + - BOOST_WGET_OPTIONS='--no-check-certificate' + - VCPKG_DIR=${CACHE_DIR}/vcpkg + - USE_CCACHE=true + - CCACHE_BASEDIR=${TRAVIS_HOME}" + - CCACHE_NOHASHDIR=true + - CCACHE_DIR=${CACHE_DIR}/ccache - APP=validator-keys - APP_ARGS=--unittest - - VERBOSE_BUILD=true - - GCC_VER=7 - -addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-xenial-7 - packages: - - gcc-7 - - g++-7 - - gcc-8 - - g++-8 - - python-software-properties - - protobuf-compiler - - libprotobuf-dev - - libssl-dev - - libstdc++6 - - binutils-gold - - gdb - - texinfo - - cmake - - lcov - - llvm-7 - - clang-7 matrix: - include: + fast_finish: true + allow_failures: + # TODO these need more investigation + # + - name: asan, clang-8 + # there are a number of UBs caught currently that need triage + - name: ubsan, clang-8 + # The Windows build will fail if any of the dependencies fail, but + # allow the rest of the builds to continue. + - stage: windep-ssl + - stage: windep-grpc + - stage: windep-libarchive + - stage: windep-boost + # I give up + - name: windows, vc2019 - - compiler: gcc - env: - - BUILD_TYPE=Debug - - CMAKE_EXTRA_ARGS=-Dcoverage=ON + exclude: + - stage: windep-grpc + - stage: windep-libarchive - - compiler: clang + include: + # debug builds + - &linux + stage: build + if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ + compiler: gcc-8 + name: gcc-8, debug env: - - BUILD_TYPE=Debug - - - compiler: gcc + - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8" + - BUILD_TYPE=Debug + script: + - sudo chmod -R a+rw ${CACHE_DIR} + - ccache -s + - travis_wait ${MAX_TIME_MIN} ci/ubuntu/build-in-docker.sh + - ccache -s + - <<: *linux + compiler: clang-8 + name: clang-8, debug env: - - BUILD_TYPE=Release - - - compiler: clang + - MATRIX_EVAL="CC=clang-8 && CXX=clang++-8" + - BUILD_TYPE=Debug + # coverage builds + - <<: *linux + if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_cov/ + compiler: gcc-8 + name: coverage, gcc-8 env: - - BUILD_TYPE=Release - - - compiler: gcc + - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8" + - BUILD_TYPE=Debug + - CMAKE_ADD="-Dcoverage=ON" + - TARGET=coverage_report + - SKIP_TESTS=true + - <<: *linux + if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_cov/ + compiler: clang-8 + name: coverage, clang-8 env: - - BUILD_TYPE=Debug - - CMAKE_EXTRA_ARGS=-Drippled_tag=master - - - compiler: gcc + - MATRIX_EVAL="CC=clang-8 && CXX=clang++-8" + - BUILD_TYPE=Debug + - CMAKE_ADD="-Dcoverage=ON" + - TARGET=coverage_report + - SKIP_TESTS=true + # nounity + - <<: *linux + if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_nounity/ + compiler: gcc-8 + name: non-unity, gcc-8 + env: + - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8" + - BUILD_TYPE=Debug + - CMAKE_ADD="-Dunity=OFF" + - <<: *linux + if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_nounity/ + compiler: clang-8 + name: non-unity, clang-8 + env: + - MATRIX_EVAL="CC=clang-8 && CXX=clang++-8" + - BUILD_TYPE=Debug + - CMAKE_ADD="-Dunity=OFF" + # release builds + - <<: *linux + if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_release/ + compiler: gcc-8 + name: gcc-8, release + env: + - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8" + - BUILD_TYPE=Release + - CMAKE_ADD="-Dassert=ON -Dunity=OFF" + - <<: *linux + if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_release/ + compiler: clang-8 + name: clang-8, release + env: + - MATRIX_EVAL="CC=clang-8 && CXX=clang++-8" + - BUILD_TYPE=Release + - CMAKE_ADD="-Dassert=ON" + # asan + - <<: *linux + if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_san/ + compiler: clang-8 + name: asan, clang-8 env: - - BUILD_TYPE=Debug - - CMAKE_EXTRA_ARGS=-Drippled_tag=develop + - MATRIX_EVAL="CC=clang-8 && CXX=clang++-8" + - BUILD_TYPE=Release + - CMAKE_ADD="-Dsan=address" + - ASAN_OPTIONS="print_stats=true:atexit=true" + #- LSAN_OPTIONS="verbosity=1:log_threads=1" + - PARALLEL_TESTS=false + # ubsan + - <<: *linux + if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_san/ + compiler: clang-8 + name: ubsan, clang-8 + env: + - MATRIX_EVAL="CC=clang-8 && CXX=clang++-8" + - BUILD_TYPE=Release + - CMAKE_ADD="-Dsan=undefined" + # once we can run clean under ubsan, add halt_on_error=1 to options below + - UBSAN_OPTIONS="print_stacktrace=1:report_error_type=1" + - PARALLEL_TESTS=false + # tsan + # current tsan failure *might* be related to: + # https://github.com/google/sanitizers/issues/1104 + # but we can't get it to run, so leave it disabled for now + # - <<: *linux + # if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_linux/ OR commit_message =~ /travis_run_san/ + # compiler: clang-8 + # name: tsan, clang-8 + # env: + # - MATRIX_EVAL="CC=clang-8 && CXX=clang++-8" + # - BUILD_TYPE=Release + # - CMAKE_ADD="-Dsan=thread" + # - TSAN_OPTIONS="history_size=3 external_symbolizer_path=/usr/bin/llvm-symbolizer verbosity=1" + # - PARALLEL_TESTS=false + # dynamic lib builds + - <<: *linux + compiler: gcc-8 + name: non-static, gcc-8 + env: + - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8" + - BUILD_TYPE=Debug + - CMAKE_ADD="-Dstatic=OFF" + - <<: *linux + compiler: gcc-8 + name: non-static + BUILD_SHARED_LIBS, gcc-8 + env: + - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8" + - BUILD_TYPE=Debug + - CMAKE_ADD="-Dstatic=OFF -DBUILD_SHARED_LIBS=ON" + # makefile + - <<: *linux + compiler: gcc-8 + name: makefile generator, gcc-8 + env: + - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8" + - BUILD_TYPE=Debug + - NINJA_BUILD=false + # misc alternative compilers + - <<: *linux + compiler: gcc-7 + name: gcc-7 + env: + - MATRIX_EVAL="CC=gcc-7 && CXX=g++-7" + - BUILD_TYPE=Debug + - <<: *linux + compiler: gcc-9 + name: gcc-9 + env: + - MATRIX_EVAL="CC=gcc-9 && CXX=g++-9" + - BUILD_TYPE=Debug + - <<: *linux + compiler: clang-7 + name: clang-7 + env: + - MATRIX_EVAL="CC=clang-7 && CXX=clang++-7" + - BUILD_TYPE=Debug + - <<: *linux + compiler: clang-9 + name: clang-9 + env: + - MATRIX_EVAL="CC=clang-9 && CXX=clang++-9" + - BUILD_TYPE=Debug + # verify build with min version of cmake + - <<: *linux + compiler: gcc-8 + name: min cmake version + env: + - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8" + - BUILD_TYPE=Debug + - CMAKE_EXE=/opt/local/cmake-3.11/bin/cmake + - SKIP_TESTS=true + # macos + - &macos + if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_mac/ + stage: build + os: osx + osx_image: xcode10.3 + name: xcode10, debug + env: + # put NIH in non-cache location since it seems to + # cause failures when homebrew updates + - NIH_CACHE_ROOT=${TRAVIS_BUILD_DIR}/nih_c + - BLD_CONFIG=Debug + - TEST_EXTRA_ARGS="" + - BOOST_ROOT=${CACHE_DIR}/boost_1_70_0 + - >- + CMAKE_ADD=" + -DBOOST_ROOT=${BOOST_ROOT}/_INSTALLED_ + -DBoost_ARCHITECTURE=-x64 + -DBoost_NO_SYSTEM_PATHS=ON + -DCMAKE_VERBOSE_MAKEFILE=ON" + addons: + homebrew: + packages: + - protobuf + - grpc + - pkg-config + - bash + - ninja + - cmake + - wget + - zstd + - libarchive + - openssl@1.1 + update: true + install: + - export OPENSSL_ROOT=$(brew --prefix openssl@1.1) + - travis_wait ${MAX_TIME_MIN} ci/shared/install_boost.sh + - brew uninstall --ignore-dependencies boost + script: + - mkdir -p build.macos && cd build.macos + - cmake -G Ninja ${CMAKE_EXTRA_ARGS} -DCMAKE_BUILD_TYPE=${BLD_CONFIG} .. + || cat $(pwd)/CMakeFiles/CMakeOutput.log $(pwd)/CMakeFiles/CMakeError.log + - travis_wait ${MAX_TIME_MIN} cmake --build . --parallel --verbose + - ./validator-keys --unittest ${TEST_EXTRA_ARGS} + - <<: *macos + name: xcode10, release + before_script: + - export BLD_CONFIG=Release + - export CMAKE_EXTRA_ARGS="${CMAKE_EXTRA_ARGS} -Dassert=ON" + - <<: *macos + osx_image: xcode11.2 + name: xcode11, debug + # windows + - &windows + if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_win/ + os: windows + env: + # put NIH in a non-cached location until + # we come up with a way to stabilize that + # cache on windows (minimize incremental changes) + - CACHE_NAME=win_01 + - NIH_CACHE_ROOT=${TRAVIS_BUILD_DIR}/nih_c + - VCPKG_DEFAULT_TRIPLET="x64-windows-static" + - MATRIX_EVAL="CC=cl.exe && CXX=cl.exe" + - BOOST_ROOT=${CACHE_DIR}/boost_1_70 + - >- + CMAKE_ADD=" + -DCMAKE_PREFIX_PATH=${BOOST_ROOT}/_INSTALLED_ + -DBOOST_ROOT=${BOOST_ROOT}/_INSTALLED_ + -DBoost_ROOT=${BOOST_ROOT}/_INSTALLED_ + -DBoost_DIR=${BOOST_ROOT}/_INSTALLED_/lib/cmake/Boost-1.70.0 + -DBoost_COMPILER=vc141 + -DCMAKE_VERBOSE_MAKEFILE=ON + -DCMAKE_TOOLCHAIN_FILE=${VCPKG_DIR}/scripts/buildsystems/vcpkg.cmake + -DVCPKG_TARGET_TRIPLET=x64-windows-static" + stage: windep-ssl + name: prereq-ssl + install: + - choco upgrade cmake.install + - choco install ninja visualstudio2017-workload-vctools -y + script: + - df -h + - travis_wait ${MAX_TIME_MIN} ci/windows/install-vcpkg.sh openssl + - <<: *windows + stage: windep-grpc + name: prereq-grpc + script: + - travis_wait ${MAX_TIME_MIN} ci/windows/install-vcpkg.sh grpc + - <<: *windows + stage: windep-libarchive + name: prereq-libarchive + script: + - travis_wait ${MAX_TIME_MIN} ci/windows/install-vcpkg.sh libarchive[lz4] + # TBD consider rocksdb via vcpkg if/when we can build with the + # vcpkg version + # - travis_wait ${MAX_TIME_MIN} ci/windows/install-vcpkg.sh rocksdb[snappy,lz4,zlib] + - <<: *windows + stage: windep-boost + name: prereq-boost + install: + - choco upgrade cmake.install + - choco install ninja visualstudio2017-workload-vctools -y + - choco install visualstudio2019buildtools visualstudio2019community visualstudio2019-workload-vctools -y + script: + - export BOOST_TOOLSET=msvc-14.1 + - travis_wait ${MAX_TIME_MIN} ci/shared/install_boost.sh + - &windows-bld + <<: *windows + # stage: winbuild1 + stage: build + name: windows, debug + before_script: + - export BLD_CONFIG=Debug + script: + - df -h + - . ./ci/windows/setup-msvc.sh + - mkdir -p build.ms && cd build.ms + # - find ${BOOST_ROOT} -type d + - ls -l ${BOOST_ROOT}/_INSTALLED_/lib/cmake/ + - cmake -G Ninja ${CMAKE_EXTRA_ARGS} -DCMAKE_BUILD_TYPE=${BLD_CONFIG} .. + || cat $(pwd)/CMakeFiles/CMakeOutput.log $(pwd)/CMakeFiles/CMakeError.log + - travis_wait ${MAX_TIME_MIN} cmake --build . --parallel --verbose + - pwd + - ls -l + # override num procs to force single unit test job + - export NUM_PROCESSORS=1 + - travis_wait ${MAX_TIME_MIN} ./validator-keys.exe --unittest + - <<: *windows-bld + # stage: winbuild2 + name: windows, release + before_script: + - export BLD_CONFIG=Release + - <<: *windows-bld + # stage: winbuild3 + name: windows, visual studio, debug + script: + - mkdir -p build.ms && cd build.ms + - cmake -G "Visual Studio 15 2017 Win64" ${CMAKE_EXTRA_ARGS} .. + - export DESTDIR=${PWD}/_installed_ + - travis_wait ${MAX_TIME_MIN} cmake --build . --parallel --verbose --config ${BLD_CONFIG} --target validator-keys + # override num procs to force single unit test job + - export NUM_PROCESSORS=1 + - >- + travis_wait ${MAX_TIME_MIN} "./Debug/validator-keys.exe" --unittest + - <<: *windows-bld + # stage: winbuild4 + name: windows, vc2019 + install: + - choco upgrade cmake.install + - choco install ninja -y + - choco install visualstudio2019buildtools visualstudio2019community visualstudio2019-workload-vctools -y + before_script: + - export BLD_CONFIG=Release + # we want to use the boost build from cache, which was built using the + # vs2017 compiler so we need to specify the Boost_COMPILER. BUT, we + # can't use the cmake config files generated by boost b/c they are + # broken for Boost_COMPILER override, so we need to specify both + # Boost_NO_BOOST_CMAKE and a slightly different Boost_COMPILER string + # to make the legacy find module work for us. If the cmake configs are + # fixed in the future, it should be possible to remove these + # workarounds. + - export CMAKE_EXTRA_ARGS="${CMAKE_EXTRA_ARGS} -DBoost_NO_BOOST_CMAKE=ON -DBoost_COMPILER=-vc141" + - echo ${CMAKE_EXTRA_ARGS} + +before_cache: + - if [ $(uname) = "Linux" ] ; then SUDO="sudo"; else SUDO=""; fi + - cd ${TRAVIS_HOME} + - if [ -f cache_ignore.tar ] ; then $SUDO tar xvf cache_ignore.tar; fi + - cd ${TRAVIS_BUILD_DIR} cache: + timeout: 900 directories: - - $BOOST_ROOT + - $CACHE_DIR before_install: - - ( ci/install-dependencies.sh ) - - export PATH=/usr/local/bin:$PATH - -script: - - ci/build-and-test.sh + # NUM_PROCESSORS was set to 1 due to problems in parallel launch of unit tests on Mac platform + - if [ "$(uname)" = "Darwin" ] ; then export NUM_PROCESSORS=1; else export NUM_PROCESSORS=$(nproc); fi + - echo "NUM PROC is ${NUM_PROCESSORS}" + - if [ "$(uname)" = "Linux" ] ; then docker pull ${DOCKER_IMAGE}; fi + - if [ "${MATRIX_EVAL}" != "" ] ; then eval "${MATRIX_EVAL}"; fi + - if [ "${CMAKE_ADD}" != "" ] ; then export CMAKE_EXTRA_ARGS="${CMAKE_EXTRA_ARGS} ${CMAKE_ADD}"; fi + - ci/ubuntu/travis-cache-start.sh notifications: - email: - false + email: false diff --git a/Builds/CMake/KeysCov.cmake b/Builds/CMake/KeysCov.cmake new file mode 100644 index 0000000..09f0b8e --- /dev/null +++ b/Builds/CMake/KeysCov.cmake @@ -0,0 +1,104 @@ +#[===================================================================[ + coverage report target + + Copied from rippled https://github.com/ripple/rippled/blob/develop/Builds/CMake/RippledCov.cmake +#]===================================================================] + +if (coverage) + if (is_clang) + if (APPLE) + execute_process (COMMAND xcrun -f llvm-profdata + OUTPUT_VARIABLE LLVM_PROFDATA + OUTPUT_STRIP_TRAILING_WHITESPACE) + else () + find_program (LLVM_PROFDATA llvm-profdata) + endif () + if (NOT LLVM_PROFDATA) + message (WARNING "unable to find llvm-profdata - skipping coverage_report target") + endif () + + if (APPLE) + execute_process (COMMAND xcrun -f llvm-cov + OUTPUT_VARIABLE LLVM_COV + OUTPUT_STRIP_TRAILING_WHITESPACE) + else () + find_program (LLVM_COV llvm-cov) + endif () + if (NOT LLVM_COV) + message (WARNING "unable to find llvm-cov - skipping coverage_report target") + endif () + + set (extract_pattern "") + if (coverage_core_only) + set (extract_pattern "${CMAKE_CURRENT_SOURCE_DIR}/src/") + endif () + + if (LLVM_COV AND LLVM_PROFDATA) + add_custom_target (coverage_report + USES_TERMINAL + COMMAND ${CMAKE_COMMAND} -E echo "Generating coverage - results will be in ${CMAKE_BINARY_DIR}/coverage/index.html." + COMMAND ${CMAKE_COMMAND} -E echo "Running validator-keys tests." + COMMAND validator-keys --unittest$<$:=${coverage_test}> + COMMAND ${LLVM_PROFDATA} + merge -sparse default.profraw -o rip.profdata + COMMAND ${CMAKE_COMMAND} -E echo "Summary of coverage:" + COMMAND ${LLVM_COV} + report -instr-profile=rip.profdata + $ ${extract_pattern} + # generate html report + COMMAND ${LLVM_COV} + show -format=html -output-dir=${CMAKE_BINARY_DIR}/coverage + -instr-profile=rip.profdata + $ ${extract_pattern} + BYPRODUCTS coverage/index.html) + endif () + elseif (is_gcc) + find_program (LCOV lcov) + if (NOT LCOV) + message (WARNING "unable to find lcov - skipping coverage_report target") + endif () + + find_program (GENHTML genhtml) + if (NOT GENHTML) + message (WARNING "unable to find genhtml - skipping coverage_report target") + endif () + + set (extract_pattern "*") + if (coverage_core_only) + set (extract_pattern "*/src/*") + endif () + + if (LCOV AND GENHTML) + add_custom_target (coverage_report + USES_TERMINAL + COMMAND ${CMAKE_COMMAND} -E echo "Generating coverage- results will be in ${CMAKE_BINARY_DIR}/coverage/index.html." + # create baseline info file + COMMAND ${LCOV} + --no-external -d "${CMAKE_CURRENT_SOURCE_DIR}" -c -d . -i -o baseline.info + | grep -v "ignoring data for external file" + # run tests + COMMAND ${CMAKE_COMMAND} -E echo "Running validator-keys tests for coverage report." + COMMAND validator-keys --unittest$<$:=${coverage_test}> + # Create test coverage data file + COMMAND ${LCOV} + --no-external -d "${CMAKE_CURRENT_SOURCE_DIR}" -c -d . -o tests.info + | grep -v "ignoring data for external file" + # Combine baseline and test coverage data + COMMAND ${LCOV} + -a baseline.info -a tests.info -o lcov-all.info + # extract our files + COMMAND ${LCOV} + -e lcov-all.info "${extract_pattern}" -o lcov.info + COMMAND ${CMAKE_COMMAND} -E echo "Summary of coverage:" + COMMAND ${LCOV} --summary lcov.info + # generate HTML report + COMMAND ${GENHTML} + -o ${CMAKE_BINARY_DIR}/coverage lcov.info + BYPRODUCTS coverage/index.html) + endif () + else() + message(STATUS "Coverage: neither clang nor gcc") + endif () +else() + message(STATUS "Coverage disabled") +endif () diff --git a/Builds/CMake/KeysInterface.cmake b/Builds/CMake/KeysInterface.cmake new file mode 100644 index 0000000..33140e0 --- /dev/null +++ b/Builds/CMake/KeysInterface.cmake @@ -0,0 +1,69 @@ +#[===================================================================[ + rippled compile options/settings via an interface library +#]===================================================================] + +add_library (keys_opts INTERFACE) +add_library (Keys::opts ALIAS keys_opts) +target_compile_definitions (keys_opts + INTERFACE + BOOST_ASIO_DISABLE_HANDLER_TYPE_REQUIREMENTS + $<$: + BOOST_ASIO_NO_DEPRECATED + BOOST_FILESYSTEM_NO_DEPRECATED + > + $<$>: + BOOST_COROUTINES_NO_DEPRECATION_WARNING + BOOST_BEAST_ALLOW_DEPRECATED + BOOST_FILESYSTEM_DEPRECATED + > + $<$: + USE_BEAST_HASHER + > + $<$:BEAST_NO_UNIT_TEST_INLINE=1> + $<$:BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES=1> + $<$:RIPPLE_SINGLE_IO_SERVICE_THREAD=1>) +target_compile_options (keys_opts + INTERFACE + $<$,$>:-Wsuggest-override> + $<$:-fno-omit-frame-pointer> + $<$,$>:-fprofile-arcs -ftest-coverage> + $<$,$>:-fprofile-instr-generate -fcoverage-mapping> + $<$:-pg> + $<$,$>:-p>) + +target_link_libraries (keys_opts + INTERFACE + $<$,$>:-fprofile-arcs -ftest-coverage> + $<$,$>:-fprofile-instr-generate -fcoverage-mapping> + $<$:-pg> + $<$,$>:-p>) + +if (jemalloc) + if (static) + set(JEMALLOC_USE_STATIC ON CACHE BOOL "" FORCE) + endif () + find_package (jemalloc REQUIRED) + target_compile_definitions (keys_opts INTERFACE PROFILE_JEMALLOC) + target_include_directories (keys_opts SYSTEM INTERFACE ${JEMALLOC_INCLUDE_DIRS}) + target_link_libraries (keys_opts INTERFACE ${JEMALLOC_LIBRARIES}) + get_filename_component (JEMALLOC_LIB_PATH ${JEMALLOC_LIBRARIES} DIRECTORY) + ## TODO see if we can use the BUILD_RPATH target property (is it transitive?) + set (CMAKE_BUILD_RPATH ${CMAKE_BUILD_RPATH} ${JEMALLOC_LIB_PATH}) +endif () + +if (san) + target_compile_options (keys_opts + INTERFACE + # sanitizers recommend minimum of -O1 for reasonable performance + $<$:-O1> + ${SAN_FLAG} + -fno-omit-frame-pointer) + target_compile_definitions (keys_opts + INTERFACE + $<$:SANITIZER=ASAN> + $<$:SANITIZER=TSAN> + $<$:SANITIZER=MSAN> + $<$:SANITIZER=UBSAN>) + target_link_libraries (keys_opts INTERFACE ${SAN_FLAG} ${SAN_LIB}) +endif () + diff --git a/Builds/CMake/KeysNIH.cmake b/Builds/CMake/KeysNIH.cmake new file mode 100644 index 0000000..5e9e244 --- /dev/null +++ b/Builds/CMake/KeysNIH.cmake @@ -0,0 +1,36 @@ +#[===================================================================[ + NIH prefix path..this is where we will download + and build any ExternalProjects, and they will hopefully + survive across build directory deletion (manual cleans) + + Copied from rippled https://github.com/ripple/rippled/blob/develop/Builds/CMake/RippledNIH.cmake + Only called if building directly and a Rippled installation was not found. +#]===================================================================] + +string (REGEX REPLACE "[ \\/%]+" "_" gen_for_path ${CMAKE_GENERATOR}) +string (TOLOWER ${gen_for_path} gen_for_path) +# HACK: trying to shorten paths for windows CI (which hits 260 MAXPATH easily) +# @see: https://issues.jenkins-ci.org/browse/JENKINS-38706?focusedCommentId=339847 +string (REPLACE "visual_studio" "vs" gen_for_path ${gen_for_path}) +if (NOT DEFINED NIH_CACHE_ROOT) + if (DEFINED ENV{NIH_CACHE_ROOT}) + set (NIH_CACHE_ROOT $ENV{NIH_CACHE_ROOT}) + else () + set (NIH_CACHE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/.nih_c") + endif () +endif () +set (nih_cache_path + "${NIH_CACHE_ROOT}/${gen_for_path}/${CMAKE_CXX_COMPILER_ID}_${CMAKE_CXX_COMPILER_VERSION}") +if (NOT is_multiconfig) + set (nih_cache_path "${nih_cache_path}/${CMAKE_BUILD_TYPE}") +endif () +file(TO_CMAKE_PATH "${nih_cache_path}" nih_cache_path) +message (STATUS "NIH-EP cache path: ${nih_cache_path}") +## two convenience variables: +set (ep_lib_prefix ${CMAKE_STATIC_LIBRARY_PREFIX}) +set (ep_lib_suffix ${CMAKE_STATIC_LIBRARY_SUFFIX}) + +# this is a setting for FetchContent and needs to be +# a cache variable +# https://cmake.org/cmake/help/latest/module/FetchContent.html#populating-the-content +set (FETCHCONTENT_BASE_DIR ${nih_cache_path} CACHE STRING "" FORCE) diff --git a/Builds/CMake/KeysSanity.cmake b/Builds/CMake/KeysSanity.cmake new file mode 100644 index 0000000..20a6179 --- /dev/null +++ b/Builds/CMake/KeysSanity.cmake @@ -0,0 +1,88 @@ +#[===================================================================[ + convenience variables and sanity checks +#]===================================================================] + +if (NOT ep_procs) + ProcessorCount(ep_procs) + if (ep_procs GREATER 1) + # never use more than half of cores for EP builds + math (EXPR ep_procs "${ep_procs} / 2") + message (STATUS "Using ${ep_procs} cores for ExternalProject builds.") + endif () +endif () +get_property (is_multiconfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) +if (is_multiconfig STREQUAL "NOTFOUND") + if (${CMAKE_GENERATOR} STREQUAL "Xcode" OR ${CMAKE_GENERATOR} MATCHES "^Visual Studio") + set (is_multiconfig TRUE) + endif () +endif () + +set (CMAKE_CONFIGURATION_TYPES "Debug;Release" CACHE STRING "" FORCE) +if (NOT is_multiconfig) + if (NOT CMAKE_BUILD_TYPE) + message (STATUS "Build type not specified - defaulting to Release") + set (CMAKE_BUILD_TYPE Release CACHE STRING "build type" FORCE) + elseif (NOT (CMAKE_BUILD_TYPE STREQUAL Debug OR CMAKE_BUILD_TYPE STREQUAL Release)) + # for simplicity, these are the only two config types we care about. Limiting + # the build types simplifies dealing with external project builds especially + message (FATAL_ERROR " *** Only Debug or Release build types are currently supported ***") + endif () +endif () + +get_directory_property(has_parent PARENT_DIRECTORY) +if (has_parent) + set (is_root_project OFF) +else () + set (is_root_project ON) +endif () + +if ("${CMAKE_CXX_COMPILER_ID}" MATCHES ".*Clang") # both Clang and AppleClang + set (is_clang TRUE) + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND + CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0) + message (FATAL_ERROR "This project requires clang 7 or later") + endif () + # TODO min AppleClang version check ? +elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + set (is_gcc TRUE) + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0) + message (FATAL_ERROR "This project requires GCC 7 or later") + endif () +endif () +if (CMAKE_GENERATOR STREQUAL "Xcode") + set (is_xcode TRUE) +endif () + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + set (is_linux TRUE) +else () + set (is_linux FALSE) +endif () + +if ("$ENV{CI}" STREQUAL "true" OR "$ENV{CONTINUOUS_INTEGRATION}" STREQUAL "true") + set (is_ci TRUE) +else () + set (is_ci FALSE) +endif () + +# check for in-source build and fail +if ("${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}") + message (FATAL_ERROR "Builds (in-source) are not allowed in " + "${CMAKE_CURRENT_SOURCE_DIR}. Please remove CMakeCache.txt and the CMakeFiles " + "directory from ${CMAKE_CURRENT_SOURCE_DIR} and try building in a separate directory.") +endif () + +if ("${CMAKE_GENERATOR}" MATCHES "Visual Studio" AND + NOT ("${CMAKE_GENERATOR}" MATCHES .*Win64.*)) + message (FATAL_ERROR + "Visual Studio 32-bit build is not supported. Use -G\"${CMAKE_GENERATOR} Win64\"") +endif () + +if (NOT CMAKE_SIZEOF_VOID_P EQUAL 8) + message (FATAL_ERROR "Rippled requires a 64 bit target architecture.\n" + "The most likely cause of this warning is trying to build rippled with a 32-bit OS.") +endif () + +if (APPLE AND NOT HOMEBREW) + find_program (HOMEBREW brew) +endif () diff --git a/CMakeLists.txt b/CMakeLists.txt index 1a24ccc..278ab9a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required (VERSION 3.11) - project (validator-keys-tool) + #[===========================================[ The tool depends on the xrpl_core library which is defined by the rippled @@ -8,9 +8,17 @@ project (validator-keys-tool) libs and, if not found, pulls them in with FetchContent. #]===========================================] +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake") +include(KeysNIH) + get_directory_property(has_parent PARENT_DIRECTORY) +if(coverage) + # Rippled also responds to the "coverage" flag, so clear it if it's set until + # the dependency is set up. + set(keys_coverage ${coverage}) + set(coverage OFF CACHE BOOL "gcc/clang only" FORCE) +endif() if (NOT has_parent) - project (validator-keys-tool) find_package(Ripple QUIET) if (NOT TARGET Ripple::xrpl_core) find_package(Git) @@ -52,10 +60,16 @@ if (NOT has_parent) if(NOT rippled_src_POPULATED) message (STATUS "Pausing to download rippled source...") FetchContent_Populate(rippled_src) + add_subdirectory(${rippled_src_SOURCE_DIR} ${rippled_src_BINARY_DIR}) endif() - add_subdirectory(${rippled_src_SOURCE_DIR} xrpl_core) endif () endif () +if(keys_coverage) + set(coverage ${keys_coverage} CACHE BOOL "gcc/clang only" FORCE) +endif() +include(KeysSanity) +include(KeysCov) +include(KeysInterface) add_executable (validator-keys src/ValidatorKeys.cpp @@ -64,7 +78,7 @@ add_executable (validator-keys src/test/ValidatorKeys_test.cpp src/test/ValidatorKeysTool_test.cpp) target_include_directories (validator-keys PRIVATE src) -target_link_libraries (validator-keys Ripple::xrpl_core) +target_link_libraries (validator-keys Ripple::xrpl_core Keys::opts) if (has_parent) set_target_properties (validator-keys PROPERTIES EXCLUDE_FROM_ALL ON) diff --git a/ci/build-and-test.sh b/ci/build-and-test.sh deleted file mode 100755 index 4d5d2ae..0000000 --- a/ci/build-and-test.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash -u -# We use set -e and bash with -u to bail on first non zero exit code of any -# processes launched or upon any unbound variable. -# We use set -x to print commands before running them to help with -# debugging. -set -ex -__dirname=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -echo "using CC: ${CC}" -"${CC}" --version -export CC -COMPNAME=$(basename $CC) -echo "using CXX: ${CXX:-notset}" -if [[ $CXX ]]; then - "${CXX}" --version - export CXX -fi -: ${BUILD_TYPE:=Debug} -echo "BUILD TYPE: ${BUILD_TYPE}" - -: ${APP:=validator-keys} -echo "using APP: ${APP}" - -JOBS=${NUM_PROCESSORS:-2} -if [[ ${TRAVIS:-false} != "true" ]]; then - JOBS=$((JOBS+1)) -fi - -if [ -x /usr/bin/time ] ; then - : ${TIME:="Duration: %E"} - export TIME - time=/usr/bin/time -else - time= -fi - -if [[ -z "${MAX_TIME:-}" ]] ; then - timeout_cmd="" -else - timeout_cmd="timeout ${MAX_TIME}" -fi - -echo "cmake building ${APP}" -: ${CMAKE_EXTRA_ARGS:=""} - -: ${COVERAGE:=false} -if [[ ${COVERAGE} == true ]]; then - echo "coverage option detected." - export PATH=$PATH:${LCOV_ROOT}/usr/bin -fi - -# -# allow explicit setting of the name of the build -# dir, otherwise default to the compiler.build_type -# -: "${BUILD_DIR:=${COMPNAME}.${BUILD_TYPE}}" -BUILDARGS=" -j${JOBS}" -if [[ ${VERBOSE_BUILD:-} == true ]]; then - CMAKE_EXTRA_ARGS+=" -DCMAKE_VERBOSE_MAKEFILE=ON" - # TODO: if we use a different generator, this - # option to build verbose would need to change: - BUILDARGS+=" verbose=1" -fi -if [ -d "build/${BUILD_DIR}" ]; then - rm -rf "build/${BUILD_DIR}" -fi - -mkdir -p "build/${BUILD_DIR}" -pushd "build/${BUILD_DIR}" -# generate -${time} cmake ../.. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ${CMAKE_EXTRA_ARGS} -# build -export DESTDIR=$(pwd)/_INSTALLED_ -time ${timeout_cmd} cmake --build . -- $BUILDARGS -popd -export APP_PATH="$PWD/build/${BUILD_DIR}/${APP}" -echo "using APP_PATH: ${APP_PATH}" - -# See what we've actually built -ldd ${APP_PATH} - -if [[ ${COVERAGE} == true ]]; then - # Push the results (lcov.info) to codecov - codecov -X gcov # don't even try and look for .gcov files ;) - find . -name "*.gcda" | xargs rm -f -fi - -${timeout_cmd} ${APP_PATH} ${APP_ARGS} - - diff --git a/ci/install-boost.sh b/ci/install-boost.sh deleted file mode 100644 index 15004bb..0000000 --- a/ci/install-boost.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -# Assumptions: -# 1) BOOST_ROOT and BOOST_URL are already defined, -# and contain valid values. -# 2) The last namepart of BOOST_ROOT matches the -# folder name internal to boost's .tar.gz -# When testing you can force a boost build by clearing travis caches: -# https://travis-ci.org/ripple/rippled/caches -set -e - -if [ -x /usr/bin/time ] ; then - : ${TIME:="Duration: %E"} - export TIME - time=/usr/bin/time -else - time= -fi - -if [ ! -d "$BOOST_ROOT/lib" ] -then - wget $BOOST_URL -O /tmp/boost.tar.gz - cd `dirname $BOOST_ROOT` - rm -fr ${BOOST_ROOT} - tar xzf /tmp/boost.tar.gz - cd $BOOST_ROOT && \ - $time ./bootstrap.sh --prefix=$BOOST_ROOT && \ - $time ./b2 cxxflags="-std=c++14" -j$((2*${NUM_PROCESSORS:-2})) &&\ - $time ./b2 install -else - echo "Using cached boost at $BOOST_ROOT" -fi - diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh deleted file mode 100755 index 293e2db..0000000 --- a/ci/install-dependencies.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -u -# Exit if anything fails. Echo commands to aid debugging. -set -ex - -# Target working dir - defaults to current dir. -# Can be set from caller, or in the first parameter -TWD=$( cd ${TWD:-${1:-${PWD:-$( pwd )}}}; pwd ) -echo "Target path is: $TWD" -# Override gcc version to $GCC_VER. -# Put an appropriate symlink at the front of the path. -mkdir -pv $HOME/bin -for g in gcc g++ gcov gcc-ar gcc-nm gcc-ranlib -do - test -x $( type -p ${g}-$GCC_VER ) - ln -sv $(type -p ${g}-$GCC_VER) $HOME/bin/${g} -done - -# What versions are we ACTUALLY running? -if [ -x $HOME/bin/g++ ]; then - $HOME/bin/g++ -v -else - g++ -v -fi - -pip install --user requests==2.13.0 -pip install --user https://github.com/codecov/codecov-python/archive/master.zip - -bash ci/install-boost.sh - diff --git a/ci/shared/install_boost.sh b/ci/shared/install_boost.sh new file mode 100755 index 0000000..f05e7f3 --- /dev/null +++ b/ci/shared/install_boost.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Assumptions: +# 1) BOOST_ROOT and BOOST_URL are already defined, +# and contain valid values. BOOST_URL2 may be defined +# as a fallback. BOOST_WGET_OPTIONS may be defined with +# retry options if the download(s) fail on the first try. +# 2) The last namepart of BOOST_ROOT matches the +# folder name internal to boost's .tar.gz +# When testing you can force a boost build by clearing travis caches: +# https://travis-ci.org/ripple/rippled/caches +set -exu + +odir=$(pwd) +: ${BOOST_TOOLSET:=msvc-14.1} + +if [[ -d "$BOOST_ROOT/lib" || -d "${BOOST_ROOT}/stage/lib" ]] ; then + echo "Using cached boost at $BOOST_ROOT" + exit +fi + +#fetch/unpack: +fn=$(basename -- "$BOOST_URL") +ext="${fn##*.}" +# wopt="--quiet" +wopt= +wget ${wopt} $BOOST_URL -O /tmp/boost.tar.${ext} || \ + ( [ -n "${BOOST_URL2}" ] && \ + wget ${wopt} $BOOST_URL2 -O /tmp/boost.tar.${ext} ) || \ + ( [ -n "${BOOST_WGET_OPTIONS}" ] && + ( wget ${wopt} ${BOOST_WGET_OPTIONS} $BOOST_URL -O /tmp/boost.tar.${ext} || \ + ( [ -n "${BOOST_URL2}" ] && \ + wget ${wopt} ${BOOST_WGET_OPTIONS} $BOOST_URL2 -O /tmp/boost.tar.${ext} ) + ) + ) +cd $(dirname $BOOST_ROOT) +rm -fr ${BOOST_ROOT} +mkdir ${BOOST_ROOT} +tar xf /tmp/boost.tar.${ext} -C ${BOOST_ROOT} --strip-components 1 +cd $BOOST_ROOT + +BLDARGS=() +if [[ ${BOOST_BUILD_ALL:-false} == "true" ]]; then + # we never need boost-python...so even for ALL + # option we can skip it + BLDARGS+=(--without-python) +else + BLDARGS+=(--with-chrono) + BLDARGS+=(--with-context) + BLDARGS+=(--with-coroutine) + BLDARGS+=(--with-date_time) + BLDARGS+=(--with-filesystem) + BLDARGS+=(--with-program_options) + BLDARGS+=(--with-regex) + BLDARGS+=(--with-system) + BLDARGS+=(--with-atomic) + BLDARGS+=(--with-thread) +fi +BLDARGS+=(-j$((2*${NUM_PROCESSORS:-2}))) +BLDARGS+=(--prefix=${BOOST_ROOT}/_INSTALLED_) +BLDARGS+=(-d0) # suppress messages/output + +if [[ -z ${COMSPEC:-} ]]; then + if [[ "$(uname)" == "Darwin" ]] ; then + BLDARGS+=(cxxflags="-std=c++14 -fvisibility=default") + else + BLDARGS+=(cxxflags="-std=c++14") + BLDARGS+=(runtime-link="static,shared") + fi + BLDARGS+=(--layout=tagged) + ./bootstrap.sh + ./b2 "${BLDARGS[@]}" stage + ./b2 "${BLDARGS[@]}" install +else + BLDARGS+=(runtime-link="static,shared") + BLDARGS+=(--layout=versioned) + BLDARGS+=(--toolset="${BOOST_TOOLSET}") + BLDARGS+=(address-model=64) + BLDARGS+=(architecture=x86) + BLDARGS+=(link=static) + BLDARGS+=(threading=multi) + cmd /E:ON /D /S /C"bootstrap.bat" + ./b2.exe "${BLDARGS[@]}" stage + ./b2.exe "${BLDARGS[@]}" install +fi + +if [[ ${CI:-false} == "true" ]]; then + # save some disk space...these are mostly + # obj files and don't need to be kept in CI contexts + rm -rf bin.v2 +fi + +cd $odir + diff --git a/ci/ubuntu/build-and-test.sh b/ci/ubuntu/build-and-test.sh new file mode 100755 index 0000000..1f88bf9 --- /dev/null +++ b/ci/ubuntu/build-and-test.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +set -ex + +function version_ge() { test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" == "$1"; } + +__dirname=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +echo "using CC: ${CC}" +"${CC}" --version +export CC + +COMPNAME=$(basename $CC) +echo "using CXX: ${CXX:-notset}" +if [[ $CXX ]]; then + "${CXX}" --version + export CXX +fi +: ${BUILD_TYPE:=Debug} +echo "BUILD TYPE: ${BUILD_TYPE}" + +: ${TARGET:=install} +echo "BUILD TARGET: ${TARGET}" + +JOBS=${NUM_PROCESSORS:-2} +if [[ ${TRAVIS:-false} != "true" ]]; then + JOBS=$((JOBS+1)) +fi + +if [[ ! -z "${CMAKE_EXE:-}" ]] ; then + export PATH="$(dirname ${CMAKE_EXE}):$PATH" +fi + +if [ -x /usr/bin/time ] ; then + : ${TIME:="Duration: %E"} + export TIME + time=/usr/bin/time +else + time= +fi + +echo "Building validator-keys" +: ${CMAKE_EXTRA_ARGS:=""} +if [[ ${NINJA_BUILD:-} == true ]]; then + CMAKE_EXTRA_ARGS+=" -G Ninja" +fi + +coverage=false +if [[ "${TARGET}" == "coverage_report" ]] ; then + echo "coverage option detected." + coverage=true +fi + +cmake --version +CMAKE_VER=$(cmake --version | cut -d " " -f 3 | head -1) + +# +# allow explicit setting of the name of the build +# dir, otherwise default to the compiler.build_type +# +: "${BUILD_DIR:=${COMPNAME}.${BUILD_TYPE}}" +BUILDARGS="--target ${TARGET}" +BUILDTOOLARGS="" +if version_ge $CMAKE_VER "3.12.0" ; then + BUILDARGS+=" --parallel" +fi + +if [[ ${NINJA_BUILD:-} == false ]]; then + if version_ge $CMAKE_VER "3.12.0" ; then + BUILDARGS+=" ${JOBS}" + else + BUILDTOOLARGS+=" -j ${JOBS}" + fi +fi + +if [[ ${VERBOSE_BUILD:-} == true ]]; then + CMAKE_EXTRA_ARGS+=" -DCMAKE_VERBOSE_MAKEFILE=ON" + if version_ge $CMAKE_VER "3.14.0" ; then + BUILDARGS+=" --verbose" + else + if [[ ${NINJA_BUILD:-} == false ]]; then + BUILDTOOLARGS+=" verbose=1" + else + BUILDTOOLARGS+=" -v" + fi + fi +fi + +if [[ ${USE_CCACHE:-} == true ]]; then + echo "using ccache with basedir [${CCACHE_BASEDIR:-}]" + CMAKE_EXTRA_ARGS+=" -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" +fi +if [ -d "build/${BUILD_DIR}" ]; then + rm -rf "build/${BUILD_DIR}" +fi + +mkdir -p "build/${BUILD_DIR}" +pushd "build/${BUILD_DIR}" + +# generate +${time} cmake ../.. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ${CMAKE_EXTRA_ARGS} +# build +export DESTDIR=$(pwd)/_INSTALLED_ + +${time} eval cmake --build . ${BUILDARGS} -- ${BUILDTOOLARGS} + +if [[ ${TARGET} == "docs" ]]; then + ## mimic the standard test output for docs build + ## to make controlling processes like jenkins happy + if [ -f docs/html/index.html ]; then + echo "1 case, 1 test total, 0 failures" + else + echo "1 case, 1 test total, 1 failures" + fi + exit +fi +popd + +export APP_PATH="$PWD/build/${BUILD_DIR}/validator-keys" +echo "using APP_PATH: ${APP_PATH}" + +# See what we've actually built +ldd ${APP_PATH} + +: ${APP_ARGS:="--unittest"} + +if [[ ${coverage} == true && $CC =~ ^gcc ]]; then + # Push the results (lcov.info) to codecov + codecov -X gcov # don't even try and look for .gcov files ;) + find . -name "*.gcda" | xargs rm -f +fi + +if [[ ${SKIP_TESTS:-} == true ]]; then + echo "skipping tests." + exit +fi + +ulimit -a +corepat=$(cat /proc/sys/kernel/core_pattern) +if [[ ${corepat} =~ ^[:space:]*\| ]] ; then + echo "WARNING: core pattern is piping - can't search for core files" + look_core=false +else + look_core=true + coredir=$(dirname ${corepat}) +fi +if [[ ${look_core} == true ]]; then + before=$(ls -A1 ${coredir}) +fi + +set +e +echo "Running tests for ${APP_PATH}" +${APP_PATH} ${APP_ARGS} +TEST_STAT=$? +set -e + +if [[ ${look_core} == true ]]; then + after=$(ls -A1 ${coredir}) + oIFS="${IFS}" + IFS=$'\n\r' + found_core=false + for l in $(diff -w --suppress-common-lines <(echo "$before") <(echo "$after")) ; do + if [[ "$l" =~ ^[[:space:]]*\>[[:space:]]*(.+)$ ]] ; then + corefile="${BASH_REMATCH[1]}" + echo "FOUND core dump file at '${coredir}/${corefile}'" + gdb_output=$(/bin/mktemp /tmp/gdb_output_XXXXXXXXXX.txt) + found_core=true + gdb \ + -ex "set height 0" \ + -ex "set logging file ${gdb_output}" \ + -ex "set logging on" \ + -ex "print 'versionString'" \ + -ex "thread apply all backtrace full" \ + -ex "info inferiors" \ + -ex quit \ + "$APP_PATH" \ + "${coredir}/${corefile}" &> /dev/null + + echo -e "CORE INFO: \n\n $(cat ${gdb_output}) \n\n)" + fi + done + IFS="${oIFS}" +fi + +if [[ ${found_core} == true ]]; then + exit -1 +else + exit $TEST_STAT +fi + diff --git a/ci/ubuntu/build-in-docker.sh b/ci/ubuntu/build-in-docker.sh new file mode 100755 index 0000000..c4ce532 --- /dev/null +++ b/ci/ubuntu/build-in-docker.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# run our build script in a docker container +# using travis-ci hosts +set -eux + +function join_by { local IFS="$1"; shift; echo "$*"; } + +set +x +echo "VERBOSE_BUILD=true" > /tmp/co.env +matchers=( + 'TRAVIS.*' 'CI' 'CC' 'CXX' + 'BUILD_TYPE' 'TARGET' 'MAX_TIME' + 'CODECOV.+' 'CMAKE.*' '.+_TESTS' + '.+_OPTIONS' 'NINJA.*' 'NUM_.+' + 'NIH_.+' 'BOOST.*' '.*CCACHE.*') + +matchstring=$(join_by '|' "${matchers[@]}") +echo "MATCHSTRING IS:: $matchstring" +env | grep -E "^(${matchstring})=" >> /tmp/co.env +set -x +# need to eliminate TRAVIS_CMD...don't want to pass it to the container +cat /tmp/co.env | grep -v TRAVIS_CMD > /tmp/co.env.2 +mv /tmp/co.env.2 /tmp/co.env +cat /tmp/co.env +mkdir -p -m 0777 ${TRAVIS_BUILD_DIR}/cores +echo "${TRAVIS_BUILD_DIR}/cores/%e.%p" | sudo tee /proc/sys/kernel/core_pattern +docker run \ + -t --env-file /tmp/co.env \ + -v ${TRAVIS_HOME}:${TRAVIS_HOME} \ + -w ${TRAVIS_BUILD_DIR} \ + --cap-add SYS_PTRACE \ + --ulimit "core=-1" \ + $DOCKER_IMAGE \ + /bin/bash -c 'if [[ $CC =~ ([[:alpha:]]+)-([[:digit:].]+) ]] ; then sudo update-alternatives --set ${BASH_REMATCH[1]} /usr/bin/$CC; fi; ci/ubuntu/build-and-test.sh' + + diff --git a/ci/ubuntu/travis-cache-start.sh b/ci/ubuntu/travis-cache-start.sh new file mode 100755 index 0000000..b7cfb74 --- /dev/null +++ b/ci/ubuntu/travis-cache-start.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# some cached files create churn, so save them here for +# later restoration before packing the cache +set -eux +pushd ${TRAVIS_HOME} +if [ -f cache_ignore.tar ] ; then + rm -f cache_ignore.tar +fi + +if [ -d _cache/nih_c ] ; then + find _cache/nih_c -name "build.ninja" | tar rf cache_ignore.tar --files-from - + find _cache/nih_c -name ".ninja_deps" | tar rf cache_ignore.tar --files-from - + find _cache/nih_c -name ".ninja_log" | tar rf cache_ignore.tar --files-from - + find _cache/nih_c -name "*.log" | tar rf cache_ignore.tar --files-from - + find _cache/nih_c -name "*.tlog" | tar rf cache_ignore.tar --files-from - + # show .a files in the cache, for sanity checking + find _cache/nih_c -name "*.a" -ls +fi + +if [ -d _cache/ccache ] ; then + find _cache/ccache -name "stats" | tar rf cache_ignore.tar --files-from - +fi + +if [ -f cache_ignore.tar ] ; then + tar -tf cache_ignore.tar +fi +popd + + diff --git a/ci/windows/install-vcpkg.sh b/ci/windows/install-vcpkg.sh new file mode 100755 index 0000000..b779829 --- /dev/null +++ b/ci/windows/install-vcpkg.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -exu + +: ${TRAVIS_BUILD_DIR:=""} +: ${VCPKG_DIR:=".vcpkg"} +export VCPKG_ROOT=${VCPKG_DIR} +: ${VCPKG_DEFAULT_TRIPLET:="x64-windows-static"} + +export VCPKG_DEFAULT_TRIPLET + +EXE="vcpkg" +if [[ -z ${COMSPEC:-} ]]; then + EXE="${EXE}.exe" +fi + +if [[ -d "${VCPKG_DIR}" && -x "${VCPKG_DIR}/${EXE}" && -d "${VCPKG_DIR}/installed" ]] ; then + echo "Using cached vcpkg at ${VCPKG_DIR}" + ${VCPKG_DIR}/${EXE} list +else + if [[ -d "${VCPKG_DIR}" ]] ; then + rm -rf "${VCPKG_DIR}" + fi + git clone --branch 2019.12 https://github.com/Microsoft/vcpkg.git ${VCPKG_DIR} + pushd ${VCPKG_DIR} + BSARGS=() + if [[ "$(uname)" == "Darwin" ]] ; then + BSARGS+=(--allowAppleClang) + fi + if [[ -z ${COMSPEC:-} ]]; then + chmod +x ./bootstrap-vcpkg.sh + time ./bootstrap-vcpkg.sh "${BSARGS[@]}" + else + time ./bootstrap-vcpkg.bat + fi + popd +fi + +# TODO: bring boost in this way as well ? +# NOTE: can pin specific ports to a commit/version like this: +# git checkout ports/boost +if [ $# -eq 0 ]; then + echo "No extra packages specified..." + PKGS=() +else + PKGS=( "$@" ) +fi +for LIB in "${PKGS[@]}"; do + time ${VCPKG_DIR}/${EXE} --clean-after-build install ${LIB} +done + + diff --git a/ci/windows/setup-msvc.sh b/ci/windows/setup-msvc.sh new file mode 100755 index 0000000..8d61c97 --- /dev/null +++ b/ci/windows/setup-msvc.sh @@ -0,0 +1,40 @@ + +# NOTE: must be sourced from a shell so it can export vars + +cat << BATCH > ./getenv.bat +CALL %* +ENV +BATCH + +while read line ; do + IFS='"' read x path arg <<<"${line}" + if [ -f "${path}" ] ; then + echo "FOUND: $path" + export VCINSTALLDIR=$(./getenv.bat "${path}" ${arg} | grep "^VCINSTALLDIR=" | sed -E "s/^VCINSTALLDIR=//g") + if [ "${VCINSTALLDIR}" != "" ] ; then + echo "USING ${VCINSTALLDIR}" + export LIB=$(./getenv.bat "${path}" ${arg} | grep "^LIB=" | sed -E "s/^LIB=//g") + export LIBPATH=$(./getenv.bat "${path}" ${arg} | grep "^LIBPATH=" | sed -E "s/^LIBPATH=//g") + export INCLUDE=$(./getenv.bat "${path}" ${arg} | grep "^INCLUDE=" | sed -E "s/^INCLUDE=//g") + ADDPATH=$(./getenv.bat "${path}" ${arg} | grep "^PATH=" | sed -E "s/^PATH=//g") + export PATH="${ADDPATH}:${PATH}" + break + fi + fi +done <