diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml new file mode 100644 index 00000000..153f0b9d --- /dev/null +++ b/.github/actions/build/action.yml @@ -0,0 +1,36 @@ +name: Install build prerequisites + +inputs: + cache-dir: + description: Where to put vcpkg cache + required: true + +runs: + using: "composite" + steps: + - name: Capture vcpkg revision for use in cache key + shell: bash + run: | + git -C vcpkg rev-parse HEAD > vcpkg_commit.txt + + - name: Restore cache + uses: actions/cache@v3 + with: + path: ${{ inputs.cache-dir }} + key: v01-vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} + restore-keys: | + v01-vcpkg-${{ runner.os }} + + - name: Install dependencies (macOS) + if: ${{ runner.os == 'macOS' }} + shell: bash + run: | + brew install llvm pkg-config nasm + ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" + ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" + + - name: Install dependencies (Ubuntu) + if: ${{ runner.os == 'Linux' }} + shell: bash + run: | + sudo apt-get install -y linux-headers-$(uname -r) nasm diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 2efa2f4d..afbf93a4 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -11,14 +11,8 @@ on: env: CMAKE_BUILD_PARALLEL_LEVEL: 3 CTEST_OUTPUT_ON_FAILURE: 1 - CMAKE_BUILD_DIR: ${{ github.workspace }}/build - CMAKE_BUILD_OPENSSL3_DIR: ${{ github.workspace }}/build_openssl3 - CMAKE_BUILD_BORINGSSL_DIR: ${{ github.workspace }}/build_boringssl - VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/build/cache,readwrite - VCPKG_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - VCPKG_REPO: ${{ github.workspace }}/vcpkg - CACHE_VERSION: v01 - CACHE_NAME: vcpkg + VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/vcpkg_cache,readwrite + CMAKE_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake jobs: formatting-check: @@ -34,226 +28,116 @@ jobs: include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' fallback-style: 'Mozilla' - quick-linux-interop-check: + build-and-unit-test: needs: formatting-check - name: Quick Linux Check and Interop - runs-on: ubuntu-latest - steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - # write the commit hash of vcpkg to a text file so we can use it in the - # hashFiles for cache - - run: | - git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt - - # First, attempt to pull key key, if that is not present, pull one of the - # restore-keys so we do not need to build from scratch. - # CACHE_VERSION - provide a way to reset cache - # CACHE_NAME - name of the cache in order to manage it - # matrix.os - cache per OS and version - # hashFiles - Recache if the vcpkg files change - - name: Restore Cache - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-ubuntu-latest-${{ hashFiles('vcpkg_commit.txt', 'vcpkg.json', 'alternatives/openssl_3/vcpkg.json') }} - restore-keys: | - ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-ubuntu-latest - - - name: Dependencies - run: | - sudo apt-get install -y linux-headers-$(uname -r) nasm - - - name: Restore cache - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/build/cache - key: VCPKG-BinaryCache-${{ runner.os }} - - - name: Build (OpenSSL 1.1) - run: | - cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" - cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target all - - - name: Unit Test (OpenSSL 1.1) - run: | - cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target test - - - name: Build (Interop Harness) - run: | - cd cmd/interop - cmake -B build -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" - cmake --build build - - - name: Test self-interop - run: | - make -C cmd/interop self-test - - - name: Test interop on test vectors - run: | - make -C cmd/interop interop-test - - - name: Test gRPC live interop with self - run: | - cd cmd/interop - ./grpc-self-test.sh - - - name: Build (OpenSSL 3) - run: | - cmake -B "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/openssl_3" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" - cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" - - - name: Unit Test (OpenSSL 3) - run: | - cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" --target test - - - name: Build (BoringSSL) - run: | - cmake -B "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/boringssl_1.1" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" - cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" - - - name: Unit Test (BoringSSL) - run: | - cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" --target test - - platform-sanitizer-tests: - if: github.event.pull_request.draft == false - needs: quick-linux-interop-check - name: Build and test platforms using sanitizers and clang-tidy + name: Build and test runs-on: ${{ matrix.os }} strategy: - fail-fast: false matrix: os: [windows-latest, ubuntu-latest, macos-latest] + crypto: [openssl_1.1, openssl_3, boringssl] include: - os: windows-latest - ossl3-vcpkg-dir: "alternatives\\openssl_3" - boringssl-vcpkg-dir: "alternatives\\boringssl_1.1" ctest-target: RUN_TESTS - os: ubuntu-latest - ossl3-vcpkg-dir: "alternatives/openssl_3" - boringssl-vcpkg-dir: "alternatives/boringssl_1.1" ctest-target: test - os: macos-latest - ossl3-vcpkg-dir: "alternatives/openssl_3" - boringssl-vcpkg-dir: "alternatives/boringssl_1.1" ctest-target: test + env: + BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" + CRYPTO_DIR: "./alternatives/${{ matrix.crypto }}" + steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: submodules: recursive fetch-depth: 0 - # write the commit hash of vcpkg to a text file so we can use it in the - # hashFiles for cache - - run: | - git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt - - # First, attempt to pull key key, if that is not present, pull one of the - # restore-keys so we do not need to build from scratch. - # CACHE_VERSION - provide a way to reset cache - # CACHE_NAME - name of the cache in order to manage it - # matrix.os - cache per OS and version - # hashFiles - Recache if the vcpkg files change - - name: Restore Cache - uses: actions/cache@v3 + - uses: ./.github/actions/build with: - path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'vcpkg.json', 'alternatives/openssl_3/vcpkg.json') }} - restore-keys: | - ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} + cache-dir: ${{ github.workspace }}/vcpkg_cache - - name: Dependencies (macOs) - if: ${{ matrix.os == 'macos-latest' }} + - name: Build run: | - brew install llvm pkg-config nasm - ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" - ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" + # XXX(RLB): If we do not have SANITIZERS=ON here, the Windows CI builds + # hang in the middle of unit testing. + cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON -DSANITIZERS=ON + cmake --build "${{ env.BUILD_DIR }}" - - name: Dependencies (Ubuntu) - if: ${{ matrix.os == 'ubuntu-latest' }} + - name: Unit Test run: | - sudo apt-get install -y linux-headers-$(uname -r) nasm + cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" + + interop-test: + if: github.event.pull_request.draft == false + needs: build-and-unit-test + name: Interop test + runs-on: ubuntu-latest - - name: Build (OpenSSL 1.1) - run: | - cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" - cmake --build "${{ env.CMAKE_BUILD_DIR }}" + env: + BUILD_DIR: "${RUNNER_TEMP}/build_openssl_1.1" + CRYPTO_DIR: "./alternatives/openssl_1.1" + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 - - name: Unit Test (OpenSSL 1.1) + - uses: ./.github/actions/build + with: + cache-dir: ${{ github.workspace }}/vcpkg_cache + + - name: Build run: | - cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target "${{ matrix.ctest-target}}" + cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" + cmake --build "${{ env.BUILD_DIR }}" - - name: Build (OpenSSL 3) + - name: Build (Interop Harness) run: | - cmake -B "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ matrix.ossl3-vcpkg-dir }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" - cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" + cd cmd/interop + cmake -B build + cmake --build build - - name: Unit Test (OpenSSL 3) + - name: Test self-interop run: | - cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" --target "${{ matrix.ctest-target}}" + make -C cmd/interop self-test - - name: Build (BoringSSL) + - name: Test interop on test vectors run: | - cmake -B "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ matrix.boringssl-vcpkg-dir }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" - cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" + make -C cmd/interop interop-test - - name: Unit Test (BoringSSL) + - name: Test gRPC live interop with self run: | - cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" --target "${{ matrix.ctest-target}}" + cd cmd/interop + ./grpc-self-test.sh - old-macos-compatibility: + clang-tidy: if: github.event.pull_request.draft == false - needs: quick-linux-interop-check - name: Build for older MacOS - runs-on: macos-latest + needs: build-and-unit-test + name: Build with clang-tidy + runs-on: ubuntu-latest + strategy: + matrix: + crypto: [openssl_1.1, openssl_3, boringssl] env: - CMAKE_BUILD_DIR: ${{ github.workspace }}/build - VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/build/cache,readwrite - MACOSX_DEPLOYMENT_TARGET: 10.11 + BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" + CRYPTO_DIR: "./alternatives/${{ matrix.crypto }}" steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: submodules: recursive fetch-depth: 0 - # write the commit hash of vcpkg to a text file so we can use it in the - # hashFiles for cache - - run: | - git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt - - # First, attempt to pull key key, if that is not present, pull one of the - # restore-keys so we do not need to build from scratch. - # CACHE_VERSION - provide a way to reset cache - # CACHE_NAME - name of the cache in order to manage it - # matrix.os - cache per OS and version - # hashFiles - Recache if the vcpkg files change - - name: Restore Cache - uses: actions/cache@v3 + - uses: ./.github/actions/build with: - path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest-legacy-${{ hashFiles('vcpkg_commit.txt', 'vcpkg.json', 'alternatives/openssl_3/vcpkg.json') }} - restore-keys: | - ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest-legacy - ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest + cache-dir: ${{ github.workspace }}/vcpkg_cache - - name: Dependencies + - name: Build with clang-tidy run: | - brew install llvm pkg-config - ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" - ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" - - - name: Build - run: | - cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" - cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target mlspp - + cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" \ + -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON + cmake --build "${{ env.BUILD_DIR }}" diff --git a/CMakeLists.txt b/CMakeLists.txt index c827512c..436f8e78 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.12) +cmake_minimum_required(VERSION 3.13) project(mlspp VERSION 0.1 @@ -44,6 +44,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU" add_compile_options(-Wall -pedantic -Wextra -Werror -Wmissing-declarations) elseif(MSVC) add_compile_options(/W4 /WX) + add_definitions(-DWINDOWS) # MSVC helpfully recommends safer equivalents for things like # getenv, but they are not portable. @@ -51,21 +52,16 @@ elseif(MSVC) endif() if (SANITIZERS) + message("Enabling sanitizers") + add_definitions(-DSANITIZERS) + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU") - set(SANITIZERS "-fsanitize=address -fsanitize=undefined") + add_compile_options(-fsanitize=address -fsanitize=undefined) + add_link_options(-fsanitize=address -fsanitize=undefined) elseif(MSVC) - set(SANITIZERS "/fsanitize=address") + # MSVC uses a different flag, and doesn't require passing it to the linker + add_compile_options("/fsanitize=address") endif() - - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SANITIZERS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SANITIZERS}") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${SANITIZERS}") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${SANITIZERS}") - set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${SANITIZERS}") - add_definitions(-DSANITIZERS) -elseif (SANITIZERS AND MSVC) - message("Enabling sanitizers") - add_definitions("/fsanitize=address") endif() if(CLANG_TIDY) @@ -96,6 +92,9 @@ endif() ### Dependencies ### +# Configure vcpkg to only build release libraries +set(VCPKG_BUILD_TYPE release) + # External libraries find_package(OpenSSL REQUIRED) if ( OPENSSL_FOUND ) diff --git a/alternatives/boringssl_1.1/vcpkg.json b/alternatives/boringssl/vcpkg.json similarity index 100% rename from alternatives/boringssl_1.1/vcpkg.json rename to alternatives/boringssl/vcpkg.json diff --git a/vcpkg.json b/alternatives/openssl_1.1/vcpkg.json similarity index 100% rename from vcpkg.json rename to alternatives/openssl_1.1/vcpkg.json diff --git a/lib/bytes/test/bytes.cpp b/lib/bytes/test/bytes.cpp index 38d3136c..39b1a909 100644 --- a/lib/bytes/test/bytes.cpp +++ b/lib/bytes/test/bytes.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -8,12 +9,14 @@ using namespace std::literals::string_literals; // To check that memory is safely zeroized on destroy, we have to deliberately // do a use-after-free. This will be caught by the sanitizers, so we only do it -// when sanitizers are not enabled. -#ifndef SANITIZERS +// when sanitizers are not enabled. This test is also disabled on Windows +// because it appears to cause Windows CI runs to fail. (In addition, Windows +// appears to overwrite freed buffers with 0xCD, so this test is unnecessary.) +#if !defined(SANITIZERS) && !defined(WINDOWS) TEST_CASE("Zeroization") { const auto size = size_t(1024); - const auto canary = uint8_t(0xff); + const auto canary = uint8_t(0xa0); auto vec = std::make_unique(size, canary); const auto* begin = vec->data(); @@ -21,15 +24,17 @@ TEST_CASE("Zeroization") const auto* end = begin + size; vec.reset(); + const auto snapshot = std::vector(begin, end); + std::cout << "snapshot = " << to_hex(snapshot) << std::endl; + // In principle, the memory previously owned by the vector should be all zero // at this point. However, since this is now unallocated memory, the // allocator can do with it what it wants, and may have written something to - // it when deallocating. For example, on macOS, the allocator appears to - // write a single pointer at the beginning. Assuming other platforms are not - // too different, we verify that no more than a few pointer's worth of bytes - // are non-zero. - const auto non_zero_threshold = 4 * sizeof(void*); - REQUIRE(std::count(begin, end, 0) >= size - non_zero_threshold); + // it when deallocating. macOS and Linux mostly leave the buffer alone, + // writing a couple of pointers to the beginning. So we look for the buffer + // to be basically all zero. + const auto threshold = size - 4 * sizeof(void*); + REQUIRE(std::count(snapshot.begin(), snapshot.end(), 0) >= threshold); } #endif