From f08ddb9553ab568b8c84a396b44d74de99d5a223 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 11:07:55 -0400 Subject: [PATCH 01/68] Add BoringSSL build to main CI --- .github/workflows/boring.yml | 93 ----------------------------------- .github/workflows/main_ci.yml | 32 +++++++++--- 2 files changed, 26 insertions(+), 99 deletions(-) delete mode 100644 .github/workflows/boring.yml diff --git a/.github/workflows/boring.yml b/.github/workflows/boring.yml deleted file mode 100644 index 72dee4b0..00000000 --- a/.github/workflows/boring.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: MLSPP CI (BoringSSL Test) - -on: - push: - branches: - - main - - boring - pull_request: - branches: - - main - -env: - CTEST_OUTPUT_ON_FAILURE: 1 - CMAKE_BUILD_BORINGSSL_DIR: ${{ github.workspace }}/build_boringssl - CMAKE_TEST_BORINGSSL_DIR: ${{ github.workspace }}/build_boringssl/test - VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/build/cache,readwrite - VCPKG_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - -jobs: - formatting-check: - name: Formatting Check - runs-on: ubuntu-latest - strategy: - matrix: - path: - - 'include' - - 'src' - - 'test' - - 'cmd' - - 'lib' - steps: - - uses: actions/checkout@v4 - - - name: Run clang-format style check for C/C++ programs - uses: jidicula/clang-format-action@v4.11.0 - with: - clang-format-version: '16' - check-path: ${{ matrix.path }} - fallback-style: 'Mozilla' - - platform-sanitizer-tests: - name: Build and test platforms using sanitizers and clang-tidy - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [windows-latest, ubuntu-latest, macos-latest] - include: - - os: windows-latest - boring-vcpkg-dir: "alternatives\\boringssl_1.1" - ctest-target: RUN_TESTS - - os: ubuntu-latest - boring-vcpkg-dir: "alternatives/boringssl_1.1" - ctest-target: test - - os: macos-latest - boring-vcpkg-dir: "alternatives/boringssl_1.1" - ctest-target: test - - steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - - name: Dependencies (macOs) - if: ${{ matrix.os == 'macos-latest' }} - run: | - brew install llvm pkg-config nasm - ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" - ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" - - - name: Dependencies (Ubuntu) - if: ${{ matrix.os == 'ubuntu-latest' }} - run: | - sudo apt-get install -y linux-headers-$(uname -r) nasm - - - name: Restore cache - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/build/cache - key: VCPKG-BinaryCache-${{ runner.os }} - - - name: Build (BoringSSL 1.1) - run: | - cmake -B "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ matrix.boring-vcpkg-dir }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" -DREQUIRE_BORINGSSL=1 - cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" - - - name: Unit Test (BoringSSL 1.1) - run: | - cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" --target "${{ matrix.ctest-target}}" - - diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 1e6ef406..be5053e7 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -13,8 +13,7 @@ env: CTEST_OUTPUT_ON_FAILURE: 1 CMAKE_BUILD_DIR: ${{ github.workspace }}/build CMAKE_BUILD_OPENSSL3_DIR: ${{ github.workspace }}/build_openssl3 - CMAKE_TEST_OPENSSL3_DIR: ${{ github.workspace }}/build_openssl3/test - CMAKE_TEST_DIR: ${{ github.workspace }}/build/test + CMAKE_BUILD_BORINGSSL_DIR: ${{ github.workspace }}/build_boringssl VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/build/cache,readwrite VCPKG_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake VCPKG_REPO: ${{ github.workspace }}/vcpkg @@ -120,6 +119,15 @@ jobs: - name: Unit Test (OpenSSL 3) run: | cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" --target test + + - name: Build (BoringSSL) + run: | + cmake -B "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/boringssl_1.1" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" + cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" + + - name: Unit Test (BoringSSL) + run: | + cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" --target test platform-sanitizer-tests: if: github.event.pull_request.draft == false @@ -133,12 +141,15 @@ jobs: include: - os: windows-latest ossl3-vcpkg-dir: "alternatives\\openssl_3" + boringssl-vcpkg-dir: "alternatives\\boringssl_1.1" ctest-target: RUN_TESTS - os: ubuntu-latest ossl3-vcpkg-dir: "alternatives/openssl_3" + boringssl-vcpkg-dir: "alternatives/boringssl_1.1" ctest-target: test - os: macos-latest ossl3-vcpkg-dir: "alternatives/openssl_3" + boringssl-vcpkg-dir: "alternatives/boringssl_1.1" ctest-target: test steps: @@ -179,24 +190,33 @@ jobs: run: | sudo apt-get install -y linux-headers-$(uname -r) - - name: Build (OpenSSL1.1) + - name: Build (OpenSSL 1.1) run: | cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" cmake --build "${{ env.CMAKE_BUILD_DIR }}" - - name: Unit Test (OpenSSL1.1) + - name: Unit Test (OpenSSL 1.1) run: | cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target "${{ matrix.ctest-target}}" - - name: Build (OpenSSL3) + - name: Build (OpenSSL 3) run: | cmake -B "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ matrix.ossl3-vcpkg-dir }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" - - name: Unit Test (OpenSSL3) + - name: Unit Test (OpenSSL 3) run: | cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" --target "${{ matrix.ctest-target}}" + - name: Build (BoringSSL) + run: | + cmake -B "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ matrix.boringssl-vcpkg-dir }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" + cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" + + - name: Unit Test (OpenSSL 3) + run: | + cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" --target "${{ matrix.ctest-target}}" + old-macos-compatibility: if: github.event.pull_request.draft == false needs: quick-linux-interop-check From 6206cc663e45bcd9bb22939179b9b8f90b552a89 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 11:26:51 -0400 Subject: [PATCH 02/68] Add nasm dependency --- .github/workflows/main_ci.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index be5053e7..0b6d8a14 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -72,10 +72,9 @@ jobs: restore-keys: | ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-ubuntu-latest - - name: Dependencies (Ubuntu) - if: ${{ matrix.os == 'ubuntu-latest' }} + - name: Dependencies run: | - sudo apt-get install -y linux-headers-$(uname -r) + sudo apt-get install -y linux-headers-$(uname -r) nasm - name: Restore cache uses: actions/cache@v3 @@ -181,14 +180,14 @@ jobs: - name: Dependencies (macOs) if: ${{ matrix.os == 'macos-latest' }} run: | - brew install llvm pkg-config + brew install llvm pkg-config nasm ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" - name: Dependencies (Ubuntu) if: ${{ matrix.os == 'ubuntu-latest' }} run: | - sudo apt-get install -y linux-headers-$(uname -r) + sudo apt-get install -y linux-headers-$(uname -r) nasm - name: Build (OpenSSL 1.1) run: | From e73ebb2a93b941f09eb9e489184154bf3cbba741 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 14:28:09 -0400 Subject: [PATCH 03/68] Fix typos --- .github/workflows/main_ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index ba7e1a3f..2efa2f4d 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -204,7 +204,7 @@ jobs: cmake -B "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ matrix.boringssl-vcpkg-dir }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" - - name: Unit Test (OpenSSL 3) + - name: Unit Test (BoringSSL) run: | cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" --target "${{ matrix.ctest-target}}" @@ -246,7 +246,7 @@ jobs: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest-legacy ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest - - name: dependencies + - name: Dependencies run: | brew install llvm pkg-config ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" From 6c0471f53f251ed2640accae88afe1deb507ffb4 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 15:55:47 -0400 Subject: [PATCH 04/68] CI Cleanup --- .github/workflows/main_ci.yml | 66 +++++++++---------- Makefile | 2 + .../{boringssl_1.1 => boringssl}/vcpkg.json | 0 .../openssl_1.1/vcpkg.json | 0 4 files changed, 35 insertions(+), 33 deletions(-) rename alternatives/{boringssl_1.1 => boringssl}/vcpkg.json (100%) rename vcpkg.json => alternatives/openssl_1.1/vcpkg.json (100%) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 2efa2f4d..c394b402 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -9,17 +9,23 @@ on: - main env: + CMAKE_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake CMAKE_BUILD_PARALLEL_LEVEL: 3 CTEST_OUTPUT_ON_FAILURE: 1 - CMAKE_BUILD_DIR: ${{ github.workspace }}/build - CMAKE_BUILD_OPENSSL3_DIR: ${{ github.workspace }}/build_openssl3 - CMAKE_BUILD_BORINGSSL_DIR: ${{ github.workspace }}/build_boringssl + + BUILD_DIR: ${{ github.workspace }}/build + BUILD_DIR_OPENSSL3: ${{ github.workspace }}/build_openssl3 + BUILD_DIR_BORINGSSL: ${{ github.workspace }}/build_boringssl + VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/build/cache,readwrite - VCPKG_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake VCPKG_REPO: ${{ github.workspace }}/vcpkg CACHE_VERSION: v01 CACHE_NAME: vcpkg + OPENSSL_11_VCPKG: alternatives/openssl_1.1 + OPENSSL_3_VCPKG: alternatives/openssl_3 + BORINGSSL_VCPKG: alternatives/boringssl + jobs: formatting-check: name: Formatting Check @@ -76,17 +82,17 @@ jobs: - name: Build (OpenSSL 1.1) run: | - cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" - cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target all + cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="${{ env.OPENSSL_11_VCPKG }}" + cmake --build "${{ env.BUILD_DIR }}" --target all - name: Unit Test (OpenSSL 1.1) run: | - cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target test + cmake --build "${{ env.BUILD_DIR }}" --target test - name: Build (Interop Harness) run: | cd cmd/interop - cmake -B build -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" + cmake -B build cmake --build build - name: Test self-interop @@ -104,21 +110,21 @@ jobs: - name: Build (OpenSSL 3) run: | - cmake -B "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/openssl_3" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" - cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" + cmake -B "${{ env.BUILD_DIR_OPENSSL3 }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="${{ env.OPENSSL_3_VCPKG }}" + cmake --build "${{ env.BUILD_DIR_OPENSSL3 }}" - name: Unit Test (OpenSSL 3) run: | - cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" --target test + cmake --build "${{ env.BUILD_DIR_OPENSSL3 }}" --target test - name: Build (BoringSSL) run: | - cmake -B "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/boringssl_1.1" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" - cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" + cmake -B "${{ env.BUILD_DIR_BORINGSSL }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="${{ env.BORINGSSL_VCPKG }}" + cmake --build "${{ env.BUILD_DIR_BORINGSSL }}" - name: Unit Test (BoringSSL) run: | - cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" --target test + cmake --build "${{ env.BUILD_DIR_BORINGSSL }}" --target test platform-sanitizer-tests: if: github.event.pull_request.draft == false @@ -131,16 +137,10 @@ jobs: os: [windows-latest, ubuntu-latest, macos-latest] include: - os: windows-latest - ossl3-vcpkg-dir: "alternatives\\openssl_3" - boringssl-vcpkg-dir: "alternatives\\boringssl_1.1" ctest-target: RUN_TESTS - os: ubuntu-latest - ossl3-vcpkg-dir: "alternatives/openssl_3" - boringssl-vcpkg-dir: "alternatives/boringssl_1.1" ctest-target: test - os: macos-latest - ossl3-vcpkg-dir: "alternatives/openssl_3" - boringssl-vcpkg-dir: "alternatives/boringssl_1.1" ctest-target: test steps: @@ -165,7 +165,7 @@ jobs: uses: actions/cache@v3 with: path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'vcpkg.json', 'alternatives/openssl_3/vcpkg.json') }} + key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/**/vcpkg.json') }} restore-keys: | ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} @@ -183,30 +183,30 @@ jobs: - name: Build (OpenSSL 1.1) run: | - cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" - cmake --build "${{ env.CMAKE_BUILD_DIR }}" + cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ env.OPENSSL_11_VCPKG }}" + cmake --build "${{ env.BUILD_DIR }}" - name: Unit Test (OpenSSL 1.1) run: | - cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target "${{ matrix.ctest-target}}" + cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" - name: Build (OpenSSL 3) run: | - cmake -B "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ matrix.ossl3-vcpkg-dir }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" - cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" + cmake -B "${{ env.BUILD_DIR_OPENSSL3 }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ env.OPENSSL_3_VCPKG }}" + cmake --build "${{ env.BUILD_DIR_OPENSSL3 }}" - name: Unit Test (OpenSSL 3) run: | - cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" --target "${{ matrix.ctest-target}}" + cmake --build "${{ env.BUILD_DIR_OPENSSL3 }}" --target "${{ matrix.ctest-target}}" - name: Build (BoringSSL) run: | - cmake -B "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ matrix.boringssl-vcpkg-dir }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" - cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" + cmake -B "${{ env.BUILD_DIR_BORINGSSL }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ env.OPENSSL_3_VCPKG }}" + cmake --build "${{ env.BUILD_DIR_BORINGSSL }}" - name: Unit Test (BoringSSL) run: | - cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" --target "${{ matrix.ctest-target}}" + cmake --build "${{ env.BUILD_DIR_BORINGSSL }}" --target "${{ matrix.ctest-target}}" old-macos-compatibility: if: github.event.pull_request.draft == false @@ -215,7 +215,7 @@ jobs: runs-on: macos-latest env: - CMAKE_BUILD_DIR: ${{ github.workspace }}/build + BUILD_DIR: ${{ github.workspace }}/build VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/build/cache,readwrite MACOSX_DEPLOYMENT_TARGET: 10.11 @@ -254,6 +254,6 @@ jobs: - name: Build run: | - cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" - cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target mlspp + cmake -B "${{ env.BUILD_DIR }}" + cmake --build "${{ env.BUILD_DIR }}" --target mlspp diff --git a/Makefile b/Makefile index 03514fb6..8aaaecc2 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ BUILD_DIR=build TEST_DIR=build/test CLANG_FORMAT=clang-format -i CLANG_TIDY=OFF +OPENSSL11_MANIFEST=alternatives/openssl_1.1 OPENSSL3_MANIFEST=alternatives/openssl_3 TOOLCHAIN_FILE=vcpkg/scripts/buildsystems/vcpkg.cmake @@ -26,6 +27,7 @@ dev: ${TOOLCHAIN_FILE} # Only enable testing, not clang-tidy/sanitizers; the latter make the build # too slow, and we can run them in CI cmake -B${BUILD_DIR} -DTESTING=ON -DCMAKE_BUILD_TYPE=Debug \ + -DVCPKG_MANIFEST_DIR=${OPENSSL11_MANIFEST} \ -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} dev3: ${TOOLCHAIN_FILE} diff --git a/alternatives/boringssl_1.1/vcpkg.json b/alternatives/boringssl/vcpkg.json similarity index 100% rename from alternatives/boringssl_1.1/vcpkg.json rename to alternatives/boringssl/vcpkg.json diff --git a/vcpkg.json b/alternatives/openssl_1.1/vcpkg.json similarity index 100% rename from vcpkg.json rename to alternatives/openssl_1.1/vcpkg.json From 3e42187bae795684cc996915523de3189878d53b Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 15:57:47 -0400 Subject: [PATCH 05/68] Make cache key consistent --- .github/workflows/main_ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index c394b402..fcc53262 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -66,7 +66,7 @@ jobs: uses: actions/cache@v3 with: path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-ubuntu-latest-${{ hashFiles('vcpkg_commit.txt', 'vcpkg.json', 'alternatives/openssl_3/vcpkg.json') }} + key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-ubuntu-latest-${{ hashFiles('vcpkg_commit.txt', 'alternatives/**/vcpkg.json') }} restore-keys: | ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-ubuntu-latest @@ -241,7 +241,7 @@ jobs: uses: actions/cache@v3 with: path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest-legacy-${{ hashFiles('vcpkg_commit.txt', 'vcpkg.json', 'alternatives/openssl_3/vcpkg.json') }} + key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest-legacy-${{ hashFiles('vcpkg_commit.txt', 'alternatives/**/vcpkg.json') }} restore-keys: | ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest-legacy ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest From cf428c70ea79faaf4aa5a2a46401e6be4ab9f13a Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 16:25:23 -0400 Subject: [PATCH 06/68] Make a matrix of non-clang-tidy builds --- .github/workflows/main_ci.yml | 233 ++++++++-------------------------- 1 file changed, 51 insertions(+), 182 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index fcc53262..e4d27238 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -13,128 +13,33 @@ env: CMAKE_BUILD_PARALLEL_LEVEL: 3 CTEST_OUTPUT_ON_FAILURE: 1 - BUILD_DIR: ${{ github.workspace }}/build - BUILD_DIR_OPENSSL3: ${{ github.workspace }}/build_openssl3 - BUILD_DIR_BORINGSSL: ${{ github.workspace }}/build_boringssl - - VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/build/cache,readwrite + VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/vcpkg_cache,readwrite VCPKG_REPO: ${{ github.workspace }}/vcpkg CACHE_VERSION: v01 CACHE_NAME: vcpkg - OPENSSL_11_VCPKG: alternatives/openssl_1.1 - OPENSSL_3_VCPKG: alternatives/openssl_3 - BORINGSSL_VCPKG: alternatives/boringssl - jobs: - formatting-check: - name: Formatting Check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Run clang-format style check for C/C++ programs - uses: jidicula/clang-format-action@v4.11.0 - with: - clang-format-version: 16 - include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' - fallback-style: 'Mozilla' - - quick-linux-interop-check: - needs: formatting-check - name: Quick Linux Check and Interop - runs-on: ubuntu-latest - steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - # write the commit hash of vcpkg to a text file so we can use it in the - # hashFiles for cache - - run: | - git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt - - # First, attempt to pull key key, if that is not present, pull one of the - # restore-keys so we do not need to build from scratch. - # CACHE_VERSION - provide a way to reset cache - # CACHE_NAME - name of the cache in order to manage it - # matrix.os - cache per OS and version - # hashFiles - Recache if the vcpkg files change - - name: Restore Cache - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-ubuntu-latest-${{ hashFiles('vcpkg_commit.txt', 'alternatives/**/vcpkg.json') }} - restore-keys: | - ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-ubuntu-latest - - - name: Dependencies - run: | - sudo apt-get install -y linux-headers-$(uname -r) nasm - - - name: Restore cache - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/build/cache - key: VCPKG-BinaryCache-${{ runner.os }} - - - name: Build (OpenSSL 1.1) - run: | - cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="${{ env.OPENSSL_11_VCPKG }}" - cmake --build "${{ env.BUILD_DIR }}" --target all - - - name: Unit Test (OpenSSL 1.1) - run: | - cmake --build "${{ env.BUILD_DIR }}" --target test - - - name: Build (Interop Harness) - run: | - cd cmd/interop - cmake -B build - cmake --build build - - - name: Test self-interop - run: | - make -C cmd/interop self-test - - - name: Test interop on test vectors - run: | - make -C cmd/interop interop-test - - - name: Test gRPC live interop with self - run: | - cd cmd/interop - ./grpc-self-test.sh - - - name: Build (OpenSSL 3) - run: | - cmake -B "${{ env.BUILD_DIR_OPENSSL3 }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="${{ env.OPENSSL_3_VCPKG }}" - cmake --build "${{ env.BUILD_DIR_OPENSSL3 }}" - - - name: Unit Test (OpenSSL 3) - run: | - cmake --build "${{ env.BUILD_DIR_OPENSSL3 }}" --target test - - - name: Build (BoringSSL) - run: | - cmake -B "${{ env.BUILD_DIR_BORINGSSL }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="${{ env.BORINGSSL_VCPKG }}" - cmake --build "${{ env.BUILD_DIR_BORINGSSL }}" - - - name: Unit Test (BoringSSL) - run: | - cmake --build "${{ env.BUILD_DIR_BORINGSSL }}" --target test - - platform-sanitizer-tests: - if: github.event.pull_request.draft == false - needs: quick-linux-interop-check - name: Build and test platforms using sanitizers and clang-tidy +# formatting-check: +# name: Formatting Check +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# +# - name: Run clang-format style check for C/C++ programs +# uses: jidicula/clang-format-action@v4.11.0 +# with: +# clang-format-version: 16 +# include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' +# fallback-style: 'Mozilla' + + build-and-test: +# needs: formatting-check + name: Build and Test runs-on: ${{ matrix.os }} strategy: - fail-fast: false matrix: os: [windows-latest, ubuntu-latest, macos-latest] + crypto: [openssl_1.1, openssl_3, boringssl] include: - os: windows-latest ctest-target: RUN_TESTS @@ -142,6 +47,13 @@ jobs: ctest-target: test - os: macos-latest ctest-target: test + - os: macos-latest + ctest-target: test + macosx-deployment_target: 10.11 + + env: + BUILD_DIR: ${{ env.RUNNER_TEMP }}/build_${{ matrix.crypto }} + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macosx-deployment_target }} steps: - name: Checkout repository and submodules @@ -150,26 +62,18 @@ jobs: submodules: recursive fetch-depth: 0 - # write the commit hash of vcpkg to a text file so we can use it in the - # hashFiles for cache - run: | git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt - # First, attempt to pull key key, if that is not present, pull one of the - # restore-keys so we do not need to build from scratch. - # CACHE_VERSION - provide a way to reset cache - # CACHE_NAME - name of the cache in order to manage it - # matrix.os - cache per OS and version - # hashFiles - Recache if the vcpkg files change - name: Restore Cache uses: actions/cache@v3 with: path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/**/vcpkg.json') }} + key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/**/vcpkg.json') }}-${{ matrix.macosx-deployment_target }} restore-keys: | ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} - - name: Dependencies (macOs) + - name: Dependencies (macOS) if: ${{ matrix.os == 'macos-latest' }} run: | brew install llvm pkg-config nasm @@ -181,79 +85,44 @@ jobs: run: | sudo apt-get install -y linux-headers-$(uname -r) nasm - - name: Build (OpenSSL 1.1) + - name: Build run: | - cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ env.OPENSSL_11_VCPKG }}" + cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" cmake --build "${{ env.BUILD_DIR }}" - - name: Unit Test (OpenSSL 1.1) + - name: Unit Tests run: | cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" - - name: Build (OpenSSL 3) + - name: Build (Interop Harness) run: | - cmake -B "${{ env.BUILD_DIR_OPENSSL3 }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ env.OPENSSL_3_VCPKG }}" - cmake --build "${{ env.BUILD_DIR_OPENSSL3 }}" + cd cmd/interop + cmake -B build + cmake --build build - - name: Unit Test (OpenSSL 3) + - name: Test self-interop run: | - cmake --build "${{ env.BUILD_DIR_OPENSSL3 }}" --target "${{ matrix.ctest-target}}" + make -C cmd/interop self-test - - name: Build (BoringSSL) + - name: Test interop on test vectors run: | - cmake -B "${{ env.BUILD_DIR_BORINGSSL }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ env.OPENSSL_3_VCPKG }}" - cmake --build "${{ env.BUILD_DIR_BORINGSSL }}" + make -C cmd/interop interop-test - - name: Unit Test (BoringSSL) + - name: Test gRPC live interop with self run: | - cmake --build "${{ env.BUILD_DIR_BORINGSSL }}" --target "${{ matrix.ctest-target}}" + cd cmd/interop + ./grpc-self-test.sh - old-macos-compatibility: + clang-tidy: if: github.event.pull_request.draft == false - needs: quick-linux-interop-check - name: Build for older MacOS - runs-on: macos-latest - - env: - BUILD_DIR: ${{ github.workspace }}/build - VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/build/cache,readwrite - MACOSX_DEPLOYMENT_TARGET: 10.11 + needs: build-and-test + name: Build with clang-tidy + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + crypto: [openssl_1.1, openssl_3, boringssl] + ctest-target: test steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - # write the commit hash of vcpkg to a text file so we can use it in the - # hashFiles for cache - - run: | - git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt - - # First, attempt to pull key key, if that is not present, pull one of the - # restore-keys so we do not need to build from scratch. - # CACHE_VERSION - provide a way to reset cache - # CACHE_NAME - name of the cache in order to manage it - # matrix.os - cache per OS and version - # hashFiles - Recache if the vcpkg files change - - name: Restore Cache - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest-legacy-${{ hashFiles('vcpkg_commit.txt', 'alternatives/**/vcpkg.json') }} - restore-keys: | - ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest-legacy - ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest - - - name: Dependencies - run: | - brew install llvm pkg-config - ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" - ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" - - - name: Build - run: | - cmake -B "${{ env.BUILD_DIR }}" - cmake --build "${{ env.BUILD_DIR }}" --target mlspp - + # TODO Build with clang-tidy From eba1ee4c5faa4503ae024743a7671f77cc7463c4 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 16:28:37 -0400 Subject: [PATCH 07/68] Fix validity issues in CI YAML --- .github/workflows/main_ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index e4d27238..5342a722 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -122,7 +122,13 @@ jobs: matrix: os: [ubuntu-latest] crypto: [openssl_1.1, openssl_3, boringssl] - ctest-target: test + ctest-target: [test] steps: # TODO Build with clang-tidy + - name: Checkout repository and submodules + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + From 4be26c8809dd6df4a0bec96b54333f74f72d4276 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 16:31:25 -0400 Subject: [PATCH 08/68] Comment out most of the workflow --- .github/workflows/main_ci.yml | 139 +++++++++++++++++----------------- 1 file changed, 69 insertions(+), 70 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 5342a722..efadeb34 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -62,73 +62,72 @@ jobs: submodules: recursive fetch-depth: 0 - - run: | - git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt - - - name: Restore Cache - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/**/vcpkg.json') }}-${{ matrix.macosx-deployment_target }} - restore-keys: | - ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} - - - name: Dependencies (macOS) - if: ${{ matrix.os == 'macos-latest' }} - run: | - brew install llvm pkg-config nasm - ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" - ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" - - - name: Dependencies (Ubuntu) - if: ${{ matrix.os == 'ubuntu-latest' }} - run: | - sudo apt-get install -y linux-headers-$(uname -r) nasm - - - name: Build - run: | - cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" - cmake --build "${{ env.BUILD_DIR }}" - - - name: Unit Tests - run: | - cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" - - - name: Build (Interop Harness) - run: | - cd cmd/interop - cmake -B build - cmake --build build - - - name: Test self-interop - run: | - make -C cmd/interop self-test - - - name: Test interop on test vectors - run: | - make -C cmd/interop interop-test - - - name: Test gRPC live interop with self - run: | - cd cmd/interop - ./grpc-self-test.sh - - clang-tidy: - if: github.event.pull_request.draft == false - needs: build-and-test - name: Build with clang-tidy - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - crypto: [openssl_1.1, openssl_3, boringssl] - ctest-target: [test] - - steps: - # TODO Build with clang-tidy - - name: Checkout repository and submodules - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - +# - run: | +# git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt +# +# - name: Restore Cache +# uses: actions/cache@v3 +# with: +# path: ${{ github.workspace }}/build/cache +# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/**/vcpkg.json') }}-${{ matrix.macosx-deployment_target }} +# restore-keys: | +# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} +# +# - name: Dependencies (macOS) +# if: ${{ matrix.os == 'macos-latest' }} +# run: | +# brew install llvm pkg-config nasm +# ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" +# ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" +# +# - name: Dependencies (Ubuntu) +# if: ${{ matrix.os == 'ubuntu-latest' }} +# run: | +# sudo apt-get install -y linux-headers-$(uname -r) nasm +# +# - name: Build +# run: | +# cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" +# cmake --build "${{ env.BUILD_DIR }}" +# +# - name: Unit Tests +# run: | +# cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" +# +# - name: Build (Interop Harness) +# run: | +# cd cmd/interop +# cmake -B build +# cmake --build build +# +# - name: Test self-interop +# run: | +# make -C cmd/interop self-test +# +# - name: Test interop on test vectors +# run: | +# make -C cmd/interop interop-test +# +# - name: Test gRPC live interop with self +# run: | +# cd cmd/interop +# ./grpc-self-test.sh +# +# clang-tidy: +# if: github.event.pull_request.draft == false +# needs: build-and-test +# name: Build with clang-tidy +# runs-on: ${{ matrix.os }} +# strategy: +# matrix: +# os: [ubuntu-latest] +# crypto: [openssl_1.1, openssl_3, boringssl] +# ctest-target: [test] +# +# steps: +# # TODO Build with clang-tidy +# - name: Checkout repository and submodules +# uses: actions/checkout@v4 +# with: +# submodules: recursive +# fetch-depth: 0 From 02c0ea3c47914157e9485c57d88e85a8a5c0bfa5 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 16:34:22 -0400 Subject: [PATCH 09/68] Reset to an earlier commit and comment things out --- .github/workflows/main_ci.yml | 245 +++++++++++++++++++++++++--------- 1 file changed, 185 insertions(+), 60 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index efadeb34..dc129367 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -9,11 +9,13 @@ on: - main env: - CMAKE_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake CMAKE_BUILD_PARALLEL_LEVEL: 3 CTEST_OUTPUT_ON_FAILURE: 1 - - VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/vcpkg_cache,readwrite + CMAKE_BUILD_DIR: ${{ github.workspace }}/build + CMAKE_BUILD_OPENSSL3_DIR: ${{ github.workspace }}/build_openssl3 + CMAKE_BUILD_BORINGSSL_DIR: ${{ github.workspace }}/build_boringssl + VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/build/cache,readwrite + VCPKG_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake VCPKG_REPO: ${{ github.workspace }}/vcpkg CACHE_VERSION: v01 CACHE_NAME: vcpkg @@ -31,73 +33,60 @@ jobs: # clang-format-version: 16 # include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' # fallback-style: 'Mozilla' - - build-and-test: +# +# quick-linux-interop-check: # needs: formatting-check - name: Build and Test - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest, ubuntu-latest, macos-latest] - crypto: [openssl_1.1, openssl_3, boringssl] - include: - - os: windows-latest - ctest-target: RUN_TESTS - - os: ubuntu-latest - ctest-target: test - - os: macos-latest - ctest-target: test - - os: macos-latest - ctest-target: test - macosx-deployment_target: 10.11 - - env: - BUILD_DIR: ${{ env.RUNNER_TEMP }}/build_${{ matrix.crypto }} - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macosx-deployment_target }} - - steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - +# name: Quick Linux Check and Interop +# runs-on: ubuntu-latest +# steps: +# - name: Checkout repository and submodules +# uses: actions/checkout@v4 +# with: +# submodules: recursive +# fetch-depth: 0 +# +# # write the commit hash of vcpkg to a text file so we can use it in the +# # hashFiles for cache # - run: | # git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt # +# # First, attempt to pull key key, if that is not present, pull one of the +# # restore-keys so we do not need to build from scratch. +# # CACHE_VERSION - provide a way to reset cache +# # CACHE_NAME - name of the cache in order to manage it +# # matrix.os - cache per OS and version +# # hashFiles - Recache if the vcpkg files change # - name: Restore Cache # uses: actions/cache@v3 # with: # path: ${{ github.workspace }}/build/cache -# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/**/vcpkg.json') }}-${{ matrix.macosx-deployment_target }} +# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-ubuntu-latest-${{ hashFiles('vcpkg_commit.txt', 'vcpkg.json', 'alternatives/openssl_3/vcpkg.json') }} # restore-keys: | -# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} -# -# - name: Dependencies (macOS) -# if: ${{ matrix.os == 'macos-latest' }} -# run: | -# brew install llvm pkg-config nasm -# ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" -# ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" +# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-ubuntu-latest # -# - name: Dependencies (Ubuntu) -# if: ${{ matrix.os == 'ubuntu-latest' }} +# - name: Dependencies # run: | # sudo apt-get install -y linux-headers-$(uname -r) nasm # -# - name: Build +# - name: Restore cache +# uses: actions/cache@v3 +# with: +# path: ${{ github.workspace }}/build/cache +# key: VCPKG-BinaryCache-${{ runner.os }} +# +# - name: Build (OpenSSL 1.1) # run: | -# cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" -# cmake --build "${{ env.BUILD_DIR }}" +# cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" +# cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target all # -# - name: Unit Tests +# - name: Unit Test (OpenSSL 1.1) # run: | -# cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" +# cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target test # # - name: Build (Interop Harness) # run: | # cd cmd/interop -# cmake -B build +# cmake -B build -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" # cmake --build build # # - name: Test self-interop @@ -112,22 +101,158 @@ jobs: # run: | # cd cmd/interop # ./grpc-self-test.sh +# +# - name: Build (OpenSSL 3) +# run: | +# cmake -B "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/openssl_3" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" +# cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" +# +# - name: Unit Test (OpenSSL 3) +# run: | +# cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" --target test +# +# - name: Build (BoringSSL) +# run: | +# cmake -B "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/boringssl_1.1" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" +# cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" # -# clang-tidy: +# - name: Unit Test (BoringSSL) +# run: | +# cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" --target test + + platform-sanitizer-tests: + if: github.event.pull_request.draft == false +# needs: quick-linux-interop-check + name: Build and test platforms using sanitizers and clang-tidy + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + include: + - os: windows-latest + ossl3-vcpkg-dir: "alternatives\\openssl_3" + boringssl-vcpkg-dir: "alternatives\\boringssl_1.1" + ctest-target: RUN_TESTS + - os: ubuntu-latest + ossl3-vcpkg-dir: "alternatives/openssl_3" + boringssl-vcpkg-dir: "alternatives/boringssl_1.1" + ctest-target: test + - os: macos-latest + ossl3-vcpkg-dir: "alternatives/openssl_3" + boringssl-vcpkg-dir: "alternatives/boringssl_1.1" + ctest-target: test + + steps: + - name: Checkout repository and submodules + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + # write the commit hash of vcpkg to a text file so we can use it in the + # hashFiles for cache + - run: | + git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt + + # First, attempt to pull key key, if that is not present, pull one of the + # restore-keys so we do not need to build from scratch. + # CACHE_VERSION - provide a way to reset cache + # CACHE_NAME - name of the cache in order to manage it + # matrix.os - cache per OS and version + # hashFiles - Recache if the vcpkg files change + - name: Restore Cache + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/build/cache + key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'vcpkg.json', 'alternatives/openssl_3/vcpkg.json') }} + restore-keys: | + ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} + + - name: Dependencies (macOs) + if: ${{ matrix.os == 'macos-latest' }} + run: | + brew install llvm pkg-config nasm + ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" + ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" + + - name: Dependencies (Ubuntu) + if: ${{ matrix.os == 'ubuntu-latest' }} + run: | + sudo apt-get install -y linux-headers-$(uname -r) nasm + + - name: Build (OpenSSL 1.1) + run: | + cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" + cmake --build "${{ env.CMAKE_BUILD_DIR }}" + + - name: Unit Test (OpenSSL 1.1) + run: | + cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target "${{ matrix.ctest-target}}" + + - name: Build (OpenSSL 3) + run: | + cmake -B "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ matrix.ossl3-vcpkg-dir }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" + cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" + + - name: Unit Test (OpenSSL 3) + run: | + cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" --target "${{ matrix.ctest-target}}" + + - name: Build (BoringSSL) + run: | + cmake -B "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ matrix.boringssl-vcpkg-dir }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" + cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" + + - name: Unit Test (BoringSSL) + run: | + cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" --target "${{ matrix.ctest-target}}" + +# old-macos-compatibility: # if: github.event.pull_request.draft == false -# needs: build-and-test -# name: Build with clang-tidy -# runs-on: ${{ matrix.os }} -# strategy: -# matrix: -# os: [ubuntu-latest] -# crypto: [openssl_1.1, openssl_3, boringssl] -# ctest-target: [test] +# needs: quick-linux-interop-check +# name: Build for older MacOS +# runs-on: macos-latest +# +# env: +# CMAKE_BUILD_DIR: ${{ github.workspace }}/build +# VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/build/cache,readwrite +# MACOSX_DEPLOYMENT_TARGET: 10.11 # # steps: -# # TODO Build with clang-tidy # - name: Checkout repository and submodules # uses: actions/checkout@v4 # with: # submodules: recursive # fetch-depth: 0 +# +# # write the commit hash of vcpkg to a text file so we can use it in the +# # hashFiles for cache +# - run: | +# git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt +# +# # First, attempt to pull key key, if that is not present, pull one of the +# # restore-keys so we do not need to build from scratch. +# # CACHE_VERSION - provide a way to reset cache +# # CACHE_NAME - name of the cache in order to manage it +# # matrix.os - cache per OS and version +# # hashFiles - Recache if the vcpkg files change +# - name: Restore Cache +# uses: actions/cache@v3 +# with: +# path: ${{ github.workspace }}/build/cache +# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest-legacy-${{ hashFiles('vcpkg_commit.txt', 'vcpkg.json', 'alternatives/openssl_3/vcpkg.json') }} +# restore-keys: | +# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest-legacy +# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest +# +# - name: Dependencies +# run: | +# brew install llvm pkg-config +# ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" +# ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" +# +# - name: Build +# run: | +# cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" +# cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target mlspp From cd71e798f12b110c3ae5727d057d006bc6ed0632 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 16:39:38 -0400 Subject: [PATCH 10/68] Start preparing main matrix --- .github/workflows/main_ci.yml | 165 ++-------------------------------- 1 file changed, 9 insertions(+), 156 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index dc129367..56b1a038 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -15,7 +15,7 @@ env: CMAKE_BUILD_OPENSSL3_DIR: ${{ github.workspace }}/build_openssl3 CMAKE_BUILD_BORINGSSL_DIR: ${{ github.workspace }}/build_boringssl VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/build/cache,readwrite - VCPKG_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake + CMAKE_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake VCPKG_REPO: ${{ github.workspace }}/vcpkg CACHE_VERSION: v01 CACHE_NAME: vcpkg @@ -34,96 +34,9 @@ jobs: # include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' # fallback-style: 'Mozilla' # -# quick-linux-interop-check: -# needs: formatting-check -# name: Quick Linux Check and Interop -# runs-on: ubuntu-latest -# steps: -# - name: Checkout repository and submodules -# uses: actions/checkout@v4 -# with: -# submodules: recursive -# fetch-depth: 0 -# -# # write the commit hash of vcpkg to a text file so we can use it in the -# # hashFiles for cache -# - run: | -# git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt -# -# # First, attempt to pull key key, if that is not present, pull one of the -# # restore-keys so we do not need to build from scratch. -# # CACHE_VERSION - provide a way to reset cache -# # CACHE_NAME - name of the cache in order to manage it -# # matrix.os - cache per OS and version -# # hashFiles - Recache if the vcpkg files change -# - name: Restore Cache -# uses: actions/cache@v3 -# with: -# path: ${{ github.workspace }}/build/cache -# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-ubuntu-latest-${{ hashFiles('vcpkg_commit.txt', 'vcpkg.json', 'alternatives/openssl_3/vcpkg.json') }} -# restore-keys: | -# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-ubuntu-latest -# -# - name: Dependencies -# run: | -# sudo apt-get install -y linux-headers-$(uname -r) nasm -# -# - name: Restore cache -# uses: actions/cache@v3 -# with: -# path: ${{ github.workspace }}/build/cache -# key: VCPKG-BinaryCache-${{ runner.os }} -# -# - name: Build (OpenSSL 1.1) -# run: | -# cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" -# cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target all -# -# - name: Unit Test (OpenSSL 1.1) -# run: | -# cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target test -# -# - name: Build (Interop Harness) -# run: | -# cd cmd/interop -# cmake -B build -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" -# cmake --build build -# -# - name: Test self-interop -# run: | -# make -C cmd/interop self-test -# -# - name: Test interop on test vectors -# run: | -# make -C cmd/interop interop-test -# -# - name: Test gRPC live interop with self -# run: | -# cd cmd/interop -# ./grpc-self-test.sh -# -# - name: Build (OpenSSL 3) -# run: | -# cmake -B "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/openssl_3" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" -# cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" -# -# - name: Unit Test (OpenSSL 3) -# run: | -# cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" --target test -# -# - name: Build (BoringSSL) -# run: | -# cmake -B "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/boringssl_1.1" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" -# cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" -# -# - name: Unit Test (BoringSSL) -# run: | -# cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" --target test - platform-sanitizer-tests: - if: github.event.pull_request.draft == false -# needs: quick-linux-interop-check - name: Build and test platforms using sanitizers and clang-tidy +# needs: formatting-check + name: Build and test runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -131,16 +44,10 @@ jobs: os: [windows-latest, ubuntu-latest, macos-latest] include: - os: windows-latest - ossl3-vcpkg-dir: "alternatives\\openssl_3" - boringssl-vcpkg-dir: "alternatives\\boringssl_1.1" ctest-target: RUN_TESTS - os: ubuntu-latest - ossl3-vcpkg-dir: "alternatives/openssl_3" - boringssl-vcpkg-dir: "alternatives/boringssl_1.1" ctest-target: test - os: macos-latest - ossl3-vcpkg-dir: "alternatives/openssl_3" - boringssl-vcpkg-dir: "alternatives/boringssl_1.1" ctest-target: test steps: @@ -150,22 +57,14 @@ jobs: submodules: recursive fetch-depth: 0 - # write the commit hash of vcpkg to a text file so we can use it in the - # hashFiles for cache - run: | git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt - # First, attempt to pull key key, if that is not present, pull one of the - # restore-keys so we do not need to build from scratch. - # CACHE_VERSION - provide a way to reset cache - # CACHE_NAME - name of the cache in order to manage it - # matrix.os - cache per OS and version - # hashFiles - Recache if the vcpkg files change - name: Restore Cache uses: actions/cache@v3 with: path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'vcpkg.json', 'alternatives/openssl_3/vcpkg.json') }} + key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} restore-keys: | ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} @@ -183,7 +82,7 @@ jobs: - name: Build (OpenSSL 1.1) run: | - cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" + cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/openssl_1.1" cmake --build "${{ env.CMAKE_BUILD_DIR }}" - name: Unit Test (OpenSSL 1.1) @@ -192,7 +91,7 @@ jobs: - name: Build (OpenSSL 3) run: | - cmake -B "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ matrix.ossl3-vcpkg-dir }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" + cmake -B "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/openssl_3" cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" - name: Unit Test (OpenSSL 3) @@ -201,58 +100,12 @@ jobs: - name: Build (BoringSSL) run: | - cmake -B "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="${{ matrix.boringssl-vcpkg-dir }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" + cmake -B "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/boringssl" cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" - name: Unit Test (BoringSSL) run: | cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" --target "${{ matrix.ctest-target}}" -# old-macos-compatibility: -# if: github.event.pull_request.draft == false -# needs: quick-linux-interop-check -# name: Build for older MacOS -# runs-on: macos-latest -# -# env: -# CMAKE_BUILD_DIR: ${{ github.workspace }}/build -# VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/build/cache,readwrite -# MACOSX_DEPLOYMENT_TARGET: 10.11 -# -# steps: -# - name: Checkout repository and submodules -# uses: actions/checkout@v4 -# with: -# submodules: recursive -# fetch-depth: 0 -# -# # write the commit hash of vcpkg to a text file so we can use it in the -# # hashFiles for cache -# - run: | -# git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt -# -# # First, attempt to pull key key, if that is not present, pull one of the -# # restore-keys so we do not need to build from scratch. -# # CACHE_VERSION - provide a way to reset cache -# # CACHE_NAME - name of the cache in order to manage it -# # matrix.os - cache per OS and version -# # hashFiles - Recache if the vcpkg files change -# - name: Restore Cache -# uses: actions/cache@v3 -# with: -# path: ${{ github.workspace }}/build/cache -# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest-legacy-${{ hashFiles('vcpkg_commit.txt', 'vcpkg.json', 'alternatives/openssl_3/vcpkg.json') }} -# restore-keys: | -# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest-legacy -# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-macos-latest -# -# - name: Dependencies -# run: | -# brew install llvm pkg-config -# ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" -# ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" -# -# - name: Build -# run: | -# cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_TOOLCHAIN_FILE }}" -# cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target mlspp +# TODO: Add the legacy macOS build to the main matrix +# TODO: Add a clang-tidy build From e836567159daec325ae05d9afe151a825d67e4b9 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 16:41:31 -0400 Subject: [PATCH 11/68] Run crypto in matrix --- .github/workflows/main_ci.yml | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 56b1a038..46dc866f 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -42,6 +42,7 @@ jobs: fail-fast: false matrix: os: [windows-latest, ubuntu-latest, macos-latest] + crypto: [openssl_1.1, openssl_3, boringssl] include: - os: windows-latest ctest-target: RUN_TESTS @@ -80,32 +81,14 @@ jobs: run: | sudo apt-get install -y linux-headers-$(uname -r) nasm - - name: Build (OpenSSL 1.1) + - name: Build run: | - cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/openssl_1.1" + cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" cmake --build "${{ env.CMAKE_BUILD_DIR }}" - - name: Unit Test (OpenSSL 1.1) + - name: Unit Test run: | cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target "${{ matrix.ctest-target}}" - - name: Build (OpenSSL 3) - run: | - cmake -B "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/openssl_3" - cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" - - - name: Unit Test (OpenSSL 3) - run: | - cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" --target "${{ matrix.ctest-target}}" - - - name: Build (BoringSSL) - run: | - cmake -B "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/boringssl" - cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" - - - name: Unit Test (BoringSSL) - run: | - cmake --build "${{ env.CMAKE_BUILD_BORINGSSL_DIR }}" --target "${{ matrix.ctest-target}}" - # TODO: Add the legacy macOS build to the main matrix # TODO: Add a clang-tidy build From 03593ba649ce4f69ba1f38211fc630ec62a1fcff Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 16:44:01 -0400 Subject: [PATCH 12/68] Add interop testing and legacy build --- .github/workflows/main_ci.yml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 46dc866f..7acd6bbf 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -34,7 +34,7 @@ jobs: # include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' # fallback-style: 'Mozilla' # - platform-sanitizer-tests: + build-and-test: # needs: formatting-check name: Build and test runs-on: ${{ matrix.os }} @@ -50,6 +50,12 @@ jobs: ctest-target: test - os: macos-latest ctest-target: test + - os: macos-latest + ctest-target: test + macosx-deployment-target: 10.11 + + env: + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macosx-deployment-target }} steps: - name: Checkout repository and submodules @@ -90,5 +96,24 @@ jobs: run: | cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target "${{ matrix.ctest-target}}" + - name: Build (Interop Harness) + run: | + cd cmd/interop + cmake -B build + cmake --build build + + - name: Test self-interop + run: | + make -C cmd/interop self-test + + - name: Test interop on test vectors + run: | + make -C cmd/interop interop-test + + - name: Test gRPC live interop with self + run: | + cd cmd/interop + ./grpc-self-test.sh + # TODO: Add the legacy macOS build to the main matrix # TODO: Add a clang-tidy build From 364419b3d8835483f5a1836f48be34c92cf15fad Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 16:45:18 -0400 Subject: [PATCH 13/68] Set deployment target to both off and on --- .github/workflows/main_ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 7acd6bbf..2e08c7a8 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -50,6 +50,7 @@ jobs: ctest-target: test - os: macos-latest ctest-target: test + macosx-deployment-target: - os: macos-latest ctest-target: test macosx-deployment-target: 10.11 @@ -71,7 +72,7 @@ jobs: uses: actions/cache@v3 with: path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} + key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }}-${{ matrix.macosx-deployment-target }} restore-keys: | ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} From a00fbf463e333dce402f53d7c3c60ba0a9973585 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 16:49:27 -0400 Subject: [PATCH 14/68] Vibe check --- .github/workflows/main_ci.yml | 115 ++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 54 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 2e08c7a8..2bf4347a 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -59,62 +59,69 @@ jobs: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macosx-deployment-target }} steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - - run: | - git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt - - - name: Restore Cache - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }}-${{ matrix.macosx-deployment-target }} - restore-keys: | - ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} - - - name: Dependencies (macOs) - if: ${{ matrix.os == 'macos-latest' }} - run: | - brew install llvm pkg-config nasm - ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" - ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" - - - name: Dependencies (Ubuntu) - if: ${{ matrix.os == 'ubuntu-latest' }} - run: | - sudo apt-get install -y linux-headers-$(uname -r) nasm - - - name: Build - run: | - cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" - cmake --build "${{ env.CMAKE_BUILD_DIR }}" - - - name: Unit Test - run: | - cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target "${{ matrix.ctest-target}}" - - - name: Build (Interop Harness) - run: | - cd cmd/interop - cmake -B build - cmake --build build - - - name: Test self-interop - run: | - make -C cmd/interop self-test - - - name: Test interop on test vectors + - name: Vibe check run: | - make -C cmd/interop interop-test + echo "matrix.os = ${{ matrix.os }}" + echo "matrix.crypto = ${{ matrix.crypto }}" + echo "matrix.macosx-deployment-target = ${{ matrix.macosx-deployment-target }}" + echo "MACOSX_DEPLOYMENT_TARGET = ${MACOSX_DEPLOYMENT_TARGET}" - - name: Test gRPC live interop with self - run: | - cd cmd/interop - ./grpc-self-test.sh +# - name: Checkout repository and submodules +# uses: actions/checkout@v4 +# with: +# submodules: recursive +# fetch-depth: 0 +# +# - run: | +# git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt +# +# - name: Restore Cache +# uses: actions/cache@v3 +# with: +# path: ${{ github.workspace }}/build/cache +# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }}-${{ matrix.macosx-deployment-target }} +# restore-keys: | +# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} +# +# - name: Dependencies (macOs) +# if: ${{ matrix.os == 'macos-latest' }} +# run: | +# brew install llvm pkg-config nasm +# ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" +# ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" +# +# - name: Dependencies (Ubuntu) +# if: ${{ matrix.os == 'ubuntu-latest' }} +# run: | +# sudo apt-get install -y linux-headers-$(uname -r) nasm +# +# - name: Build +# run: | +# cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" +# cmake --build "${{ env.CMAKE_BUILD_DIR }}" +# +# - name: Unit Test +# run: | +# cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target "${{ matrix.ctest-target}}" +# +# - name: Build (Interop Harness) +# run: | +# cd cmd/interop +# cmake -B build +# cmake --build build +# +# - name: Test self-interop +# run: | +# make -C cmd/interop self-test +# +# - name: Test interop on test vectors +# run: | +# make -C cmd/interop interop-test +# +# - name: Test gRPC live interop with self +# run: | +# cd cmd/interop +# ./grpc-self-test.sh # TODO: Add the legacy macOS build to the main matrix # TODO: Add a clang-tidy build From 1a1c5429716ef18e9994173a446c1744f9896dbb Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 16:55:21 -0400 Subject: [PATCH 15/68] Set a value for the latest deployment target --- .github/workflows/main_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 2bf4347a..24ad78f8 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -50,7 +50,7 @@ jobs: ctest-target: test - os: macos-latest ctest-target: test - macosx-deployment-target: + macosx-deployment-target: 10.12 # Note: Change when macos-latest changes - os: macos-latest ctest-target: test macosx-deployment-target: 10.11 From 7a2d9b055cc29bc3daaf8b8d2c471daed9cd5ee9 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 16:59:03 -0400 Subject: [PATCH 16/68] Remove the legacy macOS build --- .github/workflows/main_ci.yml | 123 +++++++++++++++------------------- 1 file changed, 54 insertions(+), 69 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 24ad78f8..6f004be0 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -50,78 +50,63 @@ jobs: ctest-target: test - os: macos-latest ctest-target: test - macosx-deployment-target: 10.12 # Note: Change when macos-latest changes - - os: macos-latest - ctest-target: test - macosx-deployment-target: 10.11 - - env: - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macosx-deployment-target }} steps: - - name: Vibe check + - name: Checkout repository and submodules + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - run: | + git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt + + - name: Restore Cache + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/build/cache + key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }}-${{ matrix.macosx-deployment-target }} + restore-keys: | + ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} + + - name: Dependencies (macOs) + if: ${{ matrix.os == 'macos-latest' }} run: | - echo "matrix.os = ${{ matrix.os }}" - echo "matrix.crypto = ${{ matrix.crypto }}" - echo "matrix.macosx-deployment-target = ${{ matrix.macosx-deployment-target }}" - echo "MACOSX_DEPLOYMENT_TARGET = ${MACOSX_DEPLOYMENT_TARGET}" + brew install llvm pkg-config nasm + ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" + ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" -# - name: Checkout repository and submodules -# uses: actions/checkout@v4 -# with: -# submodules: recursive -# fetch-depth: 0 -# -# - run: | -# git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt -# -# - name: Restore Cache -# uses: actions/cache@v3 -# with: -# path: ${{ github.workspace }}/build/cache -# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }}-${{ matrix.macosx-deployment-target }} -# restore-keys: | -# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} -# -# - name: Dependencies (macOs) -# if: ${{ matrix.os == 'macos-latest' }} -# run: | -# brew install llvm pkg-config nasm -# ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" -# ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" -# -# - name: Dependencies (Ubuntu) -# if: ${{ matrix.os == 'ubuntu-latest' }} -# run: | -# sudo apt-get install -y linux-headers-$(uname -r) nasm -# -# - name: Build -# run: | -# cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" -# cmake --build "${{ env.CMAKE_BUILD_DIR }}" -# -# - name: Unit Test -# run: | -# cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target "${{ matrix.ctest-target}}" -# -# - name: Build (Interop Harness) -# run: | -# cd cmd/interop -# cmake -B build -# cmake --build build -# -# - name: Test self-interop -# run: | -# make -C cmd/interop self-test -# -# - name: Test interop on test vectors -# run: | -# make -C cmd/interop interop-test -# -# - name: Test gRPC live interop with self -# run: | -# cd cmd/interop -# ./grpc-self-test.sh + - name: Dependencies (Ubuntu) + if: ${{ matrix.os == 'ubuntu-latest' }} + run: | + sudo apt-get install -y linux-headers-$(uname -r) nasm + + - name: Build + run: | + cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" + cmake --build "${{ env.CMAKE_BUILD_DIR }}" + + - name: Unit Test + run: | + cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target "${{ matrix.ctest-target}}" + + - name: Build (Interop Harness) + run: | + cd cmd/interop + cmake -B build + cmake --build build + + - name: Test self-interop + run: | + make -C cmd/interop self-test + + - name: Test interop on test vectors + run: | + make -C cmd/interop interop-test + + - name: Test gRPC live interop with self + run: | + cd cmd/interop + ./grpc-self-test.sh -# TODO: Add the legacy macOS build to the main matrix # TODO: Add a clang-tidy build From 858198d2d77f8f0692433f281335fb333786d9d0 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 17:09:03 -0400 Subject: [PATCH 17/68] Add clang-tidy build and normalize build directories --- .github/workflows/main_ci.yml | 56 +++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 6f004be0..9c79554b 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -39,7 +39,6 @@ jobs: name: Build and test runs-on: ${{ matrix.os }} strategy: - fail-fast: false matrix: os: [windows-latest, ubuntu-latest, macos-latest] crypto: [openssl_1.1, openssl_3, boringssl] @@ -51,6 +50,9 @@ jobs: - os: macos-latest ctest-target: test + env: + BUILD_DIR: build_${{ matrix.crypto }} + steps: - name: Checkout repository and submodules uses: actions/checkout@v4 @@ -65,7 +67,7 @@ jobs: uses: actions/cache@v3 with: path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }}-${{ matrix.macosx-deployment-target }} + key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} restore-keys: | ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} @@ -83,12 +85,12 @@ jobs: - name: Build run: | - cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" - cmake --build "${{ env.CMAKE_BUILD_DIR }}" + cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" + cmake --build "${{ env.BUILD_DIR }}" - name: Unit Test run: | - cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target "${{ matrix.ctest-target}}" + cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" - name: Build (Interop Harness) run: | @@ -109,4 +111,46 @@ jobs: cd cmd/interop ./grpc-self-test.sh -# TODO: Add a clang-tidy build +# clang-tidy: +# if: github.event.pull_request.draft == false +# needs: build-and-test +# name: Build with clang-tidy +# runs-on: ${{ matrix.os }} +# strategy: +# matrix: +# os: [ubuntu-latest] +# crypto: [openssl_1.1, openssl_3, boringssl] +# include: +# - os: ubuntu-latest +# ctest-target: test +# +# env: +# BUILD_DIR: build_${{ matrix.crypto }} +# +# steps: +# - name: Checkout repository and submodules +# uses: actions/checkout@v4 +# with: +# submodules: recursive +# fetch-depth: 0 +# +# - run: | +# git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt +# +# - name: Restore Cache +# uses: actions/cache@v3 +# with: +# path: ${{ github.workspace }}/build/cache +# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} +# restore-keys: | +# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} +# +# - name: Dependencies (Ubuntu) +# if: ${{ matrix.os == 'ubuntu-latest' }} +# run: | +# sudo apt-get install -y linux-headers-$(uname -r) nasm +# +# - name: Build +# run: | +# cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" +# cmake --build "${{ env.CMAKE_BUILD_DIR }}" From 8c9dcdc590a937f175169215818d9f7345eac655 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Fri, 27 Oct 2023 17:10:16 -0400 Subject: [PATCH 18/68] Put build directories in temp storage --- .github/workflows/main_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 9c79554b..d49b92f1 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -51,7 +51,7 @@ jobs: ctest-target: test env: - BUILD_DIR: build_${{ matrix.crypto }} + BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" steps: - name: Checkout repository and submodules From 61f7b9aa9eb1a3fada221197de03322194c5d6d7 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Sat, 28 Oct 2023 11:35:14 -0400 Subject: [PATCH 19/68] Remove interop testing --- .github/workflows/main_ci.yml | 38 ++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index d49b92f1..913a8320 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -92,25 +92,27 @@ jobs: run: | cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" - - name: Build (Interop Harness) - run: | - cd cmd/interop - cmake -B build - cmake --build build - - - name: Test self-interop - run: | - make -C cmd/interop self-test - - - name: Test interop on test vectors - run: | - make -C cmd/interop interop-test - - - name: Test gRPC live interop with self - run: | - cd cmd/interop - ./grpc-self-test.sh +# TODO(RLB): Build interop harness +# - name: Build (Interop Harness) +# run: | +# cd cmd/interop +# cmake -B build +# cmake --build build +# +# - name: Test self-interop +# run: | +# make -C cmd/interop self-test +# +# - name: Test interop on test vectors +# run: | +# make -C cmd/interop interop-test +# +# - name: Test gRPC live interop with self +# run: | +# cd cmd/interop +# ./grpc-self-test.sh +# TODO(RLB) Build with clang-tidy # clang-tidy: # if: github.event.pull_request.draft == false # needs: build-and-test From 799b0052dda2a1dec1f3af8142871713c50111a2 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Sat, 28 Oct 2023 11:56:15 -0400 Subject: [PATCH 20/68] Add composable build workflow --- .github/workflows/build.yml | 50 ++++++++++++++++++++++ .github/workflows/main_ci.yml | 80 +++++++++++++++++------------------ 2 files changed, 90 insertions(+), 40 deletions(-) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..a21f235c --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,50 @@ +name: Build MLSpp + +inputs: + crypto: + required: true + +env: + VCPKG_REPO: ${{ github.workspace }}/vcpkg + CACHE_VERSION: v01 + CACHE_NAME: vcpkg + +runs: + using: "composite" + env: + BUILD_DIR: "${RUNNER_TEMP}/build_${{ inputs.crypto }}" + + steps: + - name: Checkout repository and submodules + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - run: | + git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt + + - name: Restore Cache + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/build/cache + key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ runner.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} + restore-keys: | + ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ runner.os }} + + - name: Dependencies (macOs) + if: ${{ runner.os == 'macos-latest' }} + run: | + brew install llvm pkg-config nasm + ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" + ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" + + - name: Dependencies (Ubuntu) + if: ${{ runner.os == 'ubuntu-latest' }} + run: | + sudo apt-get install -y linux-headers-$(uname -r) nasm + + - name: Build + run: | + cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ inputs.crypto }}" + cmake --build "${{ env.BUILD_DIR }}" diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 913a8320..35a8617b 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -16,9 +16,6 @@ env: CMAKE_BUILD_BORINGSSL_DIR: ${{ github.workspace }}/build_boringssl VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/build/cache,readwrite CMAKE_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - VCPKG_REPO: ${{ github.workspace }}/vcpkg - CACHE_VERSION: v01 - CACHE_NAME: vcpkg jobs: # formatting-check: @@ -35,7 +32,7 @@ jobs: # fallback-style: 'Mozilla' # build-and-test: -# needs: formatting-check + needs: formatting-check name: Build and test runs-on: ${{ matrix.os }} strategy: @@ -50,43 +47,46 @@ jobs: - os: macos-latest ctest-target: test - env: - BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" - - steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - - run: | - git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt - - - name: Restore Cache - uses: actions/cache@v3 +# env: +# BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" +# +# steps: +# - name: Checkout repository and submodules +# uses: actions/checkout@v4 +# with: +# submodules: recursive +# fetch-depth: 0 +# +# - run: | +# git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt +# +# - name: Restore Cache +# uses: actions/cache@v3 +# with: +# path: ${{ github.workspace }}/build/cache +# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} +# restore-keys: | +# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} +# +# - name: Dependencies (macOs) +# if: ${{ matrix.os == 'macos-latest' }} +# run: | +# brew install llvm pkg-config nasm +# ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" +# ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" +# +# - name: Dependencies (Ubuntu) +# if: ${{ matrix.os == 'ubuntu-latest' }} +# run: | +# sudo apt-get install -y linux-headers-$(uname -r) nasm +# +# - name: Build +# run: | +# cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" +# cmake --build "${{ env.BUILD_DIR }}" + - uses: cisco/mlspp/.github/workflows/build.yml with: - path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} - restore-keys: | - ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} - - - name: Dependencies (macOs) - if: ${{ matrix.os == 'macos-latest' }} - run: | - brew install llvm pkg-config nasm - ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" - ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" - - - name: Dependencies (Ubuntu) - if: ${{ matrix.os == 'ubuntu-latest' }} - run: | - sudo apt-get install -y linux-headers-$(uname -r) nasm - - - name: Build - run: | - cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" - cmake --build "${{ env.BUILD_DIR }}" + crypto: ${{ matrix.crypto }} - name: Unit Test run: | From e0865cadd87a756aed5ca24697045ed8fa2256d2 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Sat, 28 Oct 2023 12:01:44 -0400 Subject: [PATCH 21/68] Debug main_ci.yml --- .github/workflows/build.yml | 50 ----------------------------------- .github/workflows/main_ci.yml | 9 ++++--- 2 files changed, 5 insertions(+), 54 deletions(-) delete mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index a21f235c..00000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Build MLSpp - -inputs: - crypto: - required: true - -env: - VCPKG_REPO: ${{ github.workspace }}/vcpkg - CACHE_VERSION: v01 - CACHE_NAME: vcpkg - -runs: - using: "composite" - env: - BUILD_DIR: "${RUNNER_TEMP}/build_${{ inputs.crypto }}" - - steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - - run: | - git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt - - - name: Restore Cache - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ runner.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} - restore-keys: | - ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ runner.os }} - - - name: Dependencies (macOs) - if: ${{ runner.os == 'macos-latest' }} - run: | - brew install llvm pkg-config nasm - ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" - ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" - - - name: Dependencies (Ubuntu) - if: ${{ runner.os == 'ubuntu-latest' }} - run: | - sudo apt-get install -y linux-headers-$(uname -r) nasm - - - name: Build - run: | - cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ inputs.crypto }}" - cmake --build "${{ env.BUILD_DIR }}" diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 35a8617b..4fa357b5 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -32,7 +32,7 @@ jobs: # fallback-style: 'Mozilla' # build-and-test: - needs: formatting-check +# needs: formatting-check name: Build and test runs-on: ${{ matrix.os }} strategy: @@ -49,8 +49,8 @@ jobs: # env: # BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" -# -# steps: + + steps: # - name: Checkout repository and submodules # uses: actions/checkout@v4 # with: @@ -84,7 +84,8 @@ jobs: # run: | # cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" # cmake --build "${{ env.BUILD_DIR }}" - - uses: cisco/mlspp/.github/workflows/build.yml + - name: Build MLSpp + uses: ./.github/actuibs/build.yml with: crypto: ${{ matrix.crypto }} From b5fb39f80d7408da67ffb5470d2c42d79fb57a4e Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Sat, 28 Oct 2023 12:04:02 -0400 Subject: [PATCH 22/68] Fix typo --- .github/workflows/main_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 4fa357b5..5ebdeb52 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -85,7 +85,7 @@ jobs: # cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" # cmake --build "${{ env.BUILD_DIR }}" - name: Build MLSpp - uses: ./.github/actuibs/build.yml + uses: ./.github/actions/build.yml with: crypto: ${{ matrix.crypto }} From 6be706adf0eeb2fb230bb690f5004f4fad34b40f Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Sat, 28 Oct 2023 12:05:52 -0400 Subject: [PATCH 23/68] Add action YAML --- .github/actions/build.yml | 50 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/actions/build.yml diff --git a/.github/actions/build.yml b/.github/actions/build.yml new file mode 100644 index 00000000..a21f235c --- /dev/null +++ b/.github/actions/build.yml @@ -0,0 +1,50 @@ +name: Build MLSpp + +inputs: + crypto: + required: true + +env: + VCPKG_REPO: ${{ github.workspace }}/vcpkg + CACHE_VERSION: v01 + CACHE_NAME: vcpkg + +runs: + using: "composite" + env: + BUILD_DIR: "${RUNNER_TEMP}/build_${{ inputs.crypto }}" + + steps: + - name: Checkout repository and submodules + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - run: | + git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt + + - name: Restore Cache + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/build/cache + key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ runner.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} + restore-keys: | + ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ runner.os }} + + - name: Dependencies (macOs) + if: ${{ runner.os == 'macos-latest' }} + run: | + brew install llvm pkg-config nasm + ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" + ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" + + - name: Dependencies (Ubuntu) + if: ${{ runner.os == 'ubuntu-latest' }} + run: | + sudo apt-get install -y linux-headers-$(uname -r) nasm + + - name: Build + run: | + cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ inputs.crypto }}" + cmake --build "${{ env.BUILD_DIR }}" From c7aa60dee5f030c604701ceac03870598988ac4c Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Sat, 28 Oct 2023 12:06:34 -0400 Subject: [PATCH 24/68] Name the file 'action.yml' --- .github/actions/{build.yml => build/action.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/actions/{build.yml => build/action.yml} (100%) diff --git a/.github/actions/build.yml b/.github/actions/build/action.yml similarity index 100% rename from .github/actions/build.yml rename to .github/actions/build/action.yml From 09adbc774fbf2f193ded4585653dd6eafb33ff5e Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Sat, 28 Oct 2023 12:07:17 -0400 Subject: [PATCH 25/68] Refer to the action without .yml --- .github/workflows/main_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 5ebdeb52..5f2b9980 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -85,7 +85,7 @@ jobs: # cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" # cmake --build "${{ env.BUILD_DIR }}" - name: Build MLSpp - uses: ./.github/actions/build.yml + uses: ./.github/actions/build with: crypto: ${{ matrix.crypto }} From c719dd6e328b3c162a28c2c81a6bafbe26aedddc Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Sat, 28 Oct 2023 12:08:11 -0400 Subject: [PATCH 26/68] Move checkout to main workflow --- .github/workflows/main_ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 5f2b9980..cc8c76a5 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -51,12 +51,12 @@ jobs: # BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" steps: -# - name: Checkout repository and submodules -# uses: actions/checkout@v4 -# with: -# submodules: recursive -# fetch-depth: 0 -# + - name: Checkout repository and submodules + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + # - run: | # git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt # From 3646930ae534740aede1115b7ff44298350bf6cf Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Sat, 28 Oct 2023 12:11:05 -0400 Subject: [PATCH 27/68] Move 'env' within 'runs' --- .github/actions/build/action.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index a21f235c..bfb16186 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -4,23 +4,15 @@ inputs: crypto: required: true -env: - VCPKG_REPO: ${{ github.workspace }}/vcpkg - CACHE_VERSION: v01 - CACHE_NAME: vcpkg - runs: using: "composite" env: + VCPKG_REPO: ${{ github.workspace }}/vcpkg + CACHE_VERSION: v01 + CACHE_NAME: vcpkg BUILD_DIR: "${RUNNER_TEMP}/build_${{ inputs.crypto }}" steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - run: | git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt From 8c227ade2d2c6d9c33593ce58f59d28f7481d09e Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Sat, 28 Oct 2023 12:18:03 -0400 Subject: [PATCH 28/68] Remove 'env' from action --- .github/actions/build/action.yml | 21 +++++++++++---------- .github/workflows/main_ci.yml | 5 +++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index bfb16186..9cb7adb1 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -1,28 +1,29 @@ name: Build MLSpp inputs: + build_dir: + required: true crypto: required: true runs: using: "composite" - env: - VCPKG_REPO: ${{ github.workspace }}/vcpkg - CACHE_VERSION: v01 - CACHE_NAME: vcpkg - BUILD_DIR: "${RUNNER_TEMP}/build_${{ inputs.crypto }}" steps: - run: | - git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt + git -C vcpkg rev-parse HEAD > vcpkg_commit.txt - name: Restore Cache uses: actions/cache@v3 with: - path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ runner.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} + path: ./build/cache + key: vcpkg-v1-${{ runner.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} restore-keys: | - ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ runner.os }} + vcpkg-v1-${{ runner.os }} + + - name: Check OS + run: | + echo ${{ runner.os }} - name: Dependencies (macOs) if: ${{ runner.os == 'macos-latest' }} @@ -38,5 +39,5 @@ runs: - name: Build run: | - cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ inputs.crypto }}" + cmake -B "${{ inputs.build_dir }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ inputs.crypto }}" cmake --build "${{ env.BUILD_DIR }}" diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index cc8c76a5..18405a9c 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -47,8 +47,8 @@ jobs: - os: macos-latest ctest-target: test -# env: -# BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" + env: + BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" steps: - name: Checkout repository and submodules @@ -88,6 +88,7 @@ jobs: uses: ./.github/actions/build with: crypto: ${{ matrix.crypto }} + build_dir: ${{ env.BUILD_DIR }} - name: Unit Test run: | From 2343728961cf2c9cab48be8dbba488de8cf5119a Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Sat, 28 Oct 2023 12:20:33 -0400 Subject: [PATCH 29/68] Revert action changes --- .github/actions/build/action.yml | 43 ------------------------- .github/workflows/main_ci.yml | 55 ++++++++++++++++---------------- 2 files changed, 28 insertions(+), 70 deletions(-) delete mode 100644 .github/actions/build/action.yml diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml deleted file mode 100644 index 9cb7adb1..00000000 --- a/.github/actions/build/action.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Build MLSpp - -inputs: - build_dir: - required: true - crypto: - required: true - -runs: - using: "composite" - - steps: - - run: | - git -C vcpkg rev-parse HEAD > vcpkg_commit.txt - - - name: Restore Cache - uses: actions/cache@v3 - with: - path: ./build/cache - key: vcpkg-v1-${{ runner.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} - restore-keys: | - vcpkg-v1-${{ runner.os }} - - - name: Check OS - run: | - echo ${{ runner.os }} - - - name: Dependencies (macOs) - if: ${{ runner.os == 'macos-latest' }} - run: | - brew install llvm pkg-config nasm - ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" - ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" - - - name: Dependencies (Ubuntu) - if: ${{ runner.os == 'ubuntu-latest' }} - run: | - sudo apt-get install -y linux-headers-$(uname -r) nasm - - - name: Build - run: | - cmake -B "${{ inputs.build_dir }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ inputs.crypto }}" - cmake --build "${{ env.BUILD_DIR }}" diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 18405a9c..90bd48a6 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -57,33 +57,34 @@ jobs: submodules: recursive fetch-depth: 0 -# - run: | -# git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt -# -# - name: Restore Cache -# uses: actions/cache@v3 -# with: -# path: ${{ github.workspace }}/build/cache -# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} -# restore-keys: | -# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} -# -# - name: Dependencies (macOs) -# if: ${{ matrix.os == 'macos-latest' }} -# run: | -# brew install llvm pkg-config nasm -# ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" -# ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" -# -# - name: Dependencies (Ubuntu) -# if: ${{ matrix.os == 'ubuntu-latest' }} -# run: | -# sudo apt-get install -y linux-headers-$(uname -r) nasm -# -# - name: Build -# run: | -# cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" -# cmake --build "${{ env.BUILD_DIR }}" + - run: | + git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt + + - name: Restore Cache + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/build/cache + key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} + restore-keys: | + ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} + + - name: Dependencies (macOs) + if: ${{ matrix.os == 'macos-latest' }} + run: | + brew install llvm pkg-config nasm + ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" + ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" + + - name: Dependencies (Ubuntu) + if: ${{ matrix.os == 'ubuntu-latest' }} + run: | + sudo apt-get install -y linux-headers-$(uname -r) nasm + + - name: Build + run: | + cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" + cmake --build "${{ env.BUILD_DIR }}" + - name: Build MLSpp uses: ./.github/actions/build with: From b4c334e56aa036225f8baff6b0693c42dbc9f860 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Sat, 28 Oct 2023 12:22:28 -0400 Subject: [PATCH 30/68] Really revert --- .github/workflows/main_ci.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 90bd48a6..913a8320 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -16,6 +16,9 @@ env: CMAKE_BUILD_BORINGSSL_DIR: ${{ github.workspace }}/build_boringssl VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/build/cache,readwrite CMAKE_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake + VCPKG_REPO: ${{ github.workspace }}/vcpkg + CACHE_VERSION: v01 + CACHE_NAME: vcpkg jobs: # formatting-check: @@ -85,12 +88,6 @@ jobs: cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" cmake --build "${{ env.BUILD_DIR }}" - - name: Build MLSpp - uses: ./.github/actions/build - with: - crypto: ${{ matrix.crypto }} - build_dir: ${{ env.BUILD_DIR }} - - name: Unit Test run: | cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" From 2f73ea0e6c2e85063779d79aeb4cfe6f8ca239e7 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Sat, 28 Oct 2023 12:26:48 -0400 Subject: [PATCH 31/68] Add interop and clang-tidy builds --- .github/workflows/main_ci.yml | 197 ++++++++++++++++++++-------------- 1 file changed, 118 insertions(+), 79 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 913a8320..3ae83652 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -34,21 +34,76 @@ jobs: # include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' # fallback-style: 'Mozilla' # - build-and-test: +# build-and-unit-test: # needs: formatting-check - name: Build and test +# name: Build and test +# runs-on: ${{ matrix.os }} +# strategy: +# matrix: +# os: [windows-latest, ubuntu-latest, macos-latest] +# crypto: [openssl_1.1, openssl_3, boringssl] +# include: +# - os: windows-latest +# ctest-target: RUN_TESTS +# - os: ubuntu-latest +# ctest-target: test +# - os: macos-latest +# ctest-target: test +# +# env: +# BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" +# +# steps: +# - name: Checkout repository and submodules +# uses: actions/checkout@v4 +# with: +# submodules: recursive +# fetch-depth: 0 +# +# - run: | +# git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt +# +# - name: Restore Cache +# uses: actions/cache@v3 +# with: +# path: ${{ github.workspace }}/build/cache +# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} +# restore-keys: | +# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} +# +# - name: Dependencies (macOs) +# if: ${{ matrix.os == 'macos-latest' }} +# run: | +# brew install llvm pkg-config nasm +# ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" +# ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" +# +# - name: Dependencies (Ubuntu) +# if: ${{ matrix.os == 'ubuntu-latest' }} +# run: | +# sudo apt-get install -y linux-headers-$(uname -r) nasm +# +# - name: Build +# run: | +# cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" +# cmake --build "${{ env.BUILD_DIR }}" +# +# - name: Unit Test +# run: | +# cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" + + interop-test: + if: github.event.pull_request.draft == false +# needs: build-and-test + name: Interop test runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest, ubuntu-latest, macos-latest] - crypto: [openssl_1.1, openssl_3, boringssl] + os: [ubuntu-latest] + crypto: [openssl_1.1] include: - - os: windows-latest - ctest-target: RUN_TESTS - os: ubuntu-latest ctest-target: test - - os: macos-latest - ctest-target: test env: BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" @@ -71,13 +126,6 @@ jobs: restore-keys: | ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} - - name: Dependencies (macOs) - if: ${{ matrix.os == 'macos-latest' }} - run: | - brew install llvm pkg-config nasm - ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" - ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" - - name: Dependencies (Ubuntu) if: ${{ matrix.os == 'ubuntu-latest' }} run: | @@ -88,71 +136,62 @@ jobs: cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" cmake --build "${{ env.BUILD_DIR }}" - - name: Unit Test + - name: Build (Interop Harness) run: | - cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" + cd cmd/interop + cmake -B build + cmake --build build -# TODO(RLB): Build interop harness -# - name: Build (Interop Harness) -# run: | -# cd cmd/interop -# cmake -B build -# cmake --build build -# -# - name: Test self-interop -# run: | -# make -C cmd/interop self-test -# -# - name: Test interop on test vectors -# run: | -# make -C cmd/interop interop-test -# -# - name: Test gRPC live interop with self -# run: | -# cd cmd/interop -# ./grpc-self-test.sh + - name: Test self-interop + run: | + make -C cmd/interop self-test + + - name: Test interop on test vectors + run: | + make -C cmd/interop interop-test -# TODO(RLB) Build with clang-tidy -# clang-tidy: -# if: github.event.pull_request.draft == false + - name: Test gRPC live interop with self + run: | + cd cmd/interop + ./grpc-self-test.sh + + clang-tidy: + if: github.event.pull_request.draft == false # needs: build-and-test -# name: Build with clang-tidy -# runs-on: ${{ matrix.os }} -# strategy: -# matrix: -# os: [ubuntu-latest] -# crypto: [openssl_1.1, openssl_3, boringssl] -# include: -# - os: ubuntu-latest -# ctest-target: test -# -# env: -# BUILD_DIR: build_${{ matrix.crypto }} -# -# steps: -# - name: Checkout repository and submodules -# uses: actions/checkout@v4 -# with: -# submodules: recursive -# fetch-depth: 0 -# -# - run: | -# git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt -# -# - name: Restore Cache -# uses: actions/cache@v3 -# with: -# path: ${{ github.workspace }}/build/cache -# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} -# restore-keys: | -# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} -# -# - name: Dependencies (Ubuntu) -# if: ${{ matrix.os == 'ubuntu-latest' }} -# run: | -# sudo apt-get install -y linux-headers-$(uname -r) nasm -# -# - name: Build -# run: | -# cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" -# cmake --build "${{ env.CMAKE_BUILD_DIR }}" + name: Build with clang-tidy + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + crypto: [openssl_1.1, openssl_3, boringssl] + + env: + BUILD_DIR: build_${{ matrix.crypto }} + + steps: + - name: Checkout repository and submodules + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - run: | + git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt + + - name: Restore Cache + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/build/cache + key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} + restore-keys: | + ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} + + - name: Dependencies (Ubuntu) + if: ${{ matrix.os == 'ubuntu-latest' }} + run: | + sudo apt-get install -y linux-headers-$(uname -r) nasm + + - name: Build + run: | + cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" + cmake --build "${{ env.CMAKE_BUILD_DIR }}" From d08ff13aab865181078d51038fd8fa6556e5efb4 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Sat, 28 Oct 2023 13:35:17 -0400 Subject: [PATCH 32/68] Take off the brakes --- .github/workflows/main_ci.yml | 146 +++++++++++++++++----------------- 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 3ae83652..8d6c9d42 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -21,80 +21,80 @@ env: CACHE_NAME: vcpkg jobs: -# formatting-check: -# name: Formatting Check -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# -# - name: Run clang-format style check for C/C++ programs -# uses: jidicula/clang-format-action@v4.11.0 -# with: -# clang-format-version: 16 -# include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' -# fallback-style: 'Mozilla' -# -# build-and-unit-test: -# needs: formatting-check -# name: Build and test -# runs-on: ${{ matrix.os }} -# strategy: -# matrix: -# os: [windows-latest, ubuntu-latest, macos-latest] -# crypto: [openssl_1.1, openssl_3, boringssl] -# include: -# - os: windows-latest -# ctest-target: RUN_TESTS -# - os: ubuntu-latest -# ctest-target: test -# - os: macos-latest -# ctest-target: test -# -# env: -# BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" -# -# steps: -# - name: Checkout repository and submodules -# uses: actions/checkout@v4 -# with: -# submodules: recursive -# fetch-depth: 0 -# -# - run: | -# git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt -# -# - name: Restore Cache -# uses: actions/cache@v3 -# with: -# path: ${{ github.workspace }}/build/cache -# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} -# restore-keys: | -# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} -# -# - name: Dependencies (macOs) -# if: ${{ matrix.os == 'macos-latest' }} -# run: | -# brew install llvm pkg-config nasm -# ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" -# ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" -# -# - name: Dependencies (Ubuntu) -# if: ${{ matrix.os == 'ubuntu-latest' }} -# run: | -# sudo apt-get install -y linux-headers-$(uname -r) nasm -# -# - name: Build -# run: | -# cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" -# cmake --build "${{ env.BUILD_DIR }}" -# -# - name: Unit Test -# run: | -# cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" + formatting-check: + name: Formatting Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run clang-format style check for C/C++ programs + uses: jidicula/clang-format-action@v4.11.0 + with: + clang-format-version: 16 + include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' + fallback-style: 'Mozilla' + + build-and-unit-test: + needs: formatting-check + name: Build and test + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + crypto: [openssl_1.1, openssl_3, boringssl] + include: + - os: windows-latest + ctest-target: RUN_TESTS + - os: ubuntu-latest + ctest-target: test + - os: macos-latest + ctest-target: test + + env: + BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" + + steps: + - name: Checkout repository and submodules + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - run: | + git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt + + - name: Restore Cache + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/build/cache + key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} + restore-keys: | + ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} + + - name: Dependencies (macOs) + if: ${{ matrix.os == 'macos-latest' }} + run: | + brew install llvm pkg-config nasm + ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" + ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" + + - name: Dependencies (Ubuntu) + if: ${{ matrix.os == 'ubuntu-latest' }} + run: | + sudo apt-get install -y linux-headers-$(uname -r) nasm + + - name: Build + run: | + cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" + cmake --build "${{ env.BUILD_DIR }}" + + - name: Unit Test + run: | + cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" interop-test: if: github.event.pull_request.draft == false -# needs: build-and-test + needs: build-and-test name: Interop test runs-on: ${{ matrix.os }} strategy: @@ -133,7 +133,7 @@ jobs: - name: Build run: | - cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" + cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" cmake --build "${{ env.BUILD_DIR }}" - name: Build (Interop Harness) @@ -157,7 +157,7 @@ jobs: clang-tidy: if: github.event.pull_request.draft == false -# needs: build-and-test + needs: build-and-test name: Build with clang-tidy runs-on: ${{ matrix.os }} strategy: From a7a5142ff258b48df639752e39618a5388cab7f4 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Sat, 28 Oct 2023 15:24:42 -0400 Subject: [PATCH 33/68] Fix broken references --- .github/workflows/main_ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 8d6c9d42..01b98684 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -94,7 +94,7 @@ jobs: interop-test: if: github.event.pull_request.draft == false - needs: build-and-test + needs: build-and-unit-test name: Interop test runs-on: ${{ matrix.os }} strategy: @@ -157,7 +157,7 @@ jobs: clang-tidy: if: github.event.pull_request.draft == false - needs: build-and-test + needs: build-and-unit-test name: Build with clang-tidy runs-on: ${{ matrix.os }} strategy: From e849c30eea8ba21e9a0a57fb176d2c9e1d55e570 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Sun, 29 Oct 2023 15:38:35 -0400 Subject: [PATCH 34/68] Disable Windows build --- .github/workflows/main_ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 01b98684..e92f0613 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -40,11 +40,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest, ubuntu-latest, macos-latest] +# os: [windows-latest, ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest] crypto: [openssl_1.1, openssl_3, boringssl] include: - - os: windows-latest - ctest-target: RUN_TESTS +# - os: windows-latest +# ctest-target: RUN_TESTS - os: ubuntu-latest ctest-target: test - os: macos-latest From 54e213c8fc8adfad617a20745140f415176fc300 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 14:14:49 -0400 Subject: [PATCH 35/68] Start moving build tasks to a composite action --- .github/actions/build/action.yml | 17 +++ .github/workflows/main_ci.yml | 241 ++++++++++++++++--------------- 2 files changed, 138 insertions(+), 120 deletions(-) create mode 100644 .github/actions/build/action.yml diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml new file mode 100644 index 00000000..60aa8641 --- /dev/null +++ b/.github/actions/build/action.yml @@ -0,0 +1,17 @@ +name: Build MLSpp + +inputs: + who-to-greet: # id of input + description: 'Who to greet' + required: true + default: 'World' + +runs: + using: "composite" + steps: + - name: Determine vcpkg commit + shell: bash + run: | + git -C vcpkg rev-parse HEAD > vcpkg_commit.txt + + diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index e92f0613..cdaaa71b 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -21,35 +21,36 @@ env: CACHE_NAME: vcpkg jobs: - formatting-check: - name: Formatting Check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Run clang-format style check for C/C++ programs - uses: jidicula/clang-format-action@v4.11.0 - with: - clang-format-version: 16 - include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' - fallback-style: 'Mozilla' +# formatting-check: +# name: Formatting Check +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# +# - name: Run clang-format style check for C/C++ programs +# uses: jidicula/clang-format-action@v4.11.0 +# with: +# clang-format-version: 16 +# include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' +# fallback-style: 'Mozilla' build-and-unit-test: - needs: formatting-check +# needs: formatting-check name: Build and test runs-on: ${{ matrix.os }} strategy: matrix: # os: [windows-latest, ubuntu-latest, macos-latest] - os: [ubuntu-latest, macos-latest] - crypto: [openssl_1.1, openssl_3, boringssl] +# crypto: [openssl_1.1, openssl_3, boringssl] + os: [ubuntu-latest] + crypto: [openssl_1.1] include: # - os: windows-latest # ctest-target: RUN_TESTS - os: ubuntu-latest ctest-target: test - - os: macos-latest - ctest-target: test +# - os: macos-latest +# ctest-target: test env: BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" @@ -93,106 +94,106 @@ jobs: run: | cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" - interop-test: - if: github.event.pull_request.draft == false - needs: build-and-unit-test - name: Interop test - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - crypto: [openssl_1.1] - include: - - os: ubuntu-latest - ctest-target: test - - env: - BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" - - steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - - run: | - git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt - - - name: Restore Cache - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} - restore-keys: | - ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} - - - name: Dependencies (Ubuntu) - if: ${{ matrix.os == 'ubuntu-latest' }} - run: | - sudo apt-get install -y linux-headers-$(uname -r) nasm - - - name: Build - run: | - cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" - cmake --build "${{ env.BUILD_DIR }}" - - - name: Build (Interop Harness) - run: | - cd cmd/interop - cmake -B build - cmake --build build - - - name: Test self-interop - run: | - make -C cmd/interop self-test - - - name: Test interop on test vectors - run: | - make -C cmd/interop interop-test - - - name: Test gRPC live interop with self - run: | - cd cmd/interop - ./grpc-self-test.sh - - clang-tidy: - if: github.event.pull_request.draft == false - needs: build-and-unit-test - name: Build with clang-tidy - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - crypto: [openssl_1.1, openssl_3, boringssl] - - env: - BUILD_DIR: build_${{ matrix.crypto }} - - steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - - run: | - git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt - - - name: Restore Cache - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} - restore-keys: | - ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} - - - name: Dependencies (Ubuntu) - if: ${{ matrix.os == 'ubuntu-latest' }} - run: | - sudo apt-get install -y linux-headers-$(uname -r) nasm - - - name: Build - run: | - cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" - cmake --build "${{ env.CMAKE_BUILD_DIR }}" +# interop-test: +# if: github.event.pull_request.draft == false +# needs: build-and-unit-test +# name: Interop test +# runs-on: ${{ matrix.os }} +# strategy: +# matrix: +# os: [ubuntu-latest] +# crypto: [openssl_1.1] +# include: +# - os: ubuntu-latest +# ctest-target: test +# +# env: +# BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" +# +# steps: +# - name: Checkout repository and submodules +# uses: actions/checkout@v4 +# with: +# submodules: recursive +# fetch-depth: 0 +# +# - run: | +# git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt +# +# - name: Restore Cache +# uses: actions/cache@v3 +# with: +# path: ${{ github.workspace }}/build/cache +# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} +# restore-keys: | +# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} +# +# - name: Dependencies (Ubuntu) +# if: ${{ matrix.os == 'ubuntu-latest' }} +# run: | +# sudo apt-get install -y linux-headers-$(uname -r) nasm +# +# - name: Build +# run: | +# cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" +# cmake --build "${{ env.BUILD_DIR }}" +# +# - name: Build (Interop Harness) +# run: | +# cd cmd/interop +# cmake -B build +# cmake --build build +# +# - name: Test self-interop +# run: | +# make -C cmd/interop self-test +# +# - name: Test interop on test vectors +# run: | +# make -C cmd/interop interop-test +# +# - name: Test gRPC live interop with self +# run: | +# cd cmd/interop +# ./grpc-self-test.sh +# +# clang-tidy: +# if: github.event.pull_request.draft == false +# needs: build-and-unit-test +# name: Build with clang-tidy +# runs-on: ${{ matrix.os }} +# strategy: +# matrix: +# os: [ubuntu-latest] +# crypto: [openssl_1.1, openssl_3, boringssl] +# +# env: +# BUILD_DIR: build_${{ matrix.crypto }} +# +# steps: +# - name: Checkout repository and submodules +# uses: actions/checkout@v4 +# with: +# submodules: recursive +# fetch-depth: 0 +# +# - run: | +# git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt +# +# - name: Restore Cache +# uses: actions/cache@v3 +# with: +# path: ${{ github.workspace }}/build/cache +# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} +# restore-keys: | +# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} +# +# - name: Dependencies (Ubuntu) +# if: ${{ matrix.os == 'ubuntu-latest' }} +# run: | +# sudo apt-get install -y linux-headers-$(uname -r) nasm +# +# - name: Build +# run: | +# cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" +# cmake --build "${{ env.CMAKE_BUILD_DIR }}" From 0117556a1c4f75ae2b7a0d7b1942b4e541f46589 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 14:16:18 -0400 Subject: [PATCH 36/68] Actually use composite action --- .github/workflows/main_ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index cdaaa71b..79227061 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -62,8 +62,7 @@ jobs: submodules: recursive fetch-depth: 0 - - run: | - git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt + - uses: ./.github/actions/build - name: Restore Cache uses: actions/cache@v3 From bee7467c34c31292ca556314849d3d2e7a7e83bb Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 14:52:18 -0400 Subject: [PATCH 37/68] Understand runner.os --- .github/actions/build/action.yml | 15 ++++++-- .github/workflows/main_ci.yml | 60 ++++++++++++++++---------------- 2 files changed, 42 insertions(+), 33 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 60aa8641..f95b5c3b 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -1,10 +1,15 @@ name: Build MLSpp inputs: - who-to-greet: # id of input - description: 'Who to greet' + cache_dir: + description: Where to put vcpkg cache + required: true + crypto_dir: + description: Where to find the vcpkg manifest for the underlying crypto library + required: true + build_dir: + description: Where to put build artifacts required: true - default: 'World' runs: using: "composite" @@ -14,4 +19,8 @@ runs: run: | git -C vcpkg rev-parse HEAD > vcpkg_commit.txt + - name: Determine OS + shell: bash + run: | + echo ${{ runner.os }} diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 79227061..3dd9acc3 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -40,9 +40,9 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: -# os: [windows-latest, ubuntu-latest, macos-latest] + os: [windows-latest, ubuntu-latest, macos-latest] # crypto: [openssl_1.1, openssl_3, boringssl] - os: [ubuntu-latest] +# os: [ubuntu-latest] crypto: [openssl_1.1] include: # - os: windows-latest @@ -64,34 +64,34 @@ jobs: - uses: ./.github/actions/build - - name: Restore Cache - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} - restore-keys: | - ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} - - - name: Dependencies (macOs) - if: ${{ matrix.os == 'macos-latest' }} - run: | - brew install llvm pkg-config nasm - ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" - ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" - - - name: Dependencies (Ubuntu) - if: ${{ matrix.os == 'ubuntu-latest' }} - run: | - sudo apt-get install -y linux-headers-$(uname -r) nasm - - - name: Build - run: | - cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" - cmake --build "${{ env.BUILD_DIR }}" - - - name: Unit Test - run: | - cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" +# - name: Restore Cache +# uses: actions/cache@v3 +# with: +# path: ${{ github.workspace }}/build/cache +# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} +# restore-keys: | +# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} +# +# - name: Dependencies (macOs) +# if: ${{ matrix.os == 'macos-latest' }} +# run: | +# brew install llvm pkg-config nasm +# ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" +# ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" +# +# - name: Dependencies (Ubuntu) +# if: ${{ matrix.os == 'ubuntu-latest' }} +# run: | +# sudo apt-get install -y linux-headers-$(uname -r) nasm +# +# - name: Build +# run: | +# cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" +# cmake --build "${{ env.BUILD_DIR }}" +# +# - name: Unit Test +# run: | +# cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" # interop-test: # if: github.event.pull_request.draft == false From 5fc6e43418d02f1482904b10642b8b1821b494ed Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 14:56:42 -0400 Subject: [PATCH 38/68] Move dependencies to composite action --- .github/actions/build/action.yml | 28 +++++++++---------- .github/workflows/main_ci.yml | 48 ++++++++++++-------------------- 2 files changed, 32 insertions(+), 44 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index f95b5c3b..f6d0b9ea 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -1,26 +1,26 @@ -name: Build MLSpp +name: Install build prerequisites inputs: - cache_dir: - description: Where to put vcpkg cache - required: true - crypto_dir: - description: Where to find the vcpkg manifest for the underlying crypto library - required: true - build_dir: - description: Where to put build artifacts - required: true +# cache_dir: +# description: Where to put vcpkg cache +# required: true runs: using: "composite" steps: - - name: Determine vcpkg commit + - name: Restore vcpkg cache shell: bash run: | git -C vcpkg rev-parse HEAD > vcpkg_commit.txt - - name: Determine OS - shell: bash + - name: Dependencies (macOS) + if: ${{ runner.os == 'macOS' }} run: | - echo ${{ runner.os }} + brew install llvm pkg-config nasm + ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" + ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" + - name: Dependencies (Ubuntu) + if: ${{ runner.os == 'Linux' }} + run: | + sudo apt-get install -y linux-headers-$(uname -r) nasm diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 3dd9acc3..19caa3db 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -40,9 +40,9 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest, ubuntu-latest, macos-latest] +# os: [windows-latest, ubuntu-latest, macos-latest] # crypto: [openssl_1.1, openssl_3, boringssl] -# os: [ubuntu-latest] + os: [ubuntu-latest] crypto: [openssl_1.1] include: # - os: windows-latest @@ -64,34 +64,22 @@ jobs: - uses: ./.github/actions/build -# - name: Restore Cache -# uses: actions/cache@v3 -# with: -# path: ${{ github.workspace }}/build/cache -# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} -# restore-keys: | -# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} -# -# - name: Dependencies (macOs) -# if: ${{ matrix.os == 'macos-latest' }} -# run: | -# brew install llvm pkg-config nasm -# ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" -# ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" -# -# - name: Dependencies (Ubuntu) -# if: ${{ matrix.os == 'ubuntu-latest' }} -# run: | -# sudo apt-get install -y linux-headers-$(uname -r) nasm -# -# - name: Build -# run: | -# cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" -# cmake --build "${{ env.BUILD_DIR }}" -# -# - name: Unit Test -# run: | -# cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" + - name: Restore Cache + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/build/cache + key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} + restore-keys: | + ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} + + - name: Build + run: | + cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" + cmake --build "${{ env.BUILD_DIR }}" + + - name: Unit Test + run: | + cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" # interop-test: # if: github.event.pull_request.draft == false From 791a9ba8dc83c06216499e232024da6822f23aa1 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 14:58:37 -0400 Subject: [PATCH 39/68] Action typos --- .github/actions/build/action.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index f6d0b9ea..5d20fd2b 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -1,8 +1,8 @@ name: Install build prerequisites inputs: -# cache_dir: -# description: Where to put vcpkg cache + cache_dir: + description: Where to put vcpkg cache # required: true runs: @@ -15,6 +15,7 @@ runs: - name: Dependencies (macOS) if: ${{ runner.os == 'macOS' }} + shell: bash run: | brew install llvm pkg-config nasm ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" @@ -22,5 +23,6 @@ runs: - name: Dependencies (Ubuntu) if: ${{ runner.os == 'Linux' }} + shell: bash run: | sudo apt-get install -y linux-headers-$(uname -r) nasm From 170e4539bdc4aa901994c18810ba0df2fc403859 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 15:02:45 -0400 Subject: [PATCH 40/68] Move cache control to action --- .github/actions/build/action.yml | 10 +++++++++- .github/workflows/main_ci.yml | 10 ++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 5d20fd2b..3ea26f96 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -3,7 +3,7 @@ name: Install build prerequisites inputs: cache_dir: description: Where to put vcpkg cache -# required: true + required: true runs: using: "composite" @@ -13,6 +13,14 @@ runs: run: | git -C vcpkg rev-parse HEAD > vcpkg_commit.txt + - name: Restore Cache + uses: actions/cache@v3 + with: + path: ${{ inputs.cache_dir }} + key: v01-vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} + restore-keys: | + v01-vcpkg-${{ runner.os }} + - name: Dependencies (macOS) if: ${{ runner.os == 'macOS' }} shell: bash diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 19caa3db..52b65927 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -14,7 +14,7 @@ env: CMAKE_BUILD_DIR: ${{ github.workspace }}/build CMAKE_BUILD_OPENSSL3_DIR: ${{ github.workspace }}/build_openssl3 CMAKE_BUILD_BORINGSSL_DIR: ${{ github.workspace }}/build_boringssl - VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/build/cache,readwrite + VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/vcpkg_cache,readwrite CMAKE_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake VCPKG_REPO: ${{ github.workspace }}/vcpkg CACHE_VERSION: v01 @@ -63,14 +63,8 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/build - - - name: Restore Cache - uses: actions/cache@v3 with: - path: ${{ github.workspace }}/build/cache - key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} - restore-keys: | - ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} + cache-dir: ${{ github.workspace }}/vcpkg_cache - name: Build run: | From 07d8c0f880af0f105925f46f160284671a4fc55f Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 15:06:04 -0400 Subject: [PATCH 41/68] Typo in input name --- .github/actions/build/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 3ea26f96..2a8697b1 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -1,7 +1,7 @@ name: Install build prerequisites inputs: - cache_dir: + cache-dir: description: Where to put vcpkg cache required: true @@ -16,7 +16,7 @@ runs: - name: Restore Cache uses: actions/cache@v3 with: - path: ${{ inputs.cache_dir }} + path: ${{ inputs.cache-dir }} key: v01-vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} restore-keys: | v01-vcpkg-${{ runner.os }} From c2703d43072054f97dade39c92ffeeb0ac4de1d2 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 15:23:08 -0400 Subject: [PATCH 42/68] Restor other steps, now using composite action --- .github/actions/build/action.yml | 8 +- .github/workflows/main_ci.yml | 220 +++++++++++++------------------ 2 files changed, 98 insertions(+), 130 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 2a8697b1..153f0b9d 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -8,12 +8,12 @@ inputs: runs: using: "composite" steps: - - name: Restore vcpkg cache + - name: Capture vcpkg revision for use in cache key shell: bash run: | git -C vcpkg rev-parse HEAD > vcpkg_commit.txt - - name: Restore Cache + - name: Restore cache uses: actions/cache@v3 with: path: ${{ inputs.cache-dir }} @@ -21,7 +21,7 @@ runs: restore-keys: | v01-vcpkg-${{ runner.os }} - - name: Dependencies (macOS) + - name: Install dependencies (macOS) if: ${{ runner.os == 'macOS' }} shell: bash run: | @@ -29,7 +29,7 @@ runs: ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" - - name: Dependencies (Ubuntu) + - name: Install dependencies (Ubuntu) if: ${{ runner.os == 'Linux' }} shell: bash run: | diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 52b65927..ddcc9783 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -11,31 +11,25 @@ on: env: CMAKE_BUILD_PARALLEL_LEVEL: 3 CTEST_OUTPUT_ON_FAILURE: 1 - CMAKE_BUILD_DIR: ${{ github.workspace }}/build - CMAKE_BUILD_OPENSSL3_DIR: ${{ github.workspace }}/build_openssl3 - CMAKE_BUILD_BORINGSSL_DIR: ${{ github.workspace }}/build_boringssl VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/vcpkg_cache,readwrite CMAKE_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - VCPKG_REPO: ${{ github.workspace }}/vcpkg - CACHE_VERSION: v01 - CACHE_NAME: vcpkg jobs: -# formatting-check: -# name: Formatting Check -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# -# - name: Run clang-format style check for C/C++ programs -# uses: jidicula/clang-format-action@v4.11.0 -# with: -# clang-format-version: 16 -# include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' -# fallback-style: 'Mozilla' + formatting-check: + name: Formatting Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run clang-format style check for C/C++ programs + uses: jidicula/clang-format-action@v4.11.0 + with: + clang-format-version: 16 + include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' + fallback-style: 'Mozilla' build-and-unit-test: -# needs: formatting-check + needs: formatting-check name: Build and test runs-on: ${{ matrix.os }} strategy: @@ -54,13 +48,12 @@ jobs: env: BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" + CRYPTO_DIR: "./alternatives/${{ matrix.crypto }}" steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: submodules: recursive - fetch-depth: 0 - uses: ./.github/actions/build with: @@ -68,113 +61,88 @@ jobs: - name: Build run: | - cmake -B "${{ env.BUILD_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" + cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON cmake --build "${{ env.BUILD_DIR }}" - name: Unit Test run: | cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" -# interop-test: -# if: github.event.pull_request.draft == false -# needs: build-and-unit-test -# name: Interop test -# runs-on: ${{ matrix.os }} -# strategy: -# matrix: -# os: [ubuntu-latest] -# crypto: [openssl_1.1] -# include: -# - os: ubuntu-latest -# ctest-target: test -# -# env: -# BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" -# -# steps: -# - name: Checkout repository and submodules -# uses: actions/checkout@v4 -# with: -# submodules: recursive -# fetch-depth: 0 -# -# - run: | -# git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt -# -# - name: Restore Cache -# uses: actions/cache@v3 -# with: -# path: ${{ github.workspace }}/build/cache -# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} -# restore-keys: | -# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} -# -# - name: Dependencies (Ubuntu) -# if: ${{ matrix.os == 'ubuntu-latest' }} -# run: | -# sudo apt-get install -y linux-headers-$(uname -r) nasm -# -# - name: Build -# run: | -# cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" -# cmake --build "${{ env.BUILD_DIR }}" -# -# - name: Build (Interop Harness) -# run: | -# cd cmd/interop -# cmake -B build -# cmake --build build -# -# - name: Test self-interop -# run: | -# make -C cmd/interop self-test -# -# - name: Test interop on test vectors -# run: | -# make -C cmd/interop interop-test -# -# - name: Test gRPC live interop with self -# run: | -# cd cmd/interop -# ./grpc-self-test.sh -# -# clang-tidy: -# if: github.event.pull_request.draft == false -# needs: build-and-unit-test -# name: Build with clang-tidy -# runs-on: ${{ matrix.os }} -# strategy: -# matrix: -# os: [ubuntu-latest] -# crypto: [openssl_1.1, openssl_3, boringssl] -# -# env: -# BUILD_DIR: build_${{ matrix.crypto }} -# -# steps: -# - name: Checkout repository and submodules -# uses: actions/checkout@v4 -# with: -# submodules: recursive -# fetch-depth: 0 -# -# - run: | -# git -C ${{ env.VCPKG_REPO }} rev-parse HEAD > vcpkg_commit.txt -# -# - name: Restore Cache -# uses: actions/cache@v3 -# with: -# path: ${{ github.workspace }}/build/cache -# key: ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} -# restore-keys: | -# ${{ env.CACHE_VERSION }}-${{ env.CACHE_NAME }}-${{ matrix.os }} -# -# - name: Dependencies (Ubuntu) -# if: ${{ matrix.os == 'ubuntu-latest' }} -# run: | -# sudo apt-get install -y linux-headers-$(uname -r) nasm -# -# - name: Build -# run: | -# cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -DVCPKG_MANIFEST_DIR="alternatives/${{ matrix.crypto }}" -# cmake --build "${{ env.CMAKE_BUILD_DIR }}" + interop-test: + if: github.event.pull_request.draft == false + needs: build-and-unit-test + name: Interop test + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + crypto: [openssl_1.1] + include: + - os: ubuntu-latest + ctest-target: test + + env: + BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" + CRYPTO_DIR: "./alternatives/${{ matrix.crypto }}" + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: ./.github/actions/build + with: + cache-dir: ${{ github.workspace }}/vcpkg_cache + + - name: Build + run: | + cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" + cmake --build "${{ env.BUILD_DIR }}" + + - name: Build (Interop Harness) + run: | + cd cmd/interop + cmake -B build + cmake --build build + + - name: Test self-interop + run: | + make -C cmd/interop self-test + + - name: Test interop on test vectors + run: | + make -C cmd/interop interop-test + + - name: Test gRPC live interop with self + run: | + cd cmd/interop + ./grpc-self-test.sh + + clang-tidy: + if: github.event.pull_request.draft == false + needs: build-and-unit-test + name: Build with clang-tidy + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + crypto: [openssl_1.1, openssl_3, boringssl] + + env: + BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" + CRYPTO_DIR: "./alternatives/${{ matrix.crypto }}" + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: ./.github/actions/build + with: + cache-dir: ${{ github.workspace }}/vcpkg_cache + + - name: Build with clang-tidy + run: | + cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" \ + -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON + cmake --build "${{ env.BUILD_DIR }}" From c76013701174744eb8b25e4fec6d59bd27646835 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 15:25:58 -0400 Subject: [PATCH 43/68] Re-add fetch-depth --- .github/workflows/main_ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index ddcc9783..f6704d7f 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -54,6 +54,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive + fetch-depth: 0 - uses: ./.github/actions/build with: @@ -89,6 +90,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive + fetch-depth: 0 - uses: ./.github/actions/build with: @@ -136,6 +138,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive + fetch-depth: 0 - uses: ./.github/actions/build with: From 48fac67842ed7308f275b009fa406392d0c8f6f3 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 16:20:36 -0400 Subject: [PATCH 44/68] Re-enable all builds --- .github/workflows/main_ci.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index f6704d7f..865a20d3 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -34,17 +34,15 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: -# os: [windows-latest, ubuntu-latest, macos-latest] -# crypto: [openssl_1.1, openssl_3, boringssl] - os: [ubuntu-latest] - crypto: [openssl_1.1] + os: [windows-latest, ubuntu-latest, macos-latest] + crypto: [openssl_1.1, openssl_3, boringssl] include: -# - os: windows-latest -# ctest-target: RUN_TESTS + - os: windows-latest + ctest-target: RUN_TESTS - os: ubuntu-latest ctest-target: test -# - os: macos-latest -# ctest-target: test + - os: macos-latest + ctest-target: test env: BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" From 239cac68fc0a8aedf25993ffe8a3f69556d2e14a Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 17:47:22 -0400 Subject: [PATCH 45/68] Try to gain insight into Windows failure --- .github/workflows/main_ci.yml | 190 +++++++++++++++++----------------- lib/bytes/test/bytes.cpp | 4 + 2 files changed, 100 insertions(+), 94 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 865a20d3..e322ad15 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -15,26 +15,28 @@ env: CMAKE_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake jobs: - formatting-check: - name: Formatting Check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Run clang-format style check for C/C++ programs - uses: jidicula/clang-format-action@v4.11.0 - with: - clang-format-version: 16 - include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' - fallback-style: 'Mozilla' +# formatting-check: +# name: Formatting Check +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# +# - name: Run clang-format style check for C/C++ programs +# uses: jidicula/clang-format-action@v4.11.0 +# with: +# clang-format-version: 16 +# include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' +# fallback-style: 'Mozilla' build-and-unit-test: - needs: formatting-check +# needs: formatting-check name: Build and test runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest, ubuntu-latest, macos-latest] +# os: [windows-latest, ubuntu-latest, macos-latest] +# crypto: [openssl_1.1, openssl_3, boringssl] + os: [windows-latest] crypto: [openssl_1.1, openssl_3, boringssl] include: - os: windows-latest @@ -67,83 +69,83 @@ jobs: run: | cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" - interop-test: - if: github.event.pull_request.draft == false - needs: build-and-unit-test - name: Interop test - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - crypto: [openssl_1.1] - include: - - os: ubuntu-latest - ctest-target: test - - env: - BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" - CRYPTO_DIR: "./alternatives/${{ matrix.crypto }}" - - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - - uses: ./.github/actions/build - with: - cache-dir: ${{ github.workspace }}/vcpkg_cache - - - name: Build - run: | - cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" - cmake --build "${{ env.BUILD_DIR }}" - - - name: Build (Interop Harness) - run: | - cd cmd/interop - cmake -B build - cmake --build build - - - name: Test self-interop - run: | - make -C cmd/interop self-test - - - name: Test interop on test vectors - run: | - make -C cmd/interop interop-test - - - name: Test gRPC live interop with self - run: | - cd cmd/interop - ./grpc-self-test.sh - - clang-tidy: - if: github.event.pull_request.draft == false - needs: build-and-unit-test - name: Build with clang-tidy - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - crypto: [openssl_1.1, openssl_3, boringssl] - - env: - BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" - CRYPTO_DIR: "./alternatives/${{ matrix.crypto }}" - - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - - uses: ./.github/actions/build - with: - cache-dir: ${{ github.workspace }}/vcpkg_cache - - - name: Build with clang-tidy - run: | - cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" \ - -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON - cmake --build "${{ env.BUILD_DIR }}" +# interop-test: +# if: github.event.pull_request.draft == false +# needs: build-and-unit-test +# name: Interop test +# runs-on: ${{ matrix.os }} +# strategy: +# matrix: +# os: [ubuntu-latest] +# crypto: [openssl_1.1] +# include: +# - os: ubuntu-latest +# ctest-target: test +# +# env: +# BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" +# CRYPTO_DIR: "./alternatives/${{ matrix.crypto }}" +# +# steps: +# - uses: actions/checkout@v4 +# with: +# submodules: recursive +# fetch-depth: 0 +# +# - uses: ./.github/actions/build +# with: +# cache-dir: ${{ github.workspace }}/vcpkg_cache +# +# - name: Build +# run: | +# cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" +# cmake --build "${{ env.BUILD_DIR }}" +# +# - name: Build (Interop Harness) +# run: | +# cd cmd/interop +# cmake -B build +# cmake --build build +# +# - name: Test self-interop +# run: | +# make -C cmd/interop self-test +# +# - name: Test interop on test vectors +# run: | +# make -C cmd/interop interop-test +# +# - name: Test gRPC live interop with self +# run: | +# cd cmd/interop +# ./grpc-self-test.sh +# +# clang-tidy: +# if: github.event.pull_request.draft == false +# needs: build-and-unit-test +# name: Build with clang-tidy +# runs-on: ${{ matrix.os }} +# strategy: +# matrix: +# os: [ubuntu-latest] +# crypto: [openssl_1.1, openssl_3, boringssl] +# +# env: +# BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" +# CRYPTO_DIR: "./alternatives/${{ matrix.crypto }}" +# +# steps: +# - uses: actions/checkout@v4 +# with: +# submodules: recursive +# fetch-depth: 0 +# +# - uses: ./.github/actions/build +# with: +# cache-dir: ${{ github.workspace }}/vcpkg_cache +# +# - name: Build with clang-tidy +# run: | +# cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" \ +# -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON +# cmake --build "${{ env.BUILD_DIR }}" diff --git a/lib/bytes/test/bytes.cpp b/lib/bytes/test/bytes.cpp index 38d3136c..48bf9c2f 100644 --- a/lib/bytes/test/bytes.cpp +++ b/lib/bytes/test/bytes.cpp @@ -2,6 +2,7 @@ #include #include #include +#include using namespace MLS_NAMESPACE::bytes_ns; using namespace std::literals::string_literals; @@ -21,6 +22,9 @@ TEST_CASE("Zeroization") const auto* end = begin + size; vec.reset(); + const auto snapshot = std::vector(begin, end); + std::cout << "snapshot = " << to_hex(snapshot) << std::endl; + // In principle, the memory previously owned by the vector should be all zero // at this point. However, since this is now unallocated memory, the // allocator can do with it what it wants, and may have written something to From fde8f6af4255d3644988b38829ea8cb3bb5df0fc Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 17:49:18 -0400 Subject: [PATCH 46/68] Really disable non-Windows builds --- .github/workflows/main_ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index e322ad15..4b80b0b8 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -41,10 +41,10 @@ jobs: include: - os: windows-latest ctest-target: RUN_TESTS - - os: ubuntu-latest - ctest-target: test - - os: macos-latest - ctest-target: test +# - os: ubuntu-latest +# ctest-target: test +# - os: macos-latest +# ctest-target: test env: BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" From 7c859e32dc68a51974c870c22807bfc8f6d46b65 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 18:21:43 -0400 Subject: [PATCH 47/68] Change canary value --- lib/bytes/test/bytes.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bytes/test/bytes.cpp b/lib/bytes/test/bytes.cpp index 48bf9c2f..cfbfdc0b 100644 --- a/lib/bytes/test/bytes.cpp +++ b/lib/bytes/test/bytes.cpp @@ -14,7 +14,7 @@ using namespace std::literals::string_literals; TEST_CASE("Zeroization") { const auto size = size_t(1024); - const auto canary = uint8_t(0xff); + const auto canary = uint8_t(0xa0); auto vec = std::make_unique(size, canary); const auto* begin = vec->data(); From c0e52941df78fd97899840c5359800d2e935b89a Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 18:46:11 -0400 Subject: [PATCH 48/68] Introduce some Microsoft-specific behavior --- .github/workflows/main_ci.yml | 14 +++++++------- CMakeLists.txt | 3 +++ lib/bytes/include/bytes/bytes.h | 4 ++-- lib/bytes/src/bytes.cpp | 13 +++++++++++++ lib/bytes/test/bytes.cpp | 21 +++++++++++++++------ 5 files changed, 40 insertions(+), 15 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 4b80b0b8..aa0b5f1d 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -34,17 +34,17 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: -# os: [windows-latest, ubuntu-latest, macos-latest] + os: [windows-latest, ubuntu-latest, macos-latest] +# os: [windows-latest] # crypto: [openssl_1.1, openssl_3, boringssl] - os: [windows-latest] - crypto: [openssl_1.1, openssl_3, boringssl] + crypto: [openssl_1.1] include: - os: windows-latest ctest-target: RUN_TESTS -# - os: ubuntu-latest -# ctest-target: test -# - os: macos-latest -# ctest-target: test + - os: ubuntu-latest + ctest-target: test + - os: macos-latest + ctest-target: test env: BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" diff --git a/CMakeLists.txt b/CMakeLists.txt index c827512c..83c9c0f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,6 +96,9 @@ endif() ### Dependencies ### +# Configure vcpkg to only build release libraries +set(VCPKG_BUILD_TYPE release) + # External libraries find_package(OpenSSL REQUIRED) if ( OPENSSL_FOUND ) diff --git a/lib/bytes/include/bytes/bytes.h b/lib/bytes/include/bytes/bytes.h index 582701a2..33f7119b 100644 --- a/lib/bytes/include/bytes/bytes.h +++ b/lib/bytes/include/bytes/bytes.h @@ -16,8 +16,8 @@ struct bytes bytes(bytes&&) = default; bytes& operator=(bytes&&) = default; - // Zeroize on drop - ~bytes() { std::fill(_data.begin(), _data.end(), uint8_t(0)); } + // Zeroize on destruction + ~bytes(); // Mimic std::vector ctors bytes(size_t count, const uint8_t& value = 0) diff --git a/lib/bytes/src/bytes.cpp b/lib/bytes/src/bytes.cpp index 2bb2e93e..c8ea3cd0 100644 --- a/lib/bytes/src/bytes.cpp +++ b/lib/bytes/src/bytes.cpp @@ -6,8 +6,21 @@ #include #include +#if _MSC_VER +#include +#endif + namespace MLS_NAMESPACE::bytes_ns { +bytes::~bytes() +{ +#ifndef _MSC_VER + std::fill(_data.begin(), _data.end(), uint8_t(0)); +#else + SecureZeroBytes(_data.data(), _data.size()); +#endif +} + bool bytes::operator==(const bytes& other) const { diff --git a/lib/bytes/test/bytes.cpp b/lib/bytes/test/bytes.cpp index cfbfdc0b..8c986369 100644 --- a/lib/bytes/test/bytes.cpp +++ b/lib/bytes/test/bytes.cpp @@ -28,12 +28,21 @@ TEST_CASE("Zeroization") // In principle, the memory previously owned by the vector should be all zero // at this point. However, since this is now unallocated memory, the // allocator can do with it what it wants, and may have written something to - // it when deallocating. For example, on macOS, the allocator appears to - // write a single pointer at the beginning. Assuming other platforms are not - // too different, we verify that no more than a few pointer's worth of bytes - // are non-zero. - const auto non_zero_threshold = 4 * sizeof(void*); - REQUIRE(std::count(begin, end, 0) >= size - non_zero_threshold); + // it when deallocating. This means we need to vary the test per OS. +#ifndef _MSC_VER + // macOS and Linux mostly leave the buffer alone, writing a couple of pointers + // to the beginning. So we look for the buffer to be basically all zero. + REQUIRE(std::count(snapshot.begin(), snapshot.end(), 0) == size); +#else + // Windows appeares to overwrite the buffer with 0xcd, so we test for that + // behavior. Note that this is testing for zeroization within a process, + // different from the page zeroization that happens when a page of memory is + // reallocated to another process. + // + // https://stackoverflow.com/questions/18385556/does-windows-clear-memory-pages + const auto ms_sentinel = uint8_t(0xcd); + REQUIRE(std::count(snapshot.begin(), snapshot.end(), ms_sentinel) == size); +#endif } #endif From 605190044862cfdbbf8c7e84a008a824f05c0fb1 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 21:12:45 -0400 Subject: [PATCH 49/68] Re-add some tolerance --- lib/bytes/test/bytes.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/bytes/test/bytes.cpp b/lib/bytes/test/bytes.cpp index 8c986369..525c7c3e 100644 --- a/lib/bytes/test/bytes.cpp +++ b/lib/bytes/test/bytes.cpp @@ -1,8 +1,8 @@ #include #include +#include #include #include -#include using namespace MLS_NAMESPACE::bytes_ns; using namespace std::literals::string_literals; @@ -32,7 +32,8 @@ TEST_CASE("Zeroization") #ifndef _MSC_VER // macOS and Linux mostly leave the buffer alone, writing a couple of pointers // to the beginning. So we look for the buffer to be basically all zero. - REQUIRE(std::count(snapshot.begin(), snapshot.end(), 0) == size); + const auto threshold = size - 4 * sizeof(void*); + REQUIRE(std::count(snapshot.begin(), snapshot.end(), 0) >= threshold); #else // Windows appeares to overwrite the buffer with 0xcd, so we test for that // behavior. Note that this is testing for zeroization within a process, From eec77aa7b9d27dc6d1b3809deaf69d090cbc9bca Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 21:25:31 -0400 Subject: [PATCH 50/68] Fix Windows typo --- lib/bytes/src/bytes.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bytes/src/bytes.cpp b/lib/bytes/src/bytes.cpp index c8ea3cd0..7ed3dc9f 100644 --- a/lib/bytes/src/bytes.cpp +++ b/lib/bytes/src/bytes.cpp @@ -17,7 +17,7 @@ bytes::~bytes() #ifndef _MSC_VER std::fill(_data.begin(), _data.end(), uint8_t(0)); #else - SecureZeroBytes(_data.data(), _data.size()); + SecureZeroMemory(_data.data(), _data.size()); #endif } From a8d9bfa666d47242cf21e6efcf3f3c6fa52d223e Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 21:39:49 -0400 Subject: [PATCH 51/68] Skip Message Protection tests --- .github/workflows/main_ci.yml | 4 ++-- lib/mls_vectors/test/mls_vectors.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index aa0b5f1d..2684e2d0 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -34,8 +34,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest, ubuntu-latest, macos-latest] -# os: [windows-latest] +# os: [windows-latest, ubuntu-latest, macos-latest] + os: [windows-latest] # crypto: [openssl_1.1, openssl_3, boringssl] crypto: [openssl_1.1] include: diff --git a/lib/mls_vectors/test/mls_vectors.cpp b/lib/mls_vectors/test/mls_vectors.cpp index 7ff741b6..18030be3 100644 --- a/lib/mls_vectors/test/mls_vectors.cpp +++ b/lib/mls_vectors/test/mls_vectors.cpp @@ -46,7 +46,7 @@ TEST_CASE("Key Schedule") } } -TEST_CASE("Message Protection") +TEST_CASE("Message Protection" * doctest::skip(true)) { for (auto suite : supported_suites) { auto tv = MessageProtectionTestVector{ suite }; From e395a07fd38621512c05b967f03e6961df334703 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Mon, 30 Oct 2023 21:41:52 -0400 Subject: [PATCH 52/68] Don't run spurious tests --- .github/workflows/main_ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 2684e2d0..78324f4f 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -41,10 +41,10 @@ jobs: include: - os: windows-latest ctest-target: RUN_TESTS - - os: ubuntu-latest - ctest-target: test - - os: macos-latest - ctest-target: test +# - os: ubuntu-latest +# ctest-target: test +# - os: macos-latest +# ctest-target: test env: BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" From 8f5180016e936c45cfd0c684edd207b17a461adc Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Tue, 31 Oct 2023 10:27:06 -0400 Subject: [PATCH 53/68] Only run interop tests when testing interop --- .github/workflows/main_ci.yml | 4 ++++ Makefile | 7 +------ lib/mls_vectors/test/CMakeLists.txt | 6 +++--- lib/mls_vectors/test/mls_vectors.cpp | 2 +- test/messages.cpp | 8 -------- test/treekem.cpp | 12 ------------ 6 files changed, 9 insertions(+), 30 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 78324f4f..b24edfc7 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -101,6 +101,10 @@ jobs: # cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" # cmake --build "${{ env.BUILD_DIR }}" # +# - name: Unit tests for test vector logic +# run: | +# ./${{ env.BUILD_DIR }}/lib/mls_vectors/test/mls_vectors_test +# # - name: Build (Interop Harness) # run: | # cd cmd/interop diff --git a/Makefile b/Makefile index 8aaaecc2..3b29e655 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ TOOLCHAIN_FILE=vcpkg/scripts/buildsystems/vcpkg.cmake .PHONY: all dev dev3 test ctest dtest dbtest libs test-libs test-all everything ci ci3 clean cclean format all: ${BUILD_DIR} - cmake --build ${BUILD_DIR} --target mlspp + cmake --build ${BUILD_DIR} ${BUILD_DIR}: CMakeLists.txt test/CMakeLists.txt cmake -B${BUILD_DIR} . @@ -52,19 +52,14 @@ libs: ${BUILD_DIR} cmake --build ${BUILD_DIR} --target bytes cmake --build ${BUILD_DIR} --target hpke cmake --build ${BUILD_DIR} --target tls_syntax - cmake --build ${BUILD_DIR} --target mls_vectors test-libs: ${BUILD_DIR} cmake --build ${BUILD_DIR} --target lib/bytes/test cmake --build ${BUILD_DIR} --target lib/hpke/test cmake --build ${BUILD_DIR} --target lib/tls_syntax/test - cmake --build ${BUILD_DIR} --target lib/mls_vectors/test test-all: test-libs ctest -everything: ${BUILD_DIR} - cmake --build ${BUILD_DIR} - ci: ${TOOLCHAIN_FILE} cmake -B ${BUILD_DIR} -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON \ -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} diff --git a/lib/mls_vectors/test/CMakeLists.txt b/lib/mls_vectors/test/CMakeLists.txt index 6256ddbf..f973a9ca 100644 --- a/lib/mls_vectors/test/CMakeLists.txt +++ b/lib/mls_vectors/test/CMakeLists.txt @@ -10,6 +10,6 @@ add_executable(${TEST_APP_NAME} ${TEST_SOURCES}) add_dependencies(${TEST_APP_NAME} ${CURRENT_LIB_NAME} bytes tls_syntax) target_link_libraries(${TEST_APP_NAME} ${CURRENT_LIB_NAME} doctest::doctest) -# Enable CTest -include(doctest) -doctest_discover_tests(${TEST_APP_NAME} ADD_LABELS 0) +# Note that other modules enable CTest at this stage. We do not enable CTest +# for the test vector logic because these tests take quite a bit of time to run, +# and are better run as part of interop testing. diff --git a/lib/mls_vectors/test/mls_vectors.cpp b/lib/mls_vectors/test/mls_vectors.cpp index 18030be3..7ff741b6 100644 --- a/lib/mls_vectors/test/mls_vectors.cpp +++ b/lib/mls_vectors/test/mls_vectors.cpp @@ -46,7 +46,7 @@ TEST_CASE("Key Schedule") } } -TEST_CASE("Message Protection" * doctest::skip(true)) +TEST_CASE("Message Protection") { for (auto suite : supported_suites) { auto tv = MessageProtectionTestVector{ suite }; diff --git a/test/messages.cpp b/test/messages.cpp index 033c9fac..6fed2017 100644 --- a/test/messages.cpp +++ b/test/messages.cpp @@ -155,11 +155,3 @@ TEST_CASE("Messages Interop") auto result = tv.verify(); REQUIRE(result == std::nullopt); } - -TEST_CASE("Message Protection Interop") -{ - for (auto suite : all_supported_suites) { - auto tv = MessageProtectionTestVector{ suite }; - REQUIRE(tv.verify() == std::nullopt); - } -} diff --git a/test/treekem.cpp b/test/treekem.cpp index aba87d82..d6ba0bcd 100644 --- a/test/treekem.cpp +++ b/test/treekem.cpp @@ -2,10 +2,8 @@ #include #include #include -#include using namespace MLS_NAMESPACE; -using namespace mls_vectors; class TreeKEMTest { @@ -265,13 +263,3 @@ TEST_CASE_FIXTURE(TreeKEMTest, "TreeKEM encap/decap") } } } - -TEST_CASE("TreeKEM Interop") -{ - for (auto suite : all_supported_suites) { - for (auto structure : treekem_test_tree_structures) { - auto tv = TreeKEMTestVector{ suite, structure }; - REQUIRE(tv.verify() == std::nullopt); - } - } -} From 804613884dcf00784c4da34db82707d4f0a1d550 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Wed, 1 Nov 2023 16:19:46 -0400 Subject: [PATCH 54/68] Manually run tests to provide more output --- .github/workflows/main_ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index b24edfc7..a129bf50 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -67,7 +67,8 @@ jobs: - name: Unit Test run: | - cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" + ./${{ env.BUILD_DIR }}/test/mlspp_test -s +# cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" # interop-test: # if: github.event.pull_request.draft == false From 495731ad9d576df1140cfddbb8dd2a8b0b369d2c Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Wed, 1 Nov 2023 17:22:47 -0400 Subject: [PATCH 55/68] Save the cache to speed up future builds --- .github/workflows/main_ci.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index a129bf50..26676d6b 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -65,9 +65,14 @@ jobs: cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON cmake --build "${{ env.BUILD_DIR }}" - - name: Unit Test - run: | - ./${{ env.BUILD_DIR }}/test/mlspp_test -s + - uses: actions/checkout/save@v3 + with: + path: ${{ github.workspace }}/vcpkg_cache + key: v01-vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} + +# - name: Unit Test +# run: | +# ./${{ env.BUILD_DIR }}/test/mlspp_test -s # cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" # interop-test: From 892761b261fc78983aaa62ee35a0451eb46020a3 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Wed, 1 Nov 2023 17:23:56 -0400 Subject: [PATCH 56/68] Typo --- .github/workflows/main_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 26676d6b..ecd12a04 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -65,7 +65,7 @@ jobs: cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON cmake --build "${{ env.BUILD_DIR }}" - - uses: actions/checkout/save@v3 + - uses: actions/cache/save@v3 with: path: ${{ github.workspace }}/vcpkg_cache key: v01-vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} From 9c31d3a05158eae7b19fc309d7d3f5cd4feb4189 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Wed, 1 Nov 2023 17:42:57 -0400 Subject: [PATCH 57/68] Manually run MLSpp unit tests --- .github/workflows/main_ci.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index ecd12a04..426dd486 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -65,14 +65,9 @@ jobs: cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON cmake --build "${{ env.BUILD_DIR }}" - - uses: actions/cache/save@v3 - with: - path: ${{ github.workspace }}/vcpkg_cache - key: v01-vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg_commit.txt', 'alternatives/*/vcpkg.json') }} - -# - name: Unit Test -# run: | -# ./${{ env.BUILD_DIR }}/test/mlspp_test -s + - name: Unit Test + run: | + .\${{ env.BUILD_DIR }}\test\mlspp_test -s # cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" # interop-test: From 98f1ad7f66bd9a03b5f351bf7788fe71f7d14eea Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Thu, 2 Nov 2023 09:37:42 -0400 Subject: [PATCH 58/68] Revert to ctest; add SANITIZERS=ON --- .github/workflows/main_ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 426dd486..6fa33685 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -62,13 +62,12 @@ jobs: - name: Build run: | - cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON + cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON -DSANITIZERS=ON cmake --build "${{ env.BUILD_DIR }}" - name: Unit Test run: | - .\${{ env.BUILD_DIR }}\test\mlspp_test -s -# cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" + cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" # interop-test: # if: github.event.pull_request.draft == false From c8271dbad5aec434c3bae38a2f9c2ee6a7e89a35 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Thu, 2 Nov 2023 09:45:41 -0400 Subject: [PATCH 59/68] Re-enable interop tests --- lib/mls_vectors/test/CMakeLists.txt | 6 +++--- test/messages.cpp | 8 ++++++++ test/treekem.cpp | 12 ++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/lib/mls_vectors/test/CMakeLists.txt b/lib/mls_vectors/test/CMakeLists.txt index f973a9ca..6256ddbf 100644 --- a/lib/mls_vectors/test/CMakeLists.txt +++ b/lib/mls_vectors/test/CMakeLists.txt @@ -10,6 +10,6 @@ add_executable(${TEST_APP_NAME} ${TEST_SOURCES}) add_dependencies(${TEST_APP_NAME} ${CURRENT_LIB_NAME} bytes tls_syntax) target_link_libraries(${TEST_APP_NAME} ${CURRENT_LIB_NAME} doctest::doctest) -# Note that other modules enable CTest at this stage. We do not enable CTest -# for the test vector logic because these tests take quite a bit of time to run, -# and are better run as part of interop testing. +# Enable CTest +include(doctest) +doctest_discover_tests(${TEST_APP_NAME} ADD_LABELS 0) diff --git a/test/messages.cpp b/test/messages.cpp index 6fed2017..033c9fac 100644 --- a/test/messages.cpp +++ b/test/messages.cpp @@ -155,3 +155,11 @@ TEST_CASE("Messages Interop") auto result = tv.verify(); REQUIRE(result == std::nullopt); } + +TEST_CASE("Message Protection Interop") +{ + for (auto suite : all_supported_suites) { + auto tv = MessageProtectionTestVector{ suite }; + REQUIRE(tv.verify() == std::nullopt); + } +} diff --git a/test/treekem.cpp b/test/treekem.cpp index d6ba0bcd..aba87d82 100644 --- a/test/treekem.cpp +++ b/test/treekem.cpp @@ -2,8 +2,10 @@ #include #include #include +#include using namespace MLS_NAMESPACE; +using namespace mls_vectors; class TreeKEMTest { @@ -263,3 +265,13 @@ TEST_CASE_FIXTURE(TreeKEMTest, "TreeKEM encap/decap") } } } + +TEST_CASE("TreeKEM Interop") +{ + for (auto suite : all_supported_suites) { + for (auto structure : treekem_test_tree_structures) { + auto tv = TreeKEMTestVector{ suite, structure }; + REQUIRE(tv.verify() == std::nullopt); + } + } +} From b043ae102059c106d5313103674e6f2ec99214a0 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Thu, 2 Nov 2023 10:20:18 -0400 Subject: [PATCH 60/68] Revert more changes --- CMakeLists.txt | 1 + Makefile | 9 ++++++--- lib/bytes/include/bytes/bytes.h | 4 ++-- lib/bytes/src/bytes.cpp | 13 ------------- lib/bytes/test/bytes.cpp | 22 ++++++---------------- 5 files changed, 15 insertions(+), 34 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 83c9c0f1..91fa8e23 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,6 +44,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU" add_compile_options(-Wall -pedantic -Wextra -Werror -Wmissing-declarations) elseif(MSVC) add_compile_options(/W4 /WX) + add_definitions(-DWINDOWS) # MSVC helpfully recommends safer equivalents for things like # getenv, but they are not portable. diff --git a/Makefile b/Makefile index 3b29e655..03514fb6 100644 --- a/Makefile +++ b/Makefile @@ -8,14 +8,13 @@ BUILD_DIR=build TEST_DIR=build/test CLANG_FORMAT=clang-format -i CLANG_TIDY=OFF -OPENSSL11_MANIFEST=alternatives/openssl_1.1 OPENSSL3_MANIFEST=alternatives/openssl_3 TOOLCHAIN_FILE=vcpkg/scripts/buildsystems/vcpkg.cmake .PHONY: all dev dev3 test ctest dtest dbtest libs test-libs test-all everything ci ci3 clean cclean format all: ${BUILD_DIR} - cmake --build ${BUILD_DIR} + cmake --build ${BUILD_DIR} --target mlspp ${BUILD_DIR}: CMakeLists.txt test/CMakeLists.txt cmake -B${BUILD_DIR} . @@ -27,7 +26,6 @@ dev: ${TOOLCHAIN_FILE} # Only enable testing, not clang-tidy/sanitizers; the latter make the build # too slow, and we can run them in CI cmake -B${BUILD_DIR} -DTESTING=ON -DCMAKE_BUILD_TYPE=Debug \ - -DVCPKG_MANIFEST_DIR=${OPENSSL11_MANIFEST} \ -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} dev3: ${TOOLCHAIN_FILE} @@ -52,14 +50,19 @@ libs: ${BUILD_DIR} cmake --build ${BUILD_DIR} --target bytes cmake --build ${BUILD_DIR} --target hpke cmake --build ${BUILD_DIR} --target tls_syntax + cmake --build ${BUILD_DIR} --target mls_vectors test-libs: ${BUILD_DIR} cmake --build ${BUILD_DIR} --target lib/bytes/test cmake --build ${BUILD_DIR} --target lib/hpke/test cmake --build ${BUILD_DIR} --target lib/tls_syntax/test + cmake --build ${BUILD_DIR} --target lib/mls_vectors/test test-all: test-libs ctest +everything: ${BUILD_DIR} + cmake --build ${BUILD_DIR} + ci: ${TOOLCHAIN_FILE} cmake -B ${BUILD_DIR} -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON \ -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} diff --git a/lib/bytes/include/bytes/bytes.h b/lib/bytes/include/bytes/bytes.h index 33f7119b..582701a2 100644 --- a/lib/bytes/include/bytes/bytes.h +++ b/lib/bytes/include/bytes/bytes.h @@ -16,8 +16,8 @@ struct bytes bytes(bytes&&) = default; bytes& operator=(bytes&&) = default; - // Zeroize on destruction - ~bytes(); + // Zeroize on drop + ~bytes() { std::fill(_data.begin(), _data.end(), uint8_t(0)); } // Mimic std::vector ctors bytes(size_t count, const uint8_t& value = 0) diff --git a/lib/bytes/src/bytes.cpp b/lib/bytes/src/bytes.cpp index 7ed3dc9f..2bb2e93e 100644 --- a/lib/bytes/src/bytes.cpp +++ b/lib/bytes/src/bytes.cpp @@ -6,21 +6,8 @@ #include #include -#if _MSC_VER -#include -#endif - namespace MLS_NAMESPACE::bytes_ns { -bytes::~bytes() -{ -#ifndef _MSC_VER - std::fill(_data.begin(), _data.end(), uint8_t(0)); -#else - SecureZeroMemory(_data.data(), _data.size()); -#endif -} - bool bytes::operator==(const bytes& other) const { diff --git a/lib/bytes/test/bytes.cpp b/lib/bytes/test/bytes.cpp index 525c7c3e..111d63bb 100644 --- a/lib/bytes/test/bytes.cpp +++ b/lib/bytes/test/bytes.cpp @@ -9,8 +9,9 @@ using namespace std::literals::string_literals; // To check that memory is safely zeroized on destroy, we have to deliberately // do a use-after-free. This will be caught by the sanitizers, so we only do it -// when sanitizers are not enabled. -#ifndef SANITIZERS +// when sanitizers are not enabled. This test is also disabled on Windows +// because the use-after-free causes Windows CI runs to fail. +#if !defined(SANITIZERS) || defined(WINDOWS) TEST_CASE("Zeroization") { const auto size = size_t(1024); @@ -28,22 +29,11 @@ TEST_CASE("Zeroization") // In principle, the memory previously owned by the vector should be all zero // at this point. However, since this is now unallocated memory, the // allocator can do with it what it wants, and may have written something to - // it when deallocating. This means we need to vary the test per OS. -#ifndef _MSC_VER - // macOS and Linux mostly leave the buffer alone, writing a couple of pointers - // to the beginning. So we look for the buffer to be basically all zero. + // it when deallocating. macOS and Linux mostly leave the buffer alone, + // writing a couple of pointers to the beginning. So we look for the buffer + // to be basically all zero. const auto threshold = size - 4 * sizeof(void*); REQUIRE(std::count(snapshot.begin(), snapshot.end(), 0) >= threshold); -#else - // Windows appeares to overwrite the buffer with 0xcd, so we test for that - // behavior. Note that this is testing for zeroization within a process, - // different from the page zeroization that happens when a page of memory is - // reallocated to another process. - // - // https://stackoverflow.com/questions/18385556/does-windows-clear-memory-pages - const auto ms_sentinel = uint8_t(0xcd); - REQUIRE(std::count(snapshot.begin(), snapshot.end(), ms_sentinel) == size); -#endif } #endif From 5d5b52ac5b9ed08db28094b7e8d32a0dc159e906 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Thu, 2 Nov 2023 13:04:05 -0400 Subject: [PATCH 61/68] Fix Windows check --- lib/bytes/test/bytes.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/bytes/test/bytes.cpp b/lib/bytes/test/bytes.cpp index 111d63bb..39b1a909 100644 --- a/lib/bytes/test/bytes.cpp +++ b/lib/bytes/test/bytes.cpp @@ -10,8 +10,9 @@ using namespace std::literals::string_literals; // To check that memory is safely zeroized on destroy, we have to deliberately // do a use-after-free. This will be caught by the sanitizers, so we only do it // when sanitizers are not enabled. This test is also disabled on Windows -// because the use-after-free causes Windows CI runs to fail. -#if !defined(SANITIZERS) || defined(WINDOWS) +// because it appears to cause Windows CI runs to fail. (In addition, Windows +// appears to overwrite freed buffers with 0xCD, so this test is unnecessary.) +#if !defined(SANITIZERS) && !defined(WINDOWS) TEST_CASE("Zeroization") { const auto size = size_t(1024); From 82183b2aea89826641c7154b2d6f1cfd10f39859 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Thu, 2 Nov 2023 13:22:11 -0400 Subject: [PATCH 62/68] Remove sanitizers --- .github/workflows/main_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 6fa33685..b24edfc7 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -62,7 +62,7 @@ jobs: - name: Build run: | - cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON -DSANITIZERS=ON + cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON cmake --build "${{ env.BUILD_DIR }}" - name: Unit Test From c04949f9a2eb362793a9a1e2c71e924e8b2bf78a Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Thu, 2 Nov 2023 13:40:37 -0400 Subject: [PATCH 63/68] Re-add sanitizers --- .github/workflows/main_ci.yml | 2 +- CMakeLists.txt | 21 ++++++++------------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index b24edfc7..6fa33685 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -62,7 +62,7 @@ jobs: - name: Build run: | - cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON + cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON -DSANITIZERS=ON cmake --build "${{ env.BUILD_DIR }}" - name: Unit Test diff --git a/CMakeLists.txt b/CMakeLists.txt index 91fa8e23..436f8e78 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.12) +cmake_minimum_required(VERSION 3.13) project(mlspp VERSION 0.1 @@ -52,21 +52,16 @@ elseif(MSVC) endif() if (SANITIZERS) + message("Enabling sanitizers") + add_definitions(-DSANITIZERS) + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU") - set(SANITIZERS "-fsanitize=address -fsanitize=undefined") + add_compile_options(-fsanitize=address -fsanitize=undefined) + add_link_options(-fsanitize=address -fsanitize=undefined) elseif(MSVC) - set(SANITIZERS "/fsanitize=address") + # MSVC uses a different flag, and doesn't require passing it to the linker + add_compile_options("/fsanitize=address") endif() - - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SANITIZERS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SANITIZERS}") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${SANITIZERS}") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${SANITIZERS}") - set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${SANITIZERS}") - add_definitions(-DSANITIZERS) -elseif (SANITIZERS AND MSVC) - message("Enabling sanitizers") - add_definitions("/fsanitize=address") endif() if(CLANG_TIDY) From a0798184ab2cc859772ca580c90862a4f6ab18fd Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Thu, 2 Nov 2023 13:53:21 -0400 Subject: [PATCH 64/68] Revert "Re-add sanitizers" This reverts commit c04949f9a2eb362793a9a1e2c71e924e8b2bf78a. --- .github/workflows/main_ci.yml | 2 +- CMakeLists.txt | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 6fa33685..b24edfc7 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -62,7 +62,7 @@ jobs: - name: Build run: | - cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON -DSANITIZERS=ON + cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON cmake --build "${{ env.BUILD_DIR }}" - name: Unit Test diff --git a/CMakeLists.txt b/CMakeLists.txt index 436f8e78..91fa8e23 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.13) +cmake_minimum_required(VERSION 3.12) project(mlspp VERSION 0.1 @@ -52,16 +52,21 @@ elseif(MSVC) endif() if (SANITIZERS) - message("Enabling sanitizers") - add_definitions(-DSANITIZERS) - if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU") - add_compile_options(-fsanitize=address -fsanitize=undefined) - add_link_options(-fsanitize=address -fsanitize=undefined) + set(SANITIZERS "-fsanitize=address -fsanitize=undefined") elseif(MSVC) - # MSVC uses a different flag, and doesn't require passing it to the linker - add_compile_options("/fsanitize=address") + set(SANITIZERS "/fsanitize=address") endif() + + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SANITIZERS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SANITIZERS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${SANITIZERS}") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${SANITIZERS}") + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${SANITIZERS}") + add_definitions(-DSANITIZERS) +elseif (SANITIZERS AND MSVC) + message("Enabling sanitizers") + add_definitions("/fsanitize=address") endif() if(CLANG_TIDY) From 0c3d792802ff6e1710572a1f603c8d4db052f8e1 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Thu, 2 Nov 2023 13:53:40 -0400 Subject: [PATCH 65/68] Revert "Remove sanitizers" This reverts commit 82183b2aea89826641c7154b2d6f1cfd10f39859. --- .github/workflows/main_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index b24edfc7..6fa33685 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -62,7 +62,7 @@ jobs: - name: Build run: | - cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON + cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON -DSANITIZERS=ON cmake --build "${{ env.BUILD_DIR }}" - name: Unit Test From ff99638b9a0ee15265b2671b83a01f5d1cde9627 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Thu, 2 Nov 2023 13:58:10 -0400 Subject: [PATCH 66/68] Revert "Revert "Remove sanitizers"" This reverts commit 0c3d792802ff6e1710572a1f603c8d4db052f8e1. --- .github/workflows/main_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 6fa33685..b24edfc7 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -62,7 +62,7 @@ jobs: - name: Build run: | - cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON -DSANITIZERS=ON + cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON cmake --build "${{ env.BUILD_DIR }}" - name: Unit Test From 03554b956c08c6e163ec638d240f8607e856d813 Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Thu, 2 Nov 2023 13:58:16 -0400 Subject: [PATCH 67/68] Revert "Revert "Re-add sanitizers"" This reverts commit a0798184ab2cc859772ca580c90862a4f6ab18fd. --- .github/workflows/main_ci.yml | 2 +- CMakeLists.txt | 21 ++++++++------------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index b24edfc7..6fa33685 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -62,7 +62,7 @@ jobs: - name: Build run: | - cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON + cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON -DSANITIZERS=ON cmake --build "${{ env.BUILD_DIR }}" - name: Unit Test diff --git a/CMakeLists.txt b/CMakeLists.txt index 91fa8e23..436f8e78 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.12) +cmake_minimum_required(VERSION 3.13) project(mlspp VERSION 0.1 @@ -52,21 +52,16 @@ elseif(MSVC) endif() if (SANITIZERS) + message("Enabling sanitizers") + add_definitions(-DSANITIZERS) + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU") - set(SANITIZERS "-fsanitize=address -fsanitize=undefined") + add_compile_options(-fsanitize=address -fsanitize=undefined) + add_link_options(-fsanitize=address -fsanitize=undefined) elseif(MSVC) - set(SANITIZERS "/fsanitize=address") + # MSVC uses a different flag, and doesn't require passing it to the linker + add_compile_options("/fsanitize=address") endif() - - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SANITIZERS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SANITIZERS}") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${SANITIZERS}") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${SANITIZERS}") - set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${SANITIZERS}") - add_definitions(-DSANITIZERS) -elseif (SANITIZERS AND MSVC) - message("Enabling sanitizers") - add_definitions("/fsanitize=address") endif() if(CLANG_TIDY) From 13dca8ad635b2a8207f2f7554ffdeacc6b8ac39e Mon Sep 17 00:00:00 2001 From: Richard Barnes Date: Thu, 2 Nov 2023 14:02:25 -0400 Subject: [PATCH 68/68] Re-enable other tests --- .github/workflows/main_ci.yml | 198 ++++++++++++++++------------------ 1 file changed, 93 insertions(+), 105 deletions(-) diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 6fa33685..afbf93a4 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -15,36 +15,34 @@ env: CMAKE_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake jobs: -# formatting-check: -# name: Formatting Check -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# -# - name: Run clang-format style check for C/C++ programs -# uses: jidicula/clang-format-action@v4.11.0 -# with: -# clang-format-version: 16 -# include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' -# fallback-style: 'Mozilla' + formatting-check: + name: Formatting Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run clang-format style check for C/C++ programs + uses: jidicula/clang-format-action@v4.11.0 + with: + clang-format-version: 16 + include-regex: '^\./(src|include|test|cmd)/.*\.(cpp|h)$' + fallback-style: 'Mozilla' build-and-unit-test: -# needs: formatting-check + needs: formatting-check name: Build and test runs-on: ${{ matrix.os }} strategy: matrix: -# os: [windows-latest, ubuntu-latest, macos-latest] - os: [windows-latest] -# crypto: [openssl_1.1, openssl_3, boringssl] - crypto: [openssl_1.1] + os: [windows-latest, ubuntu-latest, macos-latest] + crypto: [openssl_1.1, openssl_3, boringssl] include: - os: windows-latest ctest-target: RUN_TESTS -# - os: ubuntu-latest -# ctest-target: test -# - os: macos-latest -# ctest-target: test + - os: ubuntu-latest + ctest-target: test + - os: macos-latest + ctest-target: test env: BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" @@ -62,6 +60,8 @@ jobs: - name: Build run: | + # XXX(RLB): If we do not have SANITIZERS=ON here, the Windows CI builds + # hang in the middle of unit testing. cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -DTESTING=ON -DSANITIZERS=ON cmake --build "${{ env.BUILD_DIR }}" @@ -69,87 +69,75 @@ jobs: run: | cmake --build "${{ env.BUILD_DIR }}" --target "${{ matrix.ctest-target}}" -# interop-test: -# if: github.event.pull_request.draft == false -# needs: build-and-unit-test -# name: Interop test -# runs-on: ${{ matrix.os }} -# strategy: -# matrix: -# os: [ubuntu-latest] -# crypto: [openssl_1.1] -# include: -# - os: ubuntu-latest -# ctest-target: test -# -# env: -# BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" -# CRYPTO_DIR: "./alternatives/${{ matrix.crypto }}" -# -# steps: -# - uses: actions/checkout@v4 -# with: -# submodules: recursive -# fetch-depth: 0 -# -# - uses: ./.github/actions/build -# with: -# cache-dir: ${{ github.workspace }}/vcpkg_cache -# -# - name: Build -# run: | -# cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" -# cmake --build "${{ env.BUILD_DIR }}" -# -# - name: Unit tests for test vector logic -# run: | -# ./${{ env.BUILD_DIR }}/lib/mls_vectors/test/mls_vectors_test -# -# - name: Build (Interop Harness) -# run: | -# cd cmd/interop -# cmake -B build -# cmake --build build -# -# - name: Test self-interop -# run: | -# make -C cmd/interop self-test -# -# - name: Test interop on test vectors -# run: | -# make -C cmd/interop interop-test -# -# - name: Test gRPC live interop with self -# run: | -# cd cmd/interop -# ./grpc-self-test.sh -# -# clang-tidy: -# if: github.event.pull_request.draft == false -# needs: build-and-unit-test -# name: Build with clang-tidy -# runs-on: ${{ matrix.os }} -# strategy: -# matrix: -# os: [ubuntu-latest] -# crypto: [openssl_1.1, openssl_3, boringssl] -# -# env: -# BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" -# CRYPTO_DIR: "./alternatives/${{ matrix.crypto }}" -# -# steps: -# - uses: actions/checkout@v4 -# with: -# submodules: recursive -# fetch-depth: 0 -# -# - uses: ./.github/actions/build -# with: -# cache-dir: ${{ github.workspace }}/vcpkg_cache -# -# - name: Build with clang-tidy -# run: | -# cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" \ -# -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON -# cmake --build "${{ env.BUILD_DIR }}" + interop-test: + if: github.event.pull_request.draft == false + needs: build-and-unit-test + name: Interop test + runs-on: ubuntu-latest + + env: + BUILD_DIR: "${RUNNER_TEMP}/build_openssl_1.1" + CRYPTO_DIR: "./alternatives/openssl_1.1" + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - uses: ./.github/actions/build + with: + cache-dir: ${{ github.workspace }}/vcpkg_cache + + - name: Build + run: | + cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" + cmake --build "${{ env.BUILD_DIR }}" + + - name: Build (Interop Harness) + run: | + cd cmd/interop + cmake -B build + cmake --build build + + - name: Test self-interop + run: | + make -C cmd/interop self-test + + - name: Test interop on test vectors + run: | + make -C cmd/interop interop-test + + - name: Test gRPC live interop with self + run: | + cd cmd/interop + ./grpc-self-test.sh + + clang-tidy: + if: github.event.pull_request.draft == false + needs: build-and-unit-test + name: Build with clang-tidy + runs-on: ubuntu-latest + strategy: + matrix: + crypto: [openssl_1.1, openssl_3, boringssl] + + env: + BUILD_DIR: "${RUNNER_TEMP}/build_${{ matrix.crypto }}" + CRYPTO_DIR: "./alternatives/${{ matrix.crypto }}" + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - uses: ./.github/actions/build + with: + cache-dir: ${{ github.workspace }}/vcpkg_cache + + - name: Build with clang-tidy + run: | + cmake -B "${{ env.BUILD_DIR }}" -DVCPKG_MANIFEST_DIR="${{ env.CRYPTO_DIR }}" \ + -DTESTING=ON -DCLANG_TIDY=ON -DSANITIZERS=ON + cmake --build "${{ env.BUILD_DIR }}"