diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cff6fd6b..d9dcc4b8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -191,9 +191,14 @@ jobs: working-directory: verification run: | cmake -DNUNAVUT_EXTRA_GENERATOR_ARGS="--enable-override-variable-array-capacity;--embed-auditing-info" --preset config-clang-native-cpp-20 - cmake --build --preset build-clang-native-cpp-20-debugcov + cmake --build --preset build-clang-native-cpp-20-debugcov --target cov_all cmake -DNUNAVUT_EXTRA_GENERATOR_ARGS="--enable-override-variable-array-capacity;--embed-auditing-info" --preset config-clang-native-c-11 - cmake --build --preset build-clang-native-c-11-debugcov + cmake --build --preset build-clang-native-c-11-debugcov --target cov_all + - name: upload-verification-coverage-reports + uses: actions/upload-artifact@v4 + with: + name: verification-coverage-reports + path: verification/build/DebugCov/coverage/* language-verification-python: runs-on: ubuntu-latest diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index fa0ff906..1d28a009 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -278,7 +278,15 @@ After configuring you can also use Ninja directly:: ninja -t targets To obtain coverage information for the verification suite (not the Python code), -build the `cov_all` target and inspect the output under the `coverage` directory. +build the `cov_all` target and inspect the output under the `coverage` directory:: + + cmake --build --preset build-clang-native-c-11-debug --target cov_all + +.. warning:: + + When switching between gcc and clang you must do a full clean of your repo if you previously ran the coverage + build. For example ``git clean -xdf`` or clone a new repo in a different folder. Each compiler suite leaves + different byproducts that may interfere with the coverage tools in the other suite. While we strongly encourage you to use the cmake presets, the CMakeLists.txt for the verification suite is driven by three variables you can set in your environment or pass into cmake if using cmake directly: diff --git a/verification/CMakePresets.json b/verification/CMakePresets.json index b8192499..b8d26f50 100644 --- a/verification/CMakePresets.json +++ b/verification/CMakePresets.json @@ -272,337 +272,337 @@ ], "buildPresets": [ { - "name": "build-clang-native-c-11-debugasan", + "name": "build-clang-native-c-11-debug", "configurePreset": "config-clang-native-c-11", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cpp-14-debugasan", + "name": "build-clang-native-cpp-14-debug", "configurePreset": "config-clang-native-cpp-14", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cetl-14-17-debugasan", + "name": "build-clang-native-cetl-14-17-debug", "configurePreset": "config-clang-native-cetl-14-17", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cpp-17-debugasan", + "name": "build-clang-native-cpp-17-debug", "configurePreset": "config-clang-native-cpp-17", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cpp-17-pmr-debugasan", + "name": "build-clang-native-cpp-17-pmr-debug", "configurePreset": "config-clang-native-cpp-17-pmr", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cpp-20-debugasan", + "name": "build-clang-native-cpp-20-debug", "configurePreset": "config-clang-native-cpp-20", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cpp-20-pmr-debugasan", + "name": "build-clang-native-cpp-20-pmr-debug", "configurePreset": "config-clang-native-cpp-20-pmr", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-c-11-debugasan", + "name": "build-gcc-native-c-11-debug", "configurePreset": "config-gcc-native-c-11", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cpp-14-debugasan", + "name": "build-gcc-native-cpp-14-debug", "configurePreset": "config-gcc-native-cpp-14", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cetl-14-17-debugasan", + "name": "build-gcc-native-cetl-14-17-debug", "configurePreset": "config-gcc-native-cetl-14-17", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cpp-17-debugasan", + "name": "build-gcc-native-cpp-17-debug", "configurePreset": "config-gcc-native-cpp-17", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cpp-17-pmr-debugasan", + "name": "build-gcc-native-cpp-17-pmr-debug", "configurePreset": "config-gcc-native-cpp-17-pmr", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cpp-20-debugasan", + "name": "build-gcc-native-cpp-20-debug", "configurePreset": "config-gcc-native-cpp-20", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cpp-20-pmr-debugasan", + "name": "build-gcc-native-cpp-20-pmr-debug", "configurePreset": "config-gcc-native-cpp-20-pmr", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-c-11-debugasan", + "name": "build-gcc-native32-c-11-debug", "configurePreset": "config-gcc-native32-c-11", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cpp-14-debugasan", + "name": "build-gcc-native32-cpp-14-debug", "configurePreset": "config-gcc-native32-cpp-14", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cetl-14-17-debugasan", + "name": "build-gcc-native32-cetl-14-17-debug", "configurePreset": "config-gcc-native32-cetl-14-17", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cpp-17-debugasan", + "name": "build-gcc-native32-cpp-17-debug", "configurePreset": "config-gcc-native32-cpp-17", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cpp-17-pmr-debugasan", + "name": "build-gcc-native32-cpp-17-pmr-debug", "configurePreset": "config-gcc-native32-cpp-17-pmr", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cpp-20-debugasan", + "name": "build-gcc-native32-cpp-20-debug", "configurePreset": "config-gcc-native32-cpp-20", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cpp-20-pmr-debugasan", + "name": "build-gcc-native32-cpp-20-pmr-debug", "configurePreset": "config-gcc-native32-cpp-20-pmr", - "configuration": "DebugAsan", + "configuration": "Debug", "targets": [ "test_all" ] }, { - "name": "build-clang-native-c-11-release", + "name": "build-clang-native-c-11-debugasan", "configurePreset": "config-clang-native-c-11", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cpp-14-release", + "name": "build-clang-native-cpp-14-debugasan", "configurePreset": "config-clang-native-cpp-14", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cetl-14-17-release", + "name": "build-clang-native-cetl-14-17-debugasan", "configurePreset": "config-clang-native-cetl-14-17", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cpp-17-release", + "name": "build-clang-native-cpp-17-debugasan", "configurePreset": "config-clang-native-cpp-17", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cpp-17-pmr-release", + "name": "build-clang-native-cpp-17-pmr-debugasan", "configurePreset": "config-clang-native-cpp-17-pmr", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cpp-20-release", + "name": "build-clang-native-cpp-20-debugasan", "configurePreset": "config-clang-native-cpp-20", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cpp-20-pmr-release", + "name": "build-clang-native-cpp-20-pmr-debugasan", "configurePreset": "config-clang-native-cpp-20-pmr", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-c-11-release", + "name": "build-gcc-native-c-11-debugasan", "configurePreset": "config-gcc-native-c-11", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cpp-14-release", + "name": "build-gcc-native-cpp-14-debugasan", "configurePreset": "config-gcc-native-cpp-14", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cetl-14-17-release", + "name": "build-gcc-native-cetl-14-17-debugasan", "configurePreset": "config-gcc-native-cetl-14-17", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cpp-17-release", + "name": "build-gcc-native-cpp-17-debugasan", "configurePreset": "config-gcc-native-cpp-17", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cpp-17-pmr-release", + "name": "build-gcc-native-cpp-17-pmr-debugasan", "configurePreset": "config-gcc-native-cpp-17-pmr", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cpp-20-release", + "name": "build-gcc-native-cpp-20-debugasan", "configurePreset": "config-gcc-native-cpp-20", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cpp-20-pmr-release", + "name": "build-gcc-native-cpp-20-pmr-debugasan", "configurePreset": "config-gcc-native-cpp-20-pmr", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-c-11-release", + "name": "build-gcc-native32-c-11-debugasan", "configurePreset": "config-gcc-native32-c-11", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cpp-14-release", + "name": "build-gcc-native32-cpp-14-debugasan", "configurePreset": "config-gcc-native32-cpp-14", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cetl-14-17-release", + "name": "build-gcc-native32-cetl-14-17-debugasan", "configurePreset": "config-gcc-native32-cetl-14-17", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cpp-17-release", + "name": "build-gcc-native32-cpp-17-debugasan", "configurePreset": "config-gcc-native32-cpp-17", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cpp-17-pmr-release", + "name": "build-gcc-native32-cpp-17-pmr-debugasan", "configurePreset": "config-gcc-native32-cpp-17-pmr", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cpp-20-release", + "name": "build-gcc-native32-cpp-20-debugasan", "configurePreset": "config-gcc-native32-cpp-20", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cpp-20-pmr-release", + "name": "build-gcc-native32-cpp-20-pmr-debugasan", "configurePreset": "config-gcc-native32-cpp-20-pmr", - "configuration": "Release", + "configuration": "DebugAsan", "targets": [ "test_all" ] @@ -776,169 +776,169 @@ ] }, { - "name": "build-clang-native-c-11-debug", + "name": "build-clang-native-c-11-release", "configurePreset": "config-clang-native-c-11", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cpp-14-debug", + "name": "build-clang-native-cpp-14-release", "configurePreset": "config-clang-native-cpp-14", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cetl-14-17-debug", + "name": "build-clang-native-cetl-14-17-release", "configurePreset": "config-clang-native-cetl-14-17", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cpp-17-debug", + "name": "build-clang-native-cpp-17-release", "configurePreset": "config-clang-native-cpp-17", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cpp-17-pmr-debug", + "name": "build-clang-native-cpp-17-pmr-release", "configurePreset": "config-clang-native-cpp-17-pmr", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cpp-20-debug", + "name": "build-clang-native-cpp-20-release", "configurePreset": "config-clang-native-cpp-20", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-clang-native-cpp-20-pmr-debug", + "name": "build-clang-native-cpp-20-pmr-release", "configurePreset": "config-clang-native-cpp-20-pmr", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-c-11-debug", + "name": "build-gcc-native-c-11-release", "configurePreset": "config-gcc-native-c-11", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cpp-14-debug", + "name": "build-gcc-native-cpp-14-release", "configurePreset": "config-gcc-native-cpp-14", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cetl-14-17-debug", + "name": "build-gcc-native-cetl-14-17-release", "configurePreset": "config-gcc-native-cetl-14-17", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cpp-17-debug", + "name": "build-gcc-native-cpp-17-release", "configurePreset": "config-gcc-native-cpp-17", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cpp-17-pmr-debug", + "name": "build-gcc-native-cpp-17-pmr-release", "configurePreset": "config-gcc-native-cpp-17-pmr", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cpp-20-debug", + "name": "build-gcc-native-cpp-20-release", "configurePreset": "config-gcc-native-cpp-20", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-gcc-native-cpp-20-pmr-debug", + "name": "build-gcc-native-cpp-20-pmr-release", "configurePreset": "config-gcc-native-cpp-20-pmr", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-c-11-debug", + "name": "build-gcc-native32-c-11-release", "configurePreset": "config-gcc-native32-c-11", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cpp-14-debug", + "name": "build-gcc-native32-cpp-14-release", "configurePreset": "config-gcc-native32-cpp-14", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cetl-14-17-debug", + "name": "build-gcc-native32-cetl-14-17-release", "configurePreset": "config-gcc-native32-cetl-14-17", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cpp-17-debug", + "name": "build-gcc-native32-cpp-17-release", "configurePreset": "config-gcc-native32-cpp-17", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cpp-17-pmr-debug", + "name": "build-gcc-native32-cpp-17-pmr-release", "configurePreset": "config-gcc-native32-cpp-17-pmr", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cpp-20-debug", + "name": "build-gcc-native32-cpp-20-release", "configurePreset": "config-gcc-native32-cpp-20", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] }, { - "name": "build-gcc-native32-cpp-20-pmr-debug", + "name": "build-gcc-native32-cpp-20-pmr-release", "configurePreset": "config-gcc-native32-cpp-20-pmr", - "configuration": "Debug", + "configuration": "Release", "targets": [ "test_all" ] diff --git a/verification/cmake/compiler_flag_sets/common.cmake b/verification/cmake/compiler_flag_sets/common.cmake index d60ac361..ea2071f3 100644 --- a/verification/cmake/compiler_flag_sets/common.cmake +++ b/verification/cmake/compiler_flag_sets/common.cmake @@ -95,12 +95,12 @@ add_compile_options("$<$:${ASM_FLAG_SET}>") add_compile_options("$<$:-Wno-stringop-overflow>") add_compile_options("$<$:${LOCAL_SANITIZER_OPTIONS}>") add_compile_options( - "$<$,$>:--coverage>" + "$<$,$>:--coverage>" "$<$,$>:-fno-elide-constructors>" - "$<$,$>:-fprofile-instr-generate>" - "$<$,$>:-ftest-coverage>" - "$<$,$>:-fprofile-arcs>" - "$<$,$>:-fcoverage-mapping>" + "$<$,$>:-fprofile-instr-generate>" + "$<$,$>:-ftest-coverage>" + "$<$,$>:-fprofile-arcs>" + "$<$,$>:-fcoverage-mapping>" ) add_link_options(${EXE_LINKER_FLAG_SET}) add_link_options("$<$:${LOCAL_SANITIZER_OPTIONS}>") diff --git a/verification/cmake/toolchains/clang-native.cmake b/verification/cmake/toolchains/clang-native.cmake index 4fb9aa36..f6e704b7 100644 --- a/verification/cmake/toolchains/clang-native.cmake +++ b/verification/cmake/toolchains/clang-native.cmake @@ -8,8 +8,4 @@ set(CMAKE_C_COMPILER clang CACHE FILEPATH "C compiler") set(CMAKE_CXX_COMPILER clang++ CACHE FILEPATH "C++ compiler") set(CMAKE_ASM_COMPILER clang CACHE FILEPATH "assembler") -# Signal to unit tests that the coverage data will be generated by -# clang so they should try to use llvm-cov if it's available. -# Things are weird between gcov and llvm-cov right now but this -# seems to work in our Ubuntu 18 Docker container, so...? set(NUNAVUT_USE_LLVM_COV ON CACHE BOOL "Enable gcov compatibility with lcov coverage tools.") diff --git a/verification/cmake/toolchains/gcc-native.cmake b/verification/cmake/toolchains/gcc-native.cmake index 03d7bd02..10293003 100644 --- a/verification/cmake/toolchains/gcc-native.cmake +++ b/verification/cmake/toolchains/gcc-native.cmake @@ -6,3 +6,5 @@ set(CMAKE_C_COMPILER gcc CACHE FILEPATH "C compiler") set(CMAKE_CXX_COMPILER g++ CACHE FILEPATH "C++ compiler") set(CMAKE_ASM_COMPILER gcc CACHE FILEPATH "assembler") + +set(NUNAVUT_USE_LLVM_COV OFF CACHE BOOL "Enable gcov compatibility with lcov coverage tools.") diff --git a/verification/preset_generator.py b/verification/preset_generator.py index 3da851ef..b09fdd12 100755 --- a/verification/preset_generator.py +++ b/verification/preset_generator.py @@ -214,7 +214,7 @@ def find_configuration_types(args: argparse.Namespace, hidden_presets: list[dict "No CMAKE_CONFIGURATION_TYPES found in common configurations and CMAKE_DEFAULT_BUILD_TYPE is not set." ) - return list(configuration_types) if configuration_types else list(default_build_types) + return sorted(configuration_types) if configuration_types else sorted(default_build_types) def update_hidden_configure_presets(args: argparse.Namespace, configure_presets: dict) -> list[dict]: @@ -316,7 +316,7 @@ def main() -> int: if not validate_json_schema(args, json_presets): return 1 - if args.clean >= 1: + if args.clean is not None and args.clean >= 1: # clean level 1 deletes all existing buildPresets json_presets["buildPresets"] = []