diff --git a/.github/workflows/_unittest.yml b/.github/workflows/_unittest.yml index cfad82d949..c96840e359 100644 --- a/.github/workflows/_unittest.yml +++ b/.github/workflows/_unittest.yml @@ -37,6 +37,7 @@ jobs: # Run pytest with coverage pytest -n auto --cov=./ --cov-report=xml # Run gtest + LLVM_PROFDATA=llvm-profdata-12 LLVM_COV=llvm-cov-12 \ test/run_oss_cpp_tests.sh macos: @@ -66,4 +67,5 @@ jobs: # Run pytest with coverage ${CONDA_RUN} pytest -n auto --cov=./ --cov-report=xml # Run gtest + LLVM_PROFDATA="xcrun llvm-profdata" LLVM_COV="xcrun llvm-cov" \ ${CONDA_RUN} test/run_oss_cpp_tests.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b4c31a131..63fdb4782e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -343,6 +343,23 @@ else() set(CMAKE_TOOLCHAIN_ANDROID OFF) endif() +# Add code coverage flags to supported compilers +if(EXECUTORCH_USE_CPP_CODE_COVERAGE) + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + string(APPEND CMAKE_C_FLAGS " --coverage -fprofile-abs-path") + string(APPEND CMAKE_CXX_FLAGS " --coverage -fprofile-abs-path") + elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + string(APPEND CMAKE_C_FLAGS " -fprofile-instr-generate -fcoverage-mapping") + string(APPEND CMAKE_CXX_FLAGS + " -fprofile-instr-generate -fcoverage-mapping" + ) + else() + message(ERROR + "Code coverage for compiler ${CMAKE_CXX_COMPILER_ID} is unsupported" + ) + endif() +endif() + # EXECUTORCH_BUILD_HOST_TARGETS: Option to control the building of host-only # tools like `flatc`, along with example executables like `executor_runner` and # libraries that it uses, like `gflags`. Disabling this can be helpful when diff --git a/build/Test.cmake b/build/Test.cmake index e6b2af33e4..3e0cad1339 100644 --- a/build/Test.cmake +++ b/build/Test.cmake @@ -17,6 +17,19 @@ # It should also be cmake-lint clean. # +# Add code coverage flags to supported compilers +if(EXECUTORCH_USE_CPP_CODE_COVERAGE) + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + string(APPEND CMAKE_C_FLAGS " --coverage -fprofile-abs-path") + string(APPEND CMAKE_CXX_FLAGS " --coverage -fprofile-abs-path") + elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + string(APPEND CMAKE_C_FLAGS " -fprofile-instr-generate -fcoverage-mapping") + string(APPEND CMAKE_CXX_FLAGS " -fprofile-instr-generate -fcoverage-mapping") + else() + message(ERROR "Code coverage for compiler ${CMAKE_CXX_COMPILER_ID} is unsupported") + endif() +endif() + # A helper function to generate a gtest cxx executable target # @param target_name: name for the executable # @param SOURCES : test sources to be compiled. Sometimes diff --git a/test/run_oss_cpp_tests.sh b/test/run_oss_cpp_tests.sh index 846cc5b61c..acdafba63c 100755 --- a/test/run_oss_cpp_tests.sh +++ b/test/run_oss_cpp_tests.sh @@ -14,9 +14,18 @@ set -ex +if [[ $(uname) == "Darwin" ]]; then + export LLVM_PROFDATA="${LLVM_PROFDATA:-xcrun llvm-profdata}" + export LLVM_COV="${LLVM_COV:-xcrun llvm-cov}" +elif [[ $(uname) == "Linux" ]]; then + export LLVM_PROFDATA="${LLVM_PROFDATA:-llvm-profdata}" + export LLVM_COV="${LLVM_COV:-llvm-cov}" +fi + build_executorch() { cmake . \ -DCMAKE_INSTALL_PREFIX=cmake-out \ + -DEXECUTORCH_USE_CPP_CODE_COVERAGE=ON \ -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \ -Bcmake-out cmake --build cmake-out -j9 --target install @@ -35,17 +44,24 @@ build_and_run_test() { local test_dir=$1 cmake "${test_dir}" \ -DCMAKE_INSTALL_PREFIX=cmake-out \ + -DEXECUTORCH_USE_CPP_CODE_COVERAGE=ON \ -DCMAKE_PREFIX_PATH="$(pwd)/third-party/googletest/build" \ -Bcmake-out/"${test_dir}" cmake --build cmake-out/"${test_dir}" -j9 for t in cmake-out/"${test_dir}"/*test; do if [ -e "$t" ]; then - ./"$t"; + LLVM_PROFILE_FILE="cmake-out/$(basename $t).profraw" ./"$t"; + TEST_BINARY_LIST="${TEST_BINARY_LIST} -object $t" fi done } +report_coverage() { + "${LLVM_PROFDATA}" merge -sparse cmake-out/*.profraw -o cmake-out/merged.profdata + "${LLVM_COV}" report -instr-profile=cmake-out/merged.profdata $TEST_BINARY_LIST +} + probe_tests() { # This function finds the set of directories that contain C++ tests # CMakeLists.txt rules, that are buildable using build_and_run_test @@ -79,3 +95,5 @@ if [ -z "$1" ]; then else build_and_run_test "$1" fi + +report_coverage || true diff --git a/test/utils/generate_gtest_cmakelists.py b/test/utils/generate_gtest_cmakelists.py index e0a7d19903..4fe6976949 100644 --- a/test/utils/generate_gtest_cmakelists.py +++ b/test/utils/generate_gtest_cmakelists.py @@ -73,4 +73,3 @@ def read_config_json(json_path): ) else: print(f"Please run cmake-format -i {path_to_root}/CMakeLists.txt") - print("Note: Please update test/run_oss_cpp_tests.sh")