From 82663ac2b33b69dbb9fdbfecb298ed366697c8e4 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Wed, 29 May 2024 13:57:12 -0700 Subject: [PATCH] Collect OSS GTest code coverage (#3734) Summary: Use LLVM coverage tool to collect coverage. We print it to log for now (https://productionresultssa7.blob.core.windows.net/actions-results/36302d2c-4406-4404-88c1-7b3e63331252/workflow-job-run-00b9f9ca-16b3-535c-a3c3-70113f2f2d24/logs/job/job-logs.txt?rsct=text%2Fplain&se=2024-05-28T21%3A48%3A03Z&sig=xgur5uWUJVMNRxkwkm%2BSeqCFFjsoYcnQKtFksKHcLHo%3D&sp=r&spr=https&sr=b&st=2024-05-28T21%3A37%3A58Z&sv=2021-12-02 and search for `Branches Missed Branches Cover`) Later we can process the data in a separate workflow. Pull Request resolved: https://github.com/pytorch/executorch/pull/3734 Reviewed By: huydhn Differential Revision: D57792685 Pulled By: kirklandsign fbshipit-source-id: 22942cdb012276d2541ff9fbff7f07014b87197b --- .github/workflows/_unittest.yml | 2 ++ CMakeLists.txt | 17 +++++++++++++++++ build/Test.cmake | 13 +++++++++++++ test/run_oss_cpp_tests.sh | 20 +++++++++++++++++++- test/utils/generate_gtest_cmakelists.py | 1 - 5 files changed, 51 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_unittest.yml b/.github/workflows/_unittest.yml index cfad82d949..c96840e359 100644 --- a/.github/workflows/_unittest.yml +++ b/.github/workflows/_unittest.yml @@ -37,6 +37,7 @@ jobs: # Run pytest with coverage pytest -n auto --cov=./ --cov-report=xml # Run gtest + LLVM_PROFDATA=llvm-profdata-12 LLVM_COV=llvm-cov-12 \ test/run_oss_cpp_tests.sh macos: @@ -66,4 +67,5 @@ jobs: # Run pytest with coverage ${CONDA_RUN} pytest -n auto --cov=./ --cov-report=xml # Run gtest + LLVM_PROFDATA="xcrun llvm-profdata" LLVM_COV="xcrun llvm-cov" \ ${CONDA_RUN} test/run_oss_cpp_tests.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b4c31a131..63fdb4782e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -343,6 +343,23 @@ else() set(CMAKE_TOOLCHAIN_ANDROID OFF) endif() +# Add code coverage flags to supported compilers +if(EXECUTORCH_USE_CPP_CODE_COVERAGE) + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + string(APPEND CMAKE_C_FLAGS " --coverage -fprofile-abs-path") + string(APPEND CMAKE_CXX_FLAGS " --coverage -fprofile-abs-path") + elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + string(APPEND CMAKE_C_FLAGS " -fprofile-instr-generate -fcoverage-mapping") + string(APPEND CMAKE_CXX_FLAGS + " -fprofile-instr-generate -fcoverage-mapping" + ) + else() + message(ERROR + "Code coverage for compiler ${CMAKE_CXX_COMPILER_ID} is unsupported" + ) + endif() +endif() + # EXECUTORCH_BUILD_HOST_TARGETS: Option to control the building of host-only # tools like `flatc`, along with example executables like `executor_runner` and # libraries that it uses, like `gflags`. Disabling this can be helpful when diff --git a/build/Test.cmake b/build/Test.cmake index e6b2af33e4..3e0cad1339 100644 --- a/build/Test.cmake +++ b/build/Test.cmake @@ -17,6 +17,19 @@ # It should also be cmake-lint clean. # +# Add code coverage flags to supported compilers +if(EXECUTORCH_USE_CPP_CODE_COVERAGE) + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + string(APPEND CMAKE_C_FLAGS " --coverage -fprofile-abs-path") + string(APPEND CMAKE_CXX_FLAGS " --coverage -fprofile-abs-path") + elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + string(APPEND CMAKE_C_FLAGS " -fprofile-instr-generate -fcoverage-mapping") + string(APPEND CMAKE_CXX_FLAGS " -fprofile-instr-generate -fcoverage-mapping") + else() + message(ERROR "Code coverage for compiler ${CMAKE_CXX_COMPILER_ID} is unsupported") + endif() +endif() + # A helper function to generate a gtest cxx executable target # @param target_name: name for the executable # @param SOURCES : test sources to be compiled. Sometimes diff --git a/test/run_oss_cpp_tests.sh b/test/run_oss_cpp_tests.sh index 846cc5b61c..acdafba63c 100755 --- a/test/run_oss_cpp_tests.sh +++ b/test/run_oss_cpp_tests.sh @@ -14,9 +14,18 @@ set -ex +if [[ $(uname) == "Darwin" ]]; then + export LLVM_PROFDATA="${LLVM_PROFDATA:-xcrun llvm-profdata}" + export LLVM_COV="${LLVM_COV:-xcrun llvm-cov}" +elif [[ $(uname) == "Linux" ]]; then + export LLVM_PROFDATA="${LLVM_PROFDATA:-llvm-profdata}" + export LLVM_COV="${LLVM_COV:-llvm-cov}" +fi + build_executorch() { cmake . \ -DCMAKE_INSTALL_PREFIX=cmake-out \ + -DEXECUTORCH_USE_CPP_CODE_COVERAGE=ON \ -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \ -Bcmake-out cmake --build cmake-out -j9 --target install @@ -35,17 +44,24 @@ build_and_run_test() { local test_dir=$1 cmake "${test_dir}" \ -DCMAKE_INSTALL_PREFIX=cmake-out \ + -DEXECUTORCH_USE_CPP_CODE_COVERAGE=ON \ -DCMAKE_PREFIX_PATH="$(pwd)/third-party/googletest/build" \ -Bcmake-out/"${test_dir}" cmake --build cmake-out/"${test_dir}" -j9 for t in cmake-out/"${test_dir}"/*test; do if [ -e "$t" ]; then - ./"$t"; + LLVM_PROFILE_FILE="cmake-out/$(basename $t).profraw" ./"$t"; + TEST_BINARY_LIST="${TEST_BINARY_LIST} -object $t" fi done } +report_coverage() { + "${LLVM_PROFDATA}" merge -sparse cmake-out/*.profraw -o cmake-out/merged.profdata + "${LLVM_COV}" report -instr-profile=cmake-out/merged.profdata $TEST_BINARY_LIST +} + probe_tests() { # This function finds the set of directories that contain C++ tests # CMakeLists.txt rules, that are buildable using build_and_run_test @@ -79,3 +95,5 @@ if [ -z "$1" ]; then else build_and_run_test "$1" fi + +report_coverage || true diff --git a/test/utils/generate_gtest_cmakelists.py b/test/utils/generate_gtest_cmakelists.py index e0a7d19903..4fe6976949 100644 --- a/test/utils/generate_gtest_cmakelists.py +++ b/test/utils/generate_gtest_cmakelists.py @@ -73,4 +73,3 @@ def read_config_json(json_path): ) else: print(f"Please run cmake-format -i {path_to_root}/CMakeLists.txt") - print("Note: Please update test/run_oss_cpp_tests.sh")