Collect OSS GTest code coverage (pytorch#3734)

Summary:
Use LLVM coverage tool to collect coverage. We print it to log for now (https://productionresultssa7.blob.core.windows.net/actions-results/36302d2c-4406-4404-88c1-7b3e63331252/workflow-job-run-00b9f9ca-16b3-535c-a3c3-70113f2f2d24/logs/job/job-logs.txt?rsct=text%2Fplain&se=2024-05-28T21%3A48%3A03Z&sig=xgur5uWUJVMNRxkwkm%2BSeqCFFjsoYcnQKtFksKHcLHo%3D&sp=r&spr=https&sr=b&st=2024-05-28T21%3A37%3A58Z&sv=2021-12-02 and search for `Branches   Missed Branches     Cover`)

Later we can process the data in a separate workflow.

Pull Request resolved: pytorch#3734

Reviewed By: huydhn

Differential Revision: D57792685

Pulled By: kirklandsign

fbshipit-source-id: 22942cdb012276d2541ff9fbff7f07014b87197b

.github/workflows/_unittest.yml

@@ -37,6 +37,7 @@ jobs:
         # Run pytest with coverage
         pytest -n auto --cov=./ --cov-report=xml
         # Run gtest
+        LLVM_PROFDATA=llvm-profdata-12 LLVM_COV=llvm-cov-12 \
         test/run_oss_cpp_tests.sh
 
   macos:
@@ -66,4 +67,5 @@ jobs:
         # Run pytest with coverage
         ${CONDA_RUN} pytest -n auto --cov=./ --cov-report=xml
         # Run gtest
+        LLVM_PROFDATA="xcrun llvm-profdata" LLVM_COV="xcrun llvm-cov" \
         ${CONDA_RUN} test/run_oss_cpp_tests.sh

CMakeLists.txt

@@ -343,6 +343,23 @@ else()
   set(CMAKE_TOOLCHAIN_ANDROID OFF)
 endif()
 
+# Add code coverage flags to supported compilers
+if(EXECUTORCH_USE_CPP_CODE_COVERAGE)
+  if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+    string(APPEND CMAKE_C_FLAGS " --coverage -fprofile-abs-path")
+    string(APPEND CMAKE_CXX_FLAGS " --coverage -fprofile-abs-path")
+  elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
+    string(APPEND CMAKE_C_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
+    string(APPEND CMAKE_CXX_FLAGS
+           " -fprofile-instr-generate -fcoverage-mapping"
+    )
+  else()
+    message(ERROR
+            "Code coverage for compiler ${CMAKE_CXX_COMPILER_ID} is unsupported"
+    )
+  endif()
+endif()
+
 # EXECUTORCH_BUILD_HOST_TARGETS: Option to control the building of host-only
 # tools like `flatc`, along with example executables like `executor_runner` and
 # libraries that it uses, like `gflags`. Disabling this can be helpful when

build/Test.cmake

@@ -17,6 +17,19 @@
 # It should also be cmake-lint clean.
 #
 
+# Add code coverage flags to supported compilers
+if(EXECUTORCH_USE_CPP_CODE_COVERAGE)
+  if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+    string(APPEND CMAKE_C_FLAGS  " --coverage -fprofile-abs-path")
+    string(APPEND CMAKE_CXX_FLAGS  " --coverage -fprofile-abs-path")
+  elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
+    string(APPEND CMAKE_C_FLAGS  " -fprofile-instr-generate -fcoverage-mapping")
+    string(APPEND CMAKE_CXX_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
+  else()
+    message(ERROR "Code coverage for compiler ${CMAKE_CXX_COMPILER_ID} is unsupported")
+  endif()
+endif()
+
 # A helper function to generate a gtest cxx executable target
 # @param target_name: name for the executable
 # @param SOURCES <list_of_sources>: test sources to be compiled. Sometimes

test/run_oss_cpp_tests.sh

@@ -14,9 +14,18 @@
 
 set -ex
 
+if [[ $(uname) == "Darwin" ]]; then
+  export LLVM_PROFDATA="${LLVM_PROFDATA:-xcrun llvm-profdata}"
+  export LLVM_COV="${LLVM_COV:-xcrun llvm-cov}"
+elif [[ $(uname) == "Linux" ]]; then
+  export LLVM_PROFDATA="${LLVM_PROFDATA:-llvm-profdata}"
+  export LLVM_COV="${LLVM_COV:-llvm-cov}"
+fi
+
 build_executorch() {
   cmake . \
     -DCMAKE_INSTALL_PREFIX=cmake-out \
+    -DEXECUTORCH_USE_CPP_CODE_COVERAGE=ON \
     -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
     -Bcmake-out
   cmake --build cmake-out -j9 --target install
@@ -35,17 +44,24 @@ build_and_run_test() {
   local test_dir=$1
   cmake "${test_dir}" \
     -DCMAKE_INSTALL_PREFIX=cmake-out \
+    -DEXECUTORCH_USE_CPP_CODE_COVERAGE=ON \
     -DCMAKE_PREFIX_PATH="$(pwd)/third-party/googletest/build" \
     -Bcmake-out/"${test_dir}"
   cmake --build cmake-out/"${test_dir}" -j9
 
   for t in cmake-out/"${test_dir}"/*test; do
     if [ -e "$t" ]; then
-      ./"$t";
+      LLVM_PROFILE_FILE="cmake-out/$(basename $t).profraw" ./"$t";
+      TEST_BINARY_LIST="${TEST_BINARY_LIST} -object $t"
     fi
   done
 }
 
+report_coverage() {
+  "${LLVM_PROFDATA}" merge -sparse cmake-out/*.profraw -o cmake-out/merged.profdata
+  "${LLVM_COV}" report -instr-profile=cmake-out/merged.profdata $TEST_BINARY_LIST
+}
+
 probe_tests() {
   # This function finds the set of directories that contain C++ tests
   # CMakeLists.txt rules, that are buildable using build_and_run_test
@@ -79,3 +95,5 @@ if [ -z "$1" ]; then
 else
   build_and_run_test "$1"
 fi
+
+report_coverage || true

test/utils/generate_gtest_cmakelists.py

@@ -73,4 +73,3 @@ def read_config_json(json_path):
             )
         else:
             print(f"Please run cmake-format -i {path_to_root}/CMakeLists.txt")
-    print("Note: Please update test/run_oss_cpp_tests.sh")