diff --git a/README.md b/README.md index f9daee72..1084d996 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,9 @@ For more information about versioning handle of this project, check following [f - [fetch_fastdds_manual](multiplatform/fetch_fastdds_manual/action.yml) - Download Fast DDS and its eProsima dependencies setting the specific version of each repository. +- [flakiness_report](multiplatform/flakiness_report/action.yml) + - Generate a flakiness report from historical tests results. + - [generate_dependency_artifact](multiplatform/generate_dependency_artifact/action.yml) - Build a project and upload the installed objects as an artifact. diff --git a/external/upload-artifact/action.yml b/external/upload-artifact/action.yml index 8d1971dc..4441d49f 100644 --- a/external/upload-artifact/action.yml +++ b/external/upload-artifact/action.yml @@ -24,6 +24,13 @@ inputs: Minimum 1 day. Maximum 90 days unless changed from the repository settings page. + overwrite: + description: > + If true, an artifact with a matching name will be deleted before a new one is uploaded. + If false, the action will fail if an artifact with the given name already exists. + Does not fail if the artifact does not exist. + Optional. Default is 'false' + default: 'false' runs: using: composite @@ -46,3 +53,4 @@ runs: path: ${{ inputs.path }} if-no-files-found: ${{ inputs.if-no-files-found }} retention-days: ${{ inputs.retention-days }} + overwrite: ${{ inputs.overwrite }} diff --git a/macos/install_colcon/action.yml b/macos/install_colcon/action.yml index bc4c7ec8..db1628b4 100644 --- a/macos/install_colcon/action.yml +++ b/macos/install_colcon/action.yml @@ -6,7 +6,7 @@ runs: steps: - name: Install colcon - uses: eProsima/eProsima-CI/macos/install_python_packages@main + uses: eProsima/eProsima-CI/macos/install_python_packages@feature/detect_flaky_tests with: packages: 'setuptools==58.3.0 colcon-common-extensions colcon-mixin' upgrade: true diff --git a/multiplatform/asan_build_test/action.yml b/multiplatform/asan_build_test/action.yml index 5bb2b5a0..e9ffb9f3 100644 --- a/multiplatform/asan_build_test/action.yml +++ b/multiplatform/asan_build_test/action.yml @@ -43,7 +43,7 @@ runs: - name: Build and test id: build_and_test - uses: eProsima/eProsima-CI/multiplatform/colcon_build_test@main + uses: eProsima/eProsima-CI/multiplatform/colcon_build_test@feature/detect_flaky_tests with: packages_names: ${{ inputs.packages_names }} colcon_meta_file: ${{ inputs.colcon_meta_file }} diff --git a/multiplatform/clang_build_test/action.yml b/multiplatform/clang_build_test/action.yml index 030e81b9..a7adcd4f 100644 --- a/multiplatform/clang_build_test/action.yml +++ b/multiplatform/clang_build_test/action.yml @@ -32,7 +32,7 @@ runs: steps: - name: Build - uses: eProsima/eProsima-CI/multiplatform/colcon_build@main + uses: eProsima/eProsima-CI/multiplatform/colcon_build@feature/detect_flaky_tests with: colcon_meta_file: ${{ inputs.colcon_meta_file }} colcon_build_args: '--packages-up-to ${{ inputs.packages_names }}' @@ -41,6 +41,6 @@ runs: workspace_dependencies: ${{ inputs.workspace_dependencies }} - name: Test clang - uses: eProsima/eProsima-CI/multiplatform/clang_tidy_check@main + uses: eProsima/eProsima-CI/multiplatform/clang_tidy_check@feature/detect_flaky_tests with: packages_names: ${{ inputs.packages_names }} diff --git a/multiplatform/clang_tidy_check/action.yml b/multiplatform/clang_tidy_check/action.yml index a9b2a565..d79f7aca 100644 --- a/multiplatform/clang_tidy_check/action.yml +++ b/multiplatform/clang_tidy_check/action.yml @@ -23,7 +23,7 @@ runs: steps: - name: Run in ubuntu - uses: eProsima/eProsima-CI/ubuntu/clang_tidy_check@main + uses: eProsima/eProsima-CI/ubuntu/clang_tidy_check@feature/detect_flaky_tests if: runner.os == 'Linux' with: build_directory: ${{ inputs.build_directory }} @@ -31,7 +31,7 @@ runs: result_file_name: ${{ inputs.result_file_name }} - name: Run in windows - uses: eProsima/eProsima-CI/windows/clang_tidy_check@main + uses: eProsima/eProsima-CI/windows/clang_tidy_check@feature/detect_flaky_tests if: runner.os == 'Windows' with: build_directory: ${{ inputs.build_directory }} diff --git a/multiplatform/colcon_build/action.yml b/multiplatform/colcon_build/action.yml index 21e06023..45e2a7fc 100644 --- a/multiplatform/colcon_build/action.yml +++ b/multiplatform/colcon_build/action.yml @@ -48,7 +48,7 @@ runs: steps: - name: Run in ubuntu or macOS - uses: eProsima/eProsima-CI/ubuntu/colcon_build@main + uses: eProsima/eProsima-CI/ubuntu/colcon_build@feature/detect_flaky_tests if: runner.os == 'Linux' || runner.os == 'macOS' with: colcon_meta_file: ${{ inputs.colcon_meta_file }} @@ -61,7 +61,7 @@ runs: cmake_build_type: ${{ inputs.cmake_build_type }} - name: Run in windows - uses: eProsima/eProsima-CI/windows/colcon_build@main + uses: eProsima/eProsima-CI/windows/colcon_build@feature/detect_flaky_tests if: runner.os == 'Windows' with: colcon_meta_file: ${{ inputs.colcon_meta_file }} diff --git a/multiplatform/colcon_build_test/action.yml b/multiplatform/colcon_build_test/action.yml index cbd44158..fa12560f 100644 --- a/multiplatform/colcon_build_test/action.yml +++ b/multiplatform/colcon_build_test/action.yml @@ -54,7 +54,7 @@ runs: steps: - name: Build - uses: eProsima/eProsima-CI/multiplatform/colcon_build@main + uses: eProsima/eProsima-CI/multiplatform/colcon_build@feature/detect_flaky_tests with: colcon_meta_file: ${{ inputs.colcon_meta_file }} colcon_build_args: '--packages-up-to ${{ inputs.packages_names }}' @@ -65,7 +65,7 @@ runs: - name: Test id: test - uses: eProsima/eProsima-CI/multiplatform/colcon_test@main + uses: eProsima/eProsima-CI/multiplatform/colcon_test@feature/detect_flaky_tests with: colcon_meta_file: ${{ inputs.colcon_meta_file }} workspace: ${{ inputs.workspace }} diff --git a/multiplatform/colcon_build_test_flaky/action.yml b/multiplatform/colcon_build_test_flaky/action.yml index 23b33094..04f684d0 100644 --- a/multiplatform/colcon_build_test_flaky/action.yml +++ b/multiplatform/colcon_build_test_flaky/action.yml @@ -42,7 +42,7 @@ runs: steps: - name: Build - uses: eProsima/eProsima-CI/multiplatform/colcon_build@main + uses: eProsima/eProsima-CI/multiplatform/colcon_build@feature/detect_flaky_tests with: colcon_meta_file: ${{ inputs.colcon_meta_file }} colcon_build_args: '--packages-up-to ${{ inputs.packages_names }}' @@ -52,7 +52,7 @@ runs: - name: Test id: test - uses: eProsima/eProsima-CI/multiplatform/colcon_test@main + uses: eProsima/eProsima-CI/multiplatform/colcon_test@feature/detect_flaky_tests with: colcon_meta_file: ${{ inputs.colcon_meta_file }} workspace: ${{ inputs.workspace }} diff --git a/multiplatform/colcon_test/action.yml b/multiplatform/colcon_test/action.yml index 1787cf44..ff959318 100644 --- a/multiplatform/colcon_test/action.yml +++ b/multiplatform/colcon_test/action.yml @@ -60,7 +60,7 @@ runs: - name: Run in ubuntu or macOS id: test_ubuntu_mac - uses: eProsima/eProsima-CI/ubuntu/colcon_test@main + uses: eProsima/eProsima-CI/ubuntu/colcon_test@feature/detect_flaky_tests if: runner.os == 'Linux' || runner.os == 'macOS' with: colcon_meta_file: ${{ inputs.colcon_meta_file }} @@ -75,7 +75,7 @@ runs: - name: Run in windows id: test_windows - uses: eProsima/eProsima-CI/windows/colcon_test@main + uses: eProsima/eProsima-CI/windows/colcon_test@feature/detect_flaky_tests if: runner.os == 'Windows' with: colcon_meta_file: ${{ inputs.colcon_meta_file }} diff --git a/multiplatform/download_dependency/action.yml b/multiplatform/download_dependency/action.yml index 66fd69d0..2e2d1f34 100644 --- a/multiplatform/download_dependency/action.yml +++ b/multiplatform/download_dependency/action.yml @@ -47,7 +47,7 @@ runs: # However, looking for the id previously allows to call this action without secret_token - name: Download dependency - uses: eProsima/eProsima-CI/external/action-download-artifact@main + uses: eProsima/eProsima-CI/external/action-download-artifact@feature/detect_flaky_tests with: name: ${{ inputs.artifact_name }} workflow: ${{ inputs.workflow_source }} diff --git a/multiplatform/fetch_ddspipe_manual/action.yml b/multiplatform/fetch_ddspipe_manual/action.yml index cdf1ad9c..1443a754 100644 --- a/multiplatform/fetch_ddspipe_manual/action.yml +++ b/multiplatform/fetch_ddspipe_manual/action.yml @@ -20,14 +20,14 @@ runs: steps: - name: Run in ubuntu - uses: eProsima/eProsima-CI/ubuntu/fetch_ddspipe_manual@main + uses: eProsima/eProsima-CI/ubuntu/fetch_ddspipe_manual@feature/detect_flaky_tests if: runner.os == 'Linux' with: ddspipe_branch: ${{ inputs.ddspipe_branch }} destination_workspace: ${{ inputs.destination_workspace }} - name: Run in windows - uses: eProsima/eProsima-CI/windows/fetch_ddspipe_manual@main + uses: eProsima/eProsima-CI/windows/fetch_ddspipe_manual@feature/detect_flaky_tests if: runner.os == 'Windows' with: ddspipe_branch: ${{ inputs.ddspipe_branch }} diff --git a/multiplatform/fetch_dev_utils_manual/action.yml b/multiplatform/fetch_dev_utils_manual/action.yml index f213c184..c2d83769 100644 --- a/multiplatform/fetch_dev_utils_manual/action.yml +++ b/multiplatform/fetch_dev_utils_manual/action.yml @@ -20,14 +20,14 @@ runs: steps: - name: Run in ubuntu - uses: eProsima/eProsima-CI/ubuntu/fetch_dev_utils_manual@main + uses: eProsima/eProsima-CI/ubuntu/fetch_dev_utils_manual@feature/detect_flaky_tests if: runner.os == 'Linux' with: dev_utils_branch: ${{ inputs.dev_utils_branch }} destination_workspace: ${{ inputs.destination_workspace }} - name: Run in windows - uses: eProsima/eProsima-CI/windows/fetch_dev_utils_manual@main + uses: eProsima/eProsima-CI/windows/fetch_dev_utils_manual@feature/detect_flaky_tests if: runner.os == 'Windows' with: dev_utils_branch: ${{ inputs.dev_utils_branch }} diff --git a/multiplatform/fetch_fastdds_manual/action.yml b/multiplatform/fetch_fastdds_manual/action.yml index 01a00aba..b8adafb5 100644 --- a/multiplatform/fetch_fastdds_manual/action.yml +++ b/multiplatform/fetch_fastdds_manual/action.yml @@ -28,7 +28,7 @@ runs: steps: - name: Run in ubuntu - uses: eProsima/eProsima-CI/ubuntu/fetch_fastdds_manual@main + uses: eProsima/eProsima-CI/ubuntu/fetch_fastdds_manual@feature/detect_flaky_tests if: runner.os == 'Linux' with: foonathan_memory_vendor_branch: ${{ inputs.foonathan_memory_vendor_branch }} @@ -37,7 +37,7 @@ runs: destination_workspace: ${{ inputs.destination_workspace }} - name: Run in windows - uses: eProsima/eProsima-CI/windows/fetch_fastdds_manual@main + uses: eProsima/eProsima-CI/windows/fetch_fastdds_manual@feature/detect_flaky_tests if: runner.os == 'Windows' with: foonathan_memory_vendor_branch: ${{ inputs.foonathan_memory_vendor_branch }} diff --git a/multiplatform/flakiness_report/action.yml b/multiplatform/flakiness_report/action.yml new file mode 100644 index 00000000..a179020b --- /dev/null +++ b/multiplatform/flakiness_report/action.yml @@ -0,0 +1,58 @@ +name: 'flakiness_report' +description: 'Generate a flakiness report from test results and historical data' + +inputs: + junit_reports_dir: + description: 'Path to dir with new jUnit reports' + required: true + + junit_archive_artifact: + description: > + Name of the artifact containing the historical data. Also used for uploading a JSON report + with the flaky tests as an artifact. name ${junit_archive_artifact}_json. + required: true + + print_report: + description: 'Whether to print the report (Default: True)' + required: false + default: 'True' + + window_size: + description: 'Number of days to consider for the flakiness report' + required: false + default: '30' + + fail_on_flaky_tests: + description: 'Whether to fail the workflow if there are flaky tests' + required: false + default: 'True' + + github_token: + description: 'GitHub token' + required: true + +runs: + using: composite + steps: + + - name: Run in ubuntu or macOS + if: runner.os == 'Linux' || runner.os == 'macOS' + uses: eProsima/eProsima-CI/ubuntu/flakiness_report@feature/detect_flaky_tests + with: + junit_reports_dir: ${{ inputs.junit_reports_dir }} + junit_archive_artifact: ${{ inputs.junit_archive_artifact }} + print_report: ${{ inputs.print_report }} + window_size: ${{ inputs.window_size }} + fail_on_flaky_tests: ${{ inputs.fail_on_flaky_tests }} + github_token: ${{ inputs.github_token }} + + - name: Run in windows + if: runner.os == 'Windows' + uses: eProsima/eProsima-CI/windows/flakiness_report@feature/detect_flaky_tests + with: + junit_reports_dir: ${{ inputs.junit_reports_dir }} + junit_archive_artifact: ${{ inputs.junit_archive_artifact }} + print_report: ${{ inputs.print_report }} + window_size: ${{ inputs.window_size }} + fail_on_flaky_tests: ${{ inputs.fail_on_flaky_tests }} + github_token: ${{ inputs.github_token }} diff --git a/multiplatform/generate_dependency_artifact/action.yml b/multiplatform/generate_dependency_artifact/action.yml index 9e9f2b63..c25e263e 100644 --- a/multiplatform/generate_dependency_artifact/action.yml +++ b/multiplatform/generate_dependency_artifact/action.yml @@ -36,13 +36,13 @@ runs: steps: - name: Fetch repositories - uses: eProsima/eProsima-CI/multiplatform/vcs_import@main + uses: eProsima/eProsima-CI/multiplatform/vcs_import@feature/detect_flaky_tests with: vcs_repos_file: ${{ inputs.vcs_repos_file }} destination_workspace: ${{ inputs.workspace }}/src - name: Build workspace - uses: eProsima/eProsima-CI/multiplatform/colcon_build@main + uses: eProsima/eProsima-CI/multiplatform/colcon_build@feature/detect_flaky_tests with: colcon_meta_file: ${{ inputs.colcon_meta_file }} workspace: ${{ inputs.workspace }} @@ -50,7 +50,7 @@ runs: cmake_build_type: ${{ inputs.cmake_build_type }} - name: Upload binaries - uses: eProsima/eProsima-CI/external/upload-artifact@main + uses: eProsima/eProsima-CI/external/upload-artifact@feature/detect_flaky_tests with: name: ${{ inputs.artifact_name }} path: ${{ inputs.workspace }}/install diff --git a/multiplatform/get_configurations_from_repo/action.yml b/multiplatform/get_configurations_from_repo/action.yml index 3b58a67a..75a2dcdc 100644 --- a/multiplatform/get_configurations_from_repo/action.yml +++ b/multiplatform/get_configurations_from_repo/action.yml @@ -38,7 +38,7 @@ runs: steps: - name: Get colcon.meta file - uses: eProsima/eProsima-CI/multiplatform/get_file_from_repo@main + uses: eProsima/eProsima-CI/multiplatform/get_file_from_repo@feature/detect_flaky_tests with: source_repository_branch: ${{ inputs.source_repository_branch }} source_repository: ${{ inputs.source_repository }} @@ -46,7 +46,7 @@ runs: file_result: ${{ inputs.colcon_meta_file_result }} - name: Get dependencies.repos file - uses: eProsima/eProsima-CI/multiplatform/get_file_from_repo@main + uses: eProsima/eProsima-CI/multiplatform/get_file_from_repo@feature/detect_flaky_tests with: source_repository_branch: ${{ inputs.source_repository_branch }} source_repository: ${{ inputs.source_repository }} diff --git a/multiplatform/get_file_from_repo/action.yml b/multiplatform/get_file_from_repo/action.yml index 687315aa..d055b8a3 100644 --- a/multiplatform/get_file_from_repo/action.yml +++ b/multiplatform/get_file_from_repo/action.yml @@ -27,7 +27,7 @@ runs: steps: - name: Run in ubuntu - uses: eProsima/eProsima-CI/ubuntu/get_file_from_repo@main + uses: eProsima/eProsima-CI/ubuntu/get_file_from_repo@feature/detect_flaky_tests if: runner.os == 'Linux' with: source_repository_branch: ${{ inputs.source_repository_branch }} @@ -36,7 +36,7 @@ runs: file_result: ${{ inputs.file_result }} - name: Run in windows - uses: eProsima/eProsima-CI/windows/get_file_from_repo@main + uses: eProsima/eProsima-CI/windows/get_file_from_repo@feature/detect_flaky_tests if: runner.os == 'Windows' with: source_repository_branch: ${{ inputs.source_repository_branch }} diff --git a/multiplatform/get_workflow_id/action.yml b/multiplatform/get_workflow_id/action.yml index 21e15ed3..122b3fd2 100644 --- a/multiplatform/get_workflow_id/action.yml +++ b/multiplatform/get_workflow_id/action.yml @@ -39,7 +39,7 @@ runs: steps: - name: Install in ubuntu - uses: eProsima/eProsima-CI/ubuntu/get_workflow_id@main + uses: eProsima/eProsima-CI/ubuntu/get_workflow_id@feature/detect_flaky_tests if: runner.os == 'Linux' with: workflow_source: ${{ inputs.workflow_source }} @@ -50,7 +50,7 @@ runs: secret_token: ${{ inputs.secret_token }} - name: Install in windows - uses: eProsima/eProsima-CI/windows/get_workflow_id@main + uses: eProsima/eProsima-CI/windows/get_workflow_id@feature/detect_flaky_tests if: runner.os == 'Windows' with: workflow_source: ${{ inputs.workflow_source }} diff --git a/multiplatform/install_colcon/action.yml b/multiplatform/install_colcon/action.yml index 0b9d2ebf..cff8f119 100644 --- a/multiplatform/install_colcon/action.yml +++ b/multiplatform/install_colcon/action.yml @@ -6,13 +6,13 @@ runs: steps: - name: Run in ubuntu - uses: eProsima/eProsima-CI/ubuntu/install_colcon@main + uses: eProsima/eProsima-CI/ubuntu/install_colcon@feature/detect_flaky_tests if: runner.os == 'Linux' - name: Run in macOS - uses: eProsima/eProsima-CI/macos/install_colcon@main + uses: eProsima/eProsima-CI/macos/install_colcon@feature/detect_flaky_tests if: runner.os == 'macOS' - name: Run in windows - uses: eProsima/eProsima-CI/windows/install_colcon@main + uses: eProsima/eProsima-CI/windows/install_colcon@feature/detect_flaky_tests if: runner.os == 'Windows' diff --git a/multiplatform/install_fastdds_dependencies/action.yml b/multiplatform/install_fastdds_dependencies/action.yml index ee0cb3ff..353299c4 100644 --- a/multiplatform/install_fastdds_dependencies/action.yml +++ b/multiplatform/install_fastdds_dependencies/action.yml @@ -12,11 +12,11 @@ runs: steps: - name: Install in ubuntu - uses: eProsima/eProsima-CI/ubuntu/install_fastdds_dependencies@main + uses: eProsima/eProsima-CI/ubuntu/install_fastdds_dependencies@feature/detect_flaky_tests if: runner.os == 'Linux' - name: Install in windows - uses: eProsima/eProsima-CI/windows/install_fastdds_dependencies@main + uses: eProsima/eProsima-CI/windows/install_fastdds_dependencies@feature/detect_flaky_tests with: cmake_build_type: ${{ inputs.cmake_build_type }} if: runner.os == 'Windows' diff --git a/multiplatform/install_gtest/action.yml b/multiplatform/install_gtest/action.yml index a04ce86f..74ffe632 100644 --- a/multiplatform/install_gtest/action.yml +++ b/multiplatform/install_gtest/action.yml @@ -18,14 +18,14 @@ runs: steps: - name: Run in ubuntu or macOS - uses: eProsima/eProsima-CI/ubuntu/install_gtest@main + uses: eProsima/eProsima-CI/ubuntu/install_gtest@feature/detect_flaky_tests if: runner.os == 'Linux' || runner.os == 'macOS' with: cmake_build_type: ${{ inputs.cmake_build_type }} version: ${{ inputs.version }} - name: Run in windows - uses: eProsima/eProsima-CI/windows/install_gtest@main + uses: eProsima/eProsima-CI/windows/install_gtest@feature/detect_flaky_tests if: runner.os == 'Windows' with: cmake_build_type: ${{ inputs.cmake_build_type }} diff --git a/multiplatform/install_openssl/action.yml b/multiplatform/install_openssl/action.yml index 8244af43..9e2097c0 100644 --- a/multiplatform/install_openssl/action.yml +++ b/multiplatform/install_openssl/action.yml @@ -6,9 +6,9 @@ runs: steps: - name: Run in ubuntu - uses: eProsima/eProsima-CI/ubuntu/install_openssl@main + uses: eProsima/eProsima-CI/ubuntu/install_openssl@feature/detect_flaky_tests if: runner.os == 'Linux' - name: Run in windows - uses: eProsima/eProsima-CI/windows/install_openssl@main + uses: eProsima/eProsima-CI/windows/install_openssl@feature/detect_flaky_tests if: runner.os == 'Windows' diff --git a/multiplatform/install_python_packages/action.yml b/multiplatform/install_python_packages/action.yml index 7a0896e8..ef141d9c 100644 --- a/multiplatform/install_python_packages/action.yml +++ b/multiplatform/install_python_packages/action.yml @@ -23,7 +23,7 @@ runs: steps: - name: Run in ubuntu - uses: eProsima/eProsima-CI/ubuntu/install_python_packages@main + uses: eProsima/eProsima-CI/ubuntu/install_python_packages@feature/detect_flaky_tests if: runner.os == 'Linux' with: packages: ${{ inputs.packages }} @@ -31,7 +31,7 @@ runs: requirements_file_name: ${{ inputs.requirements_file_name }} - name: Run in macOS - uses: eProsima/eProsima-CI/macos/install_python_packages@main + uses: eProsima/eProsima-CI/macos/install_python_packages@feature/detect_flaky_tests if: runner.os == 'macOS' with: packages: ${{ inputs.packages }} @@ -39,7 +39,7 @@ runs: requirements_file_name: ${{ inputs.requirements_file_name }} - name: Run in windows - uses: eProsima/eProsima-CI/windows/install_python_packages@main + uses: eProsima/eProsima-CI/windows/install_python_packages@feature/detect_flaky_tests if: runner.os == 'Windows' with: packages: ${{ inputs.packages }} diff --git a/multiplatform/install_yamlcpp/action.yml b/multiplatform/install_yamlcpp/action.yml index d2356780..8cf32d11 100644 --- a/multiplatform/install_yamlcpp/action.yml +++ b/multiplatform/install_yamlcpp/action.yml @@ -17,14 +17,14 @@ runs: steps: - name: Run in ubuntu - uses: eProsima/eProsima-CI/ubuntu/install_yamlcpp@main + uses: eProsima/eProsima-CI/ubuntu/install_yamlcpp@feature/detect_flaky_tests if: runner.os == 'Linux' with: cmake_build_type: ${{ inputs.cmake_build_type }} version: ${{ inputs.version }} - name: Run in windows - uses: eProsima/eProsima-CI/windows/install_yamlcpp@main + uses: eProsima/eProsima-CI/windows/install_yamlcpp@feature/detect_flaky_tests if: runner.os == 'Windows' with: cmake_build_type: ${{ inputs.cmake_build_type }} diff --git a/multiplatform/junit_summary/action.yaml b/multiplatform/junit_summary/action.yaml index da15977c..4811259a 100644 --- a/multiplatform/junit_summary/action.yaml +++ b/multiplatform/junit_summary/action.yaml @@ -26,26 +26,33 @@ inputs: required: false default: 'False' + flaky_json_report: + description: 'Path to a JSON file with the flaky tests report' + required: false + default: '' + runs: using: composite steps: - name: Run in ubuntu or macOS if: runner.os == 'Linux' || runner.os == 'macOS' - uses: eProsima/eProsima-CI/ubuntu/junit_summary@main + uses: eProsima/eProsima-CI/ubuntu/junit_summary@feature/detect_flaky_tests with: junit_reports_dir: ${{ inputs.junit_reports_dir }} print_summary: ${{ inputs.print_summary }} show_failed: ${{ inputs.show_failed }} show_disabled: ${{ inputs.show_disabled }} show_skipped: ${{ inputs.show_skipped }} + flaky_json_report: ${{ inputs.flaky_json_report }} - name: Run in windows if: runner.os == 'Windows' - uses: eProsima/eProsima-CI/windows/junit_summary@main + uses: eProsima/eProsima-CI/windows/junit_summary@feature/detect_flaky_tests with: junit_reports_dir: ${{ inputs.junit_reports_dir }} print_summary: ${{ inputs.print_summary }} show_failed: ${{ inputs.show_failed }} show_disabled: ${{ inputs.show_disabled }} show_skipped: ${{ inputs.show_skipped }} + flaky_json_report: ${{ inputs.flaky_json_report }} diff --git a/multiplatform/tsan_build_test/action.yml b/multiplatform/tsan_build_test/action.yml index 60dd1679..05b897e9 100644 --- a/multiplatform/tsan_build_test/action.yml +++ b/multiplatform/tsan_build_test/action.yml @@ -48,7 +48,7 @@ runs: # These issues were fixed in GCC 12 so we upgrade to that version. CC: gcc-12 CXX: g++-12 - uses: eProsima/eProsima-CI/multiplatform/colcon_build_test@main + uses: eProsima/eProsima-CI/multiplatform/colcon_build_test@feature/detect_flaky_tests with: packages_names: ${{ inputs.packages_names }} colcon_meta_file: ${{ inputs.colcon_meta_file }} diff --git a/multiplatform/vcs_import/action.yml b/multiplatform/vcs_import/action.yml index 24beff49..2a3d193f 100644 --- a/multiplatform/vcs_import/action.yml +++ b/multiplatform/vcs_import/action.yml @@ -22,7 +22,7 @@ runs: steps: - name: Run in ubuntu - uses: eProsima/eProsima-CI/ubuntu/vcs_import@main + uses: eProsima/eProsima-CI/ubuntu/vcs_import@feature/detect_flaky_tests if: runner.os == 'Linux' || runner.os == 'macOS' with: vcs_repos_file: ${{ inputs.vcs_repos_file }} @@ -30,7 +30,7 @@ runs: skip_existing: ${{ inputs.skip_existing }} - name: Run in windows - uses: eProsima/eProsima-CI/windows/vcs_import@main + uses: eProsima/eProsima-CI/windows/vcs_import@feature/detect_flaky_tests if: runner.os == 'Windows' with: vcs_repos_file: ${{ inputs.vcs_repos_file }} diff --git a/resources/ctest-to-junit.xsl b/resources/ctest-to-junit.xsl index 9080ff5c..6516b439 100644 --- a/resources/ctest-to-junit.xsl +++ b/resources/ctest-to-junit.xsl @@ -2,6 +2,7 @@ + @@ -15,7 +16,8 @@ failures="{$numberOfFailures}" errors="{$numberOfErrors}" skipped="{$numberOfSkipped}" - time="{$buildTime}"> + time="{$buildTime}" + timestamp="{$input_timestamp}"> diff --git a/resources/ctest2junit.py b/resources/ctest2junit.py index 414cc77c..4ee2e65e 100644 --- a/resources/ctest2junit.py +++ b/resources/ctest2junit.py @@ -57,7 +57,13 @@ def parse_options(): required=True, help='Path to output jUnit file. If the file exists, the script takes no action' ) - + required_args.add_argument( + '-t', + '--timestamp', + type=str, + required=True, + help='Timestamp' + ) return parser.parse_args() @@ -94,20 +100,22 @@ def find_ctest_report(build_dir): return ret -def translate(original_xml, xsl_file): +def translate(original_xml, xsl_file, timestamp): """ Translate an XML from one spec to another using an XSLT file. :param original_xml: The XML to translate :param xsl_file: The XSLT transformation file + :param timestamp: The timestamp to use in the transformation :return: A stream containing the translated XML """ xml = etree.parse(original_xml) xslt = etree.parse(xsl_file) transform = etree.XSLT(xslt) + try: - return str(transform(xml)) + return str(transform(xml, input_timestamp=etree.XSLT.strparam(timestamp))) except Exception as e: for error in transform.error_log: print(error.message, error.line) @@ -139,7 +147,7 @@ def write_to_file(stream, filename): ctest_report = find_ctest_report(args.build_dir) if ctest_report: - junit = translate(ctest_report, args.xslt) + junit = translate(ctest_report, args.xslt, args.timestamp) write_to_file(junit, args.output_junit) exit_code = 0 diff --git a/resources/flakiness_report.py b/resources/flakiness_report.py new file mode 100644 index 00000000..708cbe80 --- /dev/null +++ b/resources/flakiness_report.py @@ -0,0 +1,88 @@ +# Copyright 2024 Proyectos y Sistemas de Mantenimiento SL (eProsima). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Script to analyze and publish flaky tests from JUnit test history files. +""" +import argparse + +from flaky.archive import FlakyTestsArchive +from flaky.publisher import FlakyTestsMdPublisher +from flaky.publisher import FlakyTestsJSONPublisher + +def parse_options() -> argparse.Namespace: + """ + Parse the command line options. + + Returns: + argparse.Namespace: Parsed command line options. + """ + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "--junit-archive", + help="Path for a folder with JUnit xml test history files", + type=str + ) + parser.add_argument( + "--window-size", + type=int, + help="flip rate calculation window size", + required=True, + ) + parser.add_argument( + '-d', + '--delete-old-files', + action='store_true', + help='Delete old files taking window size into account.' + ) + parser.add_argument( + '-m', + '--markdown-file', + type=str, + required=False, + help='Path to markdown file.' + ) + parser.add_argument( + '-j', + '--json-file', + type=str, + required=False, + help='Path to JSON file.' + ) + return parser.parse_args() + +if __name__ == "__main__": + + try: + args = parse_options() + + archive = FlakyTestsArchive( + args.junit_archive, + args.window_size, + args.delete_old_files + ) + + if args.markdown_file: + FlakyTestsMdPublisher.publish(archive, args.markdown_file) + + if args.json_file: + FlakyTestsJSONPublisher.publish(archive, args.json_file) + + ret = 0 if archive.flaky_test_count == 0 else 1 + exit(ret) + + except Exception as e: + # Exit with 255 error so that the action can fail + print(e) + exit(255) diff --git a/resources/flaky/__init__.py b/resources/flaky/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/resources/flaky/archive.py b/resources/flaky/archive.py new file mode 100644 index 00000000..9c5f345e --- /dev/null +++ b/resources/flaky/archive.py @@ -0,0 +1,313 @@ +# Copyright 2024 Proyectos y Sistemas de Mantenimiento SL (eProsima). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import re +from datetime import datetime +from decimal import getcontext, Decimal, ROUND_UP + +import pandas as pd + +from junit.junit_utils import JUnitReport + + +class FlakyTestsArchive: + """ + Class to analyze and store flaky tests from JUnit test history files. + """ + # Regex pattern to match the timestamp part of the filename + _timestamp_rstr = r'\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}' + + def __init__(self, archive_dir: str, window_size: int, delete_old_files: bool = True) -> None: + """ + Args: + archive_dir (str): Path to the directory containing the JUnit test history files. + window_size (int): Number of test runs to consider for the fliprate calculation. + delete_old_files (bool): Whether to delete old files taking window_size into account. + """ + self._analysis = { + 'version': '1.0', + 'flaky_tests': {}, + 'failed_tests': {}, + } + self._window_size = window_size + self._junit_files = FlakyTestsArchive._find_test_results(archive_dir) + + if delete_old_files: + self._delete_old_files() + + self._analyze() + + def _analyze(self) -> None: + """ + Analyze the test history files and calculate the fliprate for each test. + + Returns: + int: Number of flaky tests found. + """ + df = pd.DataFrame() + for junit_file in self._junit_files: + junit_report = JUnitReport(junit_file) + df = pd.concat([df, junit_report.to_df()]) + + fliprate_table = FlakyTestsArchive._calculate_fliprate_table(df, self._window_size) + self._get_top_fliprates(fliprate_table, 0, 4) + + @property + def flaky_test_count(self) -> int: + return len(self._analysis['flaky_tests']) + + @staticmethod + def _find_test_results(directory: str) -> list: + """ + Find all JUnit test history files in the given directory. + + Args: + directory (str): Path to the directory containing the JUnit test history files. + + Returns: + list: List of paths to the JUnit test history files. + """ + pattern = re.compile(r'^.*test_results.*' + FlakyTestsArchive._timestamp_rstr + r'\.xml$') + + files = os.listdir(directory) + matched_files = [ + os.path.join(directory, f) for f in files if pattern.match(f) + ] + matched_files.sort(key=FlakyTestsArchive._extract_timestamp) + + return matched_files + + @staticmethod + def _extract_timestamp(file_path: str) -> datetime: + """ + Extract the timestamp from the filename. + + Args: + file_path (str): Path to the file. + + Returns: + datetime: Timestamp extracted from the filename. + """ + filename = os.path.basename(file_path) + timestamp_str = re.search(FlakyTestsArchive._timestamp_rstr, filename).group() + return datetime.strptime(timestamp_str, '%Y-%m-%dT%H-%M-%S') + + def _delete_old_files(self) -> None: + """ + Delete old files taking window_size into account. + """ + if len(self._junit_files) > self._window_size: + # Calculate the number of files to delete + files_to_delete = self._junit_files[:-self._window_size] + + # Update the list of kept files + self._junit_files = self._junit_files[-self._window_size:] + + # Delete the older files + for file in files_to_delete: + print(f'Deleting old file: {file}') + os.remove(file) + + @staticmethod + def _calculate_fliprate_table(testrun_table: pd.DataFrame, window_size: int) -> pd.DataFrame: + """ + Calculate the fliprate for each test in the testrun_table. + + Args: + testrun_table (pd.DataFrame): DataFrame with the test results. + window_size (int): Number of test runs to consider for the fliprate calculation. + + Returns: + pd.DataFrame: DataFrame with the fliprates for each test. + """ + # Apply non_overlapping_window_fliprate to each group in testrun_table + fliprates = testrun_table.groupby("test_identifier")["test_status"].apply( + lambda x: FlakyTestsArchive._non_overlapping_window_fliprate(x, window_size) + ) + + # Convert fliprates Series of DataFrames to a DataFrame + fliprate_table = fliprates.reset_index() + + # Rename the index level to "window" + fliprate_table = fliprate_table.rename(columns={"level_1": "window"}) + + # Filter out rows where flip_rate is zero + # TODO(eduponz): I'm seeing tests with 5 consecutive failures that are not showing here. + # Seems it's because they are not flaky, they are just always failing. + fliprate_table = fliprate_table[fliprate_table.flip_rate != 0] + + return fliprate_table + + @staticmethod + def _non_overlapping_window_fliprate(testruns: pd.Series, window_size: int) -> pd.Series: + """ + Calculate the fliprate for a test in a non-overlapping window. + + Args: + testruns (pd.Series): Series with the test results. + window_size (int): Number of test runs to consider for the fliprate calculation. + + Returns: + pd.Series: Series with the fliprate for the test. + """ + # Apply _calc_fliprate directly to the last selected rows + testruns_last = testruns.iloc[-window_size:] + fliprate_groups = FlakyTestsArchive._calc_fliprate(testruns_last) + + return fliprate_groups.reset_index(drop=True) + + @staticmethod + def _calc_fliprate(testruns: pd.Series) -> pd.DataFrame: + """ + Calculate the fliprate for a test. + + Args: + testruns (pd.Series): Series with the test results. + + Returns: + pd.DataFrame: DataFrame with the fliprate, consecutive failures + and consecutive passes for the test. + """ + if len(testruns) < 2: + return pd.DataFrame( + { + 'flip_rate': [0.0], + 'consecutive_failures': [0], + 'consecutive_passes': [0], + 'failures': [0], + } + ) + + first = True + previous = None + flips = 0 + consecutive_failures = 0 + consecutive_passes = 0 + failures = 0 + possible_flips = len(testruns) - 1 + + for _, val in testruns.items(): + if first: + first = False + previous = val + continue + + if val != previous: + flips += 1 + + if val != "pass": + consecutive_failures += 1 + consecutive_passes = 0 + failures += 1 + else: + consecutive_failures = 0 + consecutive_passes += 1 + + previous = val + + flip_rate = flips / possible_flips + + return pd.DataFrame( + { + 'flip_rate': [flip_rate], + 'consecutive_failures': [consecutive_failures], + 'consecutive_passes': [consecutive_passes], + 'failures': [failures], + } + ) + + def _get_top_fliprates(self, fliprate_table: pd.DataFrame, top_n: int, precision: int) -> None: + """ + Get the top N fliprates from the fliprate_table. + + Args: + fliprate_table (pd.DataFrame): DataFrame with the fliprates. + top_n (int): Number of top fliprates to get. + precision (int): Number of decimal places to round the fliprates. + """ + context = getcontext() + context.prec = precision + context.rounding = ROUND_UP + last_window_values = fliprate_table.groupby("test_identifier").last() + + if top_n != 0: + top_fliprates = last_window_values.nlargest(top_n, "flip_rate") + else : + top_fliprates = last_window_values.nlargest(len(last_window_values), "flip_rate") + + # Create a dictionary with test_identifier as keys and a tuple of flip_rate and consecutive_failures as values + results = {} + for test_id, row in top_fliprates.iterrows(): + results[test_id] = { + 'flip_rate': Decimal(row['flip_rate']), + 'consecutive_failures': int(row['consecutive_failures']), + 'consecutive_passes': int(row['consecutive_passes']), + 'failures': int(row['failures']) + } + + self._analysis['flaky_tests'] = results + + def __getitem__(self, key: str) -> any: + """ + Get the value for the given key. + + Args: + key: Key to get the value for. + + Returns: + Any: Value for the given key. + """ + return self._analysis[key] + + def __iter__(self) -> iter: + """ + Return an iterator over the FlakyTestsArchive. + + Returns: + iter: Iterator over the FlakyTestsArchive. + """ + return iter(self._analysis) + + def items(self) -> dict: + """ + Return the FlakyTestsArchive as a dictionary. + + Returns: + dict: FlakyTestsArchive as a dictionary. + """ + return self._analysis.items() + + def keys(self) -> list: + """ + Return the keys of the FlakyTestsArchive. + + Returns: + list: Keys of the FlakyTestsArchive. + """ + return self._analysis.keys() + + def values(self) -> list: + """ + Return the values of the FlakyTestsArchive. + + Returns: + list: Values of the FlakyTestsArchive. + """ + return self._analysis.values() + + def __dict__(self) -> dict: + """ + Return the FlakyTestsArchive as a dictionary. + """ + return self._analysis diff --git a/resources/flaky/publisher.py b/resources/flaky/publisher.py new file mode 100644 index 00000000..ed9ff273 --- /dev/null +++ b/resources/flaky/publisher.py @@ -0,0 +1,103 @@ +# Copyright 2024 Proyectos y Sistemas de Mantenimiento SL (eProsima). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +from decimal import Decimal + +from flaky.archive import FlakyTestsArchive + + +class FlakyTestsMdPublisher: + """ + Class to publish FlakyTestsArchive as a markdown file. + """ + @staticmethod + def publish(test_archive: FlakyTestsArchive, output_file: str) -> None: + """ + Publish the FlakyTestsArchive to a markdown file. + + Args: + test_archive (FlakyTestsArchive): Archive with the flaky tests. + output_file (str): Path to the output file. + """ + # Test report + report = '## Flaky tests\n' + + # Table header + # TODO(eduponz): Add a column for the failures/runs ratio + # |#|Flaky tests|Fliprate score %|Consecutive failures|Consecutive passes|Total failures|\n + report += '|#' + report += '|Flaky tests' + report += '|Fliprate score %' + report += '|Last consecutive failures' + report += '|Last consecutive passes' + report += '|Total failures|\n' + report += '|-|-|-|-|-|-|\n' + + analysis = dict(test_archive) + assert(analysis['version'] == '1.0') + + i = 1 + for test_name, result in analysis['flaky_tests'].items(): + report += f'| {i} ' + report += f'| {test_name} ' + report += f'| {round(result["flip_rate"], 2) * 100} ' + report += f'| {result["consecutive_failures"]} ' + report += f'| {result["consecutive_passes"]} ' + report += f'| {result["failures"]} |\n' + i += 1 + + with open(output_file, 'w') as file: + file.write(report) + + +class FlakyTestsJSONPublisher: + """ + Class to publish FlakyTestsArchive as a JSON file. + """ + + @staticmethod + def publish(test_archive: FlakyTestsArchive, output_file: str) -> None: + """ + Publish the FlakyTestsArchive to a JSON file. + + Args: + test_archive (FlakyTestsArchive): Archive with the flaky tests. + output_file (str): Path to the output file. + """ + analysis = dict(test_archive) + + with open(output_file, 'w') as file: + json.dump(analysis, file, indent=4, cls=_CustomEncoder) + + +class _CustomEncoder(json.JSONEncoder): + """ + Custom JSON encoder with support for Decimal objects. + """ + def default(self, o: object) -> str: + """ + Encode the object. + + Args: + o: Object to encode. + + Returns: + str: Encoded object. + """ + # Encode Decimal objects as strings with 2 decimal places + if isinstance(o, Decimal): + return f'{o:.2f}' + + # Leave the rest to the default encoder + return super(_CustomEncoder, self).default(o) diff --git a/resources/junit/__init__.py b/resources/junit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/resources/junit/junit_utils.py b/resources/junit/junit_utils.py new file mode 100644 index 00000000..9a28d549 --- /dev/null +++ b/resources/junit/junit_utils.py @@ -0,0 +1,122 @@ +# Copyright 2024 Proyectos y Sistemas de Mantenimiento SL (eProsima). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import xml.etree.ElementTree as ET +import pandas as pd + + +class JUnitSuite: + """ + Class to represent a JUnit test suite. + """ + def __init__(self, suite_et: ET.Element) -> None: + """ + Initialize a JUnitSuite. + + Args: + suite_et (Element): ElementTree element with the suite information. + """ + self._suite_et = suite_et + + def to_df(self) -> pd.DataFrame: + """ + Convert a JUnitSuit to a list of DataFrame entries. + + Returns: + list: List of DataFrame entries. + """ + df = pd.DataFrame() + + dataframe_entries = [] + timestamp = self._suite_et.attrib.get('timestamp') + + for testcase in self._suite_et.findall('.//testcase'): + test_identifier = testcase.attrib.get('name') + + status = testcase.attrib.get('status') + + # Convert status to "pass" if it's "passed" or "run" + if status == "passed" or status == "run": + test_status = "pass" + else: + test_status = status + if test_status == "skipped": + continue + + dataframe_entries.append( + { + "timestamp": timestamp, + "test_identifier": test_identifier, + "test_status": test_status, + } + ) + + if dataframe_entries: + df = pd.DataFrame(dataframe_entries) + df["timestamp"] = pd.to_datetime(df["timestamp"]) + df = df.set_index("timestamp") + return df.sort_index() + + return df + + +class JUnitReport: + """ + Class to represent a JUnit report. + """ + def __init__(self, filename: str) -> None: + """ + Initialize a JUnitReport. + + Args: + filename (str): Path to the JUnit report file. + """ + self._filename = filename + self._suites = JUnitReport._get_suites(self._filename) + + def to_df(self) -> pd.DataFrame: + """ + Convert a JUnitReport to a DataFrame. + + Returns: + DataFrame: DataFrame with the report information. + """ + df = pd.DataFrame() + + for suite in self._suites: + df = pd.concat([df, suite.to_df()]) + + return df + + @staticmethod + def _get_suites(filename: str) -> list: + """ + Get the suites from a JUnit report file. + + Args: + filename (str): Path to the JUnit report file. + + Returns: + list: List of JUnitSuite objects. + """ + xml = ET.parse(filename) + root = xml.getroot() + + # Root can be 'testsuites' or 'testsuite' + suites = [] + if root.tag == 'testsuite': + suites = [root] + else: + suites = root.findall('.//testsuite') + + return [JUnitSuite(suite) for suite in suites] diff --git a/resources/junit_summary.py b/resources/junit_summary.py index c6f651e7..15366027 100644 --- a/resources/junit_summary.py +++ b/resources/junit_summary.py @@ -15,16 +15,19 @@ """Script to parse the jUnit test results and create a summary.""" import argparse +import json import xml.etree.ElementTree as ET DESCRIPTION = """Script to parse the jUnit test results and create a summary""" USAGE = ('python3 junit_summary.py') -def parse_options(): +def parse_options() -> argparse.Namespace: """ Parse arguments. - :return: The arguments parsed. + + Returns: + argparse.Namespace: Parsed arguments. """ parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, @@ -72,11 +75,25 @@ def parse_options(): action='store_true', help='Show a list of skipped tests' ) + parser.add_argument( + '--flaky-json-report', + type=str, + required=False, + help='Path to flaky json report file' + ) return parser.parse_args() -def junit_report_to_dict(junit_report): - """Convert a jUnit report to a dictionary.""" +def junit_report_to_dict(junit_report: str) -> list: + """ + Parse a junit report to a dictionary. + + Args: + junit_report (str): Path to junit report. + + Returns: + list: List of dictionaries with the parsed information. + """ result = [] tree = ET.parse(junit_report) root = tree.getroot() @@ -91,8 +108,17 @@ def junit_report_to_dict(junit_report): return result -def parse_testsuite(test_suite, result): - """Parse a testsuite tag.""" +def parse_testsuite(test_suite: ET.Element, result: list) -> list: + """ + Parse a test suite tag. + + Args: + test_suite (Element): Element with the test suite information. + result (list): List with the parsed information. + + Returns: + list: List with the parsed information. + """ suite_result = { 'name': '', 'tests': '', @@ -146,14 +172,47 @@ def parse_testsuite(test_suite, result): return result -def create_md_summary(results, show_failed, show_disabled, show_skipped): - """Create Markdown summary from results.""" +def create_md_summary( + results: list, + show_failed: bool, + show_disabled: bool, + show_skipped: bool, + flaky_tests: list + ) -> str: + """ + Create Markdown summary from results. + + Args: + results (list): List of dictionaries with the parsed information. + show_failed (bool): Show failed tests. + show_disabled (bool): Show disabled tests. + show_skipped (bool): Show skipped tests. + flaky_tests (list): List of flaky tests. + + Returns: + str: Markdown summary. + """ + flaky_failures = 0 + if len(flaky_tests) != 0: + for suite in results: + for failed_test in suite['failed_tests']: + if failed_test in flaky_tests: + flaky_failures += 1 + # Test summary summary = '## Test summary\n' # Table header - summary += '|Suite|Total number of tests|Test failures|Disabled test|Skipped test|Spent time [s]|Timestamp|\n' - summary += '|-|-|-|-|-|-|-|\n' + summary += '|Suite' + summary += '|Total number of tests' + summary += '|Test failures' + summary += '|Disabled test' + summary += '|Skipped test' + summary += '|Flaky failures' + summary += '|Spent time [s]' + summary += '|Timestamp' + summary += '|\n' + summary += '|-|-|-|-|-|-|-|-|\n' # Entries for suite in results: @@ -162,6 +221,7 @@ def create_md_summary(results, show_failed, show_disabled, show_skipped): summary += f'|{suite["failures"]}' summary += f'|{suite["disabled"]}' summary += f'|{suite["skipped"]}' + summary += f'|{flaky_failures}' summary += f'|{suite["time"]}' summary += f'|{suite["timestamp"]}' summary += '|\n' @@ -195,17 +255,49 @@ def create_md_summary(results, show_failed, show_disabled, show_skipped): return summary +def get_flaky_tests(flaky_json_report: str) -> list: + """ + Get flaky tests from a flaky json report. + + Args: + flaky_json_report (str): Path to flaky json report. + + Returns: + list: List of flaky tests. Empty list if no flaky tests found. + """ + flaky_tests = [] + try: + with open(flaky_json_report, 'r') as file: + flaky_json = json.load(file) + if 'version' in flaky_json and flaky_json['version'] == '1.0': + assert('flaky_tests' in flaky_json) + assert(type(flaky_json['flaky_tests']) == dict) + flaky_tests = list(flaky_json['flaky_tests'].keys()) + except FileNotFoundError: + pass + + return flaky_tests + + if __name__ == '__main__': # Parse arguments args = parse_options() + + # Parse junit report results = junit_report_to_dict(args.junit_report) + # Get flaky tests names + flaky_tests = [] + if args.flaky_json_report: + flaky_tests = get_flaky_tests(args.flaky_json_report) + # Create summary summary = create_md_summary( results, args.show_failed, args.show_disabled, - args.show_skipped + args.show_skipped, + flaky_tests ) # Print summary if required @@ -217,8 +309,11 @@ def create_md_summary(results, show_failed, show_disabled, show_skipped): with open(args.output_file, 'a') as file: file.write(summary) - # Exit code is the number of failed tests + # Exit code is the number of non-flaky failed tests exit_code = 0 for suite in results: - exit_code += int(suite['failures']) + for failed_test in suite['failed_tests']: + if failed_test not in flaky_tests: + exit_code += 1 + exit(exit_code) diff --git a/ubuntu/colcon_test/action.yml b/ubuntu/colcon_test/action.yml index 3bfb696b..bb903ddd 100644 --- a/ubuntu/colcon_test/action.yml +++ b/ubuntu/colcon_test/action.yml @@ -59,7 +59,7 @@ runs: steps: - name: Install lxml - uses: eProsima/eProsima-CI/ubuntu/install_python_packages@main + uses: eProsima/eProsima-CI/ubuntu/install_python_packages@feature/detect_flaky_tests with: packages: lxml @@ -88,10 +88,13 @@ runs: export COLCON_TEST_META_="${{ inputs.colcon_meta_file }}" fi + FILENAME_TIMESTAMP=$(date +'%Y-%m-%dT%H-%M-%S') + JUNIT_TIMESTAMP=$(date +'%Y-%m-%dT%H:%M:%S') + for package in ${{ inputs.packages_names }}; do echo "::group::Testing ${package} ..." - test_results_file="${{ inputs.workspace }}/test_results/${package}_test_results.xml" + test_results_file="${{ inputs.workspace }}/test_results/${package}_test_results_${FILENAME_TIMESTAMP}.xml" colcon test \ --metas ${COLCON_TEST_META_} \ @@ -122,6 +125,7 @@ runs: python3 ${{ github.action_path }}/../../resources/ctest2junit.py \ --build-dir ${{ inputs.workspace }}/build/${package} \ --xslt ${{ github.action_path }}/../../resources/ctest-to-junit.xsl \ + --timestamp "${JUNIT_TIMESTAMP}" \ --output-junit ${test_results_file} echo "::endgroup::" @@ -135,7 +139,7 @@ runs: shell: bash - name: Upload test report in JUnit format - uses: eProsima/eProsima-CI/external/upload-artifact@main + uses: eProsima/eProsima-CI/external/upload-artifact@feature/detect_flaky_tests if: ${{ ! cancelled() }} with: name: ${{ inputs.test_report_artifact || format('test_report_{0}_{1}_{2}', github.workflow, github.job, join(matrix.*, '_')) }} diff --git a/ubuntu/coverage_build_test_upload/action.yml b/ubuntu/coverage_build_test_upload/action.yml index e88a4b2f..ab7616da 100644 --- a/ubuntu/coverage_build_test_upload/action.yml +++ b/ubuntu/coverage_build_test_upload/action.yml @@ -62,10 +62,10 @@ runs: steps: - name: Build - uses: eProsima/eProsima-CI/ubuntu/install_gcov@main + uses: eProsima/eProsima-CI/ubuntu/install_gcov@feature/detect_flaky_tests - name: Build - uses: eProsima/eProsima-CI/multiplatform/colcon_build_test@main + uses: eProsima/eProsima-CI/multiplatform/colcon_build_test@feature/detect_flaky_tests with: packages_names: ${{ inputs.packages_names }} colcon_meta_file: ${{ inputs.colcon_meta_file }} @@ -77,7 +77,7 @@ runs: cmake_build_type: Debug - name: Generate report - uses: eProsima/eProsima-CI/ubuntu/generate_coverage_report@main + uses: eProsima/eProsima-CI/ubuntu/generate_coverage_report@feature/detect_flaky_tests with: packages_names: ${{ inputs.packages_names }} result_file_name: ${{ inputs.codecov_result_file_path }} @@ -85,7 +85,7 @@ runs: source_code_directory: ${{ github.workspace }}/src - name: Upload coverage to Codecov - uses: eProsima/eProsima-CI/external/codecov-action@main + uses: eProsima/eProsima-CI/external/codecov-action@feature/detect_flaky_tests with: token: ${{ inputs.codecov_token }} files: ${{ inputs.codecov_result_file_path }} diff --git a/ubuntu/flakiness_report/action.yml b/ubuntu/flakiness_report/action.yml new file mode 100644 index 00000000..0eb11f16 --- /dev/null +++ b/ubuntu/flakiness_report/action.yml @@ -0,0 +1,118 @@ +name: 'flakiness_report' +description: 'Generate a flakiness report from test results and historical data' + +inputs: + junit_reports_dir: + description: 'Path to dir with new jUnit reports' + required: true + + junit_archive_artifact: + description: > + Name of the artifact containing the historical data. Also used for uploading a JSON report + with the flaky tests as an artifact. name ${junit_archive_artifact}_json. + required: true + + print_report: + description: 'Whether to print the report (Default: True)' + required: false + default: 'True' + + window_size: + description: 'Number of days to consider for the flakiness report' + required: false + default: '30' + + fail_on_flaky_tests: + description: 'Whether to fail the workflow if there are flaky tests' + required: false + default: 'True' + + github_token: + description: 'GitHub token' + required: true + +runs: + using: composite + steps: + - name: Install python dependencies + id: install_python_dependencies + uses: eProsima/eProsima-CI/ubuntu/install_python_packages@v0 + with: + packages: lxml junitparser pandas + + - name: Download jUnit archive + id: download_junit_archive + uses: eProsima/eProsima-CI/external/action-download-artifact@v0 + with: + name: ${{ inputs.junit_archive_artifact }} + path: junit_archive + github_token: ${{ inputs.github_token }} + workflow_conclusion: completed + if_no_artifact_found: ignore + + - name: Flakiness Report + id: flakiness_report + shell: bash + run: | + echo "::group::Flakiness report" + + WINDOW_SIZE_OPTION="--window-size ${{ inputs.window_size }}" + + MD_FILE_OPTION="" + if [[ "${{ inputs.print_report }}" == "True" ]] + then + MD_FILE_OPTION="--markdown-file $GITHUB_STEP_SUMMARY" + fi + + echo "Creating flakiness report..." + + mkdir -p junit_archive + cp ${{ inputs.junit_reports_dir }}/*.xml junit_archive + + echo "Generating flakiness report..." + + # Prevent the action from failing if there are flaky tests + set +e + + EXIT_CODE=0 + python3 ${{ github.action_path }}/../../resources/flakiness_report.py \ + --junit-archive junit_archive \ + ${WINDOW_SIZE_OPTION} \ + --delete-old-files \ + --json-file flaky_tests.json \ + ${MD_FILE_OPTION} + + EXIT_CODE=$? + + # Restore the action to fail on error + set -e + + echo "Flakiness report generated" + + # A 255 exit code indicates that an exception occurred in the script, we want to fail in that case + if [ "${{ inputs.fail_on_flaky_tests }}" != "True" ] && [ "$EXIT_CODE" != "255" ] + then + echo "Ignoring flaky tests failures" + EXIT_CODE=0 + fi + + echo "Flakiness report exit code: ${EXIT_CODE}" + echo "::endgroup::" + exit ${EXIT_CODE} + + - name: Upload jUnit archive + id: upload_junit_archive + if: always() + uses: eProsima/eProsima-CI/external/upload-artifact@feature/detect_flaky_tests + with: + name: ${{ inputs.junit_archive_artifact }} + path: junit_archive + overwrite: true + + - name: Upload JSON report + id: upload_json_report + if: always() + uses: eProsima/eProsima-CI/external/upload-artifact@feature/detect_flaky_tests + with: + name: ${{ inputs.junit_archive_artifact }}_json + path: flaky_tests.json diff --git a/ubuntu/install_asio/action.yml b/ubuntu/install_asio/action.yml index c192a056..4a736c9e 100644 --- a/ubuntu/install_asio/action.yml +++ b/ubuntu/install_asio/action.yml @@ -5,6 +5,6 @@ runs: steps: - name: Install in ubuntu - uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@main + uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@feature/detect_flaky_tests with: packages: libasio-dev diff --git a/ubuntu/install_colcon/action.yml b/ubuntu/install_colcon/action.yml index be008d0a..4436da37 100644 --- a/ubuntu/install_colcon/action.yml +++ b/ubuntu/install_colcon/action.yml @@ -6,7 +6,7 @@ runs: steps: - name: Install colcon - uses: eProsima/eProsima-CI/ubuntu/install_python_packages@main + uses: eProsima/eProsima-CI/ubuntu/install_python_packages@feature/detect_flaky_tests with: packages: 'setuptools==58.3.0 colcon-common-extensions colcon-mixin' upgrade: true diff --git a/ubuntu/install_documentation_requirements/action.yml b/ubuntu/install_documentation_requirements/action.yml index c7d909ac..6b0cda35 100644 --- a/ubuntu/install_documentation_requirements/action.yml +++ b/ubuntu/install_documentation_requirements/action.yml @@ -12,7 +12,7 @@ runs: steps: - name: Install docs requirements with pip - uses: eProsima/eProsima-CI/ubuntu/install_python_packages@main + uses: eProsima/eProsima-CI/ubuntu/install_python_packages@feature/detect_flaky_tests if: ${{ inputs.path_to_requirements == '' }} with: packages: \ @@ -33,7 +33,7 @@ runs: xmlschema==2.1.1 - name: Install docs requirements with pip - uses: eProsima/eProsima-CI/ubuntu/install_python_packages@main + uses: eProsima/eProsima-CI/ubuntu/install_python_packages@feature/detect_flaky_tests if: ${{ inputs.path_to_requirements != '' }} with: requirements_file_name: ${{ inputs.path_to_requirements }} diff --git a/ubuntu/install_fastdds_dependencies/action.yml b/ubuntu/install_fastdds_dependencies/action.yml index 5585c83d..7a14ea48 100644 --- a/ubuntu/install_fastdds_dependencies/action.yml +++ b/ubuntu/install_fastdds_dependencies/action.yml @@ -24,27 +24,27 @@ runs: steps: - name: Set CMake version - uses: eProsima/eProsima-CI/external/get-cmake@main + uses: eProsima/eProsima-CI/external/get-cmake@feature/detect_flaky_tests with: cmakeVersion: ${{ inputs.cmakeVersion }} ninjaVersion: ${{ inputs.ninjaVersion }} - name: Install vcs tools - uses: eProsima/eProsima-CI/ubuntu/install_python_packages@main + uses: eProsima/eProsima-CI/ubuntu/install_python_packages@feature/detect_flaky_tests with: packages: 'vcstool setuptools' - name: Install colcon - uses: eProsima/eProsima-CI/ubuntu/install_colcon@main + uses: eProsima/eProsima-CI/ubuntu/install_colcon@feature/detect_flaky_tests - name: Install GTest - uses: eProsima/eProsima-CI/ubuntu/install_gtest@main + uses: eProsima/eProsima-CI/ubuntu/install_gtest@feature/detect_flaky_tests - name: Install asio - uses: eProsima/eProsima-CI/ubuntu/install_asio@main + uses: eProsima/eProsima-CI/ubuntu/install_asio@feature/detect_flaky_tests - name: Install Tiny XML - uses: eProsima/eProsima-CI/ubuntu/install_tinyxml@main + uses: eProsima/eProsima-CI/ubuntu/install_tinyxml@feature/detect_flaky_tests - name: Install Open SSL - uses: eProsima/eProsima-CI/ubuntu/install_openssl@main + uses: eProsima/eProsima-CI/ubuntu/install_openssl@feature/detect_flaky_tests diff --git a/ubuntu/install_gcov/action.yml b/ubuntu/install_gcov/action.yml index e60f8576..9ffc35c4 100644 --- a/ubuntu/install_gcov/action.yml +++ b/ubuntu/install_gcov/action.yml @@ -7,11 +7,11 @@ runs: # TODO: check if both lcov and gcovr are needed - name: Install in ubuntu by apt - uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@main + uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@feature/detect_flaky_tests with: packages: lcov - name: Install in ubuntu by pip - uses: eProsima/eProsima-CI/ubuntu/install_python_packages@main + uses: eProsima/eProsima-CI/ubuntu/install_python_packages@feature/detect_flaky_tests with: packages: gcovr==5.0 diff --git a/ubuntu/install_openssl/action.yml b/ubuntu/install_openssl/action.yml index 440daee0..bf94f259 100644 --- a/ubuntu/install_openssl/action.yml +++ b/ubuntu/install_openssl/action.yml @@ -5,6 +5,6 @@ runs: steps: - name: Install in ubuntu - uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@main + uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@feature/detect_flaky_tests with: packages: libssl-dev diff --git a/ubuntu/install_tinyxml/action.yml b/ubuntu/install_tinyxml/action.yml index 5b4287f8..b3679f7b 100644 --- a/ubuntu/install_tinyxml/action.yml +++ b/ubuntu/install_tinyxml/action.yml @@ -5,6 +5,6 @@ runs: steps: - name: Install in ubuntu - uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@main + uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@feature/detect_flaky_tests with: packages: libtinyxml2-dev diff --git a/ubuntu/install_uncrustify/action.yml b/ubuntu/install_uncrustify/action.yml index 29ad11fc..f1cb536d 100644 --- a/ubuntu/install_uncrustify/action.yml +++ b/ubuntu/install_uncrustify/action.yml @@ -5,6 +5,6 @@ runs: steps: - name: Install uncrustify - uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@main + uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@feature/detect_flaky_tests with: packages: uncrustify diff --git a/ubuntu/install_yamlcpp/action.yml b/ubuntu/install_yamlcpp/action.yml index 6c485de0..e8886ee1 100644 --- a/ubuntu/install_yamlcpp/action.yml +++ b/ubuntu/install_yamlcpp/action.yml @@ -18,6 +18,6 @@ runs: steps: - name: Install in ubuntu - uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@main + uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@feature/detect_flaky_tests with: packages: libyaml-cpp-dev diff --git a/ubuntu/junit_summary/action.yaml b/ubuntu/junit_summary/action.yaml index a0cb6edd..9262aa39 100644 --- a/ubuntu/junit_summary/action.yaml +++ b/ubuntu/junit_summary/action.yaml @@ -26,6 +26,11 @@ inputs: required: false default: 'False' + flaky_json_report: + description: 'Path to a JSON file with the flaky tests report' + required: false + default: '' + runs: using: composite steps: @@ -59,6 +64,12 @@ runs: SHOW_SKIPPED_OPTION="--show-skipped" fi + FLAKY_JSON_REPORT_OPTION="" + if [[ -f "${{ inputs.flaky_json_report }}" ]] + then + FLAKY_JSON_REPORT_OPTION="--flaky-json-report ${{ inputs.flaky_json_report }}" + fi + EXIT_CODE=0 for JUNIT_REPORT in ${{ inputs.junit_reports_dir }}/*.xml do @@ -68,7 +79,8 @@ runs: ${PRINT_SUMMARY_OPTION} \ ${SHOW_FAILED_OPTION} \ ${SHOW_DISABLED_OPTION} \ - ${SHOW_SKIPPED_OPTION} + ${SHOW_SKIPPED_OPTION} \ + ${FLAKY_JSON_REPORT_OPTION} EXIT_CODE=$((${EXIT_CODE} + $?)) done diff --git a/ubuntu/python_linter/action.yml b/ubuntu/python_linter/action.yml index f6c6d616..e40655b4 100644 --- a/ubuntu/python_linter/action.yml +++ b/ubuntu/python_linter/action.yml @@ -34,17 +34,17 @@ runs: steps: - name: Sync repository - uses: eProsima/eProsima-CI/external/checkout@main + uses: eProsima/eProsima-CI/external/checkout@feature/detect_flaky_tests with: path: src - name: Fetch all branches and tags - uses: eProsima/eProsima-CI/ubuntu/git_fetch_all@main + uses: eProsima/eProsima-CI/ubuntu/git_fetch_all@feature/detect_flaky_tests with: workspace: src - name: Install Flake8 - uses: eProsima/eProsima-CI/ubuntu/install_python_packages@main + uses: eProsima/eProsima-CI/ubuntu/install_python_packages@feature/detect_flaky_tests with: packages: flake8==5.0.4 flake8-quotes @@ -57,7 +57,7 @@ runs: - name: Get Git difference id: GetGitDifference - uses: eProsima/eProsima-CI/ubuntu/get_git_diff_files@main + uses: eProsima/eProsima-CI/ubuntu/get_git_diff_files@feature/detect_flaky_tests with: result_env_var: MODIFIED_FILES diff --git a/ubuntu/setup_cmake/action.yml b/ubuntu/setup_cmake/action.yml index d77b47f1..da5c23ef 100644 --- a/ubuntu/setup_cmake/action.yml +++ b/ubuntu/setup_cmake/action.yml @@ -14,10 +14,10 @@ runs: steps: - name: Get ubuntu version - uses: eProsima/eProsima-CI/ubuntu/get_platform@main + uses: eProsima/eProsima-CI/ubuntu/get_platform@feature/detect_flaky_tests - name: Upgrade CMake in Ubuntu 20.04 - uses: eProsima/eProsima-CI/external/get-cmake@main + uses: eProsima/eProsima-CI/external/get-cmake@feature/detect_flaky_tests with: cmakeVersion: '3.22.x' if: contains( ${{ env.EPROSIMA_UBUNTU_VERSION }} , '20.04' ) diff --git a/ubuntu/sphinx_docs/action.yml b/ubuntu/sphinx_docs/action.yml index 98f54a30..02a02309 100644 --- a/ubuntu/sphinx_docs/action.yml +++ b/ubuntu/sphinx_docs/action.yml @@ -55,20 +55,20 @@ runs: steps: - name: Sync repository - uses: eProsima/eProsima-CI/external/checkout@main + uses: eProsima/eProsima-CI/external/checkout@feature/detect_flaky_tests with: path: ${{ inputs.checkout_path }} - name: Install docs dependencies - uses: eProsima/eProsima-CI/ubuntu/install_documentation_requirements@main + uses: eProsima/eProsima-CI/ubuntu/install_documentation_requirements@feature/detect_flaky_tests with: path_to_requirements: ${{ inputs.path_to_requirements }} - name: Install Colcon - uses: eProsima/eProsima-CI/ubuntu/install_colcon@main + uses: eProsima/eProsima-CI/ubuntu/install_colcon@feature/detect_flaky_tests - name: Download cmake_utils artifact - uses: eProsima/eProsima-CI/multiplatform/download_dependency@main + uses: eProsima/eProsima-CI/multiplatform/download_dependency@feature/detect_flaky_tests with: artifact_name: built_dev_utils_ubuntu-22.04_Release_nightly workflow_source: build_dev_utils.yml @@ -78,19 +78,19 @@ runs: workflow_conclusion: completed - name: Build documentation with colcon - uses: eProsima/eProsima-CI/multiplatform/colcon_build@main + uses: eProsima/eProsima-CI/multiplatform/colcon_build@feature/detect_flaky_tests with: colcon_build_args: '--packages-select ${{ inputs.docs_subpackage_name }}' cmake_args: '${{ inputs.specific_cmake_args }}' workspace_dependencies: '${{ github.workspace }}/install' - name: Run colcon test of documentation - uses: eProsima/eProsima-CI/multiplatform/colcon_test@main + uses: eProsima/eProsima-CI/multiplatform/colcon_test@feature/detect_flaky_tests with: colcon_test_args: --packages-select ${{ inputs.docs_subpackage_name }} - name: Upload documentation - uses: eProsima/eProsima-CI/external/upload-artifact@main + uses: eProsima/eProsima-CI/external/upload-artifact@feature/detect_flaky_tests with: name: Documentation HTML ${{ inputs.docs_subpackage_name }} path: install/${{ inputs.docs_subpackage_name }}/docs/${{ inputs.docs_subpackage_name }}/sphinx/html/ diff --git a/ubuntu/uncrustify/action.yml b/ubuntu/uncrustify/action.yml index 3bdddf3e..5ace8080 100644 --- a/ubuntu/uncrustify/action.yml +++ b/ubuntu/uncrustify/action.yml @@ -57,20 +57,20 @@ runs: steps: - name: Sync repository - uses: eProsima/eProsima-CI/external/checkout@main + uses: eProsima/eProsima-CI/external/checkout@feature/detect_flaky_tests with: path: '${{ github.workspace }}/check' - name: Fetch all branches and tags - uses: eProsima/eProsima-CI/ubuntu/git_fetch_all@main + uses: eProsima/eProsima-CI/ubuntu/git_fetch_all@feature/detect_flaky_tests with: workspace: '${{ github.workspace }}/check' - name: Install uncrustify - uses: eProsima/eProsima-CI/ubuntu/install_uncrustify@main + uses: eProsima/eProsima-CI/ubuntu/install_uncrustify@feature/detect_flaky_tests - name: Install Colcon - uses: eProsima/eProsima-CI/ubuntu/install_colcon@main + uses: eProsima/eProsima-CI/ubuntu/install_colcon@feature/detect_flaky_tests # TODO: Change to main branch when PR is merged to allow c++ check over other files different than cpp hpp # (required for .ipp files) @@ -94,7 +94,7 @@ runs: - name: Get Git difference id: GetGitDifference - uses: eProsima/eProsima-CI/ubuntu/get_git_diff_files@main + uses: eProsima/eProsima-CI/ubuntu/get_git_diff_files@feature/detect_flaky_tests with: result_env_var: MODIFIED_FILES head_ref: origin/${GITHUB_HEAD_REF} diff --git a/versions.md b/versions.md index 3955a517..669d64d9 100644 --- a/versions.md +++ b/versions.md @@ -34,6 +34,12 @@ The [Forthcoming](#forthcoming) section includes those features added in `main` The upcoming release will include the following **features**: +- Add a `flakiness_report` action to generate a summary of flaky tests using historical data. +- Add an option `flaky_json_report` to `junit_summary` action to use the flaky tests report for comparing failing + test with flakiness data and only fail on true regressions. +- Extend `ctest2junit.py` to support applying an external timestamp to the translated JUnit XML files. +- Extend `upload-artifact` with an option to overwrite existing artifacts (default: false). + ## v0.20.0 - Add colcon meta argument to colcon test actions. diff --git a/windows/colcon_test/action.yml b/windows/colcon_test/action.yml index 531414d8..d8bc09b0 100644 --- a/windows/colcon_test/action.yml +++ b/windows/colcon_test/action.yml @@ -59,7 +59,7 @@ runs: steps: - name: Install lxml - uses: eProsima/eProsima-CI/windows/install_python_packages@main + uses: eProsima/eProsima-CI/windows/install_python_packages@feature/detect_flaky_tests with: packages: lxml @@ -94,10 +94,13 @@ runs: $exitCode = 0 $packages_names = "${{ inputs.packages_names }}" -split '\s+' + $FILENAME_TIMESTAMP = Get-Date -Format "yyyy-MM-ddTHH-mm-ss" + $JUNIT_TIMESTAMP = Get-Date -Format "yyyy-MM-ddTHH:mm:ss" + foreach ($package in $packages_names) { Write-Host "::group::Testing $package ..." - $testResultsFile = "$resultsPath\${package}_test_results.xml" + $testResultsFile = "$resultsPath\${package}_test_results_$FILENAME_TIMESTAMP.xml" $testCommand = @" colcon test @@ -128,7 +131,7 @@ runs: if (! $testResultsExist) { Write-Host "::group::Translating ${package} test report to jUnit..." - python3 ${{ github.action_path }}\..\..\resources\ctest2junit.py --build-dir "${{ inputs.workspace }}\build\${package}" --xslt ${{ github.action_path }}\..\..\resources\ctest-to-junit.xsl --output-junit $testResultsFile + python3 ${{ github.action_path }}\..\..\resources\ctest2junit.py --build-dir "${{ inputs.workspace }}\build\${package}" --xslt ${{ github.action_path }}\..\..\resources\ctest-to-junit.xsl --timestamp "$JUNIT_TIMESTAMP" --output-junit $testResultsFile Write-Host "::endgroup::" } @@ -139,7 +142,7 @@ runs: Exit $exitCode - name: Upload test report in JUnit format - uses: eProsima/eProsima-CI/external/upload-artifact@main + uses: eProsima/eProsima-CI/external/upload-artifact@feature/detect_flaky_tests if: ${{ ! cancelled() }} with: name: ${{ inputs.test_report_artifact || format('test_report_{0}_{1}_{2}', github.workflow, github.job, join(matrix.*, '_')) }} diff --git a/windows/flakiness_report/action.yml b/windows/flakiness_report/action.yml new file mode 100644 index 00000000..8c4de142 --- /dev/null +++ b/windows/flakiness_report/action.yml @@ -0,0 +1,103 @@ +name: 'flakiness_report' +description: 'Generate a flakiness report from test results and historical data' + +inputs: + junit_reports_dir: + description: 'Path to dir with new jUnit reports' + required: true + + junit_archive_artifact: + description: > + Name of the artifact containing the historical data. Also used for uploading a JSON report + with the flaky tests as an artifact. name ${junit_archive_artifact}_json. + required: true + + print_report: + description: 'Whether to print the report (Default: True)' + required: false + default: 'True' + + window_size: + description: 'Number of days to consider for the flakiness report' + required: false + default: '30' + + fail_on_flaky_tests: + description: 'Whether to fail the workflow if there are flaky tests' + required: false + default: 'True' + + github_token: + description: 'GitHub token' + required: true + +runs: + using: composite + steps: + - name: Install python dependencies + id: install_python_dependencies + uses: eProsima/eProsima-CI/ubuntu/install_python_packages@v0 + with: + packages: lxml junitparser pandas + + - name: Download jUnit archive + id: download_junit_archive + uses: eProsima/eProsima-CI/external/action-download-artifact@v0 + with: + name: ${{ inputs.junit_archive_artifact }} + path: junit_archive + github_token: ${{ inputs.github_token }} + workflow_conclusion: completed + if_no_artifact_found: ignore + + - name: Flakiness Report + id: flakiness_report + shell: pwsh + run: | + Write-Host "::group::Flakiness report" + + $MD_FILE_OPTION = $null + $SUMMARY = $null + if ("${{ inputs.print_report }}" -eq "True") { + $MD_FILE_OPTION = "--markdown-file" + $SUMMARY_FILE = "$GITHUB_STEP_SUMMARY" + } + + mkdir -p junit_archive + cp ${{ inputs.junit_reports_dir }}/*.xml junit_archive + + $EXIT_CODE=0 + python3 ${{ github.action_path }}/../../resources/flakiness_report.py ` + --junit-archive junit_archive ` + --window-size ${{ inputs.window_size }} ` + --delete-old-files ` + --json-file flaky_tests.json ` + $MD_FILE_OPTION $SUMMARY_FILE + + $EXIT_CODE = $LASTEXITCODE + + # A 255 exit code indicates that an exception occurred in the script, we want to fail in that case + if ("${{ inputs.fail_on_flaky_tests }}" -ne "True" -and $EXIT_CODE -ne 255) { + Write-Output "Ignoring flaky tests failures" + $EXIT_CODE = 0 + } + + Write-Host "::endgroup::" + exit ${EXIT_CODE} + + - name: Upload jUnit archive + id: upload_junit_archive + if: always() + uses: eProsima/eProsima-CI/external/upload-artifact@feature/detect_flaky_tests + with: + name: ${{ inputs.junit_archive_artifact }} + path: junit_archive + overwrite: true + + - name: Upload JSON report + id: upload_json_report + if: always() + uses: eProsima/eProsima-CI/external/upload-artifact@feature/detect_flaky_tests + with: + name: ${{ inputs.junit_archive_artifact }}_json + path: flaky_tests.json diff --git a/windows/install_colcon/action.yml b/windows/install_colcon/action.yml index 7d6a311c..39c82ee1 100644 --- a/windows/install_colcon/action.yml +++ b/windows/install_colcon/action.yml @@ -5,7 +5,7 @@ runs: steps: - name: Install colcon - uses: eProsima/eProsima-CI/windows/install_python_packages@main + uses: eProsima/eProsima-CI/windows/install_python_packages@feature/detect_flaky_tests with: packages: 'vcstool setuptools==58.3.0 colcon-common-extensions colcon-mixin' upgrade: true diff --git a/windows/install_fastdds_dependencies/action.yml b/windows/install_fastdds_dependencies/action.yml index c57a79db..c7380585 100644 --- a/windows/install_fastdds_dependencies/action.yml +++ b/windows/install_fastdds_dependencies/action.yml @@ -30,20 +30,20 @@ runs: steps: - name: Set CMake version - uses: eProsima/eProsima-CI/external/get-cmake@main + uses: eProsima/eProsima-CI/external/get-cmake@feature/detect_flaky_tests with: cmakeVersion: ${{ inputs.cmakeVersion }} ninjaVersion: ${{ inputs.ninjaVersion }} - name: Install colcon - uses: eProsima/eProsima-CI/windows/install_colcon@main + uses: eProsima/eProsima-CI/windows/install_colcon@feature/detect_flaky_tests - name: Install GTest - uses: eProsima/eProsima-CI/windows/install_gtest@main + uses: eProsima/eProsima-CI/windows/install_gtest@feature/detect_flaky_tests with: cmake_build_type: ${{ inputs.cmake_build_type }} - name: Install Open SSL - uses: eProsima/eprosima-CI/windows/install_openssl@main + uses: eProsima/eprosima-CI/windows/install_openssl@feature/detect_flaky_tests with: version: '3.1.1' diff --git a/windows/junit_summary/action.yaml b/windows/junit_summary/action.yaml index 6221724e..5bdc5811 100644 --- a/windows/junit_summary/action.yaml +++ b/windows/junit_summary/action.yaml @@ -26,6 +26,11 @@ inputs: required: false default: "False" + flaky_json_report: + description: 'Path to a JSON file with the flaky tests report' + required: false + default: "" + runs: using: composite steps: @@ -53,6 +58,13 @@ runs: $SHOW_SKIPPED_OPTION = "--show-skipped" } + $FLAKY_JSON_REPORT_OPTION = $null + $FLAKY_JSON_REPORT_FILE = $null + if (Test-Path "${{ inputs.flaky_json_report }}") { + $FLAKY_JSON_REPORT_OPTION = "--flaky-json-report" + $FLAKY_JSON_REPORT_FILE = "${{ inputs.flaky_json_report }}" + } + $EXIT_CODE=0 foreach ($JUNIT_REPORT in Get-ChildItem -Path ${{ inputs.junit_reports_dir }} -Filter "*.xml") { @@ -62,7 +74,8 @@ runs: $PRINT_SUMMARY_OPTION ` $SHOW_FAILED_OPTION ` $SHOW_DISABLED_OPTION ` - $SHOW_SKIPPED_OPTION + $SHOW_SKIPPED_OPTION ` + $FLAKY_JSON_REPORT_OPTION $FLAKY_JSON_REPORT_FILE $EXIT_CODE += $LASTEXITCODE }