[20843] Add action to generate flaky test reports

README.md

@@ -88,6 +88,9 @@ For more information about versioning handle of this project, check following [f
 - [fetch_fastdds_manual](multiplatform/fetch_fastdds_manual/action.yml)
   - Download Fast DDS and its eProsima dependencies setting the specific version of each repository.
 
+- [flakiness_report](multiplatform/flakiness_report/action.yml)
+  - Generate a flakiness report from historical tests results.
+
 - [generate_dependency_artifact](multiplatform/generate_dependency_artifact/action.yml)
   - Build a project and upload the installed objects as an artifact.
 

external/upload-artifact/action.yml

@@ -24,6 +24,13 @@ inputs:
 
       Minimum 1 day.
       Maximum 90 days unless changed from the repository settings page.
+  overwrite:
+    description: >
+      If true, an artifact with a matching name will be deleted before a new one is uploaded.
+      If false, the action will fail if an artifact with the given name already exists.
+      Does not fail if the artifact does not exist.
+      Optional. Default is 'false'
+    default: 'false'
 
 runs:
   using: composite
@@ -46,3 +53,4 @@ runs:
         path:               ${{ inputs.path }}
         if-no-files-found:  ${{ inputs.if-no-files-found }}
         retention-days:     ${{ inputs.retention-days }}
+        overwrite:          ${{ inputs.overwrite }}

macos/install_colcon/action.yml

@@ -6,7 +6,7 @@ runs:
   steps:
 
     - name: Install colcon
-      uses: eProsima/eProsima-CI/macos/install_python_packages@main
+      uses: eProsima/eProsima-CI/macos/install_python_packages@feature/detect_flaky_tests
       with:
         packages: 'setuptools==58.3.0 colcon-common-extensions colcon-mixin'
         upgrade: true

multiplatform/asan_build_test/action.yml

@@ -43,7 +43,7 @@ runs:
 
     - name: Build and test
       id: build_and_test
-      uses: eProsima/eProsima-CI/multiplatform/colcon_build_test@main
+      uses: eProsima/eProsima-CI/multiplatform/colcon_build_test@feature/detect_flaky_tests
       with:
         packages_names: ${{ inputs.packages_names }}
         colcon_meta_file: ${{ inputs.colcon_meta_file }}

multiplatform/clang_build_test/action.yml

@@ -32,7 +32,7 @@ runs:
   steps:
 
     - name: Build
-      uses: eProsima/eProsima-CI/multiplatform/colcon_build@main
+      uses: eProsima/eProsima-CI/multiplatform/colcon_build@feature/detect_flaky_tests
       with:
         colcon_meta_file: ${{ inputs.colcon_meta_file }}
         colcon_build_args: '--packages-up-to ${{ inputs.packages_names }}'
@@ -41,6 +41,6 @@ runs:
         workspace_dependencies: ${{ inputs.workspace_dependencies }}
 
     - name: Test clang
-      uses: eProsima/eProsima-CI/multiplatform/clang_tidy_check@main
+      uses: eProsima/eProsima-CI/multiplatform/clang_tidy_check@feature/detect_flaky_tests
       with:
         packages_names: ${{ inputs.packages_names }}

multiplatform/clang_tidy_check/action.yml

@@ -23,15 +23,15 @@ runs:
   steps:
 
     - name: Run in ubuntu
-      uses: eProsima/eProsima-CI/ubuntu/clang_tidy_check@main
+      uses: eProsima/eProsima-CI/ubuntu/clang_tidy_check@feature/detect_flaky_tests
       if: runner.os == 'Linux'
       with:
         build_directory: ${{ inputs.build_directory }}
         packages_names: ${{ inputs.packages_names }}
         result_file_name: ${{ inputs.result_file_name }}
 
     - name: Run in windows
-      uses: eProsima/eProsima-CI/windows/clang_tidy_check@main
+      uses: eProsima/eProsima-CI/windows/clang_tidy_check@feature/detect_flaky_tests
       if: runner.os == 'Windows'
       with:
         build_directory: ${{ inputs.build_directory }}

multiplatform/colcon_build/action.yml

@@ -48,7 +48,7 @@ runs:
   steps:
 
     - name: Run in ubuntu or macOS
-      uses: eProsima/eProsima-CI/ubuntu/colcon_build@main
+      uses: eProsima/eProsima-CI/ubuntu/colcon_build@feature/detect_flaky_tests
       if: runner.os == 'Linux' || runner.os == 'macOS'
       with:
         colcon_meta_file: ${{ inputs.colcon_meta_file }}
@@ -61,7 +61,7 @@ runs:
         cmake_build_type: ${{ inputs.cmake_build_type }}
 
     - name: Run in windows
-      uses: eProsima/eProsima-CI/windows/colcon_build@main
+      uses: eProsima/eProsima-CI/windows/colcon_build@feature/detect_flaky_tests
       if: runner.os == 'Windows'
       with:
         colcon_meta_file: ${{ inputs.colcon_meta_file }}

multiplatform/colcon_build_test/action.yml

@@ -54,7 +54,7 @@ runs:
   steps:
 
     - name: Build
-      uses: eProsima/eProsima-CI/multiplatform/colcon_build@main
+      uses: eProsima/eProsima-CI/multiplatform/colcon_build@feature/detect_flaky_tests
       with:
         colcon_meta_file: ${{ inputs.colcon_meta_file }}
         colcon_build_args: '--packages-up-to ${{ inputs.packages_names }}'
@@ -65,7 +65,7 @@ runs:
 
     - name: Test
       id: test
-      uses: eProsima/eProsima-CI/multiplatform/colcon_test@main
+      uses: eProsima/eProsima-CI/multiplatform/colcon_test@feature/detect_flaky_tests
       with:
         colcon_meta_file: ${{ inputs.colcon_meta_file }}
         workspace: ${{ inputs.workspace }}

multiplatform/colcon_build_test_flaky/action.yml

@@ -42,7 +42,7 @@ runs:
   steps:
 
     - name: Build
-      uses: eProsima/eProsima-CI/multiplatform/colcon_build@main
+      uses: eProsima/eProsima-CI/multiplatform/colcon_build@feature/detect_flaky_tests
       with:
         colcon_meta_file: ${{ inputs.colcon_meta_file }}
         colcon_build_args: '--packages-up-to ${{ inputs.packages_names }}'
@@ -52,7 +52,7 @@ runs:
 
     - name: Test
       id: test
-      uses: eProsima/eProsima-CI/multiplatform/colcon_test@main
+      uses: eProsima/eProsima-CI/multiplatform/colcon_test@feature/detect_flaky_tests
       with:
         colcon_meta_file: ${{ inputs.colcon_meta_file }}
         workspace: ${{ inputs.workspace }}

multiplatform/colcon_test/action.yml

@@ -60,7 +60,7 @@ runs:
 
     - name: Run in ubuntu or macOS
       id: test_ubuntu_mac
-      uses: eProsima/eProsima-CI/ubuntu/colcon_test@main
+      uses: eProsima/eProsima-CI/ubuntu/colcon_test@feature/detect_flaky_tests
       if: runner.os == 'Linux' || runner.os == 'macOS'
       with:
         colcon_meta_file: ${{ inputs.colcon_meta_file }}
@@ -75,7 +75,7 @@ runs:
 
     - name: Run in windows
       id: test_windows
-      uses: eProsima/eProsima-CI/windows/colcon_test@main
+      uses: eProsima/eProsima-CI/windows/colcon_test@feature/detect_flaky_tests
       if: runner.os == 'Windows'
       with:
         colcon_meta_file: ${{ inputs.colcon_meta_file }}

multiplatform/download_dependency/action.yml

@@ -47,7 +47,7 @@ runs:
     # However, looking for the id previously allows to call this action without secret_token
 
     - name: Download dependency
-      uses: eProsima/eProsima-CI/external/action-download-artifact@main
+      uses: eProsima/eProsima-CI/external/action-download-artifact@feature/detect_flaky_tests
       with:
         name: ${{ inputs.artifact_name }}
         workflow: ${{ inputs.workflow_source }}

multiplatform/fetch_ddspipe_manual/action.yml

@@ -20,14 +20,14 @@ runs:
   steps:
 
     - name: Run in ubuntu
-      uses: eProsima/eProsima-CI/ubuntu/fetch_ddspipe_manual@main
+      uses: eProsima/eProsima-CI/ubuntu/fetch_ddspipe_manual@feature/detect_flaky_tests
       if: runner.os == 'Linux'
       with:
         ddspipe_branch: ${{ inputs.ddspipe_branch }}
         destination_workspace: ${{ inputs.destination_workspace }}
 
     - name: Run in windows
-      uses: eProsima/eProsima-CI/windows/fetch_ddspipe_manual@main
+      uses: eProsima/eProsima-CI/windows/fetch_ddspipe_manual@feature/detect_flaky_tests
       if: runner.os == 'Windows'
       with:
         ddspipe_branch: ${{ inputs.ddspipe_branch }}

multiplatform/fetch_dev_utils_manual/action.yml

@@ -20,14 +20,14 @@ runs:
   steps:
 
     - name: Run in ubuntu
-      uses: eProsima/eProsima-CI/ubuntu/fetch_dev_utils_manual@main
+      uses: eProsima/eProsima-CI/ubuntu/fetch_dev_utils_manual@feature/detect_flaky_tests
       if: runner.os == 'Linux'
       with:
         dev_utils_branch: ${{ inputs.dev_utils_branch }}
         destination_workspace: ${{ inputs.destination_workspace }}
 
     - name: Run in windows
-      uses: eProsima/eProsima-CI/windows/fetch_dev_utils_manual@main
+      uses: eProsima/eProsima-CI/windows/fetch_dev_utils_manual@feature/detect_flaky_tests
       if: runner.os == 'Windows'
       with:
         dev_utils_branch: ${{ inputs.dev_utils_branch }}

multiplatform/fetch_fastdds_manual/action.yml

@@ -28,7 +28,7 @@ runs:
   steps:
 
     - name: Run in ubuntu
-      uses: eProsima/eProsima-CI/ubuntu/fetch_fastdds_manual@main
+      uses: eProsima/eProsima-CI/ubuntu/fetch_fastdds_manual@feature/detect_flaky_tests
       if: runner.os == 'Linux'
       with:
         foonathan_memory_vendor_branch: ${{ inputs.foonathan_memory_vendor_branch }}
@@ -37,7 +37,7 @@ runs:
         destination_workspace: ${{ inputs.destination_workspace }}
 
     - name: Run in windows
-      uses: eProsima/eProsima-CI/windows/fetch_fastdds_manual@main
+      uses: eProsima/eProsima-CI/windows/fetch_fastdds_manual@feature/detect_flaky_tests
       if: runner.os == 'Windows'
       with:
         foonathan_memory_vendor_branch: ${{ inputs.foonathan_memory_vendor_branch }}

multiplatform/flakiness_report/action.yml

@@ -0,0 +1,58 @@
+name: 'flakiness_report'
+description: 'Generate a flakiness report from test results and historical data'
+
+inputs:
+  junit_reports_dir:
+    description: 'Path to dir with new jUnit reports'
+    required: true
+
+  junit_archive_artifact:
+    description: >
+      Name of the artifact containing the historical data. Also used for uploading a JSON report
+      with the flaky tests as an artifact. name ${junit_archive_artifact}_json.
+    required: true
+
+  print_report:
+    description: 'Whether to print the report (Default: True)'
+    required: false
+    default: 'True'
+
+  window_size:
+    description: 'Number of days to consider for the flakiness report'
+    required: false
+    default: '30'
+
+  fail_on_flaky_tests:
+    description: 'Whether to fail the workflow if there are flaky tests'
+    required: false
+    default: 'True'
+
+  github_token:
+    description: 'GitHub token'
+    required: true
+
+runs:
+  using: composite
+  steps:
+
+    - name: Run in ubuntu or macOS
+      if: runner.os == 'Linux' || runner.os == 'macOS'
+      uses: eProsima/eProsima-CI/ubuntu/flakiness_report@feature/detect_flaky_tests
+      with:
+        junit_reports_dir: ${{ inputs.junit_reports_dir }}
+        junit_archive_artifact: ${{ inputs.junit_archive_artifact }}
+        print_report: ${{ inputs.print_report }}
+        window_size: ${{ inputs.window_size }}
+        fail_on_flaky_tests: ${{ inputs.fail_on_flaky_tests }}
+        github_token: ${{ inputs.github_token }}
+
+    - name: Run in windows
+      if: runner.os == 'Windows'
+      uses: eProsima/eProsima-CI/windows/flakiness_report@feature/detect_flaky_tests
+      with:
+        junit_reports_dir: ${{ inputs.junit_reports_dir }}
+        junit_archive_artifact: ${{ inputs.junit_archive_artifact }}
+        print_report: ${{ inputs.print_report }}
+        window_size: ${{ inputs.window_size }}
+        fail_on_flaky_tests: ${{ inputs.fail_on_flaky_tests }}
+        github_token: ${{ inputs.github_token }}

multiplatform/generate_dependency_artifact/action.yml

@@ -36,21 +36,21 @@ runs:
   steps:
 
     - name: Fetch repositories
-      uses: eProsima/eProsima-CI/multiplatform/vcs_import@main
+      uses: eProsima/eProsima-CI/multiplatform/vcs_import@feature/detect_flaky_tests
       with:
         vcs_repos_file: ${{ inputs.vcs_repos_file }}
         destination_workspace: ${{ inputs.workspace }}/src
 
     - name: Build workspace
-      uses: eProsima/eProsima-CI/multiplatform/colcon_build@main
+      uses: eProsima/eProsima-CI/multiplatform/colcon_build@feature/detect_flaky_tests
       with:
         colcon_meta_file: ${{ inputs.colcon_meta_file }}
         workspace: ${{ inputs.workspace }}
         workspace_dependencies: ${{ inputs.workspace_dependencies }}
         cmake_build_type: ${{ inputs.cmake_build_type }}
 
     - name: Upload binaries
-      uses: eProsima/eProsima-CI/external/upload-artifact@main
+      uses: eProsima/eProsima-CI/external/upload-artifact@feature/detect_flaky_tests
       with:
         name: ${{ inputs.artifact_name }}
         path: ${{ inputs.workspace }}/install

multiplatform/get_configurations_from_repo/action.yml

@@ -38,15 +38,15 @@ runs:
   steps:
 
     - name: Get colcon.meta file
-      uses: eProsima/eProsima-CI/multiplatform/get_file_from_repo@main
+      uses: eProsima/eProsima-CI/multiplatform/get_file_from_repo@feature/detect_flaky_tests
       with:
         source_repository_branch: ${{ inputs.source_repository_branch }}
         source_repository: ${{ inputs.source_repository }}
         file_name: ${{ inputs.colcon_meta_file_path }}
         file_result: ${{ inputs.colcon_meta_file_result }}
 
     - name: Get dependencies.repos file
-      uses: eProsima/eProsima-CI/multiplatform/get_file_from_repo@main
+      uses: eProsima/eProsima-CI/multiplatform/get_file_from_repo@feature/detect_flaky_tests
       with:
         source_repository_branch: ${{ inputs.source_repository_branch }}
         source_repository: ${{ inputs.source_repository }}

multiplatform/get_file_from_repo/action.yml

@@ -27,7 +27,7 @@ runs:
   steps:
 
     - name: Run in ubuntu
-      uses: eProsima/eProsima-CI/ubuntu/get_file_from_repo@main
+      uses: eProsima/eProsima-CI/ubuntu/get_file_from_repo@feature/detect_flaky_tests
       if: runner.os == 'Linux'
       with:
         source_repository_branch: ${{ inputs.source_repository_branch }}
@@ -36,7 +36,7 @@ runs:
         file_result: ${{ inputs.file_result }}
 
     - name: Run in windows
-      uses: eProsima/eProsima-CI/windows/get_file_from_repo@main
+      uses: eProsima/eProsima-CI/windows/get_file_from_repo@feature/detect_flaky_tests
       if: runner.os == 'Windows'
       with:
         source_repository_branch: ${{ inputs.source_repository_branch }}

multiplatform/get_workflow_id/action.yml

@@ -39,7 +39,7 @@ runs:
   steps:
 
     - name: Install in ubuntu
-      uses: eProsima/eProsima-CI/ubuntu/get_workflow_id@main
+      uses: eProsima/eProsima-CI/ubuntu/get_workflow_id@feature/detect_flaky_tests
       if: runner.os == 'Linux'
       with:
         workflow_source: ${{ inputs.workflow_source }}
@@ -50,7 +50,7 @@ runs:
         secret_token: ${{ inputs.secret_token }}
 
     - name: Install in windows
-      uses: eProsima/eProsima-CI/windows/get_workflow_id@main
+      uses: eProsima/eProsima-CI/windows/get_workflow_id@feature/detect_flaky_tests
       if: runner.os == 'Windows'
       with:
         workflow_source: ${{ inputs.workflow_source }}

multiplatform/install_colcon/action.yml

@@ -6,13 +6,13 @@ runs:
   steps:
 
     - name: Run in ubuntu
-      uses: eProsima/eProsima-CI/ubuntu/install_colcon@main
+      uses: eProsima/eProsima-CI/ubuntu/install_colcon@feature/detect_flaky_tests
       if: runner.os == 'Linux'
 
     - name: Run in macOS
-      uses: eProsima/eProsima-CI/macos/install_colcon@main
+      uses: eProsima/eProsima-CI/macos/install_colcon@feature/detect_flaky_tests
       if: runner.os == 'macOS'
 
     - name: Run in windows
-      uses: eProsima/eProsima-CI/windows/install_colcon@main
+      uses: eProsima/eProsima-CI/windows/install_colcon@feature/detect_flaky_tests
       if: runner.os == 'Windows'

multiplatform/install_fastdds_dependencies/action.yml

@@ -12,11 +12,11 @@ runs:
   steps:
 
     - name: Install in ubuntu
-      uses: eProsima/eProsima-CI/ubuntu/install_fastdds_dependencies@main
+      uses: eProsima/eProsima-CI/ubuntu/install_fastdds_dependencies@feature/detect_flaky_tests
       if: runner.os == 'Linux'
 
     - name: Install in windows
-      uses: eProsima/eProsima-CI/windows/install_fastdds_dependencies@main
+      uses: eProsima/eProsima-CI/windows/install_fastdds_dependencies@feature/detect_flaky_tests
       with:
         cmake_build_type: ${{ inputs.cmake_build_type }}
       if: runner.os == 'Windows'