From be68ba3306949ddddbe68e0da25f983a6e112428 Mon Sep 17 00:00:00 2001 From: Shawn Wu Date: Fri, 11 Oct 2024 04:18:49 +1100 Subject: [PATCH] Separate mutation tests (#2331) Run mutation tests in a separate job to speed up CI. --- .github/workflows/build.yml | 13 +--- .github/workflows/mutation-tests.yml | 71 ++++++++++++++++++ build.cake | 103 +++++++++++++-------------- build.ps1 | 1 + 4 files changed, 124 insertions(+), 64 deletions(-) create mode 100644 .github/workflows/mutation-tests.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cfcc15914f1..982d3d4b18a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,11 +24,7 @@ jobs: build: name: ${{ matrix.os }} runs-on: ${{ matrix.os }} - timeout-minutes: 60 - - env: - # HACK Running on Windows instead of Linux due to https://github.com/stryker-mutator/stryker-net/issues/2741 - RUN_MUTATION_TESTS: ${{ matrix.os_name == 'windows' && !startsWith(github.ref, 'refs/tags/') && 'true' || 'false' }} + timeout-minutes: 20 outputs: dotnet-sdk-version: ${{ steps.setup-dotnet.outputs.dotnet-version }} @@ -94,13 +90,6 @@ jobs: flags: ${{ matrix.os_name }} token: ${{ secrets.CODECOV_TOKEN }} - - name: Upload Mutation Report - if: always() && env.RUN_MUTATION_TESTS == 'true' - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: mutation-report - path: ./artifacts/mutation-report - - name: Publish NuGet packages uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: diff --git a/.github/workflows/mutation-tests.yml b/.github/workflows/mutation-tests.yml new file mode 100644 index 00000000000..ee925eae17b --- /dev/null +++ b/.github/workflows/mutation-tests.yml @@ -0,0 +1,71 @@ +name: mutation-tests + +on: + push: + branches: [ main, release/* ] + pull_request: + branches: [ main, release/* ] + workflow_dispatch: + +env: + DOTNET_CLI_TELEMETRY_OPTOUT: true + DOTNET_GENERATE_ASPNET_CERTIFICATE: false + DOTNET_NOLOGO: true + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + DOTNET_SYSTEM_CONSOLE_ALLOW_ANSI_COLOR_REDIRECTION: 1 + NUGET_XMLDOC_MODE: skip + TERM: xterm + +permissions: + contents: read + +jobs: + mutations: + name: 'mutations-${{ matrix.name }}' + runs-on: windows-latest + timeout-minutes: 60 + + strategy: + fail-fast: false + matrix: + include: + - name: core + target: Core + - name: legacy + target: Legacy + + steps: + + - name: Checkout code + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + with: + fetch-depth: 0 + + - name: Setup .NET SDKs + uses: actions/setup-dotnet@6bd8b7f7774af54e05809fcc5431931b3eb1ddee # v4.0.1 + with: + dotnet-version: | + 6.0.x + + - name: Setup .NET SDK + uses: actions/setup-dotnet@6bd8b7f7774af54e05809fcc5431931b3eb1ddee # v4.0.1 + + - name: Setup NuGet cache + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ~/.nuget/packages + key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj', '**/*.props') }} + restore-keys: ${{ runner.os }}-nuget- + + - name: Run mutation tests + shell: pwsh + env: + MUTATION_TARGET: 'Mutation${{ matrix.target }}' + run: ./build.ps1 -Target ${env:MUTATION_TARGET} + + - name: Upload Mutation Report + if: always() + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: mutation-report-${{ matrix.name }} + path: ./artifacts/mutation-report diff --git a/build.cake b/build.cake index 0b57c2700dd..13173854b06 100644 --- a/build.cake +++ b/build.cake @@ -159,56 +159,19 @@ Task("__RunTests") } }); -Task("__RunMutationTests") +Task("__RunCoreMutationTests") .Does((context) => { - var runMutationTests = EnvironmentVariable("RUN_MUTATION_TESTS") switch - { - "false" => false, - _ => true - }; - - if (!runMutationTests) - { - return; - } - - var oldDirectory = context.Environment.WorkingDirectory; - context.Environment.WorkingDirectory = MakeAbsolute(Directory("test")); - - TestProject(File("../src/Polly.Core/Polly.Core.csproj"), File("./Polly.Core.Tests/Polly.Core.Tests.csproj"), "Polly.Core.csproj"); - TestProject(File("../src/Polly.RateLimiting/Polly.RateLimiting.csproj"), File("./Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj"), "Polly.RateLimiting.csproj"); - TestProject(File("../src/Polly.Extensions/Polly.Extensions.csproj"), File("./Polly.Extensions.Tests/Polly.Extensions.Tests.csproj"), "Polly.Extensions.csproj"); - TestProject(File("../src/Polly.Testing/Polly.Testing.csproj"), File("./Polly.Testing.Tests/Polly.Testing.Tests.csproj"), "Polly.Testing.csproj"); - TestProject(File("../src/Polly/Polly.csproj"), File("./Polly.Specs/Polly.Specs.csproj"), "Polly.csproj"); - - context.Environment.WorkingDirectory = oldDirectory; - - void TestProject(FilePath proj, FilePath testProj, string project) - { - var dotNetBuildSettings = new DotNetBuildSettings - { - Configuration = "Debug", - Verbosity = DotNetVerbosity.Minimal, - NoRestore = true - }; - - DotNetBuild(proj.ToString(), dotNetBuildSettings); - - var strykerPath = Context.Tools.Resolve("Stryker.CLI.dll"); - var mutationScore = XmlPeek(proj, "/Project/PropertyGroup/MutationScore/text()", new XmlPeekSettings { SuppressWarning = true }); - var score = int.Parse(mutationScore); - - Information($"Running mutation tests for '{proj}'. Test Project: '{testProj}'"); - - var args = $"{strykerPath} --project {project} --test-project {testProj.FullPath} --break-at {score} --config-file {strykerConfig} --output {strykerOutput}/{project}"; + MutationTestProject(File("./src/Polly.Core/Polly.Core.csproj"), File("./test/Polly.Core.Tests/Polly.Core.Tests.csproj"), "Polly.Core.csproj"); + MutationTestProject(File("./src/Polly.RateLimiting/Polly.RateLimiting.csproj"), File("./test/Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj"), "Polly.RateLimiting.csproj"); + MutationTestProject(File("./src/Polly.Extensions/Polly.Extensions.csproj"), File("./test/Polly.Extensions.Tests/Polly.Extensions.Tests.csproj"), "Polly.Extensions.csproj"); + MutationTestProject(File("./src/Polly.Testing/Polly.Testing.csproj"), File("./test/Polly.Testing.Tests/Polly.Testing.Tests.csproj"), "Polly.Testing.csproj"); +}); - var result = StartProcess("dotnet", args); - if (result != 0) - { - throw new InvalidOperationException($"The mutation testing of '{project}' project failed."); - } - } +Task("__RunLegacyMutationTests") + .Does((context) => +{ + MutationTestProject(File("./src/Polly/Polly.csproj"), File("./test/Polly.Specs/Polly.Specs.csproj"), "Polly.csproj"); }); Task("__CreateNuGetPackages") @@ -252,18 +215,20 @@ Task("__ValidateDocs") } }); +Task("__CommonBuild") + .IsDependentOn("__Clean") + .IsDependentOn("__RestoreNuGetPackages") + .IsDependentOn("__ValidateDocs") + .IsDependentOn("__BuildSolutions"); + ////////////////////////////////////////////////////////////////////// // BUILD TASKS ////////////////////////////////////////////////////////////////////// Task("Build") - .IsDependentOn("__Clean") - .IsDependentOn("__RestoreNuGetPackages") - .IsDependentOn("__ValidateDocs") - .IsDependentOn("__BuildSolutions") + .IsDependentOn("__CommonBuild") .IsDependentOn("__ValidateAot") .IsDependentOn("__RunTests") - .IsDependentOn("__RunMutationTests") .IsDependentOn("__CreateNuGetPackages"); /////////////////////////////////////////////////////////////////////////////// @@ -273,6 +238,14 @@ Task("Build") Task("Default") .IsDependentOn("Build"); +Task("MutationCore") + .IsDependentOn("__CommonBuild") + .IsDependentOn("__RunCoreMutationTests"); + +Task("MutationLegacy") + .IsDependentOn("__CommonBuild") + .IsDependentOn("__RunLegacyMutationTests"); + /////////////////////////////////////////////////////////////////////////////// // EXECUTION /////////////////////////////////////////////////////////////////////////////// @@ -287,3 +260,29 @@ string ToolsExePath(string exeFileName) { var exePath = System.IO.Directory.GetFiles("./tools", exeFileName, SearchOption.AllDirectories).FirstOrDefault(); return exePath; } + +void MutationTestProject(FilePath proj, FilePath testProj, string project) +{ + var dotNetBuildSettings = new DotNetBuildSettings + { + Configuration = "Debug", + Verbosity = DotNetVerbosity.Minimal, + NoRestore = true + }; + + DotNetBuild(proj.ToString(), dotNetBuildSettings); + + var strykerPath = Context.Tools.Resolve("Stryker.CLI.dll"); + var mutationScore = XmlPeek(proj, "/Project/PropertyGroup/MutationScore/text()", new XmlPeekSettings { SuppressWarning = true }); + var score = int.Parse(mutationScore); + + Information($"Running mutation tests for '{proj}'. Test Project: '{testProj}'"); + + var args = $"{strykerPath} --project {project} --test-project {testProj.FullPath} --break-at {score} --config-file {strykerConfig} --output {strykerOutput}/{project}"; + + var result = StartProcess("dotnet", args); + if (result != 0) + { + throw new InvalidOperationException($"The mutation testing of '{project}' project failed."); + } +} diff --git a/build.ps1 b/build.ps1 index 24fa6319e45..ca1cfcdd00f 100755 --- a/build.ps1 +++ b/build.ps1 @@ -27,6 +27,7 @@ https://cakebuild.net Param( [string]$Script = "build.cake", [string]$Target = "Default", + [ValidateSet("Default", "MutationCore", "MutationLegacy")] [string]$Configuration = "Release", [ValidateSet("Quiet", "Minimal", "Normal", "Verbose", "Diagnostic")] [string]$Verbosity = "Verbose",