Separate mutation tests (#2331)

Run mutation tests in a separate job to speed up CI.
App-vNext · Oct 10, 2024 · be68ba3 · be68ba3
1 parent a50d472
commit be68ba3
Show file tree

Hide file tree

Showing 4 changed files with 124 additions and 64 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -24,11 +24,7 @@ jobs:
   build:
     name: ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 60
-
-    env:
-      # HACK Running on Windows instead of Linux due to https://github.com/stryker-mutator/stryker-net/issues/2741
-      RUN_MUTATION_TESTS: ${{ matrix.os_name == 'windows' && !startsWith(github.ref, 'refs/tags/') && 'true' || 'false' }}
+    timeout-minutes: 20
 
     outputs:
       dotnet-sdk-version: ${{ steps.setup-dotnet.outputs.dotnet-version }}
@@ -94,13 +90,6 @@ jobs:
         flags: ${{ matrix.os_name }}
         token: ${{ secrets.CODECOV_TOKEN }}
 
-    - name: Upload Mutation Report
-      if: always() && env.RUN_MUTATION_TESTS == 'true'
-      uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
-      with:
-        name: mutation-report
-        path: ./artifacts/mutation-report
-
     - name: Publish NuGet packages
       uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
       with:

diff --git a/.github/workflows/mutation-tests.yml b/.github/workflows/mutation-tests.yml
@@ -0,0 +1,71 @@
+name: mutation-tests
+
+on:
+  push:
+    branches: [ main, release/* ]
+  pull_request:
+    branches: [ main, release/* ]
+  workflow_dispatch:
+
+env:
+  DOTNET_CLI_TELEMETRY_OPTOUT: true
+  DOTNET_GENERATE_ASPNET_CERTIFICATE: false
+  DOTNET_NOLOGO: true
+  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
+  DOTNET_SYSTEM_CONSOLE_ALLOW_ANSI_COLOR_REDIRECTION: 1
+  NUGET_XMLDOC_MODE: skip
+  TERM: xterm
+
+permissions:
+  contents: read
+
+jobs:
+  mutations:
+    name: 'mutations-${{ matrix.name }}'
+    runs-on: windows-latest
+    timeout-minutes: 60
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: core
+            target: Core
+          - name: legacy
+            target: Legacy
+
+    steps:
+
+    - name: Checkout code
+      uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      with:
+        fetch-depth: 0
+
+    - name: Setup .NET SDKs
+      uses: actions/setup-dotnet@6bd8b7f7774af54e05809fcc5431931b3eb1ddee # v4.0.1
+      with:
+        dotnet-version: |
+          6.0.x
+
+    - name: Setup .NET SDK
+      uses: actions/setup-dotnet@6bd8b7f7774af54e05809fcc5431931b3eb1ddee # v4.0.1
+
+    - name: Setup NuGet cache
+      uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+      with:
+        path: ~/.nuget/packages
+        key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj', '**/*.props') }}
+        restore-keys: ${{ runner.os }}-nuget-
+
+    - name: Run mutation tests
+      shell: pwsh
+      env:
+        MUTATION_TARGET: 'Mutation${{ matrix.target }}'
+      run: ./build.ps1 -Target ${env:MUTATION_TARGET}
+
+    - name: Upload Mutation Report
+      if: always()
+      uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+      with:
+        name: mutation-report-${{ matrix.name }}
+        path: ./artifacts/mutation-report
diff --git a/build.cake b/build.cake
@@ -159,56 +159,19 @@ Task("__RunTests")
     }
 });
 
-Task("__RunMutationTests")
+Task("__RunCoreMutationTests")
     .Does((context) =>
 {
-    var runMutationTests = EnvironmentVariable("RUN_MUTATION_TESTS") switch
-    {
-        "false" => false,
-        _ => true
-    };
-
-    if (!runMutationTests)
-    {
-        return;
-    }
-
-    var oldDirectory = context.Environment.WorkingDirectory;
-    context.Environment.WorkingDirectory = MakeAbsolute(Directory("test"));
-
-    TestProject(File("../src/Polly.Core/Polly.Core.csproj"), File("./Polly.Core.Tests/Polly.Core.Tests.csproj"), "Polly.Core.csproj");
-    TestProject(File("../src/Polly.RateLimiting/Polly.RateLimiting.csproj"), File("./Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj"), "Polly.RateLimiting.csproj");
-    TestProject(File("../src/Polly.Extensions/Polly.Extensions.csproj"), File("./Polly.Extensions.Tests/Polly.Extensions.Tests.csproj"), "Polly.Extensions.csproj");
-    TestProject(File("../src/Polly.Testing/Polly.Testing.csproj"), File("./Polly.Testing.Tests/Polly.Testing.Tests.csproj"), "Polly.Testing.csproj");
-    TestProject(File("../src/Polly/Polly.csproj"), File("./Polly.Specs/Polly.Specs.csproj"), "Polly.csproj");
-
-    context.Environment.WorkingDirectory = oldDirectory;
-
-    void TestProject(FilePath proj, FilePath testProj, string project)
-    {
-        var dotNetBuildSettings = new DotNetBuildSettings
-        {
-            Configuration = "Debug",
-            Verbosity = DotNetVerbosity.Minimal,
-            NoRestore = true
-        };
-
-        DotNetBuild(proj.ToString(), dotNetBuildSettings);
-
-        var strykerPath = Context.Tools.Resolve("Stryker.CLI.dll");
-        var mutationScore = XmlPeek(proj, "/Project/PropertyGroup/MutationScore/text()", new XmlPeekSettings { SuppressWarning = true });
-        var score = int.Parse(mutationScore);
-
-        Information($"Running mutation tests for '{proj}'. Test Project: '{testProj}'");
-
-        var args = $"{strykerPath} --project {project} --test-project {testProj.FullPath} --break-at {score} --config-file {strykerConfig} --output {strykerOutput}/{project}";
+    MutationTestProject(File("./src/Polly.Core/Polly.Core.csproj"), File("./test/Polly.Core.Tests/Polly.Core.Tests.csproj"), "Polly.Core.csproj");
+    MutationTestProject(File("./src/Polly.RateLimiting/Polly.RateLimiting.csproj"), File("./test/Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj"), "Polly.RateLimiting.csproj");
+    MutationTestProject(File("./src/Polly.Extensions/Polly.Extensions.csproj"), File("./test/Polly.Extensions.Tests/Polly.Extensions.Tests.csproj"), "Polly.Extensions.csproj");
+    MutationTestProject(File("./src/Polly.Testing/Polly.Testing.csproj"), File("./test/Polly.Testing.Tests/Polly.Testing.Tests.csproj"), "Polly.Testing.csproj");
+});
 
-        var result = StartProcess("dotnet", args);
-        if (result != 0)
-        {
-            throw new InvalidOperationException($"The mutation testing of '{project}' project failed.");
-        }
-    }
+Task("__RunLegacyMutationTests")
+    .Does((context) =>
+{
+    MutationTestProject(File("./src/Polly/Polly.csproj"), File("./test/Polly.Specs/Polly.Specs.csproj"), "Polly.csproj");
 });
 
 Task("__CreateNuGetPackages")
@@ -252,18 +215,20 @@ Task("__ValidateDocs")
     }
 });
 
+Task("__CommonBuild")
+    .IsDependentOn("__Clean")
+    .IsDependentOn("__RestoreNuGetPackages")
+    .IsDependentOn("__ValidateDocs")
+    .IsDependentOn("__BuildSolutions");
+
 //////////////////////////////////////////////////////////////////////
 // BUILD TASKS
 //////////////////////////////////////////////////////////////////////
 
 Task("Build")
-    .IsDependentOn("__Clean")
-    .IsDependentOn("__RestoreNuGetPackages")
-    .IsDependentOn("__ValidateDocs")
-    .IsDependentOn("__BuildSolutions")
+    .IsDependentOn("__CommonBuild")
     .IsDependentOn("__ValidateAot")
     .IsDependentOn("__RunTests")
-    .IsDependentOn("__RunMutationTests")
     .IsDependentOn("__CreateNuGetPackages");
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -273,6 +238,14 @@ Task("Build")
 Task("Default")
     .IsDependentOn("Build");
 
+Task("MutationCore")
+    .IsDependentOn("__CommonBuild")
+    .IsDependentOn("__RunCoreMutationTests");
+
+Task("MutationLegacy")
+    .IsDependentOn("__CommonBuild")
+    .IsDependentOn("__RunLegacyMutationTests");
+
 ///////////////////////////////////////////////////////////////////////////////
 // EXECUTION
 ///////////////////////////////////////////////////////////////////////////////
@@ -287,3 +260,29 @@ string ToolsExePath(string exeFileName) {
     var exePath = System.IO.Directory.GetFiles("./tools", exeFileName, SearchOption.AllDirectories).FirstOrDefault();
     return exePath;
 }
+
+void MutationTestProject(FilePath proj, FilePath testProj, string project)
+{
+    var dotNetBuildSettings = new DotNetBuildSettings
+    {
+        Configuration = "Debug",
+        Verbosity = DotNetVerbosity.Minimal,
+        NoRestore = true
+    };
+
+    DotNetBuild(proj.ToString(), dotNetBuildSettings);
+
+    var strykerPath = Context.Tools.Resolve("Stryker.CLI.dll");
+    var mutationScore = XmlPeek(proj, "/Project/PropertyGroup/MutationScore/text()", new XmlPeekSettings { SuppressWarning = true });
+    var score = int.Parse(mutationScore);
+
+    Information($"Running mutation tests for '{proj}'. Test Project: '{testProj}'");
+
+    var args = $"{strykerPath} --project {project} --test-project {testProj.FullPath} --break-at {score} --config-file {strykerConfig} --output {strykerOutput}/{project}";
+
+    var result = StartProcess("dotnet", args);
+    if (result != 0)
+    {
+        throw new InvalidOperationException($"The mutation testing of '{project}' project failed.");
+    }
+}
diff --git a/build.ps1 b/build.ps1
@@ -27,6 +27,7 @@ https://cakebuild.net
 Param(
     [string]$Script = "build.cake",
     [string]$Target = "Default",
+    [ValidateSet("Default", "MutationCore", "MutationLegacy")]
     [string]$Configuration = "Release",
     [ValidateSet("Quiet", "Minimal", "Normal", "Verbose", "Diagnostic")]
     [string]$Verbosity = "Verbose",