add benchmarks for coverlet.core

.gitignore

@@ -315,4 +315,5 @@ coverage.*.cobertura.xml
 coverage.*.opencover.xml
 
 FolderProfile.pubxml
+BenchmarkDotNet.Artifacts/
 /NuGet.config

Directory.Build.props

@@ -15,7 +15,7 @@
     <CodeAnalysisTreatWarningsAsErrors>false</CodeAnalysisTreatWarningsAsErrors>
     <EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
     <LangVersion>preview</LangVersion>
-    <NoWarn>$(NoWarn);NU1507;NU5105;CS1591</NoWarn>
+    <NoWarn>$(NoWarn);NU1507;NU5105;CS1591;IDE0007</NoWarn>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <RestoreSources>
       https://api.nuget.org/v3/index.json;

Directory.Packages.props

@@ -8,11 +8,15 @@
     <GlobalPackageReference Include="Nerdbank.GitVersioning" Version="3.6.146" />
   </ItemGroup>
   <ItemGroup>
+    <PackageVersion Include="BenchmarkDotNet" Version="0.14.0" />
+    <PackageVersion Include="BenchmarkDotNet.TestAdapter" Version="0.14.0" />
+    <PackageVersion Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.14.0" />
+    <PackageVersion Include="DotNetConfig" Version="1.2.0" />
     <PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Build.Utilities.Core" Version="17.8.3" />
     <PackageVersion Include="Microsoft.Build.Framework" Version="17.8.3" />
     <PackageVersion Include="Microsoft.Build.Locator" Version="1.7.8" />
-    <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.8.0" />
+    <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.9.2" />
     <PackageVersion Include="Microsoft.Extensions.DependencyModel" Version="8.0.2" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
     <PackageVersion Include="Microsoft.Extensions.FileSystemGlobbing" Version="8.0.0" />
@@ -35,7 +39,7 @@
     <PackageVersion Include="NuGet.Versioning" Version="6.8.0" />
     <PackageVersion Include="Mono.Cecil" Version="0.11.6" />
     <PackageVersion Include="Moq" Version="4.20.70" />
-    <PackageVersion Include="ReportGenerator.Core" Version="5.2.1" />
+    <PackageVersion Include="ReportGenerator.Core" Version="5.3.8" />
     <!--For test issue 809 https://github.com/coverlet-coverage/coverlet/issues/809-->
     <PackageVersion Include="LinqKit.Microsoft.EntityFrameworkCore" Version="7.1.4" />
     <PackageVersion Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
@@ -45,10 +49,10 @@
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
     <PackageVersion Include="System.Reflection.Metadata" Version="8.0.0" />
     <PackageVersion Include="Tmds.ExecFunction" Version="0.8.0" />
-    <PackageVersion Include="xunit" Version="2.6.6" />
+    <PackageVersion Include="xunit" Version="2.9.0" />
     <PackageVersion Include="xunit.assemblyfixture" Version="2.2.0" />
-    <PackageVersion Include="xunit.assert" Version="2.6.6" />
-    <PackageVersion Include="xunit.runner.visualstudio" Version="2.5.6" />
+    <PackageVersion Include="xunit.assert" Version="2.9.0" />
+    <PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2" />
     <PackageVersion Include="System.Buffers" Version="4.5.1" />
     <PackageVersion Include="System.Memory" Version="4.5.5" />
     <PackageVersion Include="System.Net.Http" Version="4.3.4" />
@@ -57,4 +61,4 @@
     <PackageVersion Include="System.Text.Json" Version="8.0.5" />
     <PackageVersion Include="System.Text.RegularExpressions" Version="4.3.1" />
   </ItemGroup>
-</Project>
+</Project>

coverlet.sln

@@ -88,6 +88,8 @@ Project("{778DAE3C-4631-46EA-AA77-85C1314464D9}") = "coverlet.tests.projectsampl
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "coverlet.tests.utils", "test\coverlet.tests.utils\coverlet.tests.utils.csproj", "{0B109210-03CB-413F-888C-3023994AA384}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "coverlet.core.benchmark.tests", "test\coverlet.core.benchmark.tests\coverlet.core.benchmark.tests.csproj", "{8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "coverlet.tests.projectsample.wpf8.selfcontained", "test\coverlet.tests.projectsample.wpf8.selfcontained\coverlet.tests.projectsample.wpf8.selfcontained.csproj", "{71004336-9896-4AE5-8367-B29BB1680542}"
 EndProject
 Global
@@ -192,6 +194,10 @@ Global
 		{0B109210-03CB-413F-888C-3023994AA384}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0B109210-03CB-413F-888C-3023994AA384}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0B109210-03CB-413F-888C-3023994AA384}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD}.Release|Any CPU.Build.0 = Release|Any CPU
 		{71004336-9896-4AE5-8367-B29BB1680542}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{71004336-9896-4AE5-8367-B29BB1680542}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{71004336-9896-4AE5-8367-B29BB1680542}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -226,6 +232,7 @@ Global
 		{351A034E-E642-4DB9-A21D-F71C8151C243} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134}
 		{03400776-1F9A-4326-B927-1CA9B64B42A1} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134}
 		{0B109210-03CB-413F-888C-3023994AA384} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134}
+		{8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134}
 		{71004336-9896-4AE5-8367-B29BB1680542} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution

src/coverlet.core/Properties/AssemblyInfo.cs

@@ -11,6 +11,7 @@
 [assembly: InternalsVisibleTo("coverlet.collector, PublicKey=00240000048000009400000006020000002400005253413100040000010001003d23b9ef372215da7c81af920b919db5799fd021a1ca10b2e9e0ddac71237a29f8f6361a805a747457e561a7d616417f1870cda099486df25d580a4e11a0738293342881566254d7840e42f42fb9bfd8e8dca354df7dc68db14b2d0cd79bb2bf7afdbd62bd948d81b534cba7a326cf6ee840a1aee5dba0a1c660b30813ca99e5")]
 
 [assembly: InternalsVisibleTo("coverlet.core.tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100757cf9291d78a82e5bb58a827a3c46c2f959318327ad30d1b52e918321ffbd847fb21565b8576d2a3a24562a93e86c77a298b564a0f1b98f63d7a1441a3a8bcc206da3ed09d5dacc76e122a109a9d3ac608e21a054d667a2bae98510a1f0f653c0e6f58f42b4b3934f6012f5ec4a09b3dfd3e14d437ede1424bdb722aead64ad")]
+[assembly: InternalsVisibleTo("coverlet.core.benchmark.tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010061d9d48f9cd6a4733ea1d88bc8a09c53a3040c3446c41858781df135170e8fe4e82a6cc6d9836f070ae0a28ebd7cd6e30dc1a853b350ae08ae77f437bc9f9f3b0ef23eb9b05eea38f97edb26a2dd2d0d8b32c6335c47b32f5277621118267f1a5717233eae25a3fe126d89d14b85a7a8e07657bf681a8a82100762a42ec477aa")]
 [assembly: InternalsVisibleTo("coverlet.collector.tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100ed0ed6af9693182615b8dcadc83c918b8d36312f86cefc69539d67d4189cd1b89420e7c3871802ffef7f5ca7816c68ad856c77bf7c230cc07824d96aa5d1237eebd30e246b9a14e22695fb26b40c800f74ea96619092cbd3a5d430d6c003fc7a82e8ccd1e315b935105d9232fe9e99e8d7ff54bba6f191959338d4a3169df9b3")]
 [assembly: InternalsVisibleTo("coverlet.msbuild.tasks.tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010071b1583d63637a225f3f640252fee7130f0f3f2127d75025c1c3ee2d6dfc79a4950919268e0784d7ff54b0eadd8e4762e3e150da422e20e091eb0811d9d84e1779d5b95e349d5428aebb16e82e081bdf805926c5a9eb2094aaed9d36442de024264976a8835c7d6923047cf2f745e8f0ded2332f8980acd390f725224d976ed8")]
 [assembly: InternalsVisibleTo("coverlet.integration.tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010001d24efbe9cbc2dc49b7a3d2ae34ca37cfb69b4f450acd768a22ce5cd021c8a38ae7dc68b2809a1ac606ad531b578f192a5690b2986990cbda4dd84ec65a3a4c1c36f6d7bb18f08592b93091535eaee2f0c8e48763ed7f190db2008e1f9e0facd5c0df5aaab74febd3430e09a428a72e5e6b88357f92d78e47512d46ebdc3cbb")]

test/coverlet.core.benchmark.tests/CoverageBenchmarks.cs

@@ -0,0 +1,68 @@
+// Copyright (c) Toni Solarin-Sodara
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.IO;
+using BenchmarkDotNet.Attributes;
+using Coverlet.Core;
+using Coverlet.Core.Abstractions;
+using Coverlet.Core.Helpers;
+using Coverlet.Core.Symbols;
+using Moq;
+
+namespace coverlet.core.benchmark.tests
+{
+  [MemoryDiagnoser]
+  public class CoverageBenchmarks
+  {
+    private Coverage _coverage;
+    private readonly Mock<ILogger> _mockLogger = new();
+    private DirectoryInfo _directory;
+
+    [GlobalSetup(Target = nameof(GetCoverageBenchmark))]
+    public void GetCoverageBenchmarkSetup()
+    {
+      string module = GetType().Assembly.Location;
+      string pdb = Path.Combine(Path.GetDirectoryName(module), Path.GetFileNameWithoutExtension(module) + ".pdb");
+
+      _directory = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()));
+
+      File.Copy(module, Path.Combine(_directory.FullName, Path.GetFileName(module)), true);
+      File.Copy(pdb, Path.Combine(_directory.FullName, Path.GetFileName(pdb)), true);
+
+      // TODO: Find a way to mimick hits
+      var instrumentationHelper =
+          new InstrumentationHelper(new ProcessExitHandler(), new RetryHelper(), new FileSystem(), new Mock<ILogger>().Object,
+                                    new SourceRootTranslator(module, new Mock<ILogger>().Object, new FileSystem(), new AssemblyAdapter()));
+
+      var parameters = new CoverageParameters
+      {
+        IncludeFilters = new string[] { "[coverlet.tests.projectsample.excludedbyattribute*]*" },
+        IncludeDirectories = Array.Empty<string>(),
+        ExcludeFilters = Array.Empty<string>(),
+        ExcludedSourceFiles = Array.Empty<string>(),
+        ExcludeAttributes = Array.Empty<string>(),
+        IncludeTestAssembly = false,
+        SingleHit = false,
+        MergeWith = string.Empty,
+        UseSourceLink = false
+      };
+
+      _coverage = new Coverage(Path.Combine(_directory.FullName, Path.GetFileName(module)), parameters, _mockLogger.Object, instrumentationHelper, new FileSystem(), new SourceRootTranslator(_mockLogger.Object, new FileSystem()), new CecilSymbolHelper());
+      _coverage.PrepareModules();
+
+    }
+
+    [GlobalCleanup]
+    public void IterationCleanup()
+    {
+      _directory.Delete(true);
+    }
+
+    [Benchmark]
+    public void GetCoverageBenchmark()
+    {
+      CoverageResult result = _coverage.GetCoverageResult();
+    }
+  }
+}

test/coverlet.core.benchmark.tests/InstrumenterBenchmarks.cs

@@ -0,0 +1,97 @@
+// Copyright (c) Toni Solarin-Sodara
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Coverlet.Core.Abstractions;
+using Coverlet.Core.Helpers;
+using Coverlet.Core.Symbols;
+
+using BenchmarkDotNet.Attributes;
+using Coverlet.Core.Instrumentation;
+using Coverlet.Core;
+using System.IO;
+using Moq;
+
+namespace coverlet.core.benchmark.tests
+{
+  [MemoryDiagnoser]
+  public class InstrumenterBenchmarks
+    {
+    Mock<ILogger> _mockLogger;
+    Mock<FileSystem> _partialMockFileSystem;
+    readonly string[] _files = new[]
+    {
+                "System.Private.CoreLib.dll",
+                "System.Private.CoreLib.pdb"
+            };
+    Instrumenter _instrumenter;
+    DirectoryInfo _directory;
+    SourceRootTranslator _sourceRootTranslator;
+    CoverageParameters _parameters;
+    InstrumentationHelper _instrumentationHelper;
+
+    //[GlobalSetup(Target = nameof(InstrumenterBenchmarks))]
+    //public void InstrumenterBenchmarkSetup()
+    //{
+
+    //}
+
+    [GlobalCleanup]
+    public void IterationCleanup()
+    {
+      _directory.Delete(true);
+    }
+
+    [Benchmark]
+    public void InstrumenterBenchmark()
+    {
+      _mockLogger = new Mock<ILogger>();
+      _directory = Directory.CreateDirectory(Path.Combine(Directory.GetCurrentDirectory(), nameof(InstrumenterBenchmark)));
+
+      foreach (string file in _files)
+      {
+        File.Copy(Path.Combine(Directory.GetCurrentDirectory(), "TestAssets", file), Path.Combine(_directory.FullName, file), overwrite: true);
+      }
+
+      _partialMockFileSystem = new Mock<FileSystem>();
+      _partialMockFileSystem.CallBase = true;
+      _partialMockFileSystem.Setup(fs => fs.OpenRead(It.IsAny<string>())).Returns((string path) =>
+      {
+        if (Path.GetFileName(path.Replace(@"\", @"/")) == _files[1])
+        {
+          return File.OpenRead(Path.Combine(Path.Combine(Directory.GetCurrentDirectory(), "TestAssets"), _files[1]));
+        }
+        else
+        {
+          return File.OpenRead(path);
+        }
+      });
+      _partialMockFileSystem.Setup(fs => fs.Exists(It.IsAny<string>())).Returns((string path) =>
+      {
+        if (Path.GetFileName(path.Replace(@"\", @"/")) == _files[1])
+        {
+          return File.Exists(Path.Combine(Path.Combine(Directory.GetCurrentDirectory(), "TestAssets"), _files[1]));
+        }
+        else
+        {
+          if (path.Contains(@":\git\runtime"))
+          {
+            return true;
+          }
+          else
+          {
+            return File.Exists(path);
+          }
+        }
+      });
+      _sourceRootTranslator = new SourceRootTranslator(_mockLogger.Object, new FileSystem());
+      _parameters = new CoverageParameters();
+      _instrumentationHelper =
+          new InstrumentationHelper(new ProcessExitHandler(), new RetryHelper(), _partialMockFileSystem.Object, _mockLogger.Object, _sourceRootTranslator);
+      _instrumenter = new Instrumenter(Path.Combine(_directory.FullName, _files[0]), "_coverlet_instrumented", _parameters, _mockLogger.Object, _instrumentationHelper, _partialMockFileSystem.Object, _sourceRootTranslator, new CecilSymbolHelper());
+
+      // implement your benchmark here
+      InstrumenterResult result = _instrumenter.Instrument();
+    }
+
+  }
+}

test/coverlet.core.benchmark.tests/Program.cs

@@ -0,0 +1,32 @@
+// Copyright (c) Toni Solarin-Sodara
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Toolchains.InProcess.NoEmit;
+
+namespace coverlet.core.benchmark.tests
+{
+  public class Program
+  {
+
+    public static void Main(string[] args)
+    {
+
+      var config = DefaultConfig.Instance
+             .WithOptions(ConfigOptions.JoinSummary)
+             .AddJob(Job
+               .ShortRun
+               .WithLaunchCount(1)
+               .WithToolchain(InProcessNoEmitToolchain.Instance));
+      var summary = BenchmarkRunner.Run(new[]{
+            BenchmarkConverter.TypeToBenchmarks( typeof(CoverageBenchmarks), config),
+            BenchmarkConverter.TypeToBenchmarks( typeof(InstrumenterBenchmarks), config)
+            });
+
+      // Use this to select benchmarks from the console:
+      //var summaries = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, config);
+    }
+  }
+}

test/coverlet.core.benchmark.tests/Properties/AssemblyInfo.cs

@@ -0,0 +1,6 @@
+// Copyright (c) Toni Solarin-Sodara
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Reflection;
+
+[assembly: AssemblyKeyFile("coverlet.core.benchmark.tests.snk")]

test/coverlet.core.benchmark.tests/coverlet.core.benchmark.tests.csproj

@@ -0,0 +1,38 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+  <PropertyGroup>
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugType>pdbonly</DebugType>
+    <DebugSymbols>true</DebugSymbols>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <Optimize>true</Optimize>
+    <Configuration>Release</Configuration>
+    <IsPackable>false</IsPackable>
+    <NoWarn>$(NoWarn);CS0162</NoWarn>
+    <!-- Disable entry point generation as this project has it's own entry point -->
+    <GenerateProgramFile>false</GenerateProgramFile>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet.TestAdapter" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="Moq" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="$(RepoRoot)src\coverlet.core\coverlet.core.csproj" />
+    <ProjectReference Include="$(RepoRoot)test\coverlet.tests.utils\coverlet.tests.utils.csproj" />
+    <ProjectReference Include="$(RepoRoot)test\coverlet.tests.xunit.extensions\coverlet.tests.xunit.extensions.csproj" />
+
+    <ProjectReference Include="$(RepoRoot)test\coverlet.tests.projectsample.excludedbyattribute\coverlet.tests.projectsample.excludedbyattribute.csproj" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Update="TestAssets\System.Private.CoreLib.dll">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="TestAssets\System.Private.CoreLib.pdb">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+</Project>

test/coverlet.core.tests/Reporters/OpenCoverReporterTests.cs

@@ -3,7 +3,6 @@
 
 using System;
 using System.IO;
-using System.Linq;
 using System.Text;
 using System.Xml.Linq;
 using Coverlet.Core.Abstractions;
@@ -28,9 +27,9 @@ public void TestReport()
       string report = reporter.Report(result, new Mock<ISourceRootTranslator>().Object);
       Assert.NotEmpty(report);
       var doc = XDocument.Load(new StringReader(report));
-      Assert.Empty(doc.Descendants().Attributes("sequenceCoverage").Where(v => v.Value != "33.33"));
-      Assert.Empty(doc.Descendants().Attributes("branchCoverage").Where(v => v.Value != "25"));
-      Assert.Empty(doc.Descendants().Attributes("nPathComplexity").Where(v => v.Value != "4"));
+      Assert.DoesNotContain(doc.Descendants().Attributes("sequenceCoverage"), v => v.Value != "33.33");
+      Assert.DoesNotContain(doc.Descendants().Attributes("branchCoverage"), v => v.Value != "25");
+      Assert.DoesNotContain(doc.Descendants().Attributes("nPathComplexity"), v => v.Value != "4");
     }
 
     [Fact]

test/coverlet.integration.determisticbuild/coverlet.integration.determisticbuild.csproj

@@ -24,8 +24,8 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="xunit" Version="2.6.6" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.6" />
+    <PackageReference Include="xunit" Version="2.9.0" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
     <PackageReference Include="System.Buffers" Version="4.5.1" />
     <PackageReference Include="System.Memory" Version="4.5.5" />
     <PackageReference Include="System.Security.Cryptography.Pkcs" Version="6.0.4" />

test/coverlet.integration.template/coverlet.integration.template.csproj

@@ -11,8 +11,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="xunit" Version="2.6.6"/>
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.6">
+    <PackageReference Include="xunit" Version="2.9.0" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>