Migrate xUnit plugin. (#27)

* Migrate xUnit plugin.

* Remove redundant GitContext class

* Changed target framework for shared library, moved TestCase to shared location.

* Remove old models, simplify test case collection.

* Moved NUnitTestCasePayload to shared library.

* Rewriting TestResultsBuilder

* Fixed typo in enum name.

* Fleshed out rest of reporting.

* Added reference link for NUnit TestCase XML definition.

* Made xUnit package target netstandard2.0 instead of net6

* Moved some tests.

* Renamed to match new type name.

* Added some sample xUnit tests to publish.

* Refactoring to clean up task contention (attempting to run binary inside a task).

* Fixed async issues, testing generated JSON again.

* Fixing how we populate the name

* Removed testing code.

* Code cleanup, remove random tests.

src/Agoda.DevFeedback.Common/DevFeedbackPublisher.cs

@@ -20,7 +20,7 @@ public static void Publish<T>(string apiEndpoint, T data, DevLocalDataType devLo
                 case DevLocalDataType.Build:
                     targetEndpoint = GetApiEndpoint(apiEndpoint);
                     break;
-                case DevLocalDataType.NUint:
+                case DevLocalDataType.NUnit:
                     targetEndpoint = GetNunitApiEndpoint(apiEndpoint);
                     break;
                 default:
@@ -29,10 +29,8 @@ public static void Publish<T>(string apiEndpoint, T data, DevLocalDataType devLo
             using (var httpClient = new HttpClient())
             {
                 httpClient.Timeout = TimeSpan.FromSeconds(2);
-
                 var content = new StringContent(JsonSerializer.Serialize(data), Encoding.UTF8, "application/json");
                 var response = httpClient.PostAsync(targetEndpoint, content).Result;
-
                 response.EnsureSuccessStatusCode();
             }
         }

src/Agoda.DevFeedback.Common/DevLocalDataType.cs

@@ -3,6 +3,6 @@
     public enum DevLocalDataType
     {
         Build,
-        NUint
+        NUnit
     }
 }

src/Agoda.Tests.Metrics.NUnit/Agoda.Tests.Metrics.NUnit.csproj

@@ -10,7 +10,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <Content Include="Agoda.Testinglib.addins" >
+    <Content Include="Agoda.Testinglib.addins">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
 	  <PackageCopyToOutput>true</PackageCopyToOutput>
 	</Content>
@@ -26,5 +26,6 @@
 
 	<ItemGroup>
 	  <ProjectReference Include="..\Agoda.DevFeedback.Common\Agoda.DevFeedback.Common.csproj" />
+	  <ProjectReference Include="..\Agoda.Tests.Metrics\Agoda.Tests.Metrics.csproj" />
 	</ItemGroup>
 </Project>

src/Agoda.Tests.Metrics.NUnit/AgodaNUnitEventListener.cs

@@ -24,12 +24,12 @@ public void OnTestEvent(string report)
 
                 var gitContext = GitContextReader.GetGitContext();
 
-                var data = new NUnitTestCasePayload(
+                var data = new TestCasePayload(
                     typeof(AgodaNUnitEventListener).Assembly.GetName().Version.ToString(),
                     gitContext,
                     xmlConverter.TestCases
                 );
-                DevFeedbackPublisher.Publish(null, data, DevLocalDataType.NUint);
+                DevFeedbackPublisher.Publish(null, data, DevLocalDataType.NUnit);
             }
             catch (Exception e)
             {

src/Agoda.Tests.Metrics.Tests/Agoda.Tests.Metrics.Tests.csproj

@@ -0,0 +1,27 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+
+    <IsPackable>false</IsPackable>
+    <IsTestProject>true</IsTestProject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
+    <PackageReference Include="NSubstitute" Version="4.2.2" />
+    <PackageReference Include="NUnit" Version="3.13.3" />
+    <PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
+    <PackageReference Include="NUnit.Analyzers" Version="3.6.1" />
+    <PackageReference Include="coverlet.collector" Version="3.2.0" />
+    <PackageReference Include="Shouldly" Version="4.2.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Agoda.DevFeedback.Common\Agoda.DevFeedback.Common.csproj" />
+    <ProjectReference Include="..\Agoda.Tests.Metrics\Agoda.Tests.Metrics.csproj" />
+  </ItemGroup>
+
+</Project>

src/Agoda.Tests.Metrics.NUnit.Tests/NUnitTestCasePayloadTests.cs → src/Agoda.Tests.Metrics.Tests/TestCasePayloadTests.cs

@@ -1,17 +1,16 @@
 using Agoda.DevFeedback.Common;
 using NSubstitute;
-using NUnit.Framework;
 using Shouldly;
 
-namespace Agoda.Tests.Metrics.NUnit.Tests;
+namespace Agoda.Tests.Metrics.Tests;
 
 [TestFixture]
-public class NUnitTestCasePayloadTests
+public class TestCasePayloadTests
 {
     [Test]
     public void WhenInit_ShouldHaveDefaultVarsFromEnvironment()
     {
-        var underTest = new NUnitTestCasePayload(null,Substitute.For<GitContext>(), new List<TestCase>());
+        var underTest = new TestCasePayload(null,Substitute.For<GitContext>(), new List<TestCase>());
 
         underTest.Id.ShouldNotBeNull();
         underTest.UserName.ShouldBe(Environment.UserName);

src/Agoda.Tests.Metrics.Tests/Usings.cs

@@ -0,0 +1 @@
+global using NUnit.Framework;

src/Agoda.Tests.Metrics.xUnit.Tests/Agoda.Tests.Metrics.xUnit.Tests.csproj

@@ -0,0 +1,35 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+
+    <IsPackable>false</IsPackable>
+    <IsTestProject>true</IsTestProject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
+    <PackageReference Include="xunit" Version="2.4.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="coverlet.collector" Version="3.2.0">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Agoda.Tests.Metrics.xUnit\Agoda.Tests.Metrics.xUnit.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="xunit.runner.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

src/Agoda.Tests.Metrics.xUnit.Tests/TestFrameworkReference.cs

@@ -0,0 +1 @@
+[assembly: Xunit.TestFramework("Agoda.Tests.Metrics.xUnit.AgodaTestFramework", "Agoda.Tests.Metrics.xUnit")]

src/Agoda.Tests.Metrics.xUnit.Tests/TestSuite1.cs

@@ -0,0 +1,25 @@
+namespace Agoda.Tests.Metrics.xUnit.Tests
+{
+    public class TestSuite1
+    {
+        [Fact]
+        public void Test1()
+        {
+        }
+
+        [Fact]
+        public void Test2()
+        {
+        }
+
+        [Fact]
+        public void Test3()
+        {
+        }
+
+        [Fact]
+        public void Test4()
+        {
+        }
+    }
+}

src/Agoda.Tests.Metrics.xUnit.Tests/TestSuite2.cs

@@ -0,0 +1,35 @@
+namespace Agoda.Tests.Metrics.xUnit.Tests
+{
+    public class TestSuite2
+    {
+        [Fact]
+        public void Test1()
+        {
+        }
+
+        [Fact]
+        public void Test2()
+        {
+        }
+
+        [Fact]
+        public void Test3()
+        {
+        }
+
+        [Fact]
+        public void Test4()
+        {
+        }
+
+        [Theory]
+        [InlineData(100)]
+        [InlineData(75)]
+        [InlineData(50)]
+        [InlineData(25)]
+        public void Test5(int n)
+        {
+            Assert.True(n >= 25);
+        }
+    }
+}

src/Agoda.Tests.Metrics.xUnit.Tests/Usings.cs

@@ -0,0 +1 @@
+global using Xunit;

src/Agoda.Tests.Metrics.xUnit.Tests/xunit.runner.json

@@ -0,0 +1,4 @@
+{
+  "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
+  "diagnosticMessages": true
+}

src/Agoda.Tests.Metrics.xUnit/Agoda.Tests.Metrics.xUnit.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="xunit.abstractions" Version="2.0.3" />
+    <PackageReference Include="xunit.extensibility.core" Version="2.4.2" />
+    <PackageReference Include="xunit.extensibility.execution" Version="2.4.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Agoda.DevFeedback.Common\Agoda.DevFeedback.Common.csproj" />
+    <ProjectReference Include="..\Agoda.Tests.Metrics\Agoda.Tests.Metrics.csproj" />
+  </ItemGroup>
+
+</Project>

src/Agoda.Tests.Metrics.xUnit/AgodaAssemblyRunner.cs

@@ -0,0 +1,28 @@
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Agoda.Tests.Metrics.xUnit
+{
+    /// <summary>
+    /// Run all tests in an assembly
+    /// </summary>
+    internal class AgodaAssemblyRunner : XunitTestAssemblyRunner
+    {
+        private readonly TestResultsBuilder _builder;
+
+        public AgodaAssemblyRunner(ITestAssembly testAssembly, IEnumerable<IXunitTestCase> testCases, IMessageSink diagnosticMessageSink, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions, TestResultsBuilder builder)
+            : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions)
+        {
+            _builder = builder;
+        }
+
+        protected override Task<RunSummary> RunTestCollectionAsync(IMessageBus messageBus, ITestCollection testCollection, IEnumerable<IXunitTestCase> testCases, CancellationTokenSource cancellationTokenSource)
+        {
+            var messageBusWrapper = new AgodaMessageBus(messageBus, _builder);
+            return new XunitTestCollectionRunner(testCollection, testCases, DiagnosticMessageSink, messageBusWrapper, TestCaseOrderer, new ExceptionAggregator(Aggregator), cancellationTokenSource).RunAsync();
+        }
+    }
+}

src/Agoda.Tests.Metrics.xUnit/AgodaExecutor.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+using System.Reflection;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Agoda.Tests.Metrics.xUnit
+{
+    /// <summary>
+    /// Top level test executor
+    /// </summary>
+    internal class AgodaExecutor : XunitTestFrameworkExecutor
+    {
+        private readonly TestResultsBuilder _builder;
+
+        public AgodaExecutor(AssemblyName assemblyName, ISourceInformationProvider sourceInformationProvider, IMessageSink diagnosticMessageSink, TestResultsBuilder builder)
+            : base(assemblyName, sourceInformationProvider, diagnosticMessageSink)
+        {
+            _builder = builder;
+        }
+
+        protected override async void RunTestCases(IEnumerable<IXunitTestCase> testCases, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions)
+        {
+            try
+            {
+                var assemblyRunner = new AgodaAssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, executionMessageSink, executionOptions, _builder);
+                await assemblyRunner.RunAsync();
+            }
+            catch (Exception ex)
+            {
+                _builder.Diagnostic($"{ex.Message}");
+                _builder.Diagnostic($"{ex.StackTrace}");
+                throw;
+            }
+        }
+    }
+}

src/Agoda.Tests.Metrics.xUnit/AgodaMessageBus.cs

@@ -0,0 +1,69 @@
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Agoda.Tests.Metrics.xUnit
+{
+    internal class AgodaMessageBus : IMessageBus
+    {
+        IMessageBus _messageBus;
+        TestResultsBuilder _builder;
+
+        /// <summary>
+        /// Constructor that wraps an existing MessageBus
+        /// </summary>
+        public AgodaMessageBus(IMessageBus messageBus, TestResultsBuilder builder)
+        {
+            _messageBus = messageBus;
+            _builder = builder;
+        }
+
+        public void Dispose()
+        {
+            _messageBus.Dispose();
+        }
+
+        public bool QueueMessage(IMessageSinkMessage message)
+        {
+            switch (message)
+            {
+                case TestPassed testPassed:
+                    _builder?.ReportSuccess(
+                        testPassed.Test.TestCase.UniqueID,
+                        testPassed.Test.TestCase.TestMethod.Method.Name,
+                        $"{testPassed.Test.TestCase.TestMethod.TestClass.Class.Name}.{testPassed.Test.TestCase.TestMethod.Method.Name}",
+                        testPassed.Test.TestCase.TestMethod.Method.Name,
+                        testPassed.Test.TestCase.TestMethod.TestClass.Class.Name,
+                        (double)testPassed.ExecutionTime
+                        );
+                    break;
+                case TestFailed testFailed:
+                    _builder?.ReportFailure(
+                        testFailed.Test.TestCase.UniqueID,
+                        testFailed.Test.TestCase.TestMethod.Method.Name,
+                        $"{testFailed.Test.TestCase.TestMethod.TestClass.Class.Name}.{testFailed.Test.TestCase.TestMethod.Method.Name}",
+                        testFailed.Test.TestCase.TestMethod.Method.Name,
+                        testFailed.Test.TestCase.TestMethod.TestClass.Class.Name,
+                        (double)testFailed.ExecutionTime
+                        );
+                    break;
+                case TestSkipped testSkipped:
+                    _builder?.ReportFailure(
+                        testSkipped.Test.TestCase.UniqueID,
+                        testSkipped.Test.TestCase.TestMethod.Method.Name,
+                        $"{testSkipped.Test.TestCase.TestMethod.TestClass.Class.Name}.{testSkipped.Test.TestCase.TestMethod.Method.Name}",
+                        testSkipped.Test.TestCase.TestMethod.Method.Name,
+                        testSkipped.Test.TestCase.TestMethod.TestClass.Class.Name,
+                        (double)testSkipped.ExecutionTime
+                        );
+                    break;
+                case TestCollectionFinished testCollectionFinished:
+                    // Publish the results before passing the message on
+                    _builder.Publish();
+                    break;
+                default:
+                    break;
+            }
+            return _messageBus.QueueMessage(message);
+        }
+    }
+}