From e39bc3cdd7a8bf35473c92c689f5b971030b72d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Wed, 20 Nov 2024 10:16:35 +0100 Subject: [PATCH] Split GivenThatWeWantToRunILLink test class into multiple so Helix can run them separately Fixes https://github.com/dotnet/sdk/issues/44895 --- .../GivenThatWeWantToRunILLink.cs | 266 ++++++++++-------- 1 file changed, 141 insertions(+), 125 deletions(-) diff --git a/test/Microsoft.NET.Publish.Tests/GivenThatWeWantToRunILLink.cs b/test/Microsoft.NET.Publish.Tests/GivenThatWeWantToRunILLink.cs index 280f9199491d..c6c583240ef8 100644 --- a/test/Microsoft.NET.Publish.Tests/GivenThatWeWantToRunILLink.cs +++ b/test/Microsoft.NET.Publish.Tests/GivenThatWeWantToRunILLink.cs @@ -11,12 +11,14 @@ using Microsoft.NET.Build.Tasks; using Newtonsoft.Json.Linq; using static Microsoft.NET.Publish.Tests.PublishTestUtils; +using static Microsoft.NET.Publish.Tests.ILLinkTestUtils; namespace Microsoft.NET.Publish.Tests { - public class GivenThatWeWantToRunILLink : SdkTest + // this test class is split up arbitrarily so Helix can run tests in multiple workitems + public class GivenThatWeWantToRunILLink1 : SdkTest { - public GivenThatWeWantToRunILLink(ITestOutputHelper log) : base(log) + public GivenThatWeWantToRunILLink1(ITestOutputHelper log) : base(log) { } @@ -28,7 +30,7 @@ public void ILLink_only_runs_when_switch_is_enabled(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -42,7 +44,7 @@ public void ILLink_only_runs_when_switch_is_enabled(string targetFramework) var publishedDll = Path.Combine(publishDirectory, $"{projectName}.dll"); var unusedDll = Path.Combine(publishDirectory, $"{referenceProjectName}.dll"); - var unusedFrameworkDll = Path.Combine(publishDirectory, $"{unusedFrameworkAssembly}.dll"); + var unusedFrameworkDll = Path.Combine(publishDirectory, $"{UnusedFrameworkAssembly}.dll"); // Linker inputs are kept, including unused assemblies File.Exists(publishedDll).Should().BeTrue(); @@ -51,7 +53,7 @@ public void ILLink_only_runs_when_switch_is_enabled(string targetFramework) var depsFile = Path.Combine(publishDirectory, $"{projectName}.deps.json"); DoesDepsFileHaveAssembly(depsFile, referenceProjectName).Should().BeTrue(); - DoesDepsFileHaveAssembly(depsFile, unusedFrameworkAssembly).Should().BeTrue(); + DoesDepsFileHaveAssembly(depsFile, UnusedFrameworkAssembly).Should().BeTrue(); } [RequiresMSBuildVersionTheory("17.0.0.32901")] @@ -65,7 +67,7 @@ public void ILLink_runs_and_creates_linked_app(string targetFramework, bool refe var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName, referenceClassLibAsPackage); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName, referenceClassLibAsPackage); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework + referenceClassLibAsPackage) .WithProjectChanges(project => EnableNonFrameworkTrimming(project)); @@ -81,7 +83,7 @@ public void ILLink_runs_and_creates_linked_app(string targetFramework, bool refe var linkedDll = Path.Combine(linkedDirectory, $"{projectName}.dll"); var publishedDll = Path.Combine(publishDirectory, $"{projectName}.dll"); var unusedDll = Path.Combine(publishDirectory, $"{referenceProjectName}.dll"); - var unusedFrameworkDll = Path.Combine(publishDirectory, $"{unusedFrameworkAssembly}.dll"); + var unusedFrameworkDll = Path.Combine(publishDirectory, $"{UnusedFrameworkAssembly}.dll"); // Intermediate assembly is kept by linker and published, but not unused assemblies File.Exists(linkedDll).Should().BeTrue(); @@ -92,7 +94,7 @@ public void ILLink_runs_and_creates_linked_app(string targetFramework, bool refe var depsFile = Path.Combine(publishDirectory, $"{projectName}.deps.json"); DoesDepsFileHaveAssembly(depsFile, projectName).Should().BeTrue(); DoesDepsFileHaveAssembly(depsFile, referenceProjectName).Should().BeFalse(); - DoesDepsFileHaveAssembly(depsFile, unusedFrameworkAssembly).Should().BeFalse(); + DoesDepsFileHaveAssembly(depsFile, UnusedFrameworkAssembly).Should().BeFalse(); } [RequiresMSBuildVersionTheory("17.0.0.32901")] @@ -104,7 +106,7 @@ public void ILLink_links_simple_app_without_analysis_warnings_and_it_runs(string var projectName = "HelloWorld"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); testProject.AdditionalProperties["PublishTrimmed"] = "true"; var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework + trimMode); @@ -132,7 +134,7 @@ public void PublishTrimmed_fails_when_no_matching_pack_is_found(string targetFra var projectName = "HelloWorld"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => { @@ -161,7 +163,7 @@ public void PublishTrimmed_fails_for_unsupported_target_framework(string targetF var projectName = "HelloWorld"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); testProject.AdditionalProperties["PublishTrimmed"] = "true"; testProject.AdditionalProperties["NoWarn"] = "NETSDK1138"; // Silence warning about targeting EOL TFMs var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); @@ -194,7 +196,7 @@ public void IsTrimmable_warns_when_expected_for_not_correctly_multitargeted_libr var projectName = "HelloWorld"; var rid = EnvironmentInfo.GetCompatibleRid(targetFrameworks); - var testProject = CreateTestProjectForILLinkTesting(targetFrameworks, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFrameworks, projectName); testProject.AdditionalProperties["IsTrimmable"] = "true"; testProject.AdditionalProperties["NoWarn"] = "NETSDK1138"; // Silence warning about targeting EOL TFMs var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFrameworks) @@ -221,7 +223,7 @@ public void RequiresILLinkPack_errors_for_unsupported_target_framework(string ta var projectName = "HelloWorld"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); testProject.AdditionalProperties["_RequiresILLinkPack"] = "true"; var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); @@ -255,7 +257,7 @@ public void PrepareForILLink_can_set_IsTrimmable(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => SetMetadata(project, referenceProjectName, "IsTrimmable", "True")); @@ -280,7 +282,7 @@ public void PrepareForILLink_can_set_TrimMode(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName, referenceProjectIdentifier: targetFramework); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName, referenceProjectIdentifier: targetFramework); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => SetMetadata(project, referenceProjectName, "TrimMode", "link")); @@ -310,7 +312,7 @@ public void ILLink_respects_global_TrimMode(string targetFramework, string trimM var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName, referenceProjectIdentifier: targetFramework); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName, referenceProjectIdentifier: targetFramework); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework + trimMode) .WithProjectChanges(project => SetGlobalTrimMode(project, trimMode)) .WithProjectChanges(project => SetMetadata(project, referenceProjectName, "IsTrimmable", "True")) @@ -346,7 +348,7 @@ public void ILLink_roots_IntermediateAssembly(string targetFramework) var projectName = "HelloWorld"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => SetGlobalTrimMode(project, "link")) .WithProjectChanges(project => SetMetadata(project, projectName, "IsTrimmable", "True")); @@ -371,7 +373,7 @@ public void ILLink_respects_TrimmableAssembly(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); testProject.AddItem("TrimmableAssembly", "Include", referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); @@ -748,7 +750,7 @@ public void ILLink_errors_fail_the_build(string targetFramework) var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); // Set up a project with an invalid feature substitution, just to produce an error. - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); testProject.SourceFiles[$"{projectName}.xml"] = $@" @@ -775,6 +777,14 @@ public void ILLink_errors_fail_the_build(string targetFramework) File.Exists(linkSemaphore).Should().BeFalse(); File.Exists(publishedDll).Should().BeFalse(); } + } + + // this test class is split up arbitrarily so Helix can run tests in multiple workitems + public class GivenThatWeWantToRunILLink2 : SdkTest + { + public GivenThatWeWantToRunILLink2(ITestOutputHelper log) : base(log) + { + } [RequiresMSBuildVersionTheory("17.0.0.32901")] [MemberData(nameof(Net6Plus), MemberType = typeof(PublishTestUtils))] @@ -815,7 +825,7 @@ public void ILLink_verify_analysis_warnings_hello_world_app_trim_mode_copyused(s }); } - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(Log, Path.Combine(testAsset.TestRoot, testProject.Name)); @@ -897,7 +907,7 @@ public void ILLink_verify_analysis_warnings_framework_assemblies(string targetFr }); } - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(Log, Path.Combine(testAsset.TestRoot, testProject.Name)); @@ -916,7 +926,7 @@ public void ILLink_verify_analysis_warnings_hello_world_app_trim_mode_link(strin var projectName = "AnalysisWarningsOnHelloWorldApp"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(Log, Path.Combine(testAsset.TestRoot, testProject.Name)); @@ -932,7 +942,7 @@ public void ILLink_verify_analysis_warnings_hello_world_app_trim_mode_link_5_0() var projectName = "AnalysisWarningsOnHelloWorldApp"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(Log, Path.Combine(testAsset.TestRoot, testProject.Name)); @@ -942,61 +952,6 @@ public void ILLink_verify_analysis_warnings_hello_world_app_trim_mode_link_5_0() ValidateWarningsOnHelloWorldApp(publishCommand, result, new List(), targetFramework, rid); } - private void ValidateWarningsOnHelloWorldApp(PublishCommand publishCommand, CommandResult result, List expectedWarnings, string targetFramework, string rid, bool useRegex = false) - { - // This checks that there are no unexpected warnings, but does not cause failures for missing expected warnings. - var warnings = result.StdOut.Split('\n', '\r').Where(line => line.Contains("warning IL")); - - // This should also detect unexpected duplicates of expected warnings. - // Each expected warning string/regex matches at most one warning. - List extraWarnings = new(); - foreach (var warning in warnings) - { - bool expected = false; - for (int i = 0; i < expectedWarnings.Count; i++) - { - if ((useRegex && Regex.IsMatch(warning, expectedWarnings[i])) || - (!useRegex && warning.Contains(expectedWarnings[i]))) - { - expectedWarnings.RemoveAt(i); - expected = true; - break; - } - } - - if (!expected) - { - extraWarnings.Add(warning); - } - } - - StringBuilder errorMessage = new(); - - if (extraWarnings.Any()) - { - // Print additional information to recognize which framework assemblies are being used. - errorMessage.AppendLine($"Target framework from test: {targetFramework}"); - errorMessage.AppendLine($"Runtime identifier: {rid}"); - - // Get the array of runtime assemblies inside the publish folder. - string[] runtimeAssemblies = Directory.GetFiles(publishCommand.GetOutputDirectory(targetFramework: targetFramework, runtimeIdentifier: rid).FullName, "*.dll"); - var paths = new List(runtimeAssemblies); - var resolver = new PathAssemblyResolver(paths); - var mlc = new MetadataLoadContext(resolver, "System.Private.CoreLib"); - using (mlc) - { - Assembly assembly = mlc.LoadFromAssemblyPath(Path.Combine(publishCommand.GetOutputDirectory(targetFramework: targetFramework, runtimeIdentifier: rid).FullName, "System.Private.CoreLib.dll")); - string assemblyVersionInfo = (string)assembly.CustomAttributes.Where(ca => ca.AttributeType.Name == "AssemblyInformationalVersionAttribute").Select(ca => ca.ConstructorArguments[0].Value).FirstOrDefault(); - errorMessage.AppendLine($"Runtime Assembly Informational Version: {assemblyVersionInfo}"); - } - errorMessage.AppendLine($"The execution of a hello world app generated a diff in the number of warnings the app produces{Environment.NewLine}"); - errorMessage.AppendLine("Test output contained the following extra linker warnings:"); - foreach (var extraWarning in extraWarnings) - errorMessage.AppendLine($"+ {extraWarning}"); - } - Assert.True(!extraWarnings.Any(), errorMessage.ToString()); - } - [RequiresMSBuildVersionTheory("17.0.0.32901")] [MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))] public void TrimmingOptions_are_defaulted_correctly_on_trimmed_apps(string targetFramework) @@ -1004,7 +959,7 @@ public void TrimmingOptions_are_defaulted_correctly_on_trimmed_apps(string targe var projectName = "HelloWorld"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: projectName + targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1077,7 +1032,7 @@ public void ILLink_accepts_root_descriptor(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => EnableNonFrameworkTrimming(project)) .WithProjectChanges(project => AddRootDescriptor(project, $"{referenceProjectName}.xml")); @@ -1117,7 +1072,7 @@ public void ILLink_error_on_nonboolean_optimization_flag(string property) var targetFramework = "net5.0"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: property); var publishCommand = new PublishCommand(testAsset); @@ -1133,7 +1088,7 @@ public void ILLink_respects_feature_settings_from_host_config() var targetFramework = "net5.0"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName, + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName, // Reference the classlib to ensure its XML is processed. addAssemblyReference: true, // Set up a conditional feature substitution for the "FeatureDisabled" property @@ -1166,7 +1121,7 @@ public void ILLink_ignores_host_config_settings_with_link_false() var targetFramework = "net5.0"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName, + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName, // Reference the classlib to ensure its XML is processed. addAssemblyReference: true, // Set up a conditional feature substitution for the "FeatureDisabled" property @@ -1199,7 +1154,7 @@ public void ILLink_runs_incrementally(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1230,7 +1185,7 @@ public void ILLink_old_defaults_keep_nonframework(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1245,7 +1200,7 @@ public void ILLink_old_defaults_keep_nonframework(string targetFramework) var linkedDll = Path.Combine(linkedDirectory, $"{projectName}.dll"); var publishedDll = Path.Combine(publishDirectory, $"{projectName}.dll"); var unusedDll = Path.Combine(publishDirectory, $"{referenceProjectName}.dll"); - var unusedFrameworkDll = Path.Combine(publishDirectory, $"{unusedFrameworkAssembly}.dll"); + var unusedFrameworkDll = Path.Combine(publishDirectory, $"{UnusedFrameworkAssembly}.dll"); File.Exists(linkedDll).Should().BeTrue(); File.Exists(publishedDll).Should().BeTrue(); @@ -1255,7 +1210,7 @@ public void ILLink_old_defaults_keep_nonframework(string targetFramework) var depsFile = Path.Combine(publishDirectory, $"{projectName}.deps.json"); DoesDepsFileHaveAssembly(depsFile, projectName).Should().BeTrue(); DoesDepsFileHaveAssembly(depsFile, referenceProjectName).Should().BeTrue(); - DoesDepsFileHaveAssembly(depsFile, unusedFrameworkAssembly).Should().BeFalse(); + DoesDepsFileHaveAssembly(depsFile, UnusedFrameworkAssembly).Should().BeFalse(); } [RequiresMSBuildVersionFact("17.0.0.32901")] @@ -1266,7 +1221,7 @@ public void ILLink_net7_defaults_trim_nonframework() var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1281,7 +1236,7 @@ public void ILLink_net7_defaults_trim_nonframework() var linkedDll = Path.Combine(linkedDirectory, $"{projectName}.dll"); var publishedDll = Path.Combine(publishDirectory, $"{projectName}.dll"); var unusedDll = Path.Combine(publishDirectory, $"{referenceProjectName}.dll"); - var unusedFrameworkDll = Path.Combine(publishDirectory, $"{unusedFrameworkAssembly}.dll"); + var unusedFrameworkDll = Path.Combine(publishDirectory, $"{UnusedFrameworkAssembly}.dll"); File.Exists(linkedDll).Should().BeTrue(); File.Exists(publishedDll).Should().BeTrue(); @@ -1291,7 +1246,7 @@ public void ILLink_net7_defaults_trim_nonframework() var depsFile = Path.Combine(publishDirectory, $"{projectName}.deps.json"); DoesDepsFileHaveAssembly(depsFile, projectName).Should().BeTrue(); DoesDepsFileHaveAssembly(depsFile, referenceProjectName).Should().BeFalse(); - DoesDepsFileHaveAssembly(depsFile, unusedFrameworkAssembly).Should().BeFalse(); + DoesDepsFileHaveAssembly(depsFile, UnusedFrameworkAssembly).Should().BeFalse(); } [RequiresMSBuildVersionTheory("17.0.0.32901")] @@ -1302,7 +1257,7 @@ public void ILLink_does_not_include_leftover_artifacts_on_second_run(string targ var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName, referenceProjectIdentifier: targetFramework); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName, referenceProjectIdentifier: targetFramework); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => EnableNonFrameworkTrimming(project)) .WithProjectChanges(project => AddRootDescriptor(project, $"{referenceProjectName}.xml")); @@ -1355,7 +1310,7 @@ public void ILLink_keeps_symbols_by_default(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => EnableNonFrameworkTrimming(project)); @@ -1388,7 +1343,7 @@ public void ILLink_removes_symbols_when_debugger_support_is_disabled(string targ var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => EnableNonFrameworkTrimming(project)); @@ -1407,6 +1362,14 @@ public void ILLink_removes_symbols_when_debugger_support_is_disabled(string targ File.Exists(linkedPdb).Should().BeFalse(); File.Exists(publishedPdb).Should().BeFalse(); } + } + + // this test class is split up arbitrarily so Helix can run tests in multiple workitems + public class GivenThatWeWantToRunILLink3 : SdkTest + { + public GivenThatWeWantToRunILLink3(ITestOutputHelper log) : base(log) + { + } [RequiresMSBuildVersionTheory("17.0.0.32901")] [MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))] @@ -1416,7 +1379,7 @@ public void ILLink_accepts_option_to_remove_symbols(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => EnableNonFrameworkTrimming(project)); @@ -1444,7 +1407,7 @@ public void ILLink_symbols_option_can_override_defaults_from_debugger_support(st var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => EnableNonFrameworkTrimming(project)); @@ -1596,7 +1559,7 @@ public void ILLink_error_on_portable_app(string targetFramework) var projectName = "HelloWorld"; var referenceProjectName = "ClassLibForILLink"; - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName, setSelfContained: false); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName, setSelfContained: false); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1614,7 +1577,7 @@ public void ILLink_displays_informational_warning_up_to_net5_by_default(string t var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1631,7 +1594,7 @@ public void ILLink_displays_informational_warning_when_trim_analysis_warnings_ar var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1649,7 +1612,7 @@ public void ILLink_dont_display_informational_warning_by_default_on_net6plus(str var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1667,7 +1630,7 @@ public void ILLink_dont_display_time_awareness_message_on_incremental_build(stri var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1748,8 +1711,13 @@ public void Build_respects_PublishTrimmed_property(string targetFramework) // just setting PublishTrimmed doesn't inject the IsTrimmable attribute AssemblyInfo.Get(assemblyPath).ContainsKey("AssemblyMetadataAttribute").Should().BeFalse(); } + } + + internal static class ILLinkTestUtils + { + public static string UnusedFrameworkAssembly = "System.IO"; - private static bool DoesImageHaveMethod(string path, string methodNameToCheck) + public static bool DoesImageHaveMethod(string path, string methodNameToCheck) { using (FileStream fs = new(path, FileMode.Open, FileAccess.Read)) using (var peReader = new PEReader(fs)) @@ -1766,7 +1734,7 @@ private static bool DoesImageHaveMethod(string path, string methodNameToCheck) return false; } - private static bool DoesDepsFileHaveAssembly(string depsFilePath, string assemblyName) + public static bool DoesDepsFileHaveAssembly(string depsFilePath, string assemblyName) { DependencyContext dependencyContext; using (var fs = File.OpenRead(depsFilePath)) @@ -1780,15 +1748,7 @@ private static bool DoesDepsFileHaveAssembly(string depsFilePath, string assembl Path.GetFileName(f) == $"{assemblyName}.dll"))); } - static string unusedFrameworkAssembly = "System.IO"; - - private TestAsset GetProjectReference(TestProject project, string callingMethod, string identifier) - { - var asset = _testAssetsManager.CreateTestProject(project, callingMethod: callingMethod, identifier: identifier); - return asset; - } - - private void AddRootDescriptor(XDocument project, string rootDescriptorFileName) + public static void AddRootDescriptor(XDocument project, string rootDescriptorFileName) { var ns = project.Root.Name.Namespace; @@ -1798,7 +1758,7 @@ private void AddRootDescriptor(XDocument project, string rootDescriptorFileName) new XAttribute("Include", rootDescriptorFileName))); } - private void RemoveRootDescriptor(XDocument project) + public static void RemoveRootDescriptor(XDocument project) { var ns = project.Root.Name.Namespace; @@ -1807,7 +1767,7 @@ private void RemoveRootDescriptor(XDocument project) .First().Remove(); } - private void SetMetadata(XDocument project, string assemblyName, string key, string value) + public static void SetMetadata(XDocument project, string assemblyName, string key, string value) { var ns = project.Root.Name.Namespace; var targetName = "SetTrimmerMetadata"; @@ -1829,7 +1789,7 @@ private void SetMetadata(XDocument project, string assemblyName, string key, str new XAttribute(key, value)))); } - private void SetGlobalTrimMode(XDocument project, string trimMode) + public static void SetGlobalTrimMode(XDocument project, string trimMode) { var ns = project.Root.Name.Namespace; @@ -1839,7 +1799,7 @@ private void SetGlobalTrimMode(XDocument project, string trimMode) trimMode)); } - private void SetTrimmerDefaultAction(XDocument project, string action) + public static void SetTrimmerDefaultAction(XDocument project, string action) { var ns = project.Root.Name.Namespace; @@ -1848,7 +1808,7 @@ private void SetTrimmerDefaultAction(XDocument project, string action) properties.Add(new XElement(ns + "TrimmerDefaultAction", action)); } - private void EnableNonFrameworkTrimming(XDocument project) + public static void EnableNonFrameworkTrimming(XDocument project) { // Used to override the default linker options for testing // purposes. The default roots non-framework assemblies, @@ -1871,10 +1831,10 @@ private void EnableNonFrameworkTrimming(XDocument project) new XAttribute("Include", "@(IntermediateAssembly->'%(FullPath)')"))); } - static readonly string substitutionsFilename = "ILLink.Substitutions.xml"; - - private void AddFeatureDefinition(TestProject testProject, string assemblyName) + public static void AddFeatureDefinition(TestProject testProject, string assemblyName) { + const string substitutionsFilename = "ILLink.Substitutions.xml"; + // Add a feature definition that replaces the FeatureDisabled property when DisableFeature is true. testProject.EmbeddedResources[substitutionsFilename] = $@" @@ -1893,7 +1853,7 @@ private void AddFeatureDefinition(TestProject testProject, string assemblyName) }); } - private void AddRuntimeConfigOption(XDocument project, bool trim) + public static void AddRuntimeConfigOption(XDocument project, bool trim) { var ns = project.Root.Name.Namespace; @@ -1904,7 +1864,7 @@ private void AddRuntimeConfigOption(XDocument project, bool trim) new XAttribute("Trim", trim.ToString())))); } - private TestProject CreateTestProjectWithAnalysisWarnings(string targetFramework, string projectName, bool isExe = true) + public static TestProject CreateTestProjectWithAnalysisWarnings(string targetFramework, string projectName, bool isExe = true) { var testProject = new TestProject() { @@ -1978,7 +1938,7 @@ public override void IL_2046() {} return testProject; } - private TestProject CreateTestProjectWithIsTrimmableAttributes( + public static TestProject CreateTestProjectWithIsTrimmableAttributes( string targetFramework, string projectName) { @@ -2083,7 +2043,8 @@ public static void UnusedMethod() return testProject; } - private TestProject CreateTestProjectForILLinkTesting( + public static TestProject CreateTestProjectForILLinkTesting( + TestAssetsManager testAssetsManager, string targetFrameworks, string mainProjectName, string referenceProjectName = null, @@ -2188,7 +2149,7 @@ public static void FeatureImplementation() if (usePackageReference) { - var referenceAsset = GetProjectReference(referenceProject, callingMethod, referenceProjectIdentifier ?? targetFrameworks); + var referenceAsset = testAssetsManager.CreateTestProject(referenceProject, callingMethod, referenceProjectIdentifier ?? targetFrameworks); testProject.ReferencedProjects.Add(referenceAsset.TestProject); } else @@ -2211,7 +2172,7 @@ public static void FeatureImplementation() return testProject; } - private void CheckILLinkVersion(TestAsset testAsset, string targetFramework) + public static void CheckILLinkVersion(TestAsset testAsset, string targetFramework) { var getKnownPacks = new GetValuesCommand(testAsset, "KnownILLinkPack", GetValuesCommand.ValueType.Item, targetFramework) { @@ -2231,5 +2192,60 @@ private void CheckILLinkVersion(TestAsset testAsset, string targetFramework) var illinkVersion = Path.GetFileName(Path.GetDirectoryName(Path.GetDirectoryName(illinkTargetsPath))); illinkVersion.Should().Be(expectedVersion); } + + public static void ValidateWarningsOnHelloWorldApp(PublishCommand publishCommand, CommandResult result, List expectedWarnings, string targetFramework, string rid, bool useRegex = false) + { + // This checks that there are no unexpected warnings, but does not cause failures for missing expected warnings. + var warnings = result.StdOut.Split('\n', '\r').Where(line => line.Contains("warning IL")); + + // This should also detect unexpected duplicates of expected warnings. + // Each expected warning string/regex matches at most one warning. + List extraWarnings = new(); + foreach (var warning in warnings) + { + bool expected = false; + for (int i = 0; i < expectedWarnings.Count; i++) + { + if ((useRegex && Regex.IsMatch(warning, expectedWarnings[i])) || + (!useRegex && warning.Contains(expectedWarnings[i]))) + { + expectedWarnings.RemoveAt(i); + expected = true; + break; + } + } + + if (!expected) + { + extraWarnings.Add(warning); + } + } + + StringBuilder errorMessage = new(); + + if (extraWarnings.Any()) + { + // Print additional information to recognize which framework assemblies are being used. + errorMessage.AppendLine($"Target framework from test: {targetFramework}"); + errorMessage.AppendLine($"Runtime identifier: {rid}"); + + // Get the array of runtime assemblies inside the publish folder. + string[] runtimeAssemblies = Directory.GetFiles(publishCommand.GetOutputDirectory(targetFramework: targetFramework, runtimeIdentifier: rid).FullName, "*.dll"); + var paths = new List(runtimeAssemblies); + var resolver = new PathAssemblyResolver(paths); + var mlc = new MetadataLoadContext(resolver, "System.Private.CoreLib"); + using (mlc) + { + Assembly assembly = mlc.LoadFromAssemblyPath(Path.Combine(publishCommand.GetOutputDirectory(targetFramework: targetFramework, runtimeIdentifier: rid).FullName, "System.Private.CoreLib.dll")); + string assemblyVersionInfo = (string)assembly.CustomAttributes.Where(ca => ca.AttributeType.Name == "AssemblyInformationalVersionAttribute").Select(ca => ca.ConstructorArguments[0].Value).FirstOrDefault(); + errorMessage.AppendLine($"Runtime Assembly Informational Version: {assemblyVersionInfo}"); + } + errorMessage.AppendLine($"The execution of a hello world app generated a diff in the number of warnings the app produces{Environment.NewLine}"); + errorMessage.AppendLine("Test output contained the following extra linker warnings:"); + foreach (var extraWarning in extraWarnings) + errorMessage.AppendLine($"+ {extraWarning}"); + } + Assert.True(!extraWarnings.Any(), errorMessage.ToString()); + } } }