Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

File I/O Abstraction Part 8: Simplify module registry related components #16329

Merged
merged 7 commits into from
Feb 5, 2025
Merged

File I/O Abstraction Part 8: Simplify module registry related components #16329

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
911e334
Fix IOUri.GetPathRelativeTo
shenglol Feb 4, 2025
3fc5126
Simplify registry stuff
shenglol Feb 4, 2025
14ac380
Remove BicepFile.Dummy first iteration
shenglol Feb 4, 2025
9489153
Remove BicepFile.Dummy second iteration
shenglol Feb 4, 2025
bbe2ad1
Remove BicepFile.Dummy last iteration
shenglol Feb 5, 2025
a59a7cd
Merge branch 'main' into shenglol/file-io-abstraction-part-8
shenglol Feb 5, 2025
0e67524
Clean up more tests
shenglol Feb 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion src/Bicep.Cli/Commands/PublishCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
using Bicep.Core.FileSystem;
using Bicep.Core.Registry;
using Bicep.Core.SourceCode;
using Bicep.Core.Workspaces;
using Microsoft.Extensions.Logging;

namespace Bicep.Cli.Commands
Expand All @@ -23,19 +24,22 @@ public class PublishCommand : ICommand
private readonly BicepCompiler compiler;
private readonly IModuleDispatcher moduleDispatcher;
private readonly IFileSystem fileSystem;
private readonly ISourceFileFactory sourceFileFactory;
private readonly IOContext ioContext;

public PublishCommand(
DiagnosticLogger diagnosticLogger,
BicepCompiler compiler,
IOContext ioContext,
IModuleDispatcher moduleDispatcher,
ISourceFileFactory sourceFileFactory,
IFileSystem fileSystem)
{
this.diagnosticLogger = diagnosticLogger;
this.compiler = compiler;
this.moduleDispatcher = moduleDispatcher;
this.fileSystem = fileSystem;
this.sourceFileFactory = sourceFileFactory;
this.ioContext = ioContext;
}

Expand Down Expand Up @@ -110,7 +114,9 @@ private async Task PublishModuleAsync(ArtifactReference target, BinaryData compi

private ArtifactReference ValidateReference(string targetModuleReference, Uri targetModuleUri)
{
if (!this.moduleDispatcher.TryGetArtifactReference(ArtifactType.Module, targetModuleReference, targetModuleUri).IsSuccess(out var moduleReference, out var failureBuilder))
var dummyReferencingFile = this.sourceFileFactory.CreateBicepFile(targetModuleUri, string.Empty);

if (!this.moduleDispatcher.TryGetArtifactReference(dummyReferencingFile, ArtifactType.Module, targetModuleReference).IsSuccess(out var moduleReference, out var failureBuilder))
{
// TODO: We should probably clean up the dispatcher contract so this sort of thing isn't necessary (unless we change how target module is set in this command)
var message = failureBuilder(DiagnosticBuilder.ForDocumentStart()).Message;
Expand Down
10 changes: 9 additions & 1 deletion src/Bicep.Cli/Commands/PublishExtensionCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
using Bicep.Core.Registry.Extensions;
using Bicep.Core.Registry.Oci;
using Bicep.Core.TypeSystem;
using Bicep.Core.Workspaces;
using Microsoft.Extensions.Logging;

namespace Bicep.Cli.Commands
Expand All @@ -22,14 +23,18 @@ public class PublishExtensionCommand : ICommand
{
private readonly IModuleDispatcher moduleDispatcher;
private readonly IFileSystem fileSystem;
private readonly ISourceFileFactory sourceFileFactory;
private readonly IOContext ioContext;

public PublishExtensionCommand(
IOContext ioContext,
IModuleDispatcher moduleDispatcher,
ISourceFileFactory sourceFileFactory,
IFileSystem fileSystem)
{
this.moduleDispatcher = moduleDispatcher;
this.fileSystem = fileSystem;
this.sourceFileFactory = sourceFileFactory;
this.ioContext = ioContext;
}

Expand Down Expand Up @@ -101,7 +106,10 @@ private ArtifactReference ValidateReference(string targetReference, Uri targetUr
targetReference = Path.GetFileName(targetUri.LocalPath);
}

if (!this.moduleDispatcher.TryGetArtifactReference(ArtifactType.Extension, targetReference, targetUri).IsSuccess(out var extensionReference, out var failureBuilder))
var dummyReferencingFile = this.sourceFileFactory.CreateBicepFile(targetUri, "");


if (!this.moduleDispatcher.TryGetArtifactReference(dummyReferencingFile, ArtifactType.Extension, targetReference).IsSuccess(out var extensionReference, out var failureBuilder))
{
// TODO: We should probably clean up the dispatcher contract so this sort of thing isn't necessary (unless we change how target module is set in this command)
var message = failureBuilder(DiagnosticBuilder.ForDocumentStart()).Message;
Expand Down
11 changes: 5 additions & 6 deletions src/Bicep.Core.IntegrationTests/ExtensionRegistryTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,8 @@ public async Task Extensions_published_to_filesystem_can_be_compiled()
var tempDirectory = FileHelper.GetUniqueTestOutputPath(TestContext);
Directory.CreateDirectory(tempDirectory);

var extensionPath = Path.Combine(tempDirectory, "extension.tgz");
await RegistryHelper.PublishExtensionToRegistryAsync(services.Build(), Path.Combine(tempDirectory, extensionPath), typesTgz);

var bicepPath = Path.Combine(tempDirectory, "main.bicep");
var bicepUri = PathHelper.FilePathToFileUrl(bicepPath);
await File.WriteAllTextAsync(bicepPath, """
extension './extension.tgz'

Expand All @@ -100,7 +98,8 @@ await File.WriteAllTextAsync(bicepPath, """
}
""");

var bicepUri = PathHelper.FilePathToFileUrl(bicepPath);
var extensionPath = Path.Combine(tempDirectory, "extension.tgz");
await RegistryHelper.PublishExtensionToRegistryAsync(services.Build(), Path.Combine(tempDirectory, extensionPath), typesTgz, bicepUri);


var compiler = services.Build().GetCompiler();
Expand Down Expand Up @@ -209,9 +208,9 @@ extension nonExistent
}
""")));

var sourceUri = InMemoryFileResolver.GetFileUri("/path/bicepconfig.json");
var sourceUri = InMemoryFileResolver.GetFileUri("/path/to/main.bicep");
result.Should().HaveDiagnostics([
("BCP093", DiagnosticLevel.Error, $"File path \"./non_existent.tgz\" could not be resolved relative to \"{sourceUri.LocalPath}\"."),
("BCP093", DiagnosticLevel.Error, $"File path \"../non_existent.tgz\" could not be resolved relative to \"{sourceUri.LocalPath}\"."),
]);
}

Expand Down
28 changes: 19 additions & 9 deletions src/Bicep.Core.IntegrationTests/RegistryTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using Bicep.Core.UnitTests;
using Bicep.Core.UnitTests.Assertions;
using Bicep.Core.UnitTests.Utils;
using Bicep.Core.Workspaces;
using Bicep.IO.Abstraction;
using FluentAssertions;
using FluentAssertions.Execution;
Expand Down Expand Up @@ -165,15 +166,15 @@ public async Task ModuleRestoreContentionShouldProduceConsistentState()
.Build();

var dispatcher = services.Construct<IModuleDispatcher>();

var dummyFile = CreateDummyReferencingFile(services);
var moduleReferences = dataSet.RegistryModules.Values
.OrderBy(m => m.Metadata.Target)
.Select(m => dispatcher.TryGetModuleReference(m.Metadata.Target, RandomFileUri()).Unwrap())
.Select(m => TryGetModuleReference(dispatcher, dummyFile, m.Metadata.Target).Unwrap())
.ToImmutableList();

moduleReferences.Should().HaveCount(7);

// initially the cache should be empty
// initially the cache should be empty.
foreach (var moduleReference in moduleReferences)
{
dispatcher.GetArtifactRestoreStatus(moduleReference, out _).Should().Be(ArtifactRestoreStatus.Unknown);
Expand Down Expand Up @@ -220,11 +221,11 @@ public async Task ModuleRestoreWithStuckFileLockShouldFailAfterTimeout(IEnumerab
.Build();

var dispatcher = services.Construct<IModuleDispatcher>();
var dummyFile = CreateDummyReferencingFile(services);

var configuration = BicepTestConstants.BuiltInConfigurationWithAllAnalyzersDisabled;
var moduleReferences = moduleInfos
.OrderBy(m => m.Metadata.Target)
.Select(m => dispatcher.TryGetModuleReference(m.Metadata.Target, RandomFileUri()).IsSuccess(out var @ref) ? @ref : throw new AssertFailedException($"Invalid module target '{m.Metadata.Target}'."))
.Select(m => TryGetModuleReference(dispatcher, dummyFile, m.Metadata.Target).IsSuccess(out var @ref) ? @ref : throw new AssertFailedException($"Invalid module target '{m.Metadata.Target}'."))
.ToImmutableList();

moduleReferences.Should().HaveCount(moduleCount);
Expand Down Expand Up @@ -287,10 +288,11 @@ public async Task ForceModuleRestoreWithStuckFileLockShouldFailAfterTimeout(IEnu
.Build();

var dispatcher = services.Construct<IModuleDispatcher>();
var dummyFile = CreateDummyReferencingFile(services);

var moduleReferences = moduleInfos
.OrderBy(m => m.Metadata.Target)
.Select(m => dispatcher.TryGetModuleReference(m.Metadata.Target, RandomFileUri()).IsSuccess(out var @ref) ? @ref : throw new AssertFailedException($"Invalid module target '{m.Metadata.Target}'."))
.Select(m => TryGetModuleReference(dispatcher, dummyFile, m.Metadata.Target).IsSuccess(out var @ref) ? @ref : throw new AssertFailedException($"Invalid module target '{m.Metadata.Target}'."))
.ToImmutableList();

moduleReferences.Should().HaveCount(moduleCount);
Expand Down Expand Up @@ -360,11 +362,11 @@ public async Task ForceModuleRestoreShouldRestoreAllModules(IEnumerable<External
.Build();

var dispatcher = services.Construct<IModuleDispatcher>();
var dummyFile = CreateDummyReferencingFile(services);

var configuration = BicepTestConstants.BuiltInConfigurationWithAllAnalyzersDisabled;
var moduleReferences = moduleInfos
.OrderBy(m => m.Metadata.Target)
.Select(m => dispatcher.TryGetModuleReference(m.Metadata.Target, RandomFileUri()).IsSuccess(out var @ref) ? @ref : throw new AssertFailedException($"Invalid module target '{m.Metadata.Target}'."))
.Select(m => TryGetModuleReference(dispatcher, dummyFile, m.Metadata.Target).IsSuccess(out var @ref) ? @ref : throw new AssertFailedException($"Invalid module target '{m.Metadata.Target}'."))
.ToImmutableList();

moduleReferences.Should().HaveCount(moduleCount);
Expand Down Expand Up @@ -399,6 +401,14 @@ public static IEnumerable<object[]> GetModuleInfoData()
yield return new object[] { DataSets.Registry_LF.TemplateSpecs.Values, 2, true };
}

private static Uri RandomFileUri() => PathHelper.FilePathToFileUrl(Path.GetTempFileName());
private static BicepFile CreateDummyReferencingFile(IDependencyHelper dependencyHelper)
{
var sourceFileFactory = dependencyHelper.Construct<ISourceFileFactory>();

return sourceFileFactory.CreateBicepFile(new Uri("inmemory:///main.bicep"), "");
}

private static ResultWithDiagnosticBuilder<ArtifactReference> TryGetModuleReference(IModuleDispatcher moduleDispatcher, BicepSourceFile referencingFile, string reference) =>
moduleDispatcher.TryGetArtifactReference(referencingFile, ArtifactType.Module, reference);
}
}
14 changes: 7 additions & 7 deletions src/Bicep.Core.IntegrationTests/SourceArchiveTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ public async Task SourceArtifactId_ForExternalModulesWithSource_ShouldBeTheArtif
var sourceArchive = CreateSourceArchive(moduleDispatcher, result);

var file = sourceArchive.FindExpectedSourceFile("<cache>/br/mockregistry.io/test$module1/v1$/main.json");
file.SourceArtifact!.FullyQualifiedReference.Should().Be("br:mockregistry.io/test/module1:v1");
file.SourceArtifact!.ArtifactId.Should().Be("mockregistry.io/test/module1:v1");
file.Kind.Should().Be("armTemplate");

}
Expand Down Expand Up @@ -239,12 +239,12 @@ param p2 string
// act
var sourceArchive = CreateSourceArchive(moduleDispatcher, result);

sourceArchive.SourceFiles.Select(sf => (sf.Path, sf.SourceArtifact?.FullyQualifiedReference))
sourceArchive.SourceFiles.Select(sf => (sf.Path, sf.SourceArtifact?.ArtifactId))
.Should().BeEquivalentTo(new[] {
("main.bicep", null),
("<cache>/br/mockregistry.io/test$module1/v1$/main.json", "br:mockregistry.io/test/module1:v1"),
("<cache>/br/mockregistry.io/test$module1/v2$/main.json", "br:mockregistry.io/test/module1:v2"),
("<cache>/br/mockregistry.io/test$module2/v1$/main.json", "br:mockregistry.io/test/module2:v1"),
("<cache>/br/mockregistry.io/test$module1/v1$/main.json", "mockregistry.io/test/module1:v1"),
("<cache>/br/mockregistry.io/test$module1/v2$/main.json", "mockregistry.io/test/module1:v2"),
("<cache>/br/mockregistry.io/test$module2/v1$/main.json", "mockregistry.io/test/module2:v1"),
("local.bicep", null)
});
}
Expand Down Expand Up @@ -278,10 +278,10 @@ public async Task SourceArtifactId_ShouldIgnoreModuleRefsWithErrors()
// act
var sourceArchive = CreateSourceArchive(moduleDispatcher, result);

sourceArchive.SourceFiles.Select(sf => (sf.Path, sf.SourceArtifact?.FullyQualifiedReference))
sourceArchive.SourceFiles.Select(sf => (sf.Path, sf.SourceArtifact?.ArtifactId))
.Should().BeEquivalentTo(new[] {
("main.bicep", null),
("<cache>/br/mockregistry.io/test$module1/v1$/main.json", "br:mockregistry.io/test/module1:v1"),
("<cache>/br/mockregistry.io/test$module1/v1$/main.json", "mockregistry.io/test/module1:v1"),
});
}
}
Expand Down
29 changes: 19 additions & 10 deletions src/Bicep.Core.Samples/DataSetsExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,20 @@

using System.Collections.Immutable;
using Bicep.Core.Configuration;
using Bicep.Core.Diagnostics;
using Bicep.Core.Features;
using Bicep.Core.FileSystem;
using Bicep.Core.Modules;
using Bicep.Core.Registry;
using Bicep.Core.Registry.Oci;
using Bicep.Core.Semantics;
using Bicep.Core.Syntax;
using Bicep.Core.UnitTests;
using Bicep.Core.UnitTests.Features;
using Bicep.Core.UnitTests.Mock;
using Bicep.Core.UnitTests.Utils;
using Bicep.Core.Workspaces;
using Bicep.IO.Abstraction;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
Expand Down Expand Up @@ -56,18 +61,21 @@ public static IContainerRegistryClientFactory CreateMockRegistryClients(
ImmutableDictionary<string, DataSet.ExternalModuleInfo> registryModules,
params RepoDescriptor[] additionalClients)
{
var dispatcher = ServiceBuilder.Create(s => s.WithDisabledAnalyzersConfiguration()
var services = ServiceBuilder.Create(s => s.WithDisabledAnalyzersConfiguration()
.AddSingleton(BicepTestConstants.ClientFactory)
.AddSingleton(BicepTestConstants.TemplateSpecRepositoryFactory)
).Construct<IModuleDispatcher>();
.AddSingleton(BicepTestConstants.TemplateSpecRepositoryFactory));

var dispatcher = services.Construct<IModuleDispatcher>();
var sourceFileFactory = services.Construct<ISourceFileFactory>();
var dummyReferencingFile = sourceFileFactory.CreateBicepFile(new Uri("inmemory:///main.bicep"), "");

var clients = new List<RepoDescriptor>();

foreach (var (moduleName, publishInfo) in registryModules)
{
var target = publishInfo.Metadata.Target;

if (!dispatcher.TryGetArtifactReference(ArtifactType.Module, target, RandomFileUri()).IsSuccess(out var @ref) || @ref is not OciArtifactReference targetReference)
if (!dispatcher.TryGetArtifactReference(dummyReferencingFile, ArtifactType.Module, target).IsSuccess(out var @ref) || @ref is not OciArtifactReference targetReference)
{
throw new InvalidOperationException($"Module '{moduleName}' has an invalid target reference '{target}'. Specify a reference to an OCI artifact.");
}
Expand All @@ -86,15 +94,18 @@ public static ITemplateSpecRepositoryFactory CreateMockTemplateSpecRepositoryFac

public static ITemplateSpecRepositoryFactory CreateMockTemplateSpecRepositoryFactory(ImmutableDictionary<string, DataSet.ExternalModuleInfo> templateSpecs)
{
var dispatcher = ServiceBuilder.Create(s => s.WithDisabledAnalyzersConfiguration()
var services = ServiceBuilder.Create(s => s.WithDisabledAnalyzersConfiguration()
.AddSingleton(BicepTestConstants.ClientFactory)
.AddSingleton(BicepTestConstants.TemplateSpecRepositoryFactory)
).Construct<IModuleDispatcher>();
.AddSingleton(BicepTestConstants.TemplateSpecRepositoryFactory));

var dispatcher = services.Construct<IModuleDispatcher>();
var sourceFileFactory = services.Construct<ISourceFileFactory>();
var dummyReferencingFile = sourceFileFactory.CreateBicepFile(new Uri("inmemory:///main.bicep"), "");
var repositoryMocksBySubscription = new Dictionary<string, Mock<ITemplateSpecRepository>>();

foreach (var (moduleName, templateSpecInfo) in templateSpecs)
{
if (!dispatcher.TryGetArtifactReference(ArtifactType.Module, templateSpecInfo.Metadata.Target, RandomFileUri()).IsSuccess(out var @ref) || @ref is not TemplateSpecModuleReference reference)
if (!dispatcher.TryGetArtifactReference(dummyReferencingFile, ArtifactType.Module, templateSpecInfo.Metadata.Target).IsSuccess(out var @ref) || @ref is not TemplateSpecModuleReference reference)
{
throw new InvalidOperationException($"Module '{moduleName}' has an invalid target reference '{templateSpecInfo.Metadata.Target}'. Specify a reference to a template spec.");
}
Expand Down Expand Up @@ -130,7 +141,5 @@ await RegistryHelper.PublishModuleToRegistryAsync(
new(publishInfo.Metadata.Target, publishInfo.ModuleSource, WithSource: publishSource, DocumentationUri: null));
}
}

private static Uri RandomFileUri() => PathHelper.FilePathToFileUrl(Path.GetTempFileName());
}
}
43 changes: 0 additions & 43 deletions src/Bicep.Core.UnitTests/Assertions/OciModuleRegistryAssertions.cs
View file
Open in desktop

This file was deleted.

Loading
Loading