From f33af92add89f8239f7ce326290825cdd44dfe51 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Tue, 29 Oct 2024 17:05:53 +0800 Subject: [PATCH 1/4] chore(workflow): set code coverage threshold to 80% (#151) Resolves: #150 Signed-off-by: Lixia (Sylvia) Lei --- .github/.codecov.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/.codecov.yml b/.github/.codecov.yml index 26afe32..fae4503 100644 --- a/.github/.codecov.yml +++ b/.github/.codecov.yml @@ -15,5 +15,9 @@ coverage: status: project: default: - target: 70% + target: 80% + if_ci_failed: error + patch: + default: + target: 80% if_ci_failed: error From 6d949294aeb28801b11f530cf5298aa1fa7ed183 Mon Sep 17 00:00:00 2001 From: nhu1997 <103352838+nhu1997@users.noreply.github.com> Date: Mon, 11 Nov 2024 22:57:06 -0800 Subject: [PATCH 2/4] feat: Implement Pack Manifests (#149) Signed-off-by: nhu1997 --- .../InvalidDateTimeFormatException.cs | 36 + .../Exceptions/InvalidMediaTypeException.cs | 36 + .../MissingArtifactTypeException.cs | 36 + src/OrasProject.Oras/Oci/Descriptor.cs | 22 + src/OrasProject.Oras/PackManifestOptions.cs | 55 ++ src/OrasProject.Oras/Packer.cs | 317 ++++++++ tests/OrasProject.Oras.Tests/PackerTest.cs | 725 ++++++++++++++++++ 7 files changed, 1227 insertions(+) create mode 100644 src/OrasProject.Oras/Exceptions/InvalidDateTimeFormatException.cs create mode 100644 src/OrasProject.Oras/Exceptions/InvalidMediaTypeException.cs create mode 100644 src/OrasProject.Oras/Exceptions/MissingArtifactTypeException.cs create mode 100644 src/OrasProject.Oras/PackManifestOptions.cs create mode 100644 src/OrasProject.Oras/Packer.cs create mode 100644 tests/OrasProject.Oras.Tests/PackerTest.cs diff --git a/src/OrasProject.Oras/Exceptions/InvalidDateTimeFormatException.cs b/src/OrasProject.Oras/Exceptions/InvalidDateTimeFormatException.cs new file mode 100644 index 0000000..76f9b34 --- /dev/null +++ b/src/OrasProject.Oras/Exceptions/InvalidDateTimeFormatException.cs @@ -0,0 +1,36 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace OrasProject.Oras.Exceptions; + +/// +/// InvalidDateTimeFormatException is thrown when a time format is invalid. +/// +public class InvalidDateTimeFormatException : FormatException +{ + public InvalidDateTimeFormatException() + { + } + + public InvalidDateTimeFormatException(string? message) + : base(message) + { + } + + public InvalidDateTimeFormatException(string? message, Exception? inner) + : base(message, inner) + { + } +} diff --git a/src/OrasProject.Oras/Exceptions/InvalidMediaTypeException.cs b/src/OrasProject.Oras/Exceptions/InvalidMediaTypeException.cs new file mode 100644 index 0000000..3e34b42 --- /dev/null +++ b/src/OrasProject.Oras/Exceptions/InvalidMediaTypeException.cs @@ -0,0 +1,36 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace OrasProject.Oras.Exceptions; + +/// +/// InvalidMediaTypeException is thrown when media type is invalid. +/// +public class InvalidMediaTypeException : FormatException +{ + public InvalidMediaTypeException() + { + } + + public InvalidMediaTypeException(string? message) + : base(message) + { + } + + public InvalidMediaTypeException(string? message, Exception? inner) + : base(message, inner) + { + } +} diff --git a/src/OrasProject.Oras/Exceptions/MissingArtifactTypeException.cs b/src/OrasProject.Oras/Exceptions/MissingArtifactTypeException.cs new file mode 100644 index 0000000..61b5627 --- /dev/null +++ b/src/OrasProject.Oras/Exceptions/MissingArtifactTypeException.cs @@ -0,0 +1,36 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace OrasProject.Oras.Exceptions; + +/// +/// MissingArtifactTypeException is thrown when artifactType is not found in manifest. +/// +public class MissingArtifactTypeException : FormatException +{ + public MissingArtifactTypeException() + { + } + + public MissingArtifactTypeException(string? message) + : base(message) + { + } + + public MissingArtifactTypeException(string? message, Exception? inner) + : base(message, inner) + { + } +} diff --git a/src/OrasProject.Oras/Oci/Descriptor.cs b/src/OrasProject.Oras/Oci/Descriptor.cs index 9720e15..12fe348 100644 --- a/src/OrasProject.Oras/Oci/Descriptor.cs +++ b/src/OrasProject.Oras/Oci/Descriptor.cs @@ -11,7 +11,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text; using System.Text.Json.Serialization; namespace OrasProject.Oras.Oci; @@ -47,5 +50,24 @@ public class Descriptor [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public string? ArtifactType { get; set; } + public static Descriptor Create(Span data, string mediaType) + { + byte[] byteData = data.ToArray(); + return new Descriptor + { + MediaType = mediaType, + Digest = Content.Digest.ComputeSHA256(byteData), + Size = byteData.Length + }; + } + + public static Descriptor Empty => new() + { + MediaType = Oci.MediaType.EmptyJson, + Digest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + Size = 2, + Data = [0x7B, 0x7D] + }; + internal BasicDescriptor BasicDescriptor => new BasicDescriptor(MediaType, Digest, Size); } diff --git a/src/OrasProject.Oras/PackManifestOptions.cs b/src/OrasProject.Oras/PackManifestOptions.cs new file mode 100644 index 0000000..cd859c8 --- /dev/null +++ b/src/OrasProject.Oras/PackManifestOptions.cs @@ -0,0 +1,55 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using OrasProject.Oras.Oci; + +using System.Collections.Generic; +using System.Threading; + +namespace OrasProject.Oras; + +public struct PackManifestOptions +{ + public static PackManifestOptions None { get; } + + /// + /// Config references a configuration object for a container, by digest + /// For more details: https://github.com/opencontainers/image-spec/blob/v1.1.0/manifest.md#image-manifest-property-descriptions. + /// + public Descriptor? Config { get; set; } + + /// + /// Layers is an array of objects, and each object id a Content Descriptor (or simply Descriptor) + /// For more details: https://github.com/opencontainers/image-spec/blob/v1.1.0/manifest.md#image-manifest-property-descriptions. + /// + public IList? Layers { get; set; } + + /// + /// Subject is the subject of the manifest. + /// This option is invalid when PackManifestVersion is PackManifestVersion1_0. + /// + public Descriptor? Subject { get; set; } + + /// + /// ManifestAnnotations is OPTIONAL property. It contains arbitrary metadata for the image manifest + /// and MUST use the annotation rules + /// + public IDictionary? ManifestAnnotations { get; set; } + + /// + /// ConfigAnnotations is the annotation map of the config descriptor. + /// This option is valid only when Config is null. + /// + public IDictionary? ConfigAnnotations { get; set; } +} + diff --git a/src/OrasProject.Oras/Packer.cs b/src/OrasProject.Oras/Packer.cs new file mode 100644 index 0000000..f4816c2 --- /dev/null +++ b/src/OrasProject.Oras/Packer.cs @@ -0,0 +1,317 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using OrasProject.Oras.Oci; +using OrasProject.Oras.Content; +using OrasProject.Oras.Exceptions; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.RegularExpressions; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + + +namespace OrasProject.Oras; + +public static class Packer +{ + /// + /// ErrInvalidDateTimeFormat is returned + /// when "org.opencontainers.artifact.created" or "org.opencontainers.image.created" is provided, + /// but its value is not in RFC 3339 format. + /// Reference: https://www.rfc-editor.org/rfc/rfc3339#section-5.6 + /// + private const string _errInvalidDateTimeFormat = "invalid date and time format"; + + /// + /// ErrMissingArtifactType is returned + /// when ManifestVersion is Version1_1 and artifactType is empty + /// and the config media type is set to "application/vnd.oci.empty.v1+json". + /// + private const string _errMissingArtifactType = "missing artifact type"; + + public const string MediaTypeUnknownConfig = "application/vnd.unknown.config.v1+json"; + + public const string MediaTypeUnknownArtifact = "application/vnd.unknown.artifact.v1"; + + /// + /// ManifestVersion represents the manifest version used for PackManifest + /// + public enum ManifestVersion + { + // Version1_0 represents the OCI Image Manifest defined in image-spec v1.0.2. + // Reference: https://github.com/opencontainers/image-spec/blob/v1.0.2/manifest.md + Version1_0 = 1, + // Version1_1 represents the OCI Image Manifest defined in image-spec v1.1.0. + // Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0/manifest.md + Version1_1 = 2 + } + + /// + /// mediaTypeRegex checks the format of media types. + /// References: + /// - https://github.com/opencontainers/image-spec/blob/v1.1.0/schema/defs-descriptor.json#L7 + /// - https://datatracker.ietf.org/doc/html/rfc6838#section-4.2 + /// + private static readonly Regex _mediaTypeRegex = new Regex(@"^[A-Za-z0-9][A-Za-z0-9!#$&-^_.+]{0,126}/[A-Za-z0-9][A-Za-z0-9!#$&-^_.+]{0,126}(\+json)?$", RegexOptions.Compiled); + + /// + /// PackManifest generates an OCI Image Manifestbased on the given parameters + /// and pushes the packed manifest to a content storage/registry using pusher. + /// The version of the manifest to be packed is determined by manifestVersion + /// (Recommended value: Version1_1). + /// - If manifestVersion is Version1_1 + /// artifactType MUST NOT be empty unless PackManifestOptions.ConfigDescriptor is specified. + /// - If manifestVersion is Version1_0 + /// if PackManifestOptions.ConfigDescriptor is null, artifactType will be used as the + /// config media type; if artifactType is empty, + /// "application/vnd.unknown.config.v1+json" will be used. + /// if PackManifestOptions.ConfigDescriptor is NOT null, artifactType will be ignored. + /// + /// artifactType and PackManifestOptions.ConfigDescriptor.MediaType MUST comply with RFC 6838. + /// + /// Each time when PackManifest is called, if a time stamp is not specified, a new time + /// stamp is generated in the manifest annotations with the key ocispec.AnnotationCreated + /// (i.e. "org.opencontainers.image.created"). To make PackManifest reproducible, + /// set the key ocispec.AnnotationCreated to a fixed value in + /// opts.Annotations. The value MUST conform to RFC 3339. + /// + /// If succeeded, returns a descriptor of the packed manifest. + /// + /// + /// + /// + /// + /// + /// + /// + public static async Task PackManifestAsync( + IPushable pusher, + ManifestVersion version, + string? artifactType, + PackManifestOptions options = default, + CancellationToken cancellationToken = default) + { + switch (version) + { + case ManifestVersion.Version1_0: + return await PackManifestV1_0Async(pusher, artifactType, options, cancellationToken); + case ManifestVersion.Version1_1: + return await PackManifestV1_1Async(pusher, artifactType, options, cancellationToken); + default: + throw new NotSupportedException($"ManifestVersion({version}) is not supported"); + } + } + + /// + /// Pack version 1.0 manifest + /// + /// + /// + /// + /// + /// + /// + private static async Task PackManifestV1_0Async(IPushable pusher, string? artifactType, PackManifestOptions options = default, CancellationToken cancellationToken = default) + { + if (options.Subject != null) + { + throw new NotSupportedException("Subject is not supported for manifest version 1.0."); + } + + Descriptor configDescriptor; + + if (options.Config != null) + { + ValidateMediaType(options.Config.MediaType); + configDescriptor = options.Config; + } + else + { + if (string.IsNullOrEmpty(artifactType)) + { + artifactType = MediaTypeUnknownConfig; + } + ValidateMediaType(artifactType); + configDescriptor = await PushCustomEmptyConfigAsync(pusher, artifactType, options.ConfigAnnotations, cancellationToken); + } + + var annotations = EnsureAnnotationCreated(options.ManifestAnnotations, "org.opencontainers.image.created"); + var manifest = new Manifest + { + SchemaVersion = 2, + MediaType = Oci.MediaType.ImageManifest, + Config = configDescriptor, + Layers = options.Layers ?? new List(), + Annotations = annotations + }; + + return await PushManifestAsync(pusher, manifest, manifest.MediaType, manifest.Config.MediaType, manifest.Annotations, cancellationToken); + } + + /// + /// Pack version 1.1 manifest + /// + /// + /// + /// + /// + /// + /// + private static async Task PackManifestV1_1Async(IPushable pusher, string? artifactType, PackManifestOptions options = default, CancellationToken cancellationToken = default) + { + if (string.IsNullOrEmpty(artifactType) && (options.Config == null || options.Config.MediaType == MediaType.EmptyJson)) + { + throw new MissingArtifactTypeException(_errMissingArtifactType); + } else if (!string.IsNullOrEmpty(artifactType)) { + ValidateMediaType(artifactType); + } + + Descriptor configDescriptor; + + if (options.Config != null) + { + ValidateMediaType(options.Config.MediaType); + configDescriptor = options.Config; + } + else + { + configDescriptor = Descriptor.Empty; + options.Config = configDescriptor; + var configBytes = new byte[] { 0x7B, 0x7D }; + await PushIfNotExistAsync(pusher, configDescriptor, configBytes, cancellationToken); + } + + if (options.Layers == null || options.Layers.Count == 0) + { + options.Layers ??= new List(); + // use the empty descriptor as the single layer + options.Layers.Add(Descriptor.Empty); + } + + var annotations = EnsureAnnotationCreated(options.ManifestAnnotations, "org.opencontainers.image.created"); + + var manifest = new Manifest + { + SchemaVersion = 2, + MediaType = MediaType.ImageManifest, + ArtifactType = artifactType, + Subject = options.Subject, + Config = options.Config, + Layers = options.Layers, + Annotations = annotations + }; + + return await PushManifestAsync(pusher, manifest, manifest.MediaType, manifest.ArtifactType, manifest.Annotations, cancellationToken); + } + + /// + /// Save manifest to local or remote storage + /// + /// + /// + /// + /// + /// + /// + /// + private static async Task PushManifestAsync(IPushable pusher, object manifest, string mediaType, string? artifactType, IDictionary? annotations, CancellationToken cancellationToken = default) + { + var manifestJson = JsonSerializer.SerializeToUtf8Bytes(manifest); + var manifestDesc = Descriptor.Create(manifestJson, mediaType); + manifestDesc.ArtifactType = artifactType; + manifestDesc.Annotations = annotations; + + await pusher.PushAsync(manifestDesc, new MemoryStream(manifestJson), cancellationToken); + return manifestDesc; + } + + /// + /// Validate manifest media type + /// + /// + /// + private static void ValidateMediaType(string mediaType) + { + if (!_mediaTypeRegex.IsMatch(mediaType)) + { + throw new InvalidMediaTypeException($"{mediaType} is an invalid media type"); + } + } + + /// + /// Push an empty configure with unknown media type to storage + /// + /// + /// + /// + /// + /// + private static async Task PushCustomEmptyConfigAsync(IPushable pusher, string mediaType, IDictionary? annotations, CancellationToken cancellationToken = default) + { + var configBytes = JsonSerializer.SerializeToUtf8Bytes(new { }); + var configDescriptor = Descriptor.Create(configBytes, mediaType); + configDescriptor.Annotations = annotations; + + await PushIfNotExistAsync(pusher, configDescriptor, configBytes, cancellationToken); + return configDescriptor; + } + + /// + /// Push data to local or remote storage + /// + /// + /// + /// + /// + /// + private static async Task PushIfNotExistAsync(IPushable pusher, Descriptor descriptor, byte[] data, CancellationToken cancellationToken = default) + { + await pusher.PushAsync(descriptor, new MemoryStream(data), cancellationToken); + } + + /// + /// Validate the value of the key in annotations should have correct timestamp format. + /// If the key is missed, the key and current timestamp is added to the annotations + /// + /// + /// + /// + /// + private static IDictionary EnsureAnnotationCreated(IDictionary? annotations, string key) + { + if (annotations == null) + { + annotations = new Dictionary(); + } + + string? value; + if (annotations.TryGetValue(key, out value)) + { + if (!DateTime.TryParse(value, out _)) + { + throw new InvalidDateTimeFormatException(_errInvalidDateTimeFormat); + } + + return annotations; + } + + var copiedAnnotations = new Dictionary(annotations); + copiedAnnotations[key] = DateTime.UtcNow.ToString("o"); + + return copiedAnnotations; + } +} diff --git a/tests/OrasProject.Oras.Tests/PackerTest.cs b/tests/OrasProject.Oras.Tests/PackerTest.cs new file mode 100644 index 0000000..c656d87 --- /dev/null +++ b/tests/OrasProject.Oras.Tests/PackerTest.cs @@ -0,0 +1,725 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using OrasProject.Oras.Content; +using OrasProject.Oras.Oci; +using OrasProject.Oras.Exceptions; +using System.Text; +using System.Text.Json; +using Xunit; + +namespace OrasProject.Oras.Tests; + +public class PackerTest +{ + [Fact] + public async Task TestPackManifestImageV1_0() + { + var memoryTarget = new MemoryStore(); + + // Test PackManifest + var cancellationToken = new CancellationToken(); + var manifestVersion = Packer.ManifestVersion.Version1_0; + var artifactType = "application/vnd.test"; + var manifestDesc = await Packer.PackManifestAsync(memoryTarget, manifestVersion, artifactType, new PackManifestOptions(), cancellationToken); + Assert.NotNull(manifestDesc); + + Manifest? manifest; + var rc = await memoryTarget.FetchAsync(manifestDesc, cancellationToken); + Assert.NotNull(rc); + using (rc) + { + manifest = await JsonSerializer.DeserializeAsync(rc!); + } + Assert.NotNull(manifest); + + // Verify media type + var got = manifest?.MediaType; + Assert.Equal("application/vnd.oci.image.manifest.v1+json", got); + + // Verify config + var expectedConfigData = System.Text.Encoding.UTF8.GetBytes("{}"); + var expectedConfig = new Descriptor + { + MediaType = artifactType, + Digest = Digest.ComputeSHA256(expectedConfigData), + Size = expectedConfigData.Length + }; + var expectedConfigBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(expectedConfig)); + var incomingConfig = manifest?.Config; + var incomingConfigBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(incomingConfig)); + //Assert.True(manifest.Config.Equals(expectedConfig), $"got config = {manifest.Config}, want {expectedConfig}"); + Assert.Equal(incomingConfigBytes, expectedConfigBytes); + + // Verify layers + var expectedLayers = new List(); + Assert.True(manifest!.Layers.SequenceEqual(expectedLayers), $"got layers = {manifest.Layers}, want {expectedLayers}"); + + // Verify created time annotation + Assert.True(manifest.Annotations!.TryGetValue("org.opencontainers.image.created", out var createdTime), $"Annotation \"org.opencontainers.image.created\" not found"); + Assert.True(DateTime.TryParse(createdTime, out _), $"Error parsing created time: {createdTime}"); + + // Verify descriptor annotations + Assert.True(manifestDesc.Annotations!.SequenceEqual(manifest?.Annotations!), $"got descriptor annotations = {manifestDesc.Annotations}, want {manifest!.Annotations}"); + } + + [Fact] + public async Task TestPackManifestImageV1_0WithoutPassingOptions() + { + var memoryTarget = new MemoryStore(); + + // Test PackManifest + var manifestVersion = Packer.ManifestVersion.Version1_0; + var artifactType = "application/vnd.test"; + var manifestDesc = await Packer.PackManifestAsync(memoryTarget, manifestVersion, artifactType); + Assert.NotNull(manifestDesc); + + Manifest? manifest; + var rc = await memoryTarget.FetchAsync(manifestDesc); + Assert.NotNull(rc); + using (rc) + { + manifest = await JsonSerializer.DeserializeAsync(rc!); + } + Assert.NotNull(manifest); + + // Verify media type + var got = manifest?.MediaType; + Assert.Equal("application/vnd.oci.image.manifest.v1+json", got); + + // Verify config + var expectedConfigData = System.Text.Encoding.UTF8.GetBytes("{}"); + var expectedConfig = Descriptor.Create(expectedConfigData, artifactType); + var expectedConfigBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(expectedConfig)); + var incomingConfig = manifest?.Config; + var incomingConfigBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(incomingConfig)); + //Assert.True(manifest.Config.Equals(expectedConfig), $"got config = {manifest.Config}, want {expectedConfig}"); + Assert.Equal(incomingConfigBytes, expectedConfigBytes); + + // Verify layers + var expectedLayers = new List(); + Assert.True(manifest!.Layers.SequenceEqual(expectedLayers), $"got layers = {manifest.Layers}, want {expectedLayers}"); + + // Verify created time annotation + Assert.True(manifest.Annotations!.TryGetValue("org.opencontainers.image.created", out var createdTime), $"Annotation \"org.opencontainers.image.created\" not found"); + Assert.True(DateTime.TryParse(createdTime, out _), $"Error parsing created time: {createdTime}"); + + // Verify descriptor annotations + Assert.True(manifestDesc.Annotations!.SequenceEqual(manifest?.Annotations!), $"got descriptor annotations = {manifestDesc.Annotations}, want {manifest!.Annotations}"); + } + + [Fact] + public async Task TestPackManifestImageV1_0_WithOptions() + { + var memoryTarget = new MemoryStore(); + + // Prepare test content + var cancellationToken = new CancellationToken(); + var blobs = new List(); + var descs = new List(); + var appendBlob = (string mediaType, byte[] blob) => + { + blobs.Add(blob); + var desc = new Descriptor + { + MediaType = mediaType, + Digest = Digest.ComputeSHA256(blob), + Size = blob.Length + }; + descs.Add(desc); + }; + var generateManifest = (Descriptor config, List layers) => + { + var manifest = new Manifest + { + Config = config, + Layers = layers + }; + var manifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest)); + appendBlob(Oci.MediaType.ImageManifest, manifestBytes); + }; + var getBytes = (string data) => Encoding.UTF8.GetBytes(data); + appendBlob(Oci.MediaType.ImageConfig, getBytes("config")); // blob 0 + appendBlob(Oci.MediaType.ImageLayer, getBytes("hello world")); // blob 1 + appendBlob(Oci.MediaType.ImageLayer, getBytes("goodbye world")); // blob 2 + var layers = descs.GetRange(1, 2); + var configBytes = Encoding.UTF8.GetBytes("{}"); + var configDesc = new Descriptor + { + MediaType = "application/vnd.test.config", + Digest = Digest.ComputeSHA256(configBytes), + Size = configBytes.Length + }; + var configAnnotations = new Dictionary { { "foo", "bar" } }; + var annotations = new Dictionary + { + { "org.opencontainers.image.created", "2000-01-01T00:00:00Z" }, + { "foo", "bar" } + }; + var artifactType = "application/vnd.test"; + + // Test PackManifest with ConfigDescriptor + var opts = new PackManifestOptions + { + Config = configDesc, + Layers = layers, + ManifestAnnotations = annotations, + ConfigAnnotations = configAnnotations + }; + var manifestDesc = await Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_0, artifactType, opts, cancellationToken); + + var expectedManifest = new Manifest + { + SchemaVersion = 2, + MediaType = "application/vnd.oci.image.manifest.v1+json", + Config = configDesc, + Layers = layers, + Annotations = annotations + }; + var expectedManifestBytes = JsonSerializer.SerializeToUtf8Bytes(expectedManifest); + + using var rc = await memoryTarget.FetchAsync(manifestDesc, cancellationToken); + Assert.NotNull(rc); + var memoryStream = new MemoryStream(); + await rc.CopyToAsync(memoryStream); + var got = memoryStream.ToArray(); + Assert.Equal(expectedManifestBytes, got); + + // Verify descriptor + var expectedManifestDesc = new Descriptor + { + MediaType = expectedManifest.MediaType, + Digest = Digest.ComputeSHA256(expectedManifestBytes), + Size = expectedManifestBytes.Length + }; + expectedManifestDesc.ArtifactType = expectedManifest.Config.MediaType; + expectedManifestDesc.Annotations = expectedManifest.Annotations; + var expectedManifestDescBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(expectedManifestDesc)); + var manifestDescBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifestDesc)); + Assert.Equal(expectedManifestDescBytes, manifestDescBytes); + + // Test PackManifest without ConfigDescriptor + opts = new PackManifestOptions + { + Layers = layers, + ManifestAnnotations = annotations, + ConfigAnnotations = configAnnotations + }; + manifestDesc = await Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_0, artifactType, opts, cancellationToken); + + var expectedConfigDesc = new Descriptor + { + MediaType = artifactType, + Digest = Digest.ComputeSHA256(configBytes), + Annotations = configAnnotations, + Size = configBytes.Length + }; + expectedManifest = new Manifest + { + SchemaVersion = 2, + MediaType = "application/vnd.oci.image.manifest.v1+json", + Config = expectedConfigDesc, + Layers = layers, + Annotations = annotations + }; + expectedManifestBytes = JsonSerializer.SerializeToUtf8Bytes(expectedManifest); + + using var rc2 = await memoryTarget.FetchAsync(manifestDesc, cancellationToken); + Assert.NotNull(rc2); + Manifest? manifest2 = await JsonSerializer.DeserializeAsync(rc2!); + var got2 = JsonSerializer.SerializeToUtf8Bytes(manifest2); + Assert.Equal(expectedManifestBytes, got2); + + // Verify descriptor + expectedManifestDesc = new Descriptor + { + MediaType = expectedManifest.MediaType, + Digest = Digest.ComputeSHA256(expectedManifestBytes), + Size = expectedManifestBytes.Length + }; + expectedManifestDesc.ArtifactType = expectedManifest.Config.MediaType; + expectedManifestDesc.Annotations = expectedManifest.Annotations; + Assert.Equal(JsonSerializer.SerializeToUtf8Bytes(expectedManifestDesc), JsonSerializer.SerializeToUtf8Bytes(manifestDesc)); + } + + [Fact] + public async Task TestPackManifestImageV1_0_SubjectUnsupported() + { + var memoryTarget = new MemoryStore(); + + // Prepare test content + var artifactType = "application/vnd.test"; + var subjectManifest = Encoding.UTF8.GetBytes(@"{""layers"":[]}"); + var subjectDesc = new Descriptor + { + MediaType = "application/vnd.oci.image.manifest.v1+json", + Digest = Digest.ComputeSHA256(subjectManifest), + Size = subjectManifest.Length + }; + + // Test PackManifest with ConfigDescriptor + var cancellationToken = new CancellationToken(); + var opts = new PackManifestOptions + { + Subject = subjectDesc + }; + + var exception = await Assert.ThrowsAsync(async () => + { + await Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_0, artifactType, opts, cancellationToken); + }); + + Assert.Equal("Subject is not supported for manifest version 1.0.", exception.Message); + } + + [Fact] + public async Task TestPackManifestImageV1_0_NoArtifactType() + { + var memoryTarget = new MemoryStore(); + var cancellationToken = new CancellationToken(); + + // Call PackManifest with empty artifact type + var manifestDesc = await Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_0, "", new PackManifestOptions(), cancellationToken); + + var rc = await memoryTarget.FetchAsync(manifestDesc, cancellationToken); + Assert.NotNull(rc); + Manifest? manifest = await JsonSerializer.DeserializeAsync(rc); + + // Verify artifact type and config media type + + Assert.Equal(Packer.MediaTypeUnknownConfig, manifestDesc.ArtifactType); + Assert.Equal(Packer.MediaTypeUnknownConfig, manifest!.Config.MediaType); + } + + [Fact] + public void TestPackManifestImageV1_0_InvalidMediaType() + { + var memoryTarget = new MemoryStore(); + var cancellationToken = new CancellationToken(); + + // Test invalid artifact type + valid config media type + string artifactType = "random"; + byte[] configBytes = System.Text.Encoding.UTF8.GetBytes("{}"); + var configDesc = new Descriptor + { + MediaType = "application/vnd.test.config", + Digest = Digest.ComputeSHA256(configBytes), + Size = configBytes.Length + }; + var opts = new PackManifestOptions + { + Config = configDesc + }; + + try + { + var manifestDesc = Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_0, artifactType, opts, cancellationToken); + } + catch (Exception ex) + { + Assert.Null(ex); // Expecting no exception + } + + // Test invalid config media type + valid artifact type + artifactType = "application/vnd.test"; + configDesc = new Descriptor + { + MediaType = "random", + Digest = Digest.ComputeSHA256(configBytes), + Size = configBytes.Length + }; + opts = new PackManifestOptions + { + Config = configDesc + }; + + try + { + var manifestDesc = Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_0, artifactType, opts, cancellationToken); + } + catch (Exception ex) + { + Assert.True(ex is InvalidMediaTypeException, $"Expected InvalidMediaTypeException but got {ex.GetType().Name}"); + } + } + + [Fact] + public void TestPackManifestImageV1_0_InvalidDateTimeFormat() + { + var memoryTarget = new MemoryStore(); + var cancellationToken = new CancellationToken(); + + var opts = new PackManifestOptions + { + ManifestAnnotations = new Dictionary + { + { "org.opencontainers.image.created", "2000/01/01 00:00:00" } + } + }; + + try + { + var manifestDesc = Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_0, "", opts, cancellationToken); + } + catch (Exception ex) + { + // Check if the caught exception is of type InvalidDateTimeFormatException + Assert.True(ex is InvalidDateTimeFormatException, $"Expected InvalidDateTimeFormatException but got {ex.GetType().Name}"); + } + } + + [Fact] + public async Task TestPackManifestImageV1_1() + { + var memoryTarget = new MemoryStore(); + var cancellationToken = new CancellationToken(); + + // Test PackManifest + var artifactType = "application/vnd.test"; + var manifestDesc = await Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_1, artifactType, new PackManifestOptions(), cancellationToken); + + // Fetch and decode the manifest + var rc = await memoryTarget.FetchAsync(manifestDesc, cancellationToken); + Manifest? manifest; + Assert.NotNull(rc); + using (rc) + { + manifest = await JsonSerializer.DeserializeAsync(rc); + } + + // Verify layers + var emptyConfigBytes = Encoding.UTF8.GetBytes("{}"); + var emptyJSON = Descriptor.Empty; + var expectedLayers = new List { emptyJSON }; + Assert.Equal(JsonSerializer.SerializeToUtf8Bytes(expectedLayers), JsonSerializer.SerializeToUtf8Bytes(manifest!.Layers)); + } + + [Fact] + public async Task TestPackManifestImageV1_1WithoutPassingOptions() + { + var memoryTarget = new MemoryStore(); + + // Test PackManifest + var artifactType = "application/vnd.test"; + var manifestDesc = await Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_1, artifactType); + + // Fetch and decode the manifest + var rc = await memoryTarget.FetchAsync(manifestDesc); + Manifest? manifest; + Assert.NotNull(rc); + using (rc) + { + manifest = await JsonSerializer.DeserializeAsync(rc); + } + + // Verify layers + var emptyConfigBytes = Encoding.UTF8.GetBytes("{}"); + var emptyJSON = new Descriptor + { + MediaType = "application/vnd.oci.empty.v1+json", + Digest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + Size = emptyConfigBytes.Length, + Data = emptyConfigBytes + }; + var expectedLayers = new List { emptyJSON }; + Assert.Equal(JsonSerializer.SerializeToUtf8Bytes(expectedLayers), JsonSerializer.SerializeToUtf8Bytes(manifest!.Layers)); + } + + [Fact] + public async Task TestPackManifestImageV1_1_WithOptions() + { + var memoryTarget = new MemoryStore(); + var cancellationToken = new CancellationToken(); + + // Prepare test content + byte[] hellogBytes = System.Text.Encoding.UTF8.GetBytes("hello world"); + byte[] goodbyeBytes = System.Text.Encoding.UTF8.GetBytes("goodbye world"); + var layers = new List + { + new Descriptor + { + MediaType = "test", + Data = hellogBytes, + Digest = Digest.ComputeSHA256(hellogBytes), + Size = hellogBytes.Length + }, + new Descriptor + { + MediaType = "test", + Data = goodbyeBytes, + Digest = Digest.ComputeSHA256(goodbyeBytes), + Size = goodbyeBytes.Length + } + }; + var configBytes = System.Text.Encoding.UTF8.GetBytes("config"); + var configDesc = new Descriptor + { + MediaType = "application/vnd.test", + Data = configBytes, + Digest = Digest.ComputeSHA256(configBytes), + Size = configBytes.Length + }; + var configAnnotations = new Dictionary { { "foo", "bar" } }; + var annotations = new Dictionary + { + { "org.opencontainers.image.created", "2000-01-01T00:00:00Z" }, + { "foo", "bar" } + }; + var artifactType = "application/vnd.test"; + var subjectManifest = System.Text.Encoding.UTF8.GetBytes("{\"layers\":[]}"); + var subjectDesc = new Descriptor + { + MediaType = "application/vnd.oci.image.manifest.v1+json", + Digest = Digest.ComputeSHA256(subjectManifest), + Size = subjectManifest.Length + }; + + // Test PackManifest with ConfigDescriptor + var opts = new PackManifestOptions + { + Subject = subjectDesc, + Layers = layers, + Config = configDesc, + ConfigAnnotations = configAnnotations, + ManifestAnnotations = annotations + }; + var manifestDesc = await Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_1, artifactType, opts, cancellationToken); + + var expectedManifest = new Manifest + { + SchemaVersion = 2, // Historical value, doesn't pertain to OCI or Docker version + MediaType = "application/vnd.oci.image.manifest.v1+json", + ArtifactType = artifactType, + Subject = subjectDesc, + Config = configDesc, + Layers = layers, + Annotations = annotations + }; + var expectedManifestBytes = JsonSerializer.SerializeToUtf8Bytes(expectedManifest); + using var rc = await memoryTarget.FetchAsync(manifestDesc, cancellationToken); + Manifest? manifest = await JsonSerializer.DeserializeAsync(rc); + var got = JsonSerializer.SerializeToUtf8Bytes(manifest); + Assert.Equal(expectedManifestBytes, got); + + // Verify descriptor + var expectedManifestDesc = new Descriptor + { + MediaType = expectedManifest.MediaType, + Digest = Digest.ComputeSHA256(expectedManifestBytes), + Size = expectedManifestBytes.Length + }; + expectedManifestDesc.ArtifactType = expectedManifest.Config.MediaType; + expectedManifestDesc.Annotations = expectedManifest.Annotations; + var expectedManifestDescBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(expectedManifestDesc)); + var manifestDescBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifestDesc)); + Assert.Equal(expectedManifestDescBytes, manifestDescBytes); + + // Test PackManifest with ConfigDescriptor, but without artifactType + opts = new PackManifestOptions + { + Subject = subjectDesc, + Layers = layers, + Config = configDesc, + ConfigAnnotations = configAnnotations, + ManifestAnnotations = annotations + }; + + manifestDesc = await Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_1, null, opts, cancellationToken); + expectedManifest.ArtifactType = null; + expectedManifestBytes = JsonSerializer.SerializeToUtf8Bytes(expectedManifest); + using var rc2 = await memoryTarget.FetchAsync(manifestDesc, cancellationToken); + Manifest? manifest2 = await JsonSerializer.DeserializeAsync(rc2); + var got2 = JsonSerializer.SerializeToUtf8Bytes(manifest2); + Assert.Equal(expectedManifestBytes, got2); + + expectedManifestDesc = new Descriptor + { + MediaType = expectedManifest.MediaType, + Digest = Digest.ComputeSHA256(expectedManifestBytes), + Size = expectedManifestBytes.Length + }; + expectedManifestDesc.Annotations = expectedManifest.Annotations; + Assert.Equal(JsonSerializer.SerializeToUtf8Bytes(expectedManifestDesc), JsonSerializer.SerializeToUtf8Bytes(manifestDesc)); + + // Test Pack without ConfigDescriptor + opts = new PackManifestOptions + { + Subject = subjectDesc, + Layers = layers, + ConfigAnnotations = configAnnotations, + ManifestAnnotations = annotations + }; + + manifestDesc = await Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_1, artifactType, opts, cancellationToken); + var emptyConfigBytes = Encoding.UTF8.GetBytes("{}"); + var emptyJSON = new Descriptor + { + MediaType = "application/vnd.oci.empty.v1+json", + Digest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + Size = emptyConfigBytes.Length, + Data = emptyConfigBytes + }; + var expectedConfigDesc = emptyJSON; + expectedManifest.ArtifactType = artifactType; + expectedManifest.Config = expectedConfigDesc; + expectedManifestBytes = JsonSerializer.SerializeToUtf8Bytes(expectedManifest); + using var rc3 = await memoryTarget.FetchAsync(manifestDesc, cancellationToken); + Manifest? manifest3 = await JsonSerializer.DeserializeAsync(rc3); + var got3 = JsonSerializer.SerializeToUtf8Bytes(manifest3); + Assert.Equal(expectedManifestBytes, got3); + + expectedManifestDesc = new Descriptor + { + MediaType = expectedManifest.MediaType, + Digest = Digest.ComputeSHA256(expectedManifestBytes), + Size = expectedManifestBytes.Length + }; + expectedManifestDesc.ArtifactType = artifactType; + expectedManifestDesc.Annotations = expectedManifest.Annotations; + Assert.Equal(JsonSerializer.SerializeToUtf8Bytes(expectedManifestDesc), JsonSerializer.SerializeToUtf8Bytes(manifestDesc)); + } + + [Fact] + public async Task TestPackManifestImageV1_1_NoArtifactType() + { + var memoryTarget = new MemoryStore(); + var cancellationToken = new CancellationToken(); + + // Test no artifact type and no config + try + { + var manifestDesc = await Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_1, "", new PackManifestOptions(), cancellationToken); + } + catch (Exception ex) + { + Assert.True(ex is MissingArtifactTypeException, $"Expected Artifact found in manifest without config"); + } + // Test no artifact type and config with empty media type + var emptyConfigBytes = Encoding.UTF8.GetBytes("{}"); + var opts = new PackManifestOptions + { + Config = new Descriptor + { + MediaType = "application/vnd.oci.empty.v1+json", + Digest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + Size = emptyConfigBytes.Length, + Data = emptyConfigBytes + } + }; + try + { + var manifestDesc = await Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_1, "", opts, cancellationToken); + } + catch (Exception ex) + { + // Check if the caught exception is of type InvalidDateTimeFormatException + Assert.True(ex is MissingArtifactTypeException, $"Expected Artifact found in manifest with empty config"); + } + } + + [Fact] + public void Test_PackManifestImageV1_1_InvalidMediaType() + { + var memoryTarget = new MemoryStore(); + var cancellationToken = new CancellationToken(); + + // Test invalid artifact type + valid config media type + var artifactType = "random"; + byte[] configBytes = System.Text.Encoding.UTF8.GetBytes("{}"); + var configDesc = new Descriptor + { + MediaType = "application/vnd.test.config", + Digest = Digest.ComputeSHA256(configBytes), + Size = configBytes.Length + }; + var opts = new PackManifestOptions + { + Config = configDesc + }; + + try + { + var manifestDesc = Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_1, artifactType, opts, cancellationToken); + } + catch (Exception ex) + { + Assert.Null(ex); // Expecting no exception + } + + // Test invalid config media type + valid artifact type + artifactType = "application/vnd.test"; + configDesc = new Descriptor + { + MediaType = "random", + Digest = Digest.ComputeSHA256(configBytes), + Size = configBytes.Length + }; + opts = new PackManifestOptions + { + Config = configDesc + }; + + try + { + var manifestDesc = Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_1, artifactType, opts, cancellationToken); + } + catch (Exception ex) + { + Assert.True(ex is InvalidMediaTypeException, $"Expected InvalidMediaTypeException but got {ex.GetType().Name}"); + } + } + + [Fact] + public void TestPackManifestImageV1_1_InvalidDateTimeFormat() + { + var memoryTarget = new MemoryStore(); + var cancellationToken = new CancellationToken(); + + var opts = new PackManifestOptions + { + ManifestAnnotations = new Dictionary + { + { "org.opencontainers.image.created", "2000/01/01 00:00:00" } + } + }; + + var artifactType = "application/vnd.test"; + try + { + var manifestDesc = Packer.PackManifestAsync(memoryTarget, Packer.ManifestVersion.Version1_1, artifactType, opts, cancellationToken); + } + catch (Exception ex) + { + // Check if the caught exception is of type InvalidDateTimeFormatException + Assert.True(ex is InvalidDateTimeFormatException, $"Expected InvalidDateTimeFormatException but got {ex.GetType().Name}"); + } + + } + + [Fact] + public void TestPackManifestUnsupportedPackManifestVersion() + { + var memoryTarget = new MemoryStore(); + var cancellationToken = new CancellationToken(); + + try + { + var manifestDesc = Packer.PackManifestAsync(memoryTarget, (Packer.ManifestVersion)(-1), "", new PackManifestOptions(), cancellationToken); + } + catch (Exception ex) + { + // Check if the caught exception is of type InvalidDateTimeFormatException + Assert.True(ex is NotSupportedException, $"Expected InvalidDateTimeFormatException but got {ex.GetType().Name}"); + } + } +} From e1d8dc69e923591387433140eaa51c29aab4311a Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Tue, 12 Nov 2024 23:38:36 -0300 Subject: [PATCH 3/4] feat(repository): mounting support (#152) Signed-off-by: Leonardo Chaia --- src/OrasProject.Oras/Registry/IMounter.cs | 40 +++ src/OrasProject.Oras/Registry/IRepository.cs | 2 +- .../Registry/Remote/BlobStore.cs | 135 ++++++-- .../Registry/Remote/Repository.cs | 18 ++ .../Remote/RepositoryTest.cs | 298 +++++++++++++++++- 5 files changed, 471 insertions(+), 22 deletions(-) create mode 100644 src/OrasProject.Oras/Registry/IMounter.cs diff --git a/src/OrasProject.Oras/Registry/IMounter.cs b/src/OrasProject.Oras/Registry/IMounter.cs new file mode 100644 index 0000000..e9e36be --- /dev/null +++ b/src/OrasProject.Oras/Registry/IMounter.cs @@ -0,0 +1,40 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using OrasProject.Oras.Oci; + +namespace OrasProject.Oras.Registry; + +/// +/// Mounter allows cross-repository blob mounts. +/// +public interface IMounter +{ + /// + /// Mount makes the blob with the given descriptor in fromRepo + /// available in the repository signified by the receiver. + /// + /// + /// + /// + /// + /// + Task MountAsync(Descriptor descriptor, + string fromRepository, + Func>? getContent = null, + CancellationToken cancellationToken = default); +} diff --git a/src/OrasProject.Oras/Registry/IRepository.cs b/src/OrasProject.Oras/Registry/IRepository.cs index b163e2f..41682c5 100644 --- a/src/OrasProject.Oras/Registry/IRepository.cs +++ b/src/OrasProject.Oras/Registry/IRepository.cs @@ -27,7 +27,7 @@ namespace OrasProject.Oras.Registry; /// Furthermore, this interface also provides the ability to enforce the /// separation of the blob and the manifests CASs. /// -public interface IRepository : ITarget, IReferenceFetchable, IReferencePushable, IDeletable, ITagListable +public interface IRepository : ITarget, IReferenceFetchable, IReferencePushable, IDeletable, ITagListable, IMounter { /// /// Blobs provides access to the blob CAS only, which contains config blobs,layers, and other generic blobs. diff --git a/src/OrasProject.Oras/Registry/Remote/BlobStore.cs b/src/OrasProject.Oras/Registry/Remote/BlobStore.cs index 52b0783..d5bba1b 100644 --- a/src/OrasProject.Oras/Registry/Remote/BlobStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/BlobStore.cs @@ -25,7 +25,7 @@ namespace OrasProject.Oras.Registry.Remote; -public class BlobStore(Repository repository) : IBlobStore +public class BlobStore(Repository repository) : IBlobStore, IMounter { public Repository Repository { get; init; } = repository; @@ -148,25 +148,7 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok url = location.IsAbsoluteUri ? location : new Uri(url, location); } - // monolithic upload - // add digest key to query string with expected digest value - var req = new HttpRequestMessage(HttpMethod.Put, new UriBuilder(url) - { - Query = $"{url.Query}&digest={HttpUtility.UrlEncode(expected.Digest)}" - }.Uri); - req.Content = new StreamContent(content); - req.Content.Headers.ContentLength = expected.Size; - - // the expected media type is ignored as in the API doc. - req.Content.Headers.ContentType = new MediaTypeHeaderValue(MediaTypeNames.Application.Octet); - - using (var response = await Repository.Options.HttpClient.SendAsync(req, cancellationToken).ConfigureAwait(false)) - { - if (response.StatusCode != HttpStatusCode.Created) - { - throw await response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(false); - } - } + await CompletePushAsync(url, expected, content, cancellationToken); } /// @@ -198,4 +180,117 @@ public async Task ResolveAsync(string reference, CancellationToken c /// public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) => await Repository.DeleteAsync(target, false, cancellationToken).ConfigureAwait(false); + + /// + /// Mounts the given descriptor from fromRepository into the blob store. + /// + /// + /// + /// + /// + /// + /// + public async Task MountAsync(Descriptor descriptor, string fromRepository, + Func>? getContent, CancellationToken cancellationToken) + { + var url = new UriFactory(Repository.Options).BuildRepositoryBlobUpload(); + var mountReq = new HttpRequestMessage(HttpMethod.Post, new UriBuilder(url) + { + Query = + $"{url.Query}&mount={HttpUtility.UrlEncode(descriptor.Digest)}&from={HttpUtility.UrlEncode(fromRepository)}" + }.Uri); + + using (var response = await Repository.Options.HttpClient.SendAsync(mountReq, cancellationToken) + .ConfigureAwait(false)) + { + switch (response.StatusCode) + { + case HttpStatusCode.Created: + // 201, layer has been mounted + response.VerifyContentDigest(descriptor.Digest); + return; + case HttpStatusCode.Accepted: + { + // 202, mounting failed. upload session has begun + var location = response.Headers.Location ?? + throw new HttpRequestException("missing location header"); + url = location.IsAbsoluteUri ? location : new Uri(url, location); + break; + } + default: + throw await response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(false); + } + } + + // From the [spec]: + // + // "If a registry does not support cross-repository mounting + // or is unable to mount the requested blob, + // it SHOULD return a 202. + // This indicates that the upload session has begun + // and that the client MAY proceed with the upload." + // + // So we need to get the content from somewhere in order to + // push it. If the caller has provided a getContent function, we + // can use that, otherwise pull the content from the source repository. + // + // [spec]: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#mounting-a-blob-from-another-repository + + async Task GetContentStream() + { + Stream stream; + if (getContent != null) + { + stream = await getContent(cancellationToken).ConfigureAwait(false); + } + else + { + var referenceOptions = repository.Options with + { + Reference = Reference.Parse(fromRepository), + }; + stream = await new Repository(referenceOptions).FetchAsync(descriptor, cancellationToken).ConfigureAwait(false); + } + + return stream; + } + + await using (var contents = await GetContentStream()) + { + await CompletePushAsync(url, descriptor, contents, cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Completes a push operation started beforehand. + /// + /// + /// + /// + /// + /// + private async Task CompletePushAsync(Uri url, Descriptor descriptor, Stream content, + CancellationToken cancellationToken) + { + // monolithic upload + // add digest key to query string with descriptor digest value + var req = new HttpRequestMessage(HttpMethod.Put, new UriBuilder(url) + { + Query = $"{url.Query}&digest={HttpUtility.UrlEncode(descriptor.Digest)}" + }.Uri); + req.Content = new StreamContent(content); + req.Content.Headers.ContentLength = descriptor.Size; + + // the descriptor media type is ignored as in the API doc. + req.Content.Headers.ContentType = new MediaTypeHeaderValue(MediaTypeNames.Application.Octet); + + using (var response = + await Repository.Options.HttpClient.SendAsync(req, cancellationToken).ConfigureAwait(false)) + { + if (response.StatusCode != HttpStatusCode.Created) + { + throw await response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(false); + } + } + } } diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index 62d73bc..085495d 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -331,4 +331,22 @@ internal Reference ParseReferenceFromContentReference(string reference) /// /// private IBlobStore BlobStore(Descriptor desc) => IsManifest(desc) ? Manifests : Blobs; + + /// + /// Mount makes the blob with the given digest in fromRepo + /// available in the repository signified by the receiver. + /// + /// This avoids the need to pull content down from fromRepo only to push it to r. + /// + /// If the registry does not implement mounting, getContent will be used to get the + /// content to push. If getContent is null, the content will be pulled from the source + /// repository. + /// + /// + /// + /// + /// + /// + public async Task MountAsync(Descriptor descriptor, string fromRepository, Func>? getContent = null, CancellationToken cancellationToken = default) + => await ((IMounter)Blobs).MountAsync(descriptor, fromRepository, getContent, cancellationToken).ConfigureAwait(false); } diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index 84df80a..5ac968d 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -18,7 +18,6 @@ using OrasProject.Oras.Oci; using OrasProject.Oras.Registry; using OrasProject.Oras.Registry.Remote; -using System.Collections.Immutable; using System.Diagnostics; using System.Net; using System.Net.Http.Headers; @@ -2281,4 +2280,301 @@ public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigest } } + + /// + /// Repository_MountAsync tests the MountAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_MountAsync() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var gotMount = 0; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var resp = new HttpResponseMessage(); + resp.RequestMessage = req; + if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/") + { + var queries = HttpUtility.ParseQueryString(req.RequestUri.Query); + if (queries["mount"] != blobDesc.Digest) + { + resp.StatusCode = HttpStatusCode.InternalServerError; + return resp; + } + if (queries["from"] != "test") + { + resp.StatusCode = HttpStatusCode.InternalServerError; + return resp; + } + gotMount++; + resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + resp.StatusCode = HttpStatusCode.Created; + return resp; + } + resp.StatusCode = HttpStatusCode.InternalServerError; + return resp; + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test2"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + + await repo.MountAsync(blobDesc, "test", null, cancellationToken); + Assert.Equal(1, gotMount); + } + + /// + /// Repository_MountAsync_Fallback tests the MountAsync method of the Repository when the server doesn't support mount query parameters. + /// + /// + [Fact] + public async Task Repository_MountAsync_Fallback() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + string sequence = ""; + byte[] gotBlob = Array.Empty(); + var uuid = "4fd53bc9-565d-4527-ab80-3e051ac4880c"; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var resp = new HttpResponseMessage(); + resp.RequestMessage = req; + if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/") + { + resp.Headers.Location = new Uri("/v2/test2/blobs/uploads/" + uuid, UriKind.Relative); + resp.StatusCode = HttpStatusCode.Accepted; + sequence += "post "; + return resp; + } + if (req.Method == HttpMethod.Get && req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) + { + resp.Content.Headers.Add("Content-Type", "application/octet-stream"); + resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + resp.Content = new ByteArrayContent(blob); + resp.StatusCode = HttpStatusCode.OK; + sequence += "get "; + return resp; + } + if (req.Method == HttpMethod.Put && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/" + uuid) + { + if (req.Content?.Headers.GetValues("Content-Type").FirstOrDefault() != "application/octet-stream") + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + if (HttpUtility.ParseQueryString(req.RequestUri.Query)["digest"] != blobDesc.Digest) + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + gotBlob = req.Content!.ReadAsByteArrayAsync(cancellationToken).Result; + resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + resp.StatusCode = HttpStatusCode.Created; + sequence += "put "; + return resp; + } + resp.StatusCode = HttpStatusCode.Forbidden; + return resp; + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test2"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + + // getContent is null + sequence = ""; + await repo.MountAsync(blobDesc, "localhost:5000/test", null, cancellationToken); + Assert.Equal(blob, gotBlob); + Assert.Equal("post get put ", sequence); + + // getContent is non-null + sequence = ""; + await repo.MountAsync(blobDesc, "localhost:5000/test", _ => Task.FromResult(new MemoryStream(blob)), cancellationToken); + Assert.Equal(blob, gotBlob); + Assert.Equal("post put ", sequence); + } + + /// + /// Repository_MountAsync_Error tests the error handling of the MountAsync method of the Repository. + /// + /// + [Fact] + public async Task Repository_MountAsync_Error() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var resp = new HttpResponseMessage(); + resp.RequestMessage = req; + if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/") + { + resp.StatusCode = HttpStatusCode.BadRequest; + resp.Content = new StringContent(@"{ ""errors"": [ { ""code"": ""NAME_UNKNOWN"", ""message"": ""some error"" } ] }"); + return resp; + } + resp.StatusCode = HttpStatusCode.InternalServerError; + return resp; + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + + var ex = await Assert.ThrowsAsync(async () => + { + await repo.MountAsync(blobDesc, "foo", null, cancellationToken); + }); + + Assert.NotNull(ex.Errors); + Assert.Single(ex.Errors); + Assert.Equal("NAME_UNKNOWN", ex.Errors[0].Code); + Assert.Equal("some error", ex.Errors[0].Message); + } + + /// + /// Repository_MountAsync_Fallback_GetContent tests the case where the server doesn't recognize mount query parameters, + /// falling back to the regular push flow, using the getContent function parameter. + /// + /// + [Fact] + public async Task Repository_MountAsync_Fallback_GetContent() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + string sequence = ""; + byte[] gotBlob = Array.Empty(); + var uuid = "4fd53bc9-565d-4527-ab80-3e051ac4880c"; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var resp = new HttpResponseMessage(); + resp.RequestMessage = req; + if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/") + { + resp.Headers.Location = new Uri("/v2/test2/blobs/uploads/" + uuid, UriKind.Relative); + resp.StatusCode = HttpStatusCode.Accepted; + sequence += "post "; + return resp; + } + if (req.Method == HttpMethod.Put && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/" + uuid) + { + if (req.Content?.Headers.GetValues("Content-Type").FirstOrDefault() != "application/octet-stream") + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + if (HttpUtility.ParseQueryString(req.RequestUri.Query)["digest"] != blobDesc.Digest) + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + gotBlob = req.Content!.ReadAsByteArrayAsync(cancellationToken).Result; + resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + resp.StatusCode = HttpStatusCode.Created; + sequence += "put "; + return resp; + } + resp.StatusCode = HttpStatusCode.Forbidden; + return resp; + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test2"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + + await repo.MountAsync(blobDesc, "test", _ => Task.FromResult(new MemoryStream(blob)), cancellationToken); + + // Assert that the blob was pushed correctly + Assert.Equal(blob, gotBlob); + // Assert that the request sequence matches the expected behavior + Assert.Equal("post put ", sequence); + } + + /// + /// Repository_MountAsync_Fallback_GetContentError tests the case where the server doesn't recognize mount query parameters, + /// falling back to the regular push flow, but the caller wants to avoid the pull/push pattern, so an error is returned from getContent. + /// + /// + [Fact] + public async Task Repository_MountAsync_Fallback_GetContentError() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + string sequence = ""; + var uuid = "4fd53bc9-565d-4527-ab80-3e051ac4880c"; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var resp = new HttpResponseMessage(); + resp.RequestMessage = req; + if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/") + { + resp.Headers.Location = new Uri("/v2/test2/blobs/uploads/" + uuid, UriKind.Relative); + resp.StatusCode = HttpStatusCode.Accepted; + sequence += "post "; + return resp; + } + resp.StatusCode = HttpStatusCode.Forbidden; + return resp; + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test2"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + + var testErr = new Exception("test error"); + var ex = await Assert.ThrowsAsync(async () => + { + await repo.MountAsync(blobDesc, "test", _ => throw testErr, cancellationToken); + }); + + Assert.Equal(testErr, ex); + Assert.Equal("post ", sequence); + } } From 3fa86d6a49cafc3aaabf0b5325ad3805c6ce1a3d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Nov 2024 02:43:01 +0000 Subject: [PATCH 4/4] build(deps): Bump System.Text.Json from 8.0.5 to 9.0.0 in /src/OrasProject.Oras (#156) --- src/OrasProject.Oras/OrasProject.Oras.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OrasProject.Oras/OrasProject.Oras.csproj b/src/OrasProject.Oras/OrasProject.Oras.csproj index b68ec65..3cbe77e 100644 --- a/src/OrasProject.Oras/OrasProject.Oras.csproj +++ b/src/OrasProject.Oras/OrasProject.Oras.csproj @@ -23,7 +23,7 @@ - +