From a42627b4add96f08a89a63f549b8f2079aed4630 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Mon, 15 Jul 2024 10:29:10 +0800 Subject: [PATCH 01/14] created manifest index subcommand Signed-off-by: Xiaoxuan Wang --- cmd/oras/root/manifest/cmd.go | 2 + cmd/oras/root/manifest/index/cmd.go | 17 +++ cmd/oras/root/manifest/index/create.go | 159 +++++++++++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 cmd/oras/root/manifest/index/cmd.go create mode 100644 cmd/oras/root/manifest/index/create.go diff --git a/cmd/oras/root/manifest/cmd.go b/cmd/oras/root/manifest/cmd.go index c68668614..2a84ab048 100644 --- a/cmd/oras/root/manifest/cmd.go +++ b/cmd/oras/root/manifest/cmd.go @@ -17,6 +17,7 @@ package manifest import ( "github.com/spf13/cobra" + "oras.land/oras/cmd/oras/root/manifest/index" ) func Cmd() *cobra.Command { @@ -30,6 +31,7 @@ func Cmd() *cobra.Command { fetchCmd(), fetchConfigCmd(), pushCmd(), + index.Cmd(), ) return cmd } diff --git a/cmd/oras/root/manifest/index/cmd.go b/cmd/oras/root/manifest/index/cmd.go new file mode 100644 index 000000000..286c8ebba --- /dev/null +++ b/cmd/oras/root/manifest/index/cmd.go @@ -0,0 +1,17 @@ +package index + +import ( + "github.com/spf13/cobra" +) + +func Cmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "index [command]", + Short: "Index operations", + } + + cmd.AddCommand( + createCmd(), + ) + return cmd +} diff --git a/cmd/oras/root/manifest/index/create.go b/cmd/oras/root/manifest/index/create.go new file mode 100644 index 000000000..08ec4d20a --- /dev/null +++ b/cmd/oras/root/manifest/index/create.go @@ -0,0 +1,159 @@ +package index + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/errdef" + "oras.land/oras-go/v2/registry" + "oras.land/oras/cmd/oras/internal/command" + oerrors "oras.land/oras/cmd/oras/internal/errors" + "oras.land/oras/cmd/oras/internal/option" + "oras.land/oras/internal/contentutil" +) + +type createOptions struct { + option.Common + option.Target + + sources []option.Target +} + +func createCmd() *cobra.Command { + var opts createOptions + cmd := &cobra.Command{ + Use: "create [flags] --repo [:|@] [...]", + Short: "create and push an index from provided manifests", + Long: `create and push an index to a repository or an OCI image layout +Example - create an index from source manifests tagged s1, s2, s3 in the repository + localhost:5000/hello, and push the index without tagging it : + oras index create --repo localhost:5000/hello s1 s2 s3 +Example - create an index from source manifests tagged s1, s2, s3 in the repository + localhost:5000/hello, and push the index with tag 'latest' : + oras index create --repo localhost:5000/hello --tag latest s1 s2 s3 +Example - create an index from source manifests using both tags and digests, + and push the index with tag 'latest' : + oras index create --repo localhost:5000/hello --tag latest s1 sha256:xxx s3 +`, + Args: cobra.MinimumNArgs(1), + PreRunE: func(cmd *cobra.Command, args []string) error { + // parse the target index reference + opts.RawReference = args[0] + repo, _, _ := strings.Cut(opts.RawReference, ":") + + // parse the source manifests + opts.sources = make([]option.Target, len(args)-1) + for i, a := range args[1:] { + var ref string + if contentutil.IsDigest(a) { + ref = fmt.Sprintf("%s@%s", repo, a) + } else { + ref = fmt.Sprintf("%s:%s", repo, a) + } + m := option.Target{RawReference: ref, Remote: opts.Remote} + if err := m.Parse(cmd); err != nil { + return err + } + opts.sources[i] = m + } + + return option.Parse(cmd, &opts) + }, + RunE: func(cmd *cobra.Command, args []string) error { + return createIndex(cmd, opts) + }, + } + + option.ApplyFlags(&opts, cmd.Flags()) + return oerrors.Command(cmd, &opts.Target) +} + +func createIndex(cmd *cobra.Command, opts createOptions) error { + ctx, logger := command.GetLogger(cmd, &opts.Common) + dst, err := opts.NewTarget(opts.Common, logger) + if err != nil { + return err + } + // we assume that the sources and the to be created index are all in the same + // repository, so no copy is needed + manifests, err := resolveSourceManifests(cmd, opts, logger) + if err != nil { + return err + } + desc, reader := packIndex(manifests) + if err := pushIndex(ctx, dst, desc, opts.Reference, reader); err != nil { + return err + } + return nil +} + +func resolveSourceManifests(cmd *cobra.Command, destOpts createOptions, logger logrus.FieldLogger) ([]ocispec.Descriptor, error) { + var resolved []ocispec.Descriptor + for _, source := range destOpts.sources { + var err error + // prepare sourceTarget target + sourceTarget, err := source.NewReadonlyTarget(cmd.Context(), destOpts.Common, logger) + if err != nil { + return []ocispec.Descriptor{}, err + } + if err := source.EnsureReferenceNotEmpty(cmd, false); err != nil { + return []ocispec.Descriptor{}, err + } + var desc ocispec.Descriptor + desc, err = oras.Resolve(cmd.Context(), sourceTarget, source.Reference, oras.DefaultResolveOptions) + if err != nil { + return []ocispec.Descriptor{}, fmt.Errorf("failed to resolve %s: %w", source.Reference, err) + } + resolved = append(resolved, desc) + } + return resolved, nil +} + +func packIndex(manifests []ocispec.Descriptor) (ocispec.Descriptor, io.Reader) { + // todo: oras-go needs PackIndex + index := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: manifests, + // todo: annotations + } + content, _ := json.Marshal(index) + desc := ocispec.Descriptor{ + Digest: digest.FromBytes(content), + MediaType: ocispec.MediaTypeImageIndex, + Size: int64(len(content)), + } + return desc, bytes.NewReader(content) +} + +func pushIndex(ctx context.Context, dst oras.Target, desc ocispec.Descriptor, ref string, content io.Reader) error { + if refPusher, ok := dst.(registry.ReferencePusher); ok { + if ref != "" { + return refPusher.PushReference(ctx, desc, content, ref) + } + } + if err := dst.Push(ctx, desc, content); err != nil { + w := errors.Unwrap(err) + if w != errdef.ErrAlreadyExists { + return err + } + } + if ref == "" { + fmt.Println("Digest of the pushed index: ", desc.Digest) + return nil + } + return dst.Tag(ctx, desc, ref) +} From 2e4dfb8adabdad332e4019eb616f8b40e567a15c Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Mon, 15 Jul 2024 12:26:52 +0800 Subject: [PATCH 02/14] auto detect platform Signed-off-by: Xiaoxuan Wang --- cmd/oras/root/manifest/index/cmd.go | 15 +++++++ cmd/oras/root/manifest/index/create.go | 55 ++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/cmd/oras/root/manifest/index/cmd.go b/cmd/oras/root/manifest/index/cmd.go index 286c8ebba..9e353e343 100644 --- a/cmd/oras/root/manifest/index/cmd.go +++ b/cmd/oras/root/manifest/index/cmd.go @@ -1,3 +1,18 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package index import ( diff --git a/cmd/oras/root/manifest/index/create.go b/cmd/oras/root/manifest/index/create.go index 08ec4d20a..4a9f42cb7 100644 --- a/cmd/oras/root/manifest/index/create.go +++ b/cmd/oras/root/manifest/index/create.go @@ -1,3 +1,18 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package index import ( @@ -15,12 +30,14 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/errdef" "oras.land/oras-go/v2/registry" "oras.land/oras/cmd/oras/internal/command" oerrors "oras.land/oras/cmd/oras/internal/errors" "oras.land/oras/cmd/oras/internal/option" "oras.land/oras/internal/contentutil" + "oras.land/oras/internal/descriptor" ) type createOptions struct { @@ -115,11 +132,49 @@ func resolveSourceManifests(cmd *cobra.Command, destOpts createOptions, logger l if err != nil { return []ocispec.Descriptor{}, fmt.Errorf("failed to resolve %s: %w", source.Reference, err) } + // detect platform information + // 1. fetch config descriptor + configDesc, err := fetchConfigDesc(cmd.Context(), sourceTarget, source.Reference) + if err != nil { + return []ocispec.Descriptor{}, err + } + // 2. fetch config content + contentBytes, err := content.FetchAll(cmd.Context(), sourceTarget, configDesc) + if err != nil { + return []ocispec.Descriptor{}, err + } + var config ocispec.Image + if err := json.Unmarshal(contentBytes, &config); err != nil { + return []ocispec.Descriptor{}, err + } + // 3. extract platform information + desc.Platform = &config.Platform + resolved = append(resolved, desc) } return resolved, nil } +func fetchConfigDesc(ctx context.Context, src oras.ReadOnlyTarget, reference string) (ocispec.Descriptor, error) { + // fetch manifest descriptor and content + fetchOpts := oras.DefaultFetchBytesOptions + manifestDesc, manifestContent, err := oras.FetchBytes(ctx, src, reference, fetchOpts) + if err != nil { + return ocispec.Descriptor{}, err + } + + if !descriptor.IsImageManifest(manifestDesc) { + return ocispec.Descriptor{}, fmt.Errorf("%q is not an image manifest and does not have a config", manifestDesc.Digest) + } + + // unmarshal manifest content to extract config descriptor + var manifest ocispec.Manifest + if err := json.Unmarshal(manifestContent, &manifest); err != nil { + return ocispec.Descriptor{}, err + } + return manifest.Config, nil +} + func packIndex(manifests []ocispec.Descriptor) (ocispec.Descriptor, io.Reader) { // todo: oras-go needs PackIndex index := ocispec.Index{ From bb60532856427d774e91300c458118b7d40264a3 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Thu, 18 Jul 2024 15:35:41 +0800 Subject: [PATCH 03/14] --add for update Signed-off-by: Xiaoxuan Wang --- cmd/oras/root/manifest/index/cmd.go | 1 + cmd/oras/root/manifest/index/create.go | 11 +- cmd/oras/root/manifest/index/update.go | 167 +++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 5 deletions(-) create mode 100644 cmd/oras/root/manifest/index/update.go diff --git a/cmd/oras/root/manifest/index/cmd.go b/cmd/oras/root/manifest/index/cmd.go index 9e353e343..db69d566a 100644 --- a/cmd/oras/root/manifest/index/cmd.go +++ b/cmd/oras/root/manifest/index/cmd.go @@ -27,6 +27,7 @@ func Cmd() *cobra.Command { cmd.AddCommand( createCmd(), + updateCmd(), ) return cmd } diff --git a/cmd/oras/root/manifest/index/create.go b/cmd/oras/root/manifest/index/create.go index 4a9f42cb7..c6c5c8240 100644 --- a/cmd/oras/root/manifest/index/create.go +++ b/cmd/oras/root/manifest/index/create.go @@ -50,18 +50,18 @@ type createOptions struct { func createCmd() *cobra.Command { var opts createOptions cmd := &cobra.Command{ - Use: "create [flags] --repo [:|@] [...]", + Use: "create [flags] [:|@] [{|}...]", Short: "create and push an index from provided manifests", Long: `create and push an index to a repository or an OCI image layout Example - create an index from source manifests tagged s1, s2, s3 in the repository localhost:5000/hello, and push the index without tagging it : - oras index create --repo localhost:5000/hello s1 s2 s3 + oras manifest index create localhost:5000/hello s1 s2 s3 Example - create an index from source manifests tagged s1, s2, s3 in the repository localhost:5000/hello, and push the index with tag 'latest' : - oras index create --repo localhost:5000/hello --tag latest s1 s2 s3 + oras manifest index create localhost:5000/hello:latest s1 s2 s3 Example - create an index from source manifests using both tags and digests, and push the index with tag 'latest' : - oras index create --repo localhost:5000/hello --tag latest s1 sha256:xxx s3 + oras manifest index create localhost:5000/hello latest s1 sha256:xxx s3 `, Args: cobra.MinimumNArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { @@ -163,8 +163,9 @@ func fetchConfigDesc(ctx context.Context, src oras.ReadOnlyTarget, reference str return ocispec.Descriptor{}, err } + // if this manifest does not have a config if !descriptor.IsImageManifest(manifestDesc) { - return ocispec.Descriptor{}, fmt.Errorf("%q is not an image manifest and does not have a config", manifestDesc.Digest) + return ocispec.Descriptor{}, nil } // unmarshal manifest content to extract config descriptor diff --git a/cmd/oras/root/manifest/index/update.go b/cmd/oras/root/manifest/index/update.go new file mode 100644 index 000000000..b5439c1f5 --- /dev/null +++ b/cmd/oras/root/manifest/index/update.go @@ -0,0 +1,167 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package index + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/spf13/cobra" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content" + "oras.land/oras/cmd/oras/internal/command" + oerrors "oras.land/oras/cmd/oras/internal/errors" + "oras.land/oras/cmd/oras/internal/option" + "oras.land/oras/internal/contentutil" +) + +type updateOptions struct { + option.Common + option.Target + + addArguments []string + removeArguments []string + addTargets []option.Target + removeTargets []option.Target +} + +func updateCmd() *cobra.Command { + var opts updateOptions + cmd := &cobra.Command{ + Use: "update", + Short: "add or remove manifests from an image index", + Long: `TBD`, + Args: cobra.MinimumNArgs(1), + PreRunE: func(cmd *cobra.Command, args []string) error { + opts.RawReference = args[0] + repo, _, _ := strings.Cut(opts.RawReference, ":") + + opts.addTargets = make([]option.Target, len(opts.addArguments)) + // parse the add manifest arguments + for i, a := range opts.addArguments { + var ref string + if contentutil.IsDigest(a) { + ref = fmt.Sprintf("%s@%s", repo, a) + } else { + ref = fmt.Sprintf("%s:%s", repo, a) + } + opts.addArguments[i] = ref + m := option.Target{RawReference: ref, Remote: opts.Remote} + if err := m.Parse(cmd); err != nil { + return err + } + opts.addTargets[i] = m + } + + return option.Parse(cmd, &opts) + }, + RunE: func(cmd *cobra.Command, args []string) error { + return updateIndex(cmd, opts) + }, + } + option.ApplyFlags(&opts, cmd.Flags()) + cmd.Flags().StringArrayVarP(&opts.addArguments, "add", "", nil, "manifests to add to the index") + cmd.Flags().StringArrayVarP(&opts.removeArguments, "remove", "", nil, "manifests to remove from the index") + + return oerrors.Command(cmd, &opts.Target) +} + +func updateIndex(cmd *cobra.Command, opts updateOptions) error { + // fetch old index + ctx, logger := command.GetLogger(cmd, &opts.Common) + oldIndex, err := opts.NewTarget(opts.Common, logger) + if err != nil { + return err + } + desc, err := oras.Resolve(ctx, oldIndex, opts.Reference, oras.DefaultResolveOptions) + if err != nil { + return fmt.Errorf("failed to resolve %s: %w", opts.Reference, err) + } + contentBytes, err := content.FetchAll(ctx, oldIndex, desc) + if err != nil { + return err + } + var index ocispec.Index + if err := json.Unmarshal(contentBytes, &index); err != nil { + return err + } + manifests := index.Manifests + + // resolve the index to add, need to get its platform information + for _, b := range opts.addTargets { + target, err := b.NewReadonlyTarget(ctx, opts.Common, logger) + if err != nil { + return err + } + if err := b.EnsureReferenceNotEmpty(cmd, false); err != nil { + return err + } + desc, err := oras.Resolve(ctx, target, b.Reference, oras.DefaultResolveOptions) + if err != nil { + return fmt.Errorf("failed to resolve %s: %w", b.Reference, err) + } + // detect platform information + // 1. fetch config descriptor + configDesc, err := fetchConfigDesc(ctx, target, b.Reference) + if err != nil { + return err + } + // 2. fetch config content + contentBytes, err := content.FetchAll(ctx, target, configDesc) + if err != nil { + return err + } + var config ocispec.Image + if err := json.Unmarshal(contentBytes, &config); err != nil { + return err + } + // 3. extract platform information + desc.Platform = &config.Platform + + manifests = append(manifests, desc) + } + + // pack the new index + newIndex := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + MediaType: ocispec.MediaTypeImageIndex, + ArtifactType: index.ArtifactType, + Manifests: manifests, + Subject: index.Subject, + Annotations: index.Annotations, + // todo: annotations + } + content, _ := json.Marshal(newIndex) + newDesc := ocispec.Descriptor{ + Digest: digest.FromBytes(content), + MediaType: ocispec.MediaTypeImageIndex, + Size: int64(len(content)), + } + reader := bytes.NewReader(content) + + // push the new index + if err := pushIndex(ctx, oldIndex, newDesc, opts.Reference, reader); err != nil { + return err + } + return nil +} From 02abf45e8bd1dbaf8ac8f323c01f0d2a65a7c108 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Thu, 18 Jul 2024 16:21:59 +0800 Subject: [PATCH 04/14] --remove for update Signed-off-by: Xiaoxuan Wang --- cmd/oras/root/manifest/index/update.go | 46 +++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/cmd/oras/root/manifest/index/update.go b/cmd/oras/root/manifest/index/update.go index b5439c1f5..986f1b5f8 100644 --- a/cmd/oras/root/manifest/index/update.go +++ b/cmd/oras/root/manifest/index/update.go @@ -71,6 +71,23 @@ func updateCmd() *cobra.Command { opts.addTargets[i] = m } + opts.removeTargets = make([]option.Target, len(opts.removeArguments)) + // parse the remove manifest arguments + for i, a := range opts.removeArguments { + var ref string + if contentutil.IsDigest(a) { + ref = fmt.Sprintf("%s@%s", repo, a) + } else { + ref = fmt.Sprintf("%s:%s", repo, a) + } + opts.removeArguments[i] = ref + m := option.Target{RawReference: ref, Remote: opts.Remote} + if err := m.Parse(cmd); err != nil { + return err + } + opts.removeTargets[i] = m + } + return option.Parse(cmd, &opts) }, RunE: func(cmd *cobra.Command, args []string) error { @@ -105,7 +122,7 @@ func updateIndex(cmd *cobra.Command, opts updateOptions) error { } manifests := index.Manifests - // resolve the index to add, need to get its platform information + // resolve the manifests to add, need to get theirs platform information for _, b := range opts.addTargets { target, err := b.NewReadonlyTarget(ctx, opts.Common, logger) if err != nil { @@ -139,6 +156,33 @@ func updateIndex(cmd *cobra.Command, opts updateOptions) error { manifests = append(manifests, desc) } + // resolve the manifests to remove + set := make(map[digest.Digest]struct{}) + for _, b := range opts.removeTargets { + target, err := b.NewReadonlyTarget(ctx, opts.Common, logger) + if err != nil { + return err + } + if err := b.EnsureReferenceNotEmpty(cmd, false); err != nil { + return err + } + desc, err := oras.Resolve(ctx, target, b.Reference, oras.DefaultResolveOptions) + if err != nil { + return fmt.Errorf("failed to resolve %s: %w", b.Reference, err) + } + set[desc.Digest] = struct{}{} + } + + pointer := len(manifests) - 1 + for i, m := range manifests { + if _, b := set[m.Digest]; b { + // swap + manifests[i] = manifests[pointer] + pointer = pointer - 1 + } + } + manifests = manifests[:pointer+1] + // pack the new index newIndex := ocispec.Index{ Versioned: specs.Versioned{ From 805bbc4602f9273f4c9f943291b8db71cc5034a7 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Fri, 19 Jul 2024 12:17:58 +0800 Subject: [PATCH 05/14] parseTargetsFromStrings() Signed-off-by: Xiaoxuan Wang --- cmd/oras/root/manifest/index/create.go | 36 +++++++++++----------- cmd/oras/root/manifest/index/update.go | 42 +++++--------------------- 2 files changed, 27 insertions(+), 51 deletions(-) diff --git a/cmd/oras/root/manifest/index/create.go b/cmd/oras/root/manifest/index/create.go index c6c5c8240..f1707ac27 100644 --- a/cmd/oras/root/manifest/index/create.go +++ b/cmd/oras/root/manifest/index/create.go @@ -65,26 +65,15 @@ Example - create an index from source manifests using both tags and digests, `, Args: cobra.MinimumNArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { - // parse the target index reference + // parse the destination index reference opts.RawReference = args[0] repo, _, _ := strings.Cut(opts.RawReference, ":") // parse the source manifests opts.sources = make([]option.Target, len(args)-1) - for i, a := range args[1:] { - var ref string - if contentutil.IsDigest(a) { - ref = fmt.Sprintf("%s@%s", repo, a) - } else { - ref = fmt.Sprintf("%s:%s", repo, a) - } - m := option.Target{RawReference: ref, Remote: opts.Remote} - if err := m.Parse(cmd); err != nil { - return err - } - opts.sources[i] = m + if err := parseTargetsFromStrings(cmd, args[1:], opts.sources, repo, opts.Remote); err != nil { + return err } - return option.Parse(cmd, &opts) }, RunE: func(cmd *cobra.Command, args []string) error { @@ -109,8 +98,22 @@ func createIndex(cmd *cobra.Command, opts createOptions) error { return err } desc, reader := packIndex(manifests) - if err := pushIndex(ctx, dst, desc, opts.Reference, reader); err != nil { - return err + return pushIndex(ctx, dst, desc, opts.Reference, reader) +} + +func parseTargetsFromStrings(cmd *cobra.Command, arguments []string, targets []option.Target, repo string, remote option.Remote) error { + for i, arg := range arguments { + var ref string + if contentutil.IsDigest(arg) { + ref = fmt.Sprintf("%s@%s", repo, arg) + } else { + ref = fmt.Sprintf("%s:%s", repo, arg) + } + m := option.Target{RawReference: ref, Remote: remote} + if err := m.Parse(cmd); err != nil { + return err + } + targets[i] = m } return nil } @@ -177,7 +180,6 @@ func fetchConfigDesc(ctx context.Context, src oras.ReadOnlyTarget, reference str } func packIndex(manifests []ocispec.Descriptor) (ocispec.Descriptor, io.Reader) { - // todo: oras-go needs PackIndex index := ocispec.Index{ Versioned: specs.Versioned{ SchemaVersion: 2, diff --git a/cmd/oras/root/manifest/index/update.go b/cmd/oras/root/manifest/index/update.go index 986f1b5f8..db7653e64 100644 --- a/cmd/oras/root/manifest/index/update.go +++ b/cmd/oras/root/manifest/index/update.go @@ -30,7 +30,6 @@ import ( "oras.land/oras/cmd/oras/internal/command" oerrors "oras.land/oras/cmd/oras/internal/errors" "oras.land/oras/cmd/oras/internal/option" - "oras.land/oras/internal/contentutil" ) type updateOptions struct { @@ -54,38 +53,16 @@ func updateCmd() *cobra.Command { opts.RawReference = args[0] repo, _, _ := strings.Cut(opts.RawReference, ":") - opts.addTargets = make([]option.Target, len(opts.addArguments)) // parse the add manifest arguments - for i, a := range opts.addArguments { - var ref string - if contentutil.IsDigest(a) { - ref = fmt.Sprintf("%s@%s", repo, a) - } else { - ref = fmt.Sprintf("%s:%s", repo, a) - } - opts.addArguments[i] = ref - m := option.Target{RawReference: ref, Remote: opts.Remote} - if err := m.Parse(cmd); err != nil { - return err - } - opts.addTargets[i] = m + opts.addTargets = make([]option.Target, len(opts.addArguments)) + if err := parseTargetsFromStrings(cmd, opts.addArguments, opts.addTargets, repo, opts.Remote); err != nil { + return err } - opts.removeTargets = make([]option.Target, len(opts.removeArguments)) // parse the remove manifest arguments - for i, a := range opts.removeArguments { - var ref string - if contentutil.IsDigest(a) { - ref = fmt.Sprintf("%s@%s", repo, a) - } else { - ref = fmt.Sprintf("%s:%s", repo, a) - } - opts.removeArguments[i] = ref - m := option.Target{RawReference: ref, Remote: opts.Remote} - if err := m.Parse(cmd); err != nil { - return err - } - opts.removeTargets[i] = m + opts.removeTargets = make([]option.Target, len(opts.removeArguments)) + if err := parseTargetsFromStrings(cmd, opts.removeArguments, opts.removeTargets, repo, opts.Remote); err != nil { + return err } return option.Parse(cmd, &opts) @@ -122,7 +99,7 @@ func updateIndex(cmd *cobra.Command, opts updateOptions) error { } manifests := index.Manifests - // resolve the manifests to add, need to get theirs platform information + // resolve the manifests to add, need to get their platform information for _, b := range opts.addTargets { target, err := b.NewReadonlyTarget(ctx, opts.Common, logger) if err != nil { @@ -204,8 +181,5 @@ func updateIndex(cmd *cobra.Command, opts updateOptions) error { reader := bytes.NewReader(content) // push the new index - if err := pushIndex(ctx, oldIndex, newDesc, opts.Reference, reader); err != nil { - return err - } - return nil + return pushIndex(ctx, oldIndex, newDesc, opts.Reference, reader) } From 65c03b94c628d56f15ce405d69d2d62093a196bd Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Fri, 19 Jul 2024 12:42:37 +0800 Subject: [PATCH 06/14] getPlatform() Signed-off-by: Xiaoxuan Wang --- cmd/oras/root/manifest/index/create.go | 34 +++++++++++++---------- cmd/oras/root/manifest/index/update.go | 38 ++++++++------------------ 2 files changed, 31 insertions(+), 41 deletions(-) diff --git a/cmd/oras/root/manifest/index/create.go b/cmd/oras/root/manifest/index/create.go index f1707ac27..ca86212e6 100644 --- a/cmd/oras/root/manifest/index/create.go +++ b/cmd/oras/root/manifest/index/create.go @@ -118,6 +118,24 @@ func parseTargetsFromStrings(cmd *cobra.Command, arguments []string, targets []o return nil } +func getPlatform(ctx context.Context, target oras.ReadOnlyTarget, reference string) (*ocispec.Platform, error) { + // fetch config descriptor + configDesc, err := fetchConfigDesc(ctx, target, reference) + if err != nil { + return &ocispec.Platform{}, err + } + // fetch config content + contentBytes, err := content.FetchAll(ctx, target, configDesc) + if err != nil { + return &ocispec.Platform{}, err + } + var config ocispec.Image + if err := json.Unmarshal(contentBytes, &config); err != nil { + return &ocispec.Platform{}, err + } + return &config.Platform, nil +} + func resolveSourceManifests(cmd *cobra.Command, destOpts createOptions, logger logrus.FieldLogger) ([]ocispec.Descriptor, error) { var resolved []ocispec.Descriptor for _, source := range destOpts.sources { @@ -135,24 +153,10 @@ func resolveSourceManifests(cmd *cobra.Command, destOpts createOptions, logger l if err != nil { return []ocispec.Descriptor{}, fmt.Errorf("failed to resolve %s: %w", source.Reference, err) } - // detect platform information - // 1. fetch config descriptor - configDesc, err := fetchConfigDesc(cmd.Context(), sourceTarget, source.Reference) + desc.Platform, err = getPlatform(cmd.Context(), sourceTarget, source.Reference) if err != nil { return []ocispec.Descriptor{}, err } - // 2. fetch config content - contentBytes, err := content.FetchAll(cmd.Context(), sourceTarget, configDesc) - if err != nil { - return []ocispec.Descriptor{}, err - } - var config ocispec.Image - if err := json.Unmarshal(contentBytes, &config); err != nil { - return []ocispec.Descriptor{}, err - } - // 3. extract platform information - desc.Platform = &config.Platform - resolved = append(resolved, desc) } return resolved, nil diff --git a/cmd/oras/root/manifest/index/update.go b/cmd/oras/root/manifest/index/update.go index db7653e64..478c427bc 100644 --- a/cmd/oras/root/manifest/index/update.go +++ b/cmd/oras/root/manifest/index/update.go @@ -79,17 +79,17 @@ func updateCmd() *cobra.Command { } func updateIndex(cmd *cobra.Command, opts updateOptions) error { - // fetch old index + // fetch the index to update, and get its manifests ctx, logger := command.GetLogger(cmd, &opts.Common) - oldIndex, err := opts.NewTarget(opts.Common, logger) + indexTarget, err := opts.NewTarget(opts.Common, logger) if err != nil { return err } - desc, err := oras.Resolve(ctx, oldIndex, opts.Reference, oras.DefaultResolveOptions) + desc, err := oras.Resolve(ctx, indexTarget, opts.Reference, oras.DefaultResolveOptions) if err != nil { return fmt.Errorf("failed to resolve %s: %w", opts.Reference, err) } - contentBytes, err := content.FetchAll(ctx, oldIndex, desc) + contentBytes, err := content.FetchAll(ctx, indexTarget, desc) if err != nil { return err } @@ -100,36 +100,22 @@ func updateIndex(cmd *cobra.Command, opts updateOptions) error { manifests := index.Manifests // resolve the manifests to add, need to get their platform information - for _, b := range opts.addTargets { - target, err := b.NewReadonlyTarget(ctx, opts.Common, logger) + for _, addTarget := range opts.addTargets { + target, err := addTarget.NewReadonlyTarget(ctx, opts.Common, logger) if err != nil { return err } - if err := b.EnsureReferenceNotEmpty(cmd, false); err != nil { + if err := addTarget.EnsureReferenceNotEmpty(cmd, false); err != nil { return err } - desc, err := oras.Resolve(ctx, target, b.Reference, oras.DefaultResolveOptions) - if err != nil { - return fmt.Errorf("failed to resolve %s: %w", b.Reference, err) - } - // detect platform information - // 1. fetch config descriptor - configDesc, err := fetchConfigDesc(ctx, target, b.Reference) + desc, err := oras.Resolve(ctx, target, addTarget.Reference, oras.DefaultResolveOptions) if err != nil { - return err + return fmt.Errorf("failed to resolve %s: %w", addTarget.Reference, err) } - // 2. fetch config content - contentBytes, err := content.FetchAll(ctx, target, configDesc) + desc.Platform, err = getPlatform(ctx, target, addTarget.Reference) if err != nil { return err } - var config ocispec.Image - if err := json.Unmarshal(contentBytes, &config); err != nil { - return err - } - // 3. extract platform information - desc.Platform = &config.Platform - manifests = append(manifests, desc) } @@ -153,7 +139,7 @@ func updateIndex(cmd *cobra.Command, opts updateOptions) error { pointer := len(manifests) - 1 for i, m := range manifests { if _, b := set[m.Digest]; b { - // swap + // swap the to-be-removed manifest to the end of slice manifests[i] = manifests[pointer] pointer = pointer - 1 } @@ -181,5 +167,5 @@ func updateIndex(cmd *cobra.Command, opts updateOptions) error { reader := bytes.NewReader(content) // push the new index - return pushIndex(ctx, oldIndex, newDesc, opts.Reference, reader) + return pushIndex(ctx, indexTarget, newDesc, opts.Reference, reader) } From 7d7cb9aa6cec3e10eacf8f24efed2fe3d5f61dc6 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Fri, 19 Jul 2024 12:52:32 +0800 Subject: [PATCH 07/14] packIndex() Signed-off-by: Xiaoxuan Wang --- cmd/oras/root/manifest/index/create.go | 12 +++++++----- cmd/oras/root/manifest/index/update.go | 25 ++----------------------- 2 files changed, 9 insertions(+), 28 deletions(-) diff --git a/cmd/oras/root/manifest/index/create.go b/cmd/oras/root/manifest/index/create.go index ca86212e6..d59d0aa50 100644 --- a/cmd/oras/root/manifest/index/create.go +++ b/cmd/oras/root/manifest/index/create.go @@ -97,7 +97,7 @@ func createIndex(cmd *cobra.Command, opts createOptions) error { if err != nil { return err } - desc, reader := packIndex(manifests) + desc, reader := packIndex(&ocispec.Index{}, manifests) return pushIndex(ctx, dst, desc, opts.Reference, reader) } @@ -183,14 +183,16 @@ func fetchConfigDesc(ctx context.Context, src oras.ReadOnlyTarget, reference str return manifest.Config, nil } -func packIndex(manifests []ocispec.Descriptor) (ocispec.Descriptor, io.Reader) { +func packIndex(oldIndex *ocispec.Index, manifests []ocispec.Descriptor) (ocispec.Descriptor, io.Reader) { index := ocispec.Index{ Versioned: specs.Versioned{ SchemaVersion: 2, }, - MediaType: ocispec.MediaTypeImageIndex, - Manifests: manifests, - // todo: annotations + MediaType: ocispec.MediaTypeImageIndex, + ArtifactType: oldIndex.ArtifactType, + Manifests: manifests, + Subject: oldIndex.Subject, + Annotations: oldIndex.Annotations, } content, _ := json.Marshal(index) desc := ocispec.Descriptor{ diff --git a/cmd/oras/root/manifest/index/update.go b/cmd/oras/root/manifest/index/update.go index 478c427bc..776e3c409 100644 --- a/cmd/oras/root/manifest/index/update.go +++ b/cmd/oras/root/manifest/index/update.go @@ -16,13 +16,11 @@ limitations under the License. package index import ( - "bytes" "encoding/json" "fmt" "strings" "github.com/opencontainers/go-digest" - "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "oras.land/oras-go/v2" @@ -144,28 +142,9 @@ func updateIndex(cmd *cobra.Command, opts updateOptions) error { pointer = pointer - 1 } } + // shrink the slice to remove the manifests manifests = manifests[:pointer+1] - // pack the new index - newIndex := ocispec.Index{ - Versioned: specs.Versioned{ - SchemaVersion: 2, - }, - MediaType: ocispec.MediaTypeImageIndex, - ArtifactType: index.ArtifactType, - Manifests: manifests, - Subject: index.Subject, - Annotations: index.Annotations, - // todo: annotations - } - content, _ := json.Marshal(newIndex) - newDesc := ocispec.Descriptor{ - Digest: digest.FromBytes(content), - MediaType: ocispec.MediaTypeImageIndex, - Size: int64(len(content)), - } - reader := bytes.NewReader(content) - - // push the new index + newDesc, reader := packIndex(&index, manifests) return pushIndex(ctx, indexTarget, newDesc, opts.Reference, reader) } From f50a429d8fb87d2901478841b3b6ffc73e5ad7b1 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Fri, 19 Jul 2024 16:32:11 +0800 Subject: [PATCH 08/14] fetchIndex() Signed-off-by: Xiaoxuan Wang --- cmd/oras/root/manifest/index/create.go | 4 +--- cmd/oras/root/manifest/index/update.go | 31 ++++++++++++++++---------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/cmd/oras/root/manifest/index/create.go b/cmd/oras/root/manifest/index/create.go index d59d0aa50..e6740af97 100644 --- a/cmd/oras/root/manifest/index/create.go +++ b/cmd/oras/root/manifest/index/create.go @@ -75,6 +75,7 @@ Example - create an index from source manifests using both tags and digests, return err } return option.Parse(cmd, &opts) + // todo: add EnsureReferenceNotEmpty somewhere }, RunE: func(cmd *cobra.Command, args []string) error { return createIndex(cmd, opts) @@ -145,9 +146,6 @@ func resolveSourceManifests(cmd *cobra.Command, destOpts createOptions, logger l if err != nil { return []ocispec.Descriptor{}, err } - if err := source.EnsureReferenceNotEmpty(cmd, false); err != nil { - return []ocispec.Descriptor{}, err - } var desc ocispec.Descriptor desc, err = oras.Resolve(cmd.Context(), sourceTarget, source.Reference, oras.DefaultResolveOptions) if err != nil { diff --git a/cmd/oras/root/manifest/index/update.go b/cmd/oras/root/manifest/index/update.go index 776e3c409..0675c7c84 100644 --- a/cmd/oras/root/manifest/index/update.go +++ b/cmd/oras/root/manifest/index/update.go @@ -16,6 +16,7 @@ limitations under the License. package index import ( + "context" "encoding/json" "fmt" "strings" @@ -64,6 +65,7 @@ func updateCmd() *cobra.Command { } return option.Parse(cmd, &opts) + // todo: add EnsureReferenceNotEmpty somewhere }, RunE: func(cmd *cobra.Command, args []string) error { return updateIndex(cmd, opts) @@ -83,18 +85,10 @@ func updateIndex(cmd *cobra.Command, opts updateOptions) error { if err != nil { return err } - desc, err := oras.Resolve(ctx, indexTarget, opts.Reference, oras.DefaultResolveOptions) - if err != nil { - return fmt.Errorf("failed to resolve %s: %w", opts.Reference, err) - } - contentBytes, err := content.FetchAll(ctx, indexTarget, desc) + index, err := fetchIndex(ctx, indexTarget, opts.Reference) if err != nil { return err } - var index ocispec.Index - if err := json.Unmarshal(contentBytes, &index); err != nil { - return err - } manifests := index.Manifests // resolve the manifests to add, need to get their platform information @@ -124,9 +118,6 @@ func updateIndex(cmd *cobra.Command, opts updateOptions) error { if err != nil { return err } - if err := b.EnsureReferenceNotEmpty(cmd, false); err != nil { - return err - } desc, err := oras.Resolve(ctx, target, b.Reference, oras.DefaultResolveOptions) if err != nil { return fmt.Errorf("failed to resolve %s: %w", b.Reference, err) @@ -148,3 +139,19 @@ func updateIndex(cmd *cobra.Command, opts updateOptions) error { newDesc, reader := packIndex(&index, manifests) return pushIndex(ctx, indexTarget, newDesc, opts.Reference, reader) } + +func fetchIndex(ctx context.Context, target oras.ReadOnlyTarget, reference string) (ocispec.Index, error) { + desc, err := oras.Resolve(ctx, target, reference, oras.DefaultResolveOptions) + if err != nil { + return ocispec.Index{}, fmt.Errorf("failed to resolve %s: %w", reference, err) + } + contentBytes, err := content.FetchAll(ctx, target, desc) + if err != nil { + return ocispec.Index{}, err + } + var index ocispec.Index + if err := json.Unmarshal(contentBytes, &index); err != nil { + return ocispec.Index{}, err + } + return index, nil +} From 8cea53999773a0818f2f5d974a27ee37202d0ffb Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Fri, 19 Jul 2024 17:03:11 +0800 Subject: [PATCH 09/14] addManifests() and removeManifests() Signed-off-by: Xiaoxuan Wang --- cmd/oras/root/manifest/index/update.go | 78 ++++++++++++++------------ 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/cmd/oras/root/manifest/index/update.go b/cmd/oras/root/manifest/index/update.go index 0675c7c84..85a9db901 100644 --- a/cmd/oras/root/manifest/index/update.go +++ b/cmd/oras/root/manifest/index/update.go @@ -23,6 +23,7 @@ import ( "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" @@ -79,7 +80,6 @@ func updateCmd() *cobra.Command { } func updateIndex(cmd *cobra.Command, opts updateOptions) error { - // fetch the index to update, and get its manifests ctx, logger := command.GetLogger(cmd, &opts.Common) indexTarget, err := opts.NewTarget(opts.Common, logger) if err != nil { @@ -89,42 +89,66 @@ func updateIndex(cmd *cobra.Command, opts updateOptions) error { if err != nil { return err } - manifests := index.Manifests + manifests, err := addManifests(ctx, opts.Common, logger, index.Manifests, opts.addTargets) + if err != nil { + return err + } + manifests, err = removeManifests(ctx, opts.Common, logger, manifests, opts.removeTargets) + if err != nil { + return err + } + newDesc, reader := packIndex(&index, manifests) + return pushIndex(ctx, indexTarget, newDesc, opts.Reference, reader) +} - // resolve the manifests to add, need to get their platform information - for _, addTarget := range opts.addTargets { - target, err := addTarget.NewReadonlyTarget(ctx, opts.Common, logger) +func fetchIndex(ctx context.Context, target oras.ReadOnlyTarget, reference string) (ocispec.Index, error) { + desc, err := oras.Resolve(ctx, target, reference, oras.DefaultResolveOptions) + if err != nil { + return ocispec.Index{}, fmt.Errorf("failed to resolve %s: %w", reference, err) + } + contentBytes, err := content.FetchAll(ctx, target, desc) + if err != nil { + return ocispec.Index{}, err + } + var index ocispec.Index + if err := json.Unmarshal(contentBytes, &index); err != nil { + return ocispec.Index{}, err + } + return index, nil +} + +func addManifests(ctx context.Context, common option.Common, logger logrus.FieldLogger, manifests []ocispec.Descriptor, targets []option.Target) ([]ocispec.Descriptor, error) { + for _, addTarget := range targets { + target, err := addTarget.NewReadonlyTarget(ctx, common, logger) if err != nil { - return err - } - if err := addTarget.EnsureReferenceNotEmpty(cmd, false); err != nil { - return err + return []ocispec.Descriptor{}, err } desc, err := oras.Resolve(ctx, target, addTarget.Reference, oras.DefaultResolveOptions) if err != nil { - return fmt.Errorf("failed to resolve %s: %w", addTarget.Reference, err) + return []ocispec.Descriptor{}, fmt.Errorf("failed to resolve %s: %w", addTarget.Reference, err) } desc.Platform, err = getPlatform(ctx, target, addTarget.Reference) if err != nil { - return err + return []ocispec.Descriptor{}, err } manifests = append(manifests, desc) } + return manifests, nil +} - // resolve the manifests to remove +func removeManifests(ctx context.Context, common option.Common, logger logrus.FieldLogger, manifests []ocispec.Descriptor, targets []option.Target) ([]ocispec.Descriptor, error) { set := make(map[digest.Digest]struct{}) - for _, b := range opts.removeTargets { - target, err := b.NewReadonlyTarget(ctx, opts.Common, logger) + for _, b := range targets { + target, err := b.NewReadonlyTarget(ctx, common, logger) if err != nil { - return err + return []ocispec.Descriptor{}, err } desc, err := oras.Resolve(ctx, target, b.Reference, oras.DefaultResolveOptions) if err != nil { - return fmt.Errorf("failed to resolve %s: %w", b.Reference, err) + return []ocispec.Descriptor{}, fmt.Errorf("failed to resolve %s: %w", b.Reference, err) } set[desc.Digest] = struct{}{} } - pointer := len(manifests) - 1 for i, m := range manifests { if _, b := set[m.Digest]; b { @@ -135,23 +159,5 @@ func updateIndex(cmd *cobra.Command, opts updateOptions) error { } // shrink the slice to remove the manifests manifests = manifests[:pointer+1] - - newDesc, reader := packIndex(&index, manifests) - return pushIndex(ctx, indexTarget, newDesc, opts.Reference, reader) -} - -func fetchIndex(ctx context.Context, target oras.ReadOnlyTarget, reference string) (ocispec.Index, error) { - desc, err := oras.Resolve(ctx, target, reference, oras.DefaultResolveOptions) - if err != nil { - return ocispec.Index{}, fmt.Errorf("failed to resolve %s: %w", reference, err) - } - contentBytes, err := content.FetchAll(ctx, target, desc) - if err != nil { - return ocispec.Index{}, err - } - var index ocispec.Index - if err := json.Unmarshal(contentBytes, &index); err != nil { - return ocispec.Index{}, err - } - return index, nil + return manifests, nil } From a6ac176cbe415b9edaf825096e65cc248d3d38c8 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Mon, 22 Jul 2024 15:07:21 +0800 Subject: [PATCH 10/14] added poc Signed-off-by: Xiaoxuan Wang --- docs/proposals/multi-arch.md | 114 +++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 docs/proposals/multi-arch.md diff --git a/docs/proposals/multi-arch.md b/docs/proposals/multi-arch.md new file mode 100644 index 000000000..2b1783fd7 --- /dev/null +++ b/docs/proposals/multi-arch.md @@ -0,0 +1,114 @@ +# Multi-arch image support for ORAS + +## Command Design + +### Overview + +- `oras manifest index create`: Create an image index from source manifests. +- `oras manifest index update`: Add/remove manifests to/from an image index. + +### Create an image index + +#### Definition + +Create an image index from source manifests. The command auto-detects platform information for each source manifests. + +> [!IMPORTANT] +> All source manifests referenced are required to exist in the same repository as the target index. + +Usage: +``` +oras manifest index create [flags] [:][...]] [{|}...] +``` + +Flags: + +- `--subject`: Add a subject manifest for the to-be-created index. +- `--oci-layout`: Set the given repository as an OCI layout. +- `--annotation`: Add annotations for the to-be-created index. +- `--annotation-file`: Add annotations for the to-be-created index and the individual source manifests. +- `--artifact-type`: Add artifact type information for the to-be-created index. +- `--output`: Output the updated manifest to a location. Auto push is disabled. + +Aliases: `pack` + +#### Examples + +Create an index from source manifests tagged `amd64`, `darwin`, `armv7` in the repository `localhost:5000/hello`, and push the index without tagging it. + +```sh +oras manifest index create localhost:5000/hello amd64 darwin armv7 +``` + +Create an index from source manifests tagged amd64, darwin, armv7 in the repository localhost:5000/hello, and push the index with tag `latest`: + +```sh +oras manifest index create localhost:5000/hello:latest amd64 darwin armv7 +``` + +Create an index from source manifests using both tags and digests, and push the index with tag `latest`: + +```sh +oras manifest index create localhost:5000/hello:latest amd64 sha256:xxx armv7 +``` + +### Update an image index + +#### Definition + +Add/Remove a manifest from an image index. The updated index will be created as a new index and the old index will not be deleted. +If the user specify the index with tags, the corresponding tags will be updated to the new index. If the old index has other tags, the remaining tags will not be updated to the new index. + +Usage: + +``` +oras manifest index update {:|@} {--add/--remove} [{|}...] +``` + +Flags: + +- `--add`: Add a manifest to the index. The manifest will be added as the last element of the index. +- `--remove`: Remove a manifest from the index. +- `--annotation`: Update annotations for the index. +- `--annotation-file`: Update annotations for the index and the individual manifests. +- `--oci-layout`: Set the target as an oci image layout. +- `--tag`: Tag the updated index. Multiple tags can be provided. +- `--output`: Output the updated manifest to a location. Auto push is disabled. + +> [!NOTE] +> One of `--add`/`--remove`/`--annotation`/`--annotation-file` should be used, as there has to be something to update. Otherwise the command does nothing. + +#### Examples + +Add one manifest and remove two manifests from an index. + +```sh +oras manifest index update localhost:5000/hello:latest --add win64 --remove sha256:xxx --remove arm64 +``` + +## Design Considerations + +### Command design: Making a subcommand group `index` under `oras manifest` + +`oras manifest index create` is chosen instead of `oras manifest create-index` with the following reasons: +* The structure of the `oras manifest index` sub command group aligns well with the existing sub command groups `oras manifest/blob/repo`. +* If in the future more index commands are needed, grouping them under the `index` group makes the manifest commands neater. Operations for other manifest types may be needed in the future, and creating new sub groups parallel to `index` looks feasible (i.e. `oras manifest image create`). + +### Combining _add manifest_ and _remove manifest_ operations as one `index update` operation + +Combining the add and remove manifest operations as one `update` command has several benefits, such as it makes less garbage and fewer request calls when doing multiple adds and removes. + +### `oras manifest index create / update` will auto detect platform information for each source manifest. + +Platform information is automatically detected from the manifest config. Specifying +platform by the user might be supported in the future. + +## FAQ + +### Should we require all the source manifests and the to-be-created index to be in the same repository? + +Yes, as allowing multiple repositories will introduce a lot of copying of missing blobs and manifests. + +### Should we automatically push the created/updated index? + +Yes, the created/updated index is automatically pushed for better user experience. From b5610fb7e067caa1700fb3ee5fabd27c93a7d879 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Tue, 23 Jul 2024 10:49:18 +0800 Subject: [PATCH 11/14] refinement Signed-off-by: Xiaoxuan Wang --- cmd/oras/root/manifest/index/create.go | 164 ++++++++----------------- cmd/oras/root/manifest/index/update.go | 80 +++++------- docs/proposals/multi-arch.md | 2 +- 3 files changed, 86 insertions(+), 160 deletions(-) diff --git a/cmd/oras/root/manifest/index/create.go b/cmd/oras/root/manifest/index/create.go index e6740af97..245c9332c 100644 --- a/cmd/oras/root/manifest/index/create.go +++ b/cmd/oras/root/manifest/index/create.go @@ -19,24 +19,16 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" - "io" - "strings" - "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" - "oras.land/oras-go/v2/errdef" - "oras.land/oras-go/v2/registry" "oras.land/oras/cmd/oras/internal/command" oerrors "oras.land/oras/cmd/oras/internal/errors" "oras.land/oras/cmd/oras/internal/option" - "oras.land/oras/internal/contentutil" "oras.land/oras/internal/descriptor" ) @@ -44,36 +36,32 @@ type createOptions struct { option.Common option.Target - sources []option.Target + sources []string } func createCmd() *cobra.Command { var opts createOptions cmd := &cobra.Command{ Use: "create [flags] [:|@] [{|}...]", - Short: "create and push an index from provided manifests", - Long: `create and push an index to a repository or an OCI image layout + Short: "Create and push an index from provided manifests", + Long: `Create and push an index to a repository or an OCI image layout + Example - create an index from source manifests tagged s1, s2, s3 in the repository - localhost:5000/hello, and push the index without tagging it : + localhost:5000/hello, and push the index without tagging it: oras manifest index create localhost:5000/hello s1 s2 s3 + Example - create an index from source manifests tagged s1, s2, s3 in the repository - localhost:5000/hello, and push the index with tag 'latest' : + localhost:5000/hello, and push the index with tag 'latest': oras manifest index create localhost:5000/hello:latest s1 s2 s3 + Example - create an index from source manifests using both tags and digests, - and push the index with tag 'latest' : + and push the index with tag 'latest': oras manifest index create localhost:5000/hello latest s1 sha256:xxx s3 `, Args: cobra.MinimumNArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { - // parse the destination index reference opts.RawReference = args[0] - repo, _, _ := strings.Cut(opts.RawReference, ":") - - // parse the source manifests - opts.sources = make([]option.Target, len(args)-1) - if err := parseTargetsFromStrings(cmd, args[1:], opts.sources, repo, opts.Remote); err != nil { - return err - } + opts.sources = args[1:] return option.Parse(cmd, &opts) // todo: add EnsureReferenceNotEmpty somewhere }, @@ -88,100 +76,60 @@ Example - create an index from source manifests using both tags and digests, func createIndex(cmd *cobra.Command, opts createOptions) error { ctx, logger := command.GetLogger(cmd, &opts.Common) - dst, err := opts.NewTarget(opts.Common, logger) + target, err := opts.NewTarget(opts.Common, logger) if err != nil { return err } // we assume that the sources and the to be created index are all in the same // repository, so no copy is needed - manifests, err := resolveSourceManifests(cmd, opts, logger) + manifests, err := resolveSourceManifests(ctx, target, opts.sources) if err != nil { return err } - desc, reader := packIndex(&ocispec.Index{}, manifests) - return pushIndex(ctx, dst, desc, opts.Reference, reader) -} - -func parseTargetsFromStrings(cmd *cobra.Command, arguments []string, targets []option.Target, repo string, remote option.Remote) error { - for i, arg := range arguments { - var ref string - if contentutil.IsDigest(arg) { - ref = fmt.Sprintf("%s@%s", repo, arg) - } else { - ref = fmt.Sprintf("%s:%s", repo, arg) - } - m := option.Target{RawReference: ref, Remote: remote} - if err := m.Parse(cmd); err != nil { - return err - } - targets[i] = m - } - return nil -} - -func getPlatform(ctx context.Context, target oras.ReadOnlyTarget, reference string) (*ocispec.Platform, error) { - // fetch config descriptor - configDesc, err := fetchConfigDesc(ctx, target, reference) + desc, content, err := packIndex(&ocispec.Index{}, manifests) if err != nil { - return &ocispec.Platform{}, err - } - // fetch config content - contentBytes, err := content.FetchAll(ctx, target, configDesc) - if err != nil { - return &ocispec.Platform{}, err - } - var config ocispec.Image - if err := json.Unmarshal(contentBytes, &config); err != nil { - return &ocispec.Platform{}, err + return err } - return &config.Platform, nil + return pushIndex(ctx, target, opts.Reference, desc, content) } -func resolveSourceManifests(cmd *cobra.Command, destOpts createOptions, logger logrus.FieldLogger) ([]ocispec.Descriptor, error) { +func resolveSourceManifests(ctx context.Context, target oras.ReadOnlyTarget, sources []string) ([]ocispec.Descriptor, error) { var resolved []ocispec.Descriptor - for _, source := range destOpts.sources { - var err error - // prepare sourceTarget target - sourceTarget, err := source.NewReadonlyTarget(cmd.Context(), destOpts.Common, logger) - if err != nil { - return []ocispec.Descriptor{}, err - } - var desc ocispec.Descriptor - desc, err = oras.Resolve(cmd.Context(), sourceTarget, source.Reference, oras.DefaultResolveOptions) + for _, source := range sources { + desc, content, err := oras.FetchBytes(ctx, target, source, oras.DefaultFetchBytesOptions) if err != nil { - return []ocispec.Descriptor{}, fmt.Errorf("failed to resolve %s: %w", source.Reference, err) + return nil, err } - desc.Platform, err = getPlatform(cmd.Context(), sourceTarget, source.Reference) - if err != nil { - return []ocispec.Descriptor{}, err + if descriptor.IsImageManifest(desc) { + desc.Platform, err = getPlatform(ctx, target, content) + if err != nil { + return nil, err + } } resolved = append(resolved, desc) } return resolved, nil } -func fetchConfigDesc(ctx context.Context, src oras.ReadOnlyTarget, reference string) (ocispec.Descriptor, error) { - // fetch manifest descriptor and content - fetchOpts := oras.DefaultFetchBytesOptions - manifestDesc, manifestContent, err := oras.FetchBytes(ctx, src, reference, fetchOpts) - if err != nil { - return ocispec.Descriptor{}, err +func getPlatform(ctx context.Context, target oras.ReadOnlyTarget, manifestBytes []byte) (*ocispec.Platform, error) { + // extract config descriptor + var manifest ocispec.Manifest + if err := json.Unmarshal(manifestBytes, &manifest); err != nil { + return nil, err } - - // if this manifest does not have a config - if !descriptor.IsImageManifest(manifestDesc) { - return ocispec.Descriptor{}, nil + // fetch config content + contentBytes, err := content.FetchAll(ctx, target, manifest.Config) + if err != nil { + return nil, err } - - // unmarshal manifest content to extract config descriptor - var manifest ocispec.Manifest - if err := json.Unmarshal(manifestContent, &manifest); err != nil { - return ocispec.Descriptor{}, err + var platform ocispec.Platform + if err := json.Unmarshal(contentBytes, &platform); err != nil { + return nil, err } - return manifest.Config, nil + return &platform, nil } -func packIndex(oldIndex *ocispec.Index, manifests []ocispec.Descriptor) (ocispec.Descriptor, io.Reader) { +func packIndex(oldIndex *ocispec.Index, manifests []ocispec.Descriptor) (ocispec.Descriptor, []byte, error) { index := ocispec.Index{ Versioned: specs.Versioned{ SchemaVersion: 2, @@ -192,30 +140,24 @@ func packIndex(oldIndex *ocispec.Index, manifests []ocispec.Descriptor) (ocispec Subject: oldIndex.Subject, Annotations: oldIndex.Annotations, } - content, _ := json.Marshal(index) - desc := ocispec.Descriptor{ - Digest: digest.FromBytes(content), - MediaType: ocispec.MediaTypeImageIndex, - Size: int64(len(content)), + indexBytes, err := json.Marshal(index) + if err != nil { + return ocispec.Descriptor{}, nil, err } - return desc, bytes.NewReader(content) + desc := content.NewDescriptorFromBytes(ocispec.MediaTypeImageIndex, indexBytes) + return desc, indexBytes, nil } -func pushIndex(ctx context.Context, dst oras.Target, desc ocispec.Descriptor, ref string, content io.Reader) error { - if refPusher, ok := dst.(registry.ReferencePusher); ok { - if ref != "" { - return refPusher.PushReference(ctx, desc, content, ref) - } - } - if err := dst.Push(ctx, desc, content); err != nil { - w := errors.Unwrap(err) - if w != errdef.ErrAlreadyExists { - return err - } - } +func pushIndex(ctx context.Context, target oras.Target, ref string, desc ocispec.Descriptor, content []byte) error { + var err error if ref == "" { - fmt.Println("Digest of the pushed index: ", desc.Digest) - return nil + err = target.Push(ctx, desc, bytes.NewReader(content)) + } else { + desc, err = oras.TagBytes(ctx, target, desc.MediaType, content, ref) } - return dst.Tag(ctx, desc, ref) + if err != nil { + return err + } + fmt.Println("Created and pushed index:", desc.Digest) + return nil } diff --git a/cmd/oras/root/manifest/index/update.go b/cmd/oras/root/manifest/index/update.go index 85a9db901..e12e6a4bd 100644 --- a/cmd/oras/root/manifest/index/update.go +++ b/cmd/oras/root/manifest/index/update.go @@ -19,17 +19,16 @@ import ( "context" "encoding/json" "fmt" - "strings" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" "oras.land/oras/cmd/oras/internal/command" oerrors "oras.land/oras/cmd/oras/internal/errors" "oras.land/oras/cmd/oras/internal/option" + "oras.land/oras/internal/descriptor" ) type updateOptions struct { @@ -38,33 +37,21 @@ type updateOptions struct { addArguments []string removeArguments []string - addTargets []option.Target - removeTargets []option.Target } func updateCmd() *cobra.Command { var opts updateOptions cmd := &cobra.Command{ - Use: "update", - Short: "add or remove manifests from an image index", - Long: `TBD`, - Args: cobra.MinimumNArgs(1), + Use: "update {:|@} {--add/--remove/--annotation/--annotation-file} [...]", + Short: "Update an image index", + Long: `Update an image index and push to the repository or OCI image layout + +Example - add one manifest and remove two manifests from an index: + oras manifest index update localhost:5000/hello:latest --add win64 --remove sha256:xxx --remove arm64 + `, + Args: cobra.MinimumNArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { opts.RawReference = args[0] - repo, _, _ := strings.Cut(opts.RawReference, ":") - - // parse the add manifest arguments - opts.addTargets = make([]option.Target, len(opts.addArguments)) - if err := parseTargetsFromStrings(cmd, opts.addArguments, opts.addTargets, repo, opts.Remote); err != nil { - return err - } - - // parse the remove manifest arguments - opts.removeTargets = make([]option.Target, len(opts.removeArguments)) - if err := parseTargetsFromStrings(cmd, opts.removeArguments, opts.removeTargets, repo, opts.Remote); err != nil { - return err - } - return option.Parse(cmd, &opts) // todo: add EnsureReferenceNotEmpty somewhere }, @@ -81,24 +68,27 @@ func updateCmd() *cobra.Command { func updateIndex(cmd *cobra.Command, opts updateOptions) error { ctx, logger := command.GetLogger(cmd, &opts.Common) - indexTarget, err := opts.NewTarget(opts.Common, logger) + target, err := opts.NewTarget(opts.Common, logger) + if err != nil { + return err + } + index, err := fetchIndex(ctx, target, opts.Reference) if err != nil { return err } - index, err := fetchIndex(ctx, indexTarget, opts.Reference) + manifests, err := addManifests(ctx, index.Manifests, target, opts.addArguments) if err != nil { return err } - manifests, err := addManifests(ctx, opts.Common, logger, index.Manifests, opts.addTargets) + manifests, err = removeManifests(ctx, manifests, target, opts.removeArguments) if err != nil { return err } - manifests, err = removeManifests(ctx, opts.Common, logger, manifests, opts.removeTargets) + desc, content, err := packIndex(&index, manifests) if err != nil { return err } - newDesc, reader := packIndex(&index, manifests) - return pushIndex(ctx, indexTarget, newDesc, opts.Reference, reader) + return pushIndex(ctx, target, opts.Reference, desc, content) } func fetchIndex(ctx context.Context, target oras.ReadOnlyTarget, reference string) (ocispec.Index, error) { @@ -117,41 +107,35 @@ func fetchIndex(ctx context.Context, target oras.ReadOnlyTarget, reference strin return index, nil } -func addManifests(ctx context.Context, common option.Common, logger logrus.FieldLogger, manifests []ocispec.Descriptor, targets []option.Target) ([]ocispec.Descriptor, error) { - for _, addTarget := range targets { - target, err := addTarget.NewReadonlyTarget(ctx, common, logger) - if err != nil { - return []ocispec.Descriptor{}, err - } - desc, err := oras.Resolve(ctx, target, addTarget.Reference, oras.DefaultResolveOptions) +func addManifests(ctx context.Context, manifests []ocispec.Descriptor, target oras.ReadOnlyTarget, adds []string) ([]ocispec.Descriptor, error) { + for _, add := range adds { + desc, content, err := oras.FetchBytes(ctx, target, add, oras.DefaultFetchBytesOptions) if err != nil { - return []ocispec.Descriptor{}, fmt.Errorf("failed to resolve %s: %w", addTarget.Reference, err) + return nil, err } - desc.Platform, err = getPlatform(ctx, target, addTarget.Reference) - if err != nil { - return []ocispec.Descriptor{}, err + if descriptor.IsImageManifest(desc) { + desc.Platform, err = getPlatform(ctx, target, content) + if err != nil { + return nil, err + } } manifests = append(manifests, desc) } return manifests, nil } -func removeManifests(ctx context.Context, common option.Common, logger logrus.FieldLogger, manifests []ocispec.Descriptor, targets []option.Target) ([]ocispec.Descriptor, error) { +func removeManifests(ctx context.Context, manifests []ocispec.Descriptor, target oras.ReadOnlyTarget, removes []string) ([]ocispec.Descriptor, error) { set := make(map[digest.Digest]struct{}) - for _, b := range targets { - target, err := b.NewReadonlyTarget(ctx, common, logger) - if err != nil { - return []ocispec.Descriptor{}, err - } - desc, err := oras.Resolve(ctx, target, b.Reference, oras.DefaultResolveOptions) + for _, rem := range removes { + desc, _, err := oras.FetchBytes(ctx, target, rem, oras.DefaultFetchBytesOptions) if err != nil { - return []ocispec.Descriptor{}, fmt.Errorf("failed to resolve %s: %w", b.Reference, err) + return nil, err } set[desc.Digest] = struct{}{} } pointer := len(manifests) - 1 for i, m := range manifests { - if _, b := set[m.Digest]; b { + if _, exists := set[m.Digest]; exists { // swap the to-be-removed manifest to the end of slice manifests[i] = manifests[pointer] pointer = pointer - 1 diff --git a/docs/proposals/multi-arch.md b/docs/proposals/multi-arch.md index 2b1783fd7..2b5e111e1 100644 --- a/docs/proposals/multi-arch.md +++ b/docs/proposals/multi-arch.md @@ -62,7 +62,7 @@ If the user specify the index with tags, the corresponding tags will be updated Usage: ``` -oras manifest index update {:|@} {--add/--remove} [{|}...] +oras manifest index update {:|@} {--add/--remove/--annotation/--annotation-file} {...} ``` Flags: From 0294e944b8891361ee5f4d45b554c77114d8f627 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Wed, 24 Jul 2024 16:07:26 +0800 Subject: [PATCH 12/14] multiple tags Signed-off-by: Xiaoxuan Wang --- cmd/oras/root/manifest/index/create.go | 32 ++++++++++++++++---------- cmd/oras/root/manifest/index/update.go | 14 ++++++++--- docs/proposals/multi-arch.md | 13 +++++++++++ 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/cmd/oras/root/manifest/index/create.go b/cmd/oras/root/manifest/index/create.go index 245c9332c..d26e0e2f3 100644 --- a/cmd/oras/root/manifest/index/create.go +++ b/cmd/oras/root/manifest/index/create.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -36,31 +37,37 @@ type createOptions struct { option.Common option.Target - sources []string + sources []string + extraRefs []string } func createCmd() *cobra.Command { var opts createOptions cmd := &cobra.Command{ - Use: "create [flags] [:|@] [{|}...]", + Use: "create [flags] [:][...]] [{|}...]", Short: "Create and push an index from provided manifests", Long: `Create and push an index to a repository or an OCI image layout -Example - create an index from source manifests tagged s1, s2, s3 in the repository +Example - create an index from source manifests tagged amd64, arm64, darwin in the repository localhost:5000/hello, and push the index without tagging it: - oras manifest index create localhost:5000/hello s1 s2 s3 + oras manifest index create localhost:5000/hello amd64 arm64 darwin -Example - create an index from source manifests tagged s1, s2, s3 in the repository +Example - create an index from source manifests tagged amd64, arm64, darwin in the repository localhost:5000/hello, and push the index with tag 'latest': - oras manifest index create localhost:5000/hello:latest s1 s2 s3 + oras manifest index create localhost:5000/hello:latest amd64 arm64 darwin Example - create an index from source manifests using both tags and digests, and push the index with tag 'latest': - oras manifest index create localhost:5000/hello latest s1 sha256:xxx s3 + oras manifest index create localhost:5000/hello latest amd64 sha256:xxx darwin + +Example - create an index and push it with multiple tags: + oras manifest index create localhost:5000/tag1, tag2, tag3 amd64 arm64 sha256:xxx `, Args: cobra.MinimumNArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { - opts.RawReference = args[0] + refs := strings.Split(args[0], ",") + opts.RawReference = refs[0] + opts.extraRefs = refs[1:] opts.sources = args[1:] return option.Parse(cmd, &opts) // todo: add EnsureReferenceNotEmpty somewhere @@ -90,7 +97,7 @@ func createIndex(cmd *cobra.Command, opts createOptions) error { if err != nil { return err } - return pushIndex(ctx, target, opts.Reference, desc, content) + return pushIndex(ctx, target, desc, content, opts.Reference, opts.extraRefs) } func resolveSourceManifests(ctx context.Context, target oras.ReadOnlyTarget, sources []string) ([]ocispec.Descriptor, error) { @@ -148,12 +155,13 @@ func packIndex(oldIndex *ocispec.Index, manifests []ocispec.Descriptor) (ocispec return desc, indexBytes, nil } -func pushIndex(ctx context.Context, target oras.Target, ref string, desc ocispec.Descriptor, content []byte) error { +func pushIndex(ctx context.Context, target oras.Target, desc ocispec.Descriptor, content []byte, ref string, extraRefs []string) error { var err error - if ref == "" { + refs := append(extraRefs, ref) + if len(refs) == 0 { err = target.Push(ctx, desc, bytes.NewReader(content)) } else { - desc, err = oras.TagBytes(ctx, target, desc.MediaType, content, ref) + desc, err = oras.TagBytesN(ctx, target, desc.MediaType, content, refs, oras.DefaultTagBytesNOptions) } if err != nil { return err diff --git a/cmd/oras/root/manifest/index/update.go b/cmd/oras/root/manifest/index/update.go index e12e6a4bd..4aadcc429 100644 --- a/cmd/oras/root/manifest/index/update.go +++ b/cmd/oras/root/manifest/index/update.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -35,6 +36,7 @@ type updateOptions struct { option.Common option.Target + extraRefs []string addArguments []string removeArguments []string } @@ -48,10 +50,16 @@ func updateCmd() *cobra.Command { Example - add one manifest and remove two manifests from an index: oras manifest index update localhost:5000/hello:latest --add win64 --remove sha256:xxx --remove arm64 - `, + +Example - update the index referenced by tag1 and tag3, and make tag1 and tag3 point to the + updated index. If the old index has other tags, they remain pointing to the old index. + oras manifest index update localhost:5000/hello:tag1,tag3 --remove sha256:xxx --remove sha256:xxx --add s390x + `, Args: cobra.MinimumNArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { - opts.RawReference = args[0] + refs := strings.Split(args[0], ",") + opts.RawReference = refs[0] + opts.extraRefs = refs[1:] return option.Parse(cmd, &opts) // todo: add EnsureReferenceNotEmpty somewhere }, @@ -88,7 +96,7 @@ func updateIndex(cmd *cobra.Command, opts updateOptions) error { if err != nil { return err } - return pushIndex(ctx, target, opts.Reference, desc, content) + return pushIndex(ctx, target, desc, content, opts.Reference, opts.extraRefs) } func fetchIndex(ctx context.Context, target oras.ReadOnlyTarget, reference string) (ocispec.Index, error) { diff --git a/docs/proposals/multi-arch.md b/docs/proposals/multi-arch.md index 2b5e111e1..3aaa2d82f 100644 --- a/docs/proposals/multi-arch.md +++ b/docs/proposals/multi-arch.md @@ -52,6 +52,12 @@ Create an index from source manifests using both tags and digests, and push the oras manifest index create localhost:5000/hello:latest amd64 sha256:xxx armv7 ``` +Create an index and push it with multiple tags: + +```sh +oras manifest index create localhost:5000/tag1, tag2, tag3 amd64 arm64 sha256:xxx +``` + ### Update an image index #### Definition @@ -86,6 +92,13 @@ Add one manifest and remove two manifests from an index. oras manifest index update localhost:5000/hello:latest --add win64 --remove sha256:xxx --remove arm64 ``` +Update the index referenced by tag1 and tag3, and make tag1 and tag3 point to the +updated index. If the old index has other tags, they remain pointing to the old index. + +```sh +oras manifest index update localhost:5000/hello:tag1,tag3 --remove sha256:xxx --remove sha256:xxx --add s390x +``` + ## Design Considerations ### Command design: Making a subcommand group `index` under `oras manifest` From fcb746b5e64534a2e30c82e65fd97b6538c786d1 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Fri, 9 Aug 2024 14:35:33 +0800 Subject: [PATCH 13/14] merge flag for update Signed-off-by: Xiaoxuan Wang --- cmd/oras/root/manifest/index/create.go | 9 ++++++--- cmd/oras/root/manifest/index/update.go | 27 ++++++++++++++++++++++++++ docs/proposals/multi-arch.md | 9 ++++++++- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/cmd/oras/root/manifest/index/create.go b/cmd/oras/root/manifest/index/create.go index d26e0e2f3..59cab7d74 100644 --- a/cmd/oras/root/manifest/index/create.go +++ b/cmd/oras/root/manifest/index/create.go @@ -157,11 +157,14 @@ func packIndex(oldIndex *ocispec.Index, manifests []ocispec.Descriptor) (ocispec func pushIndex(ctx context.Context, target oras.Target, desc ocispec.Descriptor, content []byte, ref string, extraRefs []string) error { var err error - refs := append(extraRefs, ref) - if len(refs) == 0 { + // need to refine the variable names of ref, extra ref + if ref != "" { + extraRefs = append(extraRefs, ref) + } + if len(extraRefs) == 0 { err = target.Push(ctx, desc, bytes.NewReader(content)) } else { - desc, err = oras.TagBytesN(ctx, target, desc.MediaType, content, refs, oras.DefaultTagBytesNOptions) + desc, err = oras.TagBytesN(ctx, target, desc.MediaType, content, extraRefs, oras.DefaultTagBytesNOptions) } if err != nil { return err diff --git a/cmd/oras/root/manifest/index/update.go b/cmd/oras/root/manifest/index/update.go index 4aadcc429..461ffed59 100644 --- a/cmd/oras/root/manifest/index/update.go +++ b/cmd/oras/root/manifest/index/update.go @@ -39,6 +39,7 @@ type updateOptions struct { extraRefs []string addArguments []string removeArguments []string + mergeArguments []string } func updateCmd() *cobra.Command { @@ -54,6 +55,9 @@ Example - add one manifest and remove two manifests from an index: Example - update the index referenced by tag1 and tag3, and make tag1 and tag3 point to the updated index. If the old index has other tags, they remain pointing to the old index. oras manifest index update localhost:5000/hello:tag1,tag3 --remove sha256:xxx --remove sha256:xxx --add s390x + +Example - remove a manifest and merge manifests from another two indexes. + oras manifest index update localhost:5000/hello:latest --remove sha256:xxx --merge index01 index02 `, Args: cobra.MinimumNArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { @@ -70,6 +74,7 @@ Example - update the index referenced by tag1 and tag3, and make tag1 and tag3 p option.ApplyFlags(&opts, cmd.Flags()) cmd.Flags().StringArrayVarP(&opts.addArguments, "add", "", nil, "manifests to add to the index") cmd.Flags().StringArrayVarP(&opts.removeArguments, "remove", "", nil, "manifests to remove from the index") + cmd.Flags().StringArrayVarP(&opts.mergeArguments, "merge", "", nil, "manifests to merge into the index") return oerrors.Command(cmd, &opts.Target) } @@ -88,6 +93,10 @@ func updateIndex(cmd *cobra.Command, opts updateOptions) error { if err != nil { return err } + manifests, err = mergeIndexes(ctx, manifests, target, opts.mergeArguments) + if err != nil { + return err + } manifests, err = removeManifests(ctx, manifests, target, opts.removeArguments) if err != nil { return err @@ -132,6 +141,24 @@ func addManifests(ctx context.Context, manifests []ocispec.Descriptor, target or return manifests, nil } +func mergeIndexes(ctx context.Context, manifests []ocispec.Descriptor, target oras.ReadOnlyTarget, indexes []string) ([]ocispec.Descriptor, error) { + for _, index := range indexes { + desc, content, err := oras.FetchBytes(ctx, target, index, oras.DefaultFetchBytesOptions) + if err != nil { + return nil, err + } + if desc.MediaType != ocispec.MediaTypeImageIndex { + return nil, fmt.Errorf("%s is not an image index", index) + } + var index ocispec.Index + if err := json.Unmarshal(content, &index); err != nil { + return nil, err + } + manifests = append(manifests, index.Manifests...) + } + return manifests, nil +} + func removeManifests(ctx context.Context, manifests []ocispec.Descriptor, target oras.ReadOnlyTarget, removes []string) ([]ocispec.Descriptor, error) { set := make(map[digest.Digest]struct{}) for _, rem := range removes { diff --git a/docs/proposals/multi-arch.md b/docs/proposals/multi-arch.md index 3aaa2d82f..9df1144c2 100644 --- a/docs/proposals/multi-arch.md +++ b/docs/proposals/multi-arch.md @@ -75,6 +75,7 @@ Flags: - `--add`: Add a manifest to the index. The manifest will be added as the last element of the index. - `--remove`: Remove a manifest from the index. +- `--merge`: Merge the manifests from existing indexes. Subjects, artifact types and annotations of the existing indexes will not be in the updated index. - `--annotation`: Update annotations for the index. - `--annotation-file`: Update annotations for the index and the individual manifests. - `--oci-layout`: Set the target as an oci image layout. @@ -82,7 +83,7 @@ Flags: - `--output`: Output the updated manifest to a location. Auto push is disabled. > [!NOTE] -> One of `--add`/`--remove`/`--annotation`/`--annotation-file` should be used, as there has to be something to update. Otherwise the command does nothing. +> One of `--add`/`--remove`/`--merge`/`--annotation`/`--annotation-file` should be used, as there has to be something to update. Otherwise the command does nothing. #### Examples @@ -99,6 +100,12 @@ updated index. If the old index has other tags, they remain pointing to the old oras manifest index update localhost:5000/hello:tag1,tag3 --remove sha256:xxx --remove sha256:xxx --add s390x ``` +Remove a manifest and merge manifests from another two indexes. + +```sh +oras manifest index update localhost:5000/hello:latest --remove sha256:xxx --merge index01 index02 +``` + ## Design Considerations ### Command design: Making a subcommand group `index` under `oras manifest` From e0d922ea9693dfbe512bd7d6bbc4d248721cf507 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Fri, 9 Aug 2024 14:54:07 +0800 Subject: [PATCH 14/14] --tag for update Signed-off-by: Xiaoxuan Wang --- cmd/oras/root/manifest/index/update.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/oras/root/manifest/index/update.go b/cmd/oras/root/manifest/index/update.go index 461ffed59..0c10efe80 100644 --- a/cmd/oras/root/manifest/index/update.go +++ b/cmd/oras/root/manifest/index/update.go @@ -40,6 +40,7 @@ type updateOptions struct { addArguments []string removeArguments []string mergeArguments []string + tags []string } func updateCmd() *cobra.Command { @@ -75,6 +76,7 @@ Example - remove a manifest and merge manifests from another two indexes. cmd.Flags().StringArrayVarP(&opts.addArguments, "add", "", nil, "manifests to add to the index") cmd.Flags().StringArrayVarP(&opts.removeArguments, "remove", "", nil, "manifests to remove from the index") cmd.Flags().StringArrayVarP(&opts.mergeArguments, "merge", "", nil, "manifests to merge into the index") + cmd.Flags().StringArrayVarP(&opts.tags, "tag", "", nil, "tag adds for the updated manifest") return oerrors.Command(cmd, &opts.Target) } @@ -105,7 +107,7 @@ func updateIndex(cmd *cobra.Command, opts updateOptions) error { if err != nil { return err } - return pushIndex(ctx, target, desc, content, opts.Reference, opts.extraRefs) + return pushIndex(ctx, target, desc, content, opts.Reference, append(opts.extraRefs, opts.tags...)) } func fetchIndex(ctx context.Context, target oras.ReadOnlyTarget, reference string) (ocispec.Index, error) {