From 616c3e1080e2dfcbebc503084ffe87272d36f8ef Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 23 Jan 2024 17:04:18 +0800 Subject: [PATCH] feat: support mounting blobs during copy (#1242) Signed-off-by: Billy Zha --- cmd/oras/internal/display/track/target.go | 7 + .../internal/display/track/target_test.go | 34 +---- cmd/oras/root/cp.go | 18 ++- cmd/oras/root/cp_test.go | 137 ++++++++++++++++-- test/e2e/internal/utils/match/status.go | 1 + test/e2e/suite/command/cp.go | 9 ++ 6 files changed, 166 insertions(+), 40 deletions(-) diff --git a/cmd/oras/internal/display/track/target.go b/cmd/oras/internal/display/track/target.go index 5ff07efe3..3342f7ee2 100644 --- a/cmd/oras/internal/display/track/target.go +++ b/cmd/oras/internal/display/track/target.go @@ -66,6 +66,13 @@ func NewTarget(t oras.GraphTarget, actionPrompt, donePrompt string, tty *os.File return gt, nil } +// Mount mounts a blob from a specified repository. This method is invoked only +// by the `*remote.Repository` target. +func (t *graphTarget) Mount(ctx context.Context, desc ocispec.Descriptor, fromRepo string, getContent func() (io.ReadCloser, error)) error { + mounter := t.GraphTarget.(registry.Mounter) + return mounter.Mount(ctx, desc, fromRepo, getContent) +} + // Push pushes the content to the base oras.GraphTarget with tracking. func (t *graphTarget) Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error { r, err := managedReader(content, expected, t.manager, t.actionPrompt, t.donePrompt) diff --git a/cmd/oras/internal/display/track/target_test.go b/cmd/oras/internal/display/track/target_test.go index 6622df531..fa2c19a7d 100644 --- a/cmd/oras/internal/display/track/target_test.go +++ b/cmd/oras/internal/display/track/target_test.go @@ -25,6 +25,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content/memory" + "oras.land/oras-go/v2/registry/remote" "oras.land/oras/cmd/oras/internal/display/console/testutils" ) @@ -79,34 +80,7 @@ func Test_referenceGraphTarget_PushReference(t *testing.T) { } } -func Test_referenceGraphTarget_Prompt(t *testing.T) { - // prepare - pty, device, err := testutils.NewPty() - if err != nil { - t.Fatal(err) - } - defer device.Close() - content := []byte("test") - desc := ocispec.Descriptor{ - MediaType: "application/octet-stream", - Digest: digest.FromBytes(content), - Size: int64(len(content)), - } - // test - prompt := "prompt" - target, err := NewTarget(memory.New(), "action", "done", device) - if err != nil { - t.Fatal(err) - } - m := target.(*graphTarget).manager - if err := target.Prompt(desc, prompt); err != nil { - t.Fatal(err) - } - if err := m.Close(); err != nil { - t.Fatal(err) - } - // validate - if err = testutils.MatchPty(pty, device, prompt, desc.MediaType, "100.00%", desc.Digest.String()); err != nil { - t.Fatal(err) - } +func Test_referenceGraphTarget_Mount(t *testing.T) { + target := graphTarget{GraphTarget: &remote.Repository{}} + _ = target.Mount(context.Background(), ocispec.Descriptor{}, "", nil) } diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 1ab2a78b7..6df3d2d58 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -29,6 +29,7 @@ import ( "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/registry" + "oras.land/oras-go/v2/registry/remote" "oras.land/oras-go/v2/registry/remote/auth" "oras.land/oras/cmd/oras/internal/argument" "oras.land/oras/cmd/oras/internal/display" @@ -162,8 +163,15 @@ func doCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.GraphTar promptCopying = "Copying" promptCopied = "Copied " promptSkipped = "Skipped" + promptMounted = "Mounted" ) - + srcRepo, srcIsRemote := src.(*remote.Repository) + dstRepo, dstIsRemote := dst.(*remote.Repository) + if srcIsRemote && dstIsRemote && srcRepo.Reference.Registry == dstRepo.Reference.Registry { + extendedCopyOptions.MountFrom = func(ctx context.Context, desc ocispec.Descriptor) ([]string, error) { + return []string{srcRepo.Reference.Repository}, nil + } + } if opts.TTY == nil { // none TTY output extendedCopyOptions.OnCopySkipped = func(ctx context.Context, desc ocispec.Descriptor) error { @@ -180,6 +188,10 @@ func doCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.GraphTar } return display.PrintStatus(desc, promptCopied, opts.Verbose) } + extendedCopyOptions.OnMounted = func(ctx context.Context, desc ocispec.Descriptor) error { + committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]) + return display.PrintStatus(desc, promptMounted, opts.Verbose) + } } else { // TTY output tracked, err := track.NewTarget(dst, promptCopying, promptCopied, opts.TTY) @@ -198,6 +210,10 @@ func doCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.GraphTar return tracked.Prompt(desc, promptSkipped) }) } + extendedCopyOptions.OnMounted = func(ctx context.Context, desc ocispec.Descriptor) error { + committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]) + return tracked.Prompt(desc, promptMounted) + } } var desc ocispec.Descriptor diff --git a/cmd/oras/root/cp_test.go b/cmd/oras/root/cp_test.go index 2ef53ba11..629cafb43 100644 --- a/cmd/oras/root/cp_test.go +++ b/cmd/oras/root/cp_test.go @@ -1,3 +1,5 @@ +//go:build darwin || freebsd || linux || netbsd || openbsd || solaris + /* Copyright The ORAS Authors. Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,37 +21,99 @@ import ( "bytes" "context" "fmt" + "net/http" + "net/http/httptest" + "net/url" "os" "testing" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/content/memory" + "oras.land/oras-go/v2/registry/remote" "oras.land/oras/cmd/oras/internal/display/console/testutils" ) var ( - src *memory.Store - desc ocispec.Descriptor + memStore *memory.Store + memDesc ocispec.Descriptor + manifestConent = []byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","artifactType":"application/vnd.unknown.artifact.v1","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2,"data":"e30="},"layers":[{"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2,"data":"e30="}]}`) + manifestDigest = "sha256:1bb053792feb8d8d590001c212f2defad9277e091d2aa868cde2879ff41abb1b" + configContent = []byte("{}") + configDigest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + configMediaType = "application/vnd.oci.empty.v1+json" + host string + repoFrom = "from" + repoTo = "to" ) func TestMain(m *testing.M) { - src = memory.New() + // memory store for testing + memStore = memory.New() content := []byte("test") r := bytes.NewReader(content) - desc = ocispec.Descriptor{ + memDesc = ocispec.Descriptor{ MediaType: "application/octet-stream", Digest: digest.FromBytes(content), Size: int64(len(content)), } - if err := src.Push(context.Background(), desc, r); err != nil { + if err := memStore.Push(context.Background(), memDesc, r); err != nil { fmt.Println("Setup failed:", err) os.Exit(1) } - if err := src.Tag(context.Background(), desc, desc.Digest.String()); err != nil { + if err := memStore.Tag(context.Background(), memDesc, memDesc.Digest.String()); err != nil { fmt.Println("Setup failed:", err) os.Exit(1) } + + // test server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == fmt.Sprintf("/v2/%s/manifests/%s", repoFrom, manifestDigest) && + r.Method == http.MethodHead: + w.Header().Set("Content-Type", ocispec.MediaTypeImageManifest) + w.Header().Set("Content-Length", fmt.Sprint(len(manifestConent))) + w.WriteHeader(http.StatusOK) + case r.URL.Path == fmt.Sprintf("/v2/%s/manifests/%s", repoFrom, manifestDigest) && + r.Method == http.MethodGet: + w.Header().Set("Content-Type", ocispec.MediaTypeImageManifest) + w.Header().Set("Content-Length", fmt.Sprint(len(manifestConent))) + _, _ = w.Write(manifestConent) + w.WriteHeader(http.StatusOK) + case r.URL.Path == fmt.Sprintf("/v2/%s/blobs/%s", repoFrom, configDigest) && + r.Method == http.MethodGet: + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", fmt.Sprint(len(configContent))) + _, _ = w.Write(configContent) + w.WriteHeader(http.StatusOK) + case r.URL.Path == fmt.Sprintf("/v2/%s/manifests/%s", repoTo, manifestDigest) && + r.Method == http.MethodHead: + w.WriteHeader(http.StatusNotFound) + case r.URL.Path == fmt.Sprintf("/v2/%s/blobs/%s", repoTo, configDigest) && + r.Method == http.MethodHead: + w.WriteHeader(http.StatusNotFound) + case r.URL.Path == fmt.Sprintf("/v2/%s/blobs/uploads/", repoTo) && + r.URL.Query().Get("mount") == configDigest && + r.URL.Query().Get("from") == repoFrom && + r.Method == http.MethodPost: + w.Header().Set("Location", fmt.Sprintf("/v2/%s/blobs/%s", repoTo, configDigest)) + w.WriteHeader(http.StatusCreated) + case r.URL.Path == fmt.Sprintf("/v2/%s/manifests/%s", repoTo, manifestDigest) && + r.Method == http.MethodPut: + w.WriteHeader(http.StatusCreated) + case r.URL.Path == fmt.Sprintf("/v2/%s/manifests/%s", repoTo, manifestDigest) && + r.Method == http.MethodGet: + w.Header().Set("Content-Type", ocispec.MediaTypeImageManifest) + w.Header().Set("Content-Length", fmt.Sprint(len(manifestConent))) + _, _ = w.Write(manifestConent) + w.WriteHeader(http.StatusOK) + default: + w.WriteHeader(http.StatusNotAcceptable) + } + })) + defer ts.Close() + uri, _ := url.Parse(ts.URL) + host = "localhost:" + uri.Port() m.Run() } @@ -63,15 +127,70 @@ func Test_doCopy(t *testing.T) { var opts copyOptions opts.TTY = slave opts.Verbose = true - opts.From.Reference = desc.Digest.String() + opts.From.Reference = memDesc.Digest.String() dst := memory.New() // test - _, err = doCopy(context.Background(), src, dst, &opts) + _, err = doCopy(context.Background(), memStore, dst, &opts) + if err != nil { + t.Fatal(err) + } + // validate + if err = testutils.MatchPty(pty, slave, "Copied", memDesc.MediaType, "100.00%", memDesc.Digest.String()); err != nil { + t.Fatal(err) + } +} + +func Test_doCopy_skipped(t *testing.T) { + // prepare + pty, slave, err := testutils.NewPty() + if err != nil { + t.Fatal(err) + } + defer slave.Close() + var opts copyOptions + opts.TTY = slave + opts.Verbose = true + opts.From.Reference = memDesc.Digest.String() + // test + _, err = doCopy(context.Background(), memStore, memStore, &opts) + if err != nil { + t.Fatal(err) + } + // validate + if err = testutils.MatchPty(pty, slave, "Exists", memDesc.MediaType, "100.00%", memDesc.Digest.String()); err != nil { + t.Fatal(err) + } +} + +func Test_doCopy_mounted(t *testing.T) { + // prepare + pty, slave, err := testutils.NewPty() + if err != nil { + t.Fatal(err) + } + defer slave.Close() + var opts copyOptions + opts.TTY = slave + opts.Verbose = true + opts.From.Reference = manifestDigest + // mocked repositories + from, err := remote.NewRepository(fmt.Sprintf("%s/%s", host, repoFrom)) + if err != nil { + t.Fatal(err) + } + from.PlainHTTP = true + to, err := remote.NewRepository(fmt.Sprintf("%s/%s", host, repoTo)) + if err != nil { + t.Fatal(err) + } + to.PlainHTTP = true + // test + _, err = doCopy(context.Background(), from, to, &opts) if err != nil { t.Fatal(err) } // validate - if err = testutils.MatchPty(pty, slave, "Copied", desc.MediaType, "100.00%", desc.Digest.String()); err != nil { + if err = testutils.MatchPty(pty, slave, "Mounted", configMediaType, "100.00%", configDigest); err != nil { t.Fatal(err) } } diff --git a/test/e2e/internal/utils/match/status.go b/test/e2e/internal/utils/match/status.go index b98656a22..c35ae0e1f 100644 --- a/test/e2e/internal/utils/match/status.go +++ b/test/e2e/internal/utils/match/status.go @@ -76,6 +76,7 @@ func newStateMachine(cmd string) *stateMachine { sm.addPath("Copying", "Copied") sm.addPath("Skipped") sm.addPath("Exists") + sm.addPath("Mounted") default: ginkgo.Fail("Unrecognized cmd name " + cmd) } diff --git a/test/e2e/suite/command/cp.go b/test/e2e/suite/command/cp.go index e09e6f61f..d122e9d45 100644 --- a/test/e2e/suite/command/cp.go +++ b/test/e2e/suite/command/cp.go @@ -349,6 +349,15 @@ var _ = Describe("1.1 registry users:", func() { var _ = Describe("OCI spec 1.0 registry users:", func() { When("running `cp`", func() { + It("should copy an image artifact with mounting", func() { + repo := cpTestRepo("1.0-mount") + src := RegistryRef(FallbackHost, ArtifactRepo, foobar.Tag) + dst := RegistryRef(FallbackHost, repo, "") + out := ORAS("cp", src, dst, "-v").Exec() + Expect(out).Should(gbytes.Say("Mounted fcde2b2edba5 bar")) + CompareRef(src, RegistryRef(FallbackHost, repo, foobar.Digest)) + }) + It("should copy an image artifact and its referrers from a registry to a fallback registry", func() { repo := cpTestRepo("to-fallback") stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...)