From 8a3158effd4ffda5d903733b119aada3839490c8 Mon Sep 17 00:00:00 2001 From: fengwei0328 Date: Tue, 10 Dec 2024 11:21:55 +0800 Subject: [PATCH] Fix duplicate image entries in k8s.io namespaces Signed-off-by: fengwei0328 --- cmd/nerdctl/image/image_history.go | 18 ++++---- cmd/nerdctl/image/image_list.go | 8 ++-- cmd/nerdctl/inspect/inspect.go | 4 +- go.mod | 1 + go.sum | 2 + pkg/cmd/image/list.go | 18 +++++++- pkg/cmd/image/remove.go | 20 ++++---- pkg/cmd/image/save.go | 8 ++-- pkg/cmd/image/tag.go | 4 +- pkg/idutil/imagewalker/imagewalker.go | 66 ++++++++++++++++++++++++--- pkg/imgutil/imgutil.go | 10 ++-- pkg/ipfs/image.go | 4 +- 12 files changed, 116 insertions(+), 47 deletions(-) diff --git a/cmd/nerdctl/image/image_history.go b/cmd/nerdctl/image/image_history.go index d4f4aa9bcc8..72ec1498297 100644 --- a/cmd/nerdctl/image/image_history.go +++ b/cmd/nerdctl/image/image_history.go @@ -92,22 +92,22 @@ func historyAction(cmd *cobra.Command, args []string) error { walker := &imagewalker.ImageWalker{ Client: client, - OnFound: func(ctx context.Context, found imagewalker.Found) error { + OnFound: func(ctx context.Context, found imagewalker.Found) (error, bool) { if found.MatchCount > 1 { - return fmt.Errorf("multiple IDs found with provided prefix: %s", found.Req) + return fmt.Errorf("multiple IDs found with provided prefix: %s", found.Req), false } ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() img := containerd.NewImage(client, found.Image) imageConfig, _, err := imgutil.ReadImageConfig(ctx, img) if err != nil { - return fmt.Errorf("failed to ReadImageConfig: %w", err) + return fmt.Errorf("failed to ReadImageConfig: %w", err), false } configHistories := imageConfig.History layerCounter := 0 diffIDs, err := img.RootFS(ctx) if err != nil { - return fmt.Errorf("failed to get diffIDS: %w", err) + return fmt.Errorf("failed to get diffIDS: %w", err), false } var historys []historyPrintable for _, h := range configHistories { @@ -115,7 +115,7 @@ func historyAction(cmd *cobra.Command, args []string) error { var snapshotName string if !h.EmptyLayer { if len(diffIDs) <= layerCounter { - return fmt.Errorf("too many non-empty layers in History section") + return fmt.Errorf("too many non-empty layers in History section"), false } diffIDs := diffIDs[0 : layerCounter+1] chainID := identity.ChainID(diffIDs).String() @@ -123,11 +123,11 @@ func historyAction(cmd *cobra.Command, args []string) error { s := client.SnapshotService(globalOptions.Snapshotter) stat, err := s.Stat(ctx, chainID) if err != nil { - return fmt.Errorf("failed to get stat: %w", err) + return fmt.Errorf("failed to get stat: %w", err), false } use, err := s.Usage(ctx, chainID) if err != nil { - return fmt.Errorf("failed to get usage: %w", err) + return fmt.Errorf("failed to get usage: %w", err), false } size = use.Size snapshotName = stat.Name @@ -147,9 +147,9 @@ func historyAction(cmd *cobra.Command, args []string) error { } err = printHistory(cmd, historys) if err != nil { - return fmt.Errorf("failed printHistory: %w", err) + return fmt.Errorf("failed printHistory: %w", err), false } - return nil + return nil, false }, } diff --git a/cmd/nerdctl/image/image_list.go b/cmd/nerdctl/image/image_list.go index 06566f471fd..63ba286954c 100644 --- a/cmd/nerdctl/image/image_list.go +++ b/cmd/nerdctl/image/image_list.go @@ -65,7 +65,7 @@ Properties: }) imagesCommand.Flags().Bool("digests", false, "Show digests (compatible with Docker, unlike ID)") imagesCommand.Flags().Bool("names", false, "Show image names") - imagesCommand.Flags().BoolP("all", "a", true, "(unimplemented yet, always true)") + imagesCommand.Flags().BoolP("all", "a", false, "Show all images repo, include imageID, repoTAg, repoDigest") return imagesCommand } @@ -110,6 +110,7 @@ func processImageListOptions(cmd *cobra.Command, args []string) (*types.ImageLis if err != nil { return nil, err } + all, err := cmd.Flags().GetBool("all") return &types.ImageListOptions{ GOptions: globalOptions, Quiet: quiet, @@ -119,7 +120,7 @@ func processImageListOptions(cmd *cobra.Command, args []string) (*types.ImageLis NameAndRefFilter: filters, Digests: digests, Names: names, - All: true, + All: all, Stdout: cmd.OutOrStdout(), }, nil @@ -130,9 +131,6 @@ func imagesAction(cmd *cobra.Command, args []string) error { if err != nil { return err } - if !options.All { - options.All = true - } client, ctx, cancel, err := clientutil.NewClient(cmd.Context(), options.GOptions.Namespace, options.GOptions.Address) if err != nil { diff --git a/cmd/nerdctl/inspect/inspect.go b/cmd/nerdctl/inspect/inspect.go index a91624b27f1..61ee7b5eab7 100644 --- a/cmd/nerdctl/inspect/inspect.go +++ b/cmd/nerdctl/inspect/inspect.go @@ -98,8 +98,8 @@ func inspectAction(cmd *cobra.Command, args []string) error { imagewalker := &imagewalker.ImageWalker{ Client: client, - OnFound: func(ctx context.Context, found imagewalker.Found) error { - return nil + OnFound: func(ctx context.Context, found imagewalker.Found) (error, bool) { + return nil, false }, } diff --git a/go.mod b/go.mod index 2341440f07e..06602ad8396 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/containerd/accelerated-container-image v1.2.3 github.com/containerd/cgroups/v3 v3.0.4 github.com/containerd/console v1.0.4 + github.com/containerd/containerd v1.7.23 github.com/containerd/containerd/api v1.8.0 github.com/containerd/containerd/v2 v2.0.0 github.com/containerd/continuity v0.4.5 diff --git a/go.sum b/go.sum index 2202ce91b04..e089478db1b 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,8 @@ github.com/containerd/cgroups/v3 v3.0.4 h1:2fs7l3P0Qxb1nKWuJNFiwhp2CqiKzho71DQkD github.com/containerd/cgroups/v3 v3.0.4/go.mod h1:SA5DLYnXO8pTGYiAHXz94qvLQTKfVM5GEVisn4jpins= github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro= github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= +github.com/containerd/containerd v1.7.23 h1:H2CClyUkmpKAGlhQp95g2WXHfLYc7whAuvZGBNYOOwQ= +github.com/containerd/containerd v1.7.23/go.mod h1:7QUzfURqZWCZV7RLNEn1XjUCQLEf0bkaK4GjUaZehxw= github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0= github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc= github.com/containerd/containerd/v2 v2.0.0 h1:qLDdFaAykQrIyLiqwQrNLLz95wiC36bAZVwioUwqShM= diff --git a/pkg/cmd/image/list.go b/pkg/cmd/image/list.go index 19905645c59..a8a9c0cc871 100644 --- a/pkg/cmd/image/list.go +++ b/pkg/cmd/image/list.go @@ -29,6 +29,7 @@ import ( "text/template" "time" + "github.com/distribution/reference" "github.com/docker/go-units" "github.com/opencontainers/image-spec/identity" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -128,6 +129,21 @@ type imagePrintable struct { func printImages(ctx context.Context, client *containerd.Client, imageList []images.Image, options *types.ImageListOptions) error { w := options.Stdout + var ImageList []images.Image + if options.GOptions.Namespace != "k8s.io" || options.All { + ImageList = imageList + } else { + for _, ima := range imageList { + parsed, err := reference.ParseAnyReference(ima.Name) + if err != nil { + continue + } + if _, ok := parsed.(reference.Tagged); !ok { + continue + } + ImageList = append(ImageList, ima) + } + } digestsFlag := options.Digests if options.Format == "wide" { digestsFlag = true @@ -174,7 +190,7 @@ func printImages(ctx context.Context, client *containerd.Client, imageList []ima snapshotter: containerdutil.SnapshotService(client, options.GOptions.Snapshotter), } - for _, img := range imageList { + for _, img := range ImageList { if err := printer.printImage(ctx, img); err != nil { log.G(ctx).Warn(err) } diff --git a/pkg/cmd/image/remove.go b/pkg/cmd/image/remove.go index 8075a928659..4a5c945406c 100644 --- a/pkg/cmd/image/remove.go +++ b/pkg/cmd/image/remove.go @@ -64,37 +64,37 @@ func Remove(ctx context.Context, client *containerd.Client, args []string, optio walker := &imagewalker.ImageWalker{ Client: client, - OnFound: func(ctx context.Context, found imagewalker.Found) error { + OnFound: func(ctx context.Context, found imagewalker.Found) (error, bool) { if found.NameMatchIndex == -1 { // if found multiple images, return error unless in force-mode and // there is only 1 unique image. if found.MatchCount > 1 && !(options.Force && found.UniqueImages == 1) { - return fmt.Errorf("multiple IDs found with provided prefix: %s", found.Req) + return fmt.Errorf("multiple IDs found with provided prefix: %s", found.Req), false } } else if found.NameMatchIndex != found.MatchIndex { // when there is an image with a name matching the argument but the argument is a digest short id, // the deletion process is not performed. - return nil + return nil, false } if cid, ok := runningImages[found.Image.Name]; ok { if options.Force { if err = is.Delete(ctx, found.Image.Name); err != nil { - return err + return err, false } fmt.Fprintf(options.Stdout, "Untagged: %s\n", found.Image.Name) fmt.Fprintf(options.Stdout, "Untagged: %s\n", found.Image.Target.Digest.String()) found.Image.Name = ":" if _, err = is.Create(ctx, found.Image); err != nil { - return err + return err, false } - return nil + return nil, false } - return fmt.Errorf("conflict: unable to delete %s (cannot be forced) - image is being used by running container %s", found.Req, cid) + return fmt.Errorf("conflict: unable to delete %s (cannot be forced) - image is being used by running container %s", found.Req, cid), false } if cid, ok := usedImages[found.Image.Name]; ok && !options.Force { - return fmt.Errorf("conflict: unable to delete %s (must be forced) - image is being used by stopped container %s", found.Req, cid) + return fmt.Errorf("conflict: unable to delete %s (must be forced) - image is being used by stopped container %s", found.Req, cid), false } // digests is used only for emulating human-readable output of `docker rmi` digests, err := found.Image.RootFS(ctx, cs, platforms.DefaultStrict()) @@ -103,13 +103,13 @@ func Remove(ctx context.Context, client *containerd.Client, args []string, optio } if err := is.Delete(ctx, found.Image.Name, delOpts...); err != nil { - return err + return err, false } fmt.Fprintf(options.Stdout, "Untagged: %s@%s\n", found.Image.Name, found.Image.Target.Digest) for _, digest := range digests { fmt.Fprintf(options.Stdout, "Deleted: %s\n", digest) } - return nil + return nil, true }, } diff --git a/pkg/cmd/image/save.go b/pkg/cmd/image/save.go index 815305ee80c..9c4559b5227 100644 --- a/pkg/cmd/image/save.go +++ b/pkg/cmd/image/save.go @@ -44,15 +44,15 @@ func Save(ctx context.Context, client *containerd.Client, images []string, optio savedImages := make(map[string]struct{}) walker := &imagewalker.ImageWalker{ Client: client, - OnFound: func(ctx context.Context, found imagewalker.Found) error { + OnFound: func(ctx context.Context, found imagewalker.Found) (error, bool) { if found.UniqueImages > 1 { - return fmt.Errorf("ambiguous digest ID: multiple IDs found with provided prefix %s", found.Req) + return fmt.Errorf("ambiguous digest ID: multiple IDs found with provided prefix %s", found.Req), false } // Ensure all the layers are here: https://github.com/containerd/nerdctl/issues/3425 err = EnsureAllContent(ctx, client, found.Image.Name, options.GOptions) if err != nil { - return err + return err, false } imgName := found.Image.Name @@ -61,7 +61,7 @@ func Save(ctx context.Context, client *containerd.Client, images []string, optio savedImages[imgDigest] = struct{}{} exportOpts = append(exportOpts, archive.WithImage(imageStore, imgName)) } - return nil + return nil, false }, } diff --git a/pkg/cmd/image/tag.go b/pkg/cmd/image/tag.go index 5323080f745..49f2aa2ce3e 100644 --- a/pkg/cmd/image/tag.go +++ b/pkg/cmd/image/tag.go @@ -34,11 +34,11 @@ func Tag(ctx context.Context, client *containerd.Client, options types.ImageTagO var srcName string walker := &imagewalker.ImageWalker{ Client: client, - OnFound: func(ctx context.Context, found imagewalker.Found) error { + OnFound: func(ctx context.Context, found imagewalker.Found) (error, bool) { if srcName == "" { srcName = found.Image.Name } - return nil + return nil, false }, } matchCount, err := walker.Walk(ctx, options.Source) diff --git a/pkg/idutil/imagewalker/imagewalker.go b/pkg/idutil/imagewalker/imagewalker.go index 4103711ed91..61c1b717c98 100644 --- a/pkg/idutil/imagewalker/imagewalker.go +++ b/pkg/idutil/imagewalker/imagewalker.go @@ -22,6 +22,7 @@ import ( "regexp" "strings" + "github.com/distribution/reference" "github.com/opencontainers/go-digest" containerd "github.com/containerd/containerd/v2/client" @@ -39,7 +40,7 @@ type Found struct { NameMatchIndex int // Image index with a name matching the argument for `nerdctl rmi`. } -type OnFound func(ctx context.Context, found Found) error +type OnFound func(ctx context.Context, found Found) (error, bool) type ImageWalker struct { Client *containerd.Client @@ -52,16 +53,27 @@ type ImageWalker struct { func (w *ImageWalker) Walk(ctx context.Context, req string) (int, error) { var filters []string var parsedReferenceStr string + var images_tag, images_repo []images.Image + var tag_num int + var repo string parsedReference, err := referenceutil.Parse(req) if err == nil { parsedReferenceStr = parsedReference.String() filters = append(filters, fmt.Sprintf("name==%s", parsedReferenceStr)) } + //Get the image ID , if reg == imageTag use + image, err := w.Client.GetImage(ctx, parsedReferenceStr) + if err != nil { + repo = req + } else { + repo = strings.Split(image.Target().Digest.String(), ":")[1][:12] + } + filters = append(filters, fmt.Sprintf("name==%s", req), - fmt.Sprintf("target.digest~=^sha256:%s.*$", regexp.QuoteMeta(req)), - fmt.Sprintf("target.digest~=^%s.*$", regexp.QuoteMeta(req)), + fmt.Sprintf("target.digest~=^sha256:%s.*$", regexp.QuoteMeta(repo)), + fmt.Sprintf("target.digest~=^%s.*$", regexp.QuoteMeta(repo)), ) images, err := w.Client.ImageService().List(ctx, filters...) @@ -69,12 +81,28 @@ func (w *ImageWalker) Walk(ctx context.Context, req string) (int, error) { return -1, err } - matchCount := len(images) + //Distinguish between tag and non-tag + for _, ima := range images { + ref := ima.Name + parsed, err := reference.ParseAnyReference(ref) + if err != nil { + continue + } + switch parsed.(type) { + case reference.Canonical, reference.Digested: + images_repo = append(images_repo, ima) + case reference.Tagged: + images_tag = append(images_tag, ima) + tag_num++ + } + } + + matchCount := len(images_tag) // to handle the `rmi -f` case where returned images are different but // have the same short prefix. uniqueImages := make(map[digest.Digest]bool) nameMatchIndex := -1 - for i, image := range images { + for i, image := range images_tag { uniqueImages[image.Target.Digest] = true // to get target image index for `nerdctl rmi `. if (parsedReferenceStr != "" && image.Name == parsedReferenceStr) || image.Name == req { @@ -82,7 +110,12 @@ func (w *ImageWalker) Walk(ctx context.Context, req string) (int, error) { } } - for i, img := range images { + //The matchCount count is only required if it is passed in as an image ID + if nameMatchIndex != -1 || matchCount < 1 { + matchCount = 1 + } + + for i, img := range images_tag { f := Found{ Image: img, Req: req, @@ -91,8 +124,27 @@ func (w *ImageWalker) Walk(ctx context.Context, req string) (int, error) { UniqueImages: len(uniqueImages), NameMatchIndex: nameMatchIndex, } - if e := w.OnFound(ctx, f); e != nil { + e, ok := w.OnFound(ctx, f) + if e != nil { return -1, e + } else if ok { + tag_num = tag_num - 1 + } + } + //If the corresponding imageTag does not exist, delete the repoDigests + if tag_num == 0 { + for i, img := range images_repo { + f := Found{ + Image: img, + Req: req, + MatchIndex: i, + MatchCount: 1, + UniqueImages: len(uniqueImages), + NameMatchIndex: -1, + } + if e, _ := w.OnFound(ctx, f); e != nil { + return -1, e + } } } return matchCount, nil diff --git a/pkg/imgutil/imgutil.go b/pkg/imgutil/imgutil.go index 09ca283b5dc..cb88f84525b 100644 --- a/pkg/imgutil/imgutil.go +++ b/pkg/imgutil/imgutil.go @@ -63,16 +63,16 @@ func GetExistingImage(ctx context.Context, client *containerd.Client, snapshotte var res *EnsuredImage imgwalker := &imagewalker.ImageWalker{ Client: client, - OnFound: func(ctx context.Context, found imagewalker.Found) error { + OnFound: func(ctx context.Context, found imagewalker.Found) (error, bool) { if res != nil { - return nil + return nil, false } image := containerd.NewImageWithPlatform(client, found.Image, platforms.OnlyStrict(platform)) imgConfig, err := getImageConfig(ctx, image) if err != nil { // Image found but blob not found for foreign arch // Ignore err and return nil, so that the walker can visit the next candidate. - return nil + return nil, false } res = &EnsuredImage{ Ref: found.Image.Name, @@ -83,10 +83,10 @@ func GetExistingImage(ctx context.Context, client *containerd.Client, snapshotte } if unpacked, err := image.IsUnpacked(ctx, snapshotter); err == nil && !unpacked { if err := image.Unpack(ctx, snapshotter); err != nil { - return err + return err, false } } - return nil + return nil, false }, } count, err := imgwalker.Walk(ctx, rawRef) diff --git a/pkg/ipfs/image.go b/pkg/ipfs/image.go index 84d73bda32e..f2d15da491d 100644 --- a/pkg/ipfs/image.go +++ b/pkg/ipfs/image.go @@ -108,9 +108,9 @@ func ensureContentsOfIPFSImage(ctx context.Context, client *containerd.Client, r var img images.Image walker := &imagewalker.ImageWalker{ Client: client, - OnFound: func(ctx context.Context, found imagewalker.Found) error { + OnFound: func(ctx context.Context, found imagewalker.Found) (error, bool) { img = found.Image - return nil + return nil, false }, } n, err := walker.Walk(ctx, ref)