Merge branch 'main' into kmichel-fix-heredoc-conflict

.cirrus.yml

@@ -32,7 +32,7 @@ env:
     DEBIAN_NAME: "debian-13"
 
     # Image identifiers
-    IMAGE_SUFFIX: "c20240821t171500z-f40f39d13"
+    IMAGE_SUFFIX: "c20240826t190000z-f40f39d13"
     FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}"
     DEBIAN_CACHE_IMAGE_NAME: "debian-${IMAGE_SUFFIX}"

.github/workflows/pr.yml

@@ -4,7 +4,7 @@ on:
 
 jobs:
   commit:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     # Only check commits on pull requests.
     if: github.event_name == 'pull_request'
     steps:

.packit.yaml

@@ -14,6 +14,8 @@ packages:
     specfile_path: rpm/buildah.spec
   buildah-rhel:
     specfile_path: rpm/buildah.spec
+  buildah-eln:
+    specfile_path: rpm/buildah.spec
 
 srpm_build_deps:
   - make
@@ -26,12 +28,21 @@ jobs:
       failure_comment:
         message: "Ephemeral COPR build failed. @containers/packit-build please check."
     targets:
-      fedora-development-x86_64: {}
-      fedora-development-aarch64: {}
-      fedora-latest-x86_64: {}
-      fedora-latest-aarch64: {}
-      fedora-latest-stable-x86_64: {}
-      fedora-latest-stable-aarch64: {}
+      - fedora-development-x86_64
+      - fedora-development-aarch64
+      - fedora-latest-x86_64
+      - fedora-latest-aarch64
+      - fedora-latest-stable-x86_64
+      - fedora-latest-stable-aarch64
+      - fedora-40-x86_64
+      - fedora-40-aarch64
+    enable_net: true
+
+  - job: copr_build
+    trigger: pull_request
+    packages: [buildah-eln]
+    notifications: *copr_build_failure_notification
+    targets:
       fedora-eln-x86_64:
         additional_repos:
           - "https://kojipkgs.fedoraproject.org/repos/eln-build/latest/x86_64/"
@@ -78,7 +89,7 @@ jobs:
     trigger: release
     packages: [buildah-fedora]
     update_release: false
-    dist_git_branches:
+    dist_git_branches: &fedora_targets
       - fedora-all
 
   # Sync to CentOS Stream
@@ -89,12 +100,13 @@ jobs:
     dist_git_branches:
       - c10s
 
+  # Fedora Koji build
   - job: koji_build
     trigger: commit
-    dist_git_branches:
-      - fedora-all
-
-  - job: bodhi_update
-    trigger: commit
-    dist_git_branches:
-      - fedora-branched # rawhide updates are created automatically
+    sidetag_group: podman-releases
+    # Dependents are not rpm dependencies, but the package whose bodhi update
+    # should include this package.
+    # Ref: https://packit.dev/docs/fedora-releases-guide/releasing-multiple-packages
+    dependents:
+      - podman
+    dist_git_branches: *fedora_targets

Makefile

@@ -41,7 +41,7 @@ LIBSECCOMP_COMMIT := release-2.3
 
 EXTRA_LDFLAGS ?=
 BUILDAH_LDFLAGS := $(GO_LDFLAGS) '-X main.GitCommit=$(GIT_COMMIT) -X main.buildInfo=$(SOURCE_DATE_EPOCH) -X main.cniVersion=$(CNI_COMMIT) $(EXTRA_LDFLAGS)'
-SOURCES=*.go imagebuildah/*.go bind/*.go chroot/*.go copier/*.go define/*.go docker/*.go internal/config/*.go internal/mkcw/*.go internal/mkcw/types/*.go internal/parse/*.go internal/sbom/*.go internal/source/*.go internal/tmpdir/*.go internal/*.go internal/util/*.go internal/volumes/*.go manifests/*.go pkg/blobcache/*.go pkg/chrootuser/*.go pkg/cli/*.go pkg/completion/*.go pkg/formats/*.go pkg/jail/*.go pkg/overlay/*.go pkg/parse/*.go pkg/rusage/*.go pkg/sshagent/*.go pkg/umask/*.go pkg/util/*.go pkg/volumes/*.go util/*.go
+SOURCES=*.go imagebuildah/*.go bind/*.go chroot/*.go copier/*.go define/*.go docker/*.go internal/config/*.go internal/mkcw/*.go internal/mkcw/types/*.go internal/parse/*.go internal/sbom/*.go internal/source/*.go internal/tmpdir/*.go internal/*.go internal/util/*.go internal/volumes/*.go manifests/*.go pkg/binfmt/*.go pkg/blobcache/*.go pkg/chrootuser/*.go pkg/cli/*.go pkg/completion/*.go pkg/formats/*.go pkg/jail/*.go pkg/overlay/*.go pkg/parse/*.go pkg/rusage/*.go pkg/sshagent/*.go pkg/umask/*.go pkg/util/*.go pkg/volumes/*.go util/*.go
 
 LINTFLAGS ?=
 

add.go

@@ -20,12 +20,14 @@ import (
 
 	"github.com/containers/buildah/copier"
 	"github.com/containers/buildah/define"
+	"github.com/containers/buildah/internal/tmpdir"
 	"github.com/containers/buildah/pkg/chrootuser"
 	"github.com/containers/common/pkg/retry"
 	"github.com/containers/image/v5/pkg/tlsclientconfig"
 	"github.com/containers/image/v5/types"
 	"github.com/containers/storage/pkg/fileutils"
 	"github.com/containers/storage/pkg/idtools"
+	"github.com/containers/storage/pkg/regexp"
 	"github.com/docker/go-connections/tlsconfig"
 	"github.com/hashicorp/go-multierror"
 	digest "github.com/opencontainers/go-digest"
@@ -93,9 +95,27 @@ type AddAndCopyOptions struct {
 	RetryDelay time.Duration
 }
 
-// sourceIsRemote returns true if "source" is a remote location.
+// gitURLFragmentSuffix matches fragments to use as Git reference and build
+// context from the Git repository e.g.
+//
+//	github.com/containers/buildah.git
+//	github.com/containers/buildah.git#main
+//	github.com/containers/buildah.git#v1.35.0
+var gitURLFragmentSuffix = regexp.Delayed(`\.git(?:#.+)?$`)
+
+// sourceIsGit returns true if "source" is a git location.
+func sourceIsGit(source string) bool {
+	return isURL(source) && gitURLFragmentSuffix.MatchString(source)
+}
+
+func isURL(url string) bool {
+	return strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")
+}
+
+// sourceIsRemote returns true if "source" is a remote location
+// and *not* a git repo. Certain github urls such as raw.github.* are allowed.
 func sourceIsRemote(source string) bool {
-	return strings.HasPrefix(source, "http://") || strings.HasPrefix(source, "https://")
+	return isURL(source) && !gitURLFragmentSuffix.MatchString(source)
 }
 
 // getURL writes a tar archive containing the named content
@@ -274,7 +294,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 	}
 
 	// Figure out what sorts of sources we have.
-	var localSources, remoteSources []string
+	var localSources, remoteSources, gitSources []string
 	for i, src := range sources {
 		if src == "" {
 			return errors.New("empty source location")
@@ -283,12 +303,22 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 			remoteSources = append(remoteSources, src)
 			continue
 		}
+		if sourceIsGit(src) {
+			gitSources = append(gitSources, src)
+			continue
+		}
 		if !filepath.IsAbs(src) && options.ContextDir == "" {
 			sources[i] = filepath.Join(currentDir, src)
 		}
 		localSources = append(localSources, sources[i])
 	}
 
+	// Treat git sources as a subset of remote sources
+	// differentiating only in how we fetch the two later on.
+	if len(gitSources) > 0 {
+		remoteSources = append(remoteSources, gitSources...)
+	}
+
 	// Check how many items our local source specs matched.  Each spec
 	// should have matched at least one item, otherwise we consider it an
 	// error.
@@ -320,7 +350,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 		}
 		numLocalSourceItems += len(localSourceStat.Globbed)
 	}
-	if numLocalSourceItems+len(remoteSources) == 0 {
+	if numLocalSourceItems+len(remoteSources)+len(gitSources) == 0 {
 		return fmt.Errorf("no sources %v found: %w", sources, syscall.ENOENT)
 	}
 
@@ -377,6 +407,9 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 				destCanBeFile = true
 			}
 		}
+		if len(gitSources) > 0 {
+			destMustBeDirectory = true
+		}
 	}
 
 	// We care if the destination either doesn't exist, or exists and is a
@@ -448,7 +481,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 		var multiErr *multierror.Error
 		var getErr, closeErr, renameErr, putErr error
 		var wg sync.WaitGroup
-		if sourceIsRemote(src) {
+		if sourceIsRemote(src) || sourceIsGit(src) {
 			pipeReader, pipeWriter := io.Pipe()
 			var srcDigest digest.Digest
 			if options.Checksum != "" {
@@ -457,17 +490,43 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 					return fmt.Errorf("invalid checksum flag: %w", err)
 				}
 			}
+
 			wg.Add(1)
-			go func() {
-				getErr = retry.IfNecessary(context.TODO(), func() error {
-					return getURL(src, chownFiles, mountPoint, renameTarget, pipeWriter, chmodDirsFiles, srcDigest, options.CertPath, options.InsecureSkipTLSVerify)
-				}, &retry.Options{
-					MaxRetry: options.MaxRetries,
-					Delay:    options.RetryDelay,
-				})
-				pipeWriter.Close()
-				wg.Done()
-			}()
+			if sourceIsGit(src) {
+				go func() {
+					var cloneDir string
+					cloneDir, _, getErr = define.TempDirForURL(tmpdir.GetTempDir(), "", src)
+					getOptions := copier.GetOptions{
+						UIDMap:         srcUIDMap,
+						GIDMap:         srcGIDMap,
+						Excludes:       options.Excludes,
+						ExpandArchives: extract,
+						ChownDirs:      chownDirs,
+						ChmodDirs:      chmodDirsFiles,
+						ChownFiles:     chownFiles,
+						ChmodFiles:     chmodDirsFiles,
+						StripSetuidBit: options.StripSetuidBit,
+						StripSetgidBit: options.StripSetgidBit,
+						StripStickyBit: options.StripStickyBit,
+					}
+					writer := io.WriteCloser(pipeWriter)
+					getErr = copier.Get(cloneDir, cloneDir, getOptions, []string{"."}, writer)
+					pipeWriter.Close()
+					wg.Done()
+				}()
+			} else {
+				go func() {
+					getErr = retry.IfNecessary(context.TODO(), func() error {
+						return getURL(src, chownFiles, mountPoint, renameTarget, pipeWriter, chmodDirsFiles, srcDigest, options.CertPath, options.InsecureSkipTLSVerify)
+					}, &retry.Options{
+						MaxRetry: options.MaxRetries,
+						Delay:    options.RetryDelay,
+					})
+					pipeWriter.Close()
+					wg.Done()
+				}()
+			}
+
 			wg.Add(1)
 			go func() {
 				b.ContentDigester.Start("")

chroot/run_linux.go

@@ -15,10 +15,10 @@ import (
 	"github.com/containers/buildah/copier"
 	"github.com/containers/storage/pkg/mount"
 	"github.com/containers/storage/pkg/unshare"
+	"github.com/moby/sys/capability"
 	"github.com/opencontainers/runc/libcontainer/apparmor"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
-	"github.com/syndtr/gocapability/capability"
 	"golang.org/x/sys/unix"
 )
 
@@ -180,9 +180,9 @@ func setCapabilities(spec *specs.Spec, keepCaps ...string) error {
 		capability.EFFECTIVE:   spec.Process.Capabilities.Effective,
 		capability.INHERITABLE: {},
 		capability.PERMITTED:   spec.Process.Capabilities.Permitted,
-		capability.AMBIENT:     spec.Process.Capabilities.Ambient,
+		capability.AMBIENT:     {},
 	}
-	knownCaps := capability.List()
+	knownCaps := capability.ListKnown()
 	noCap := capability.Cap(-1)
 	for capType, capList := range capMap {
 		for _, capSpec := range capList {

cmd/buildah/build.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/containers/buildah/imagebuildah"
@@ -73,7 +74,7 @@ func buildCmd(c *cobra.Command, inputArgs []string, iopts buildahcli.BuildOption
 	if c.Flag("logfile").Changed {
 		logfile, err := os.OpenFile(iopts.Logfile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
 		if err != nil {
-			return err
+			return fmt.Errorf("opening log file: %w", err)
 		}
 		iopts.Logwriter = logfile
 		defer iopts.Logwriter.Close()

cmd/buildah/manifest.go

@@ -246,7 +246,7 @@ func init() {
 	manifestPushCommand.SetUsageTemplate(UsageTemplate())
 	flags = manifestPushCommand.Flags()
 	flags.BoolVar(&manifestPushOpts.rm, "rm", false, "remove the manifest list if push succeeds")
-	flags.BoolVar(&manifestPushOpts.all, "all", false, "also push the images in the list")
+	flags.BoolVar(&manifestPushOpts.all, "all", true, "also push the images in the list")
 	flags.StringVar(&manifestPushOpts.authfile, "authfile", auth.GetDefaultAuthFile(), "path of the authentication file. Use REGISTRY_AUTH_FILE environment variable to override")
 	flags.StringVar(&manifestPushOpts.certDir, "cert-dir", "", "use certificates at the specified path to access the registry")
 	flags.StringVar(&manifestPushOpts.creds, "creds", "", "use `[username[:password]]` for accessing the registry")
@@ -432,25 +432,22 @@ func manifestAddCmd(c *cobra.Command, args []string, opts manifestAddOpts) error
 	switch len(args) {
 	case 0, 1:
 		return errors.New("At least a list image and an image or artifact to add must be specified")
-	case 2:
+	default:
 		listImageSpec = args[0]
 		if listImageSpec == "" {
-			return fmt.Errorf(`Invalid image name "%s"`, args[0])
+			return fmt.Errorf("Invalid image name %q", args[0])
 		}
 		if opts.artifact {
 			artifactSpec = args[1:]
 		} else {
+			if len(args) > 2 {
+				return errors.New("Too many arguments: expected list and image add to list")
+			}
 			imageSpec = args[1]
 			if imageSpec == "" {
-				return fmt.Errorf(`Invalid image name "%s"`, args[1])
+				return fmt.Errorf("Invalid image name %q", args[1])
 			}
 		}
-	default:
-		if opts.artifact {
-			artifactSpec = args[1:]
-		} else {
-			return errors.New("Too many arguments: expected list and image add to list")
-		}
 	}
 
 	store, err := getStore(c)

cmd/buildah/unshare.go

@@ -11,9 +11,9 @@ import (
 
 	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/unshare"
+	"github.com/moby/sys/capability"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"github.com/syndtr/gocapability/capability"
 )
 
 var (
@@ -138,7 +138,7 @@ func debugCapabilities() {
 		logrus.Errorf("error loading our current capabilities: %v", err)
 		return
 	}
-	knownCaps := capability.List()
+	knownCaps := capability.ListKnown()
 	effective := make([]string, 0, len(knownCaps))
 	for i := range knownCaps {
 		have := pid.Get(capability.EFFECTIVE, knownCaps[i])

copier/copier_linux_test.go

@@ -13,9 +13,9 @@ import (
 
 	"github.com/containers/storage/pkg/mount"
 	"github.com/containers/storage/pkg/reexec"
+	"github.com/moby/sys/capability"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/syndtr/gocapability/capability"
 	"golang.org/x/sys/unix"
 )
 

copier/xattrs.go

@@ -9,15 +9,18 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/containers/storage/pkg/unshare"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 )
 
 const (
 	xattrsSupported = true
+	imaXattr        = "security.ima"
 )
 
 var (
-	relevantAttributes    = []string{"security.capability", "security.ima", "user.*"} // the attributes that we preserve - we discard others
+	relevantAttributes    = []string{"security.capability", imaXattr, "user.*"} // the attributes that we preserve - we discard others
 	initialXattrListSize  = 64 * 1024
 	initialXattrValueSize = 64 * 1024
 )
@@ -92,7 +95,11 @@ func Lsetxattrs(path string, xattrs map[string]string) error {
 	for attribute, value := range xattrs {
 		if isRelevantXattr(attribute) {
 			if err := unix.Lsetxattr(path, attribute, []byte(value), 0); err != nil {
-				return fmt.Errorf("setting value of extended attribute %q on %q: %w", attribute, path, err)
+				if unshare.IsRootless() && attribute == imaXattr {
+					logrus.Warnf("Unable to set %q xattr on %q: %v", attribute, path, err)
+				} else {
+					return fmt.Errorf("setting value of extended attribute %q on %q: %w", attribute, path, err)
+				}
 			}
 		}
 	}