Merge pull request #225 from depot/feat/push

feat: add depot push for ephemeral registry

pkg/api/rpc.go

@@ -33,6 +33,14 @@ func NewProjectsClient() cliv1beta1connect.ProjectsServiceClient {
 	return cliv1beta1connect.NewProjectsServiceClient(http.DefaultClient, baseURL, WithUserAgent())
 }
 
+func NewPushClient() cliv1connect.PushServiceClient {
+	baseURL := os.Getenv("DEPOT_API_URL")
+	if baseURL == "" {
+		baseURL = "https://api.depot.dev"
+	}
+	return cliv1connect.NewPushServiceClient(http.DefaultClient, baseURL, WithUserAgent())
+}
+
 func WithAuthentication[T any](req *connect.Request[T], token string) *connect.Request[T] {
 	req.Header().Add("Authorization", "Bearer "+token)
 	return req

pkg/cmd/push/auth.go

@@ -0,0 +1,100 @@
+package push
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/containerd/containerd/reference"
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/containerd/containerd/remotes/docker/auth"
+	depotapi "github.com/depot/cli/pkg/api"
+	"github.com/docker/cli/cli/command"
+	configtypes "github.com/docker/cli/cli/config/types"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// GetAuthToken gets an auth token for a registry.
+// It does this by loading the local docker auth, determining the authorization schema via a HEAD request,
+// and then requesting a token from the realm.
+func GetAuthToken(ctx context.Context, dockerCli command.Cli, parsedTag *ParsedTag, manifest ocispecs.Descriptor) (*Token, error) {
+	authConfig, err := GetAuthConfig(dockerCli, parsedTag.Host)
+	if err != nil {
+		return nil, err
+	}
+
+	push := true
+	scope, err := docker.RepositoryScope(parsedTag.Refspec, push)
+	if err != nil {
+		return nil, err
+	}
+
+	challenge, err := AuthKind(ctx, parsedTag.Refspec, manifest)
+	if err != nil {
+		return nil, err
+	}
+
+	return FetchToken(ctx, authConfig, challenge, []string{scope})
+}
+
+// GetAuthConfig gets the auth config from the local docker login.
+func GetAuthConfig(dockerCli command.Cli, host string) (*configtypes.AuthConfig, error) {
+	if host == "registry-1.docker.io" {
+		host = "https://index.docker.io/v1/"
+	}
+
+	config, err := dockerCli.ConfigFile().GetAuthConfig(host)
+	if err != nil {
+		return nil, err
+	}
+	return &config, nil
+}
+
+// AuthKind tries to do a HEAD request to the manifest to try to get the WWW-Authenticate header.
+// If HEAD is not supported, it will try to get a GET.  Apparently, this is for older registries.
+func AuthKind(ctx context.Context, refspec reference.Spec, manifest ocispecs.Descriptor) (*auth.Challenge, error) {
+	// Reversing the refspec's path.Join behavior.
+	i := strings.Index(refspec.Locator, "/")
+	host, repository := refspec.Locator[:i], refspec.Locator[i+1:]
+	if host == "docker.io" {
+		host = "registry-1.docker.io"
+	}
+
+	url := fmt.Sprintf("https://%s/v2/%s/manifests/%s", host, repository, refspec.Object)
+	req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Accept", manifest.MediaType)
+	req.Header.Add("Accept", `*/*`)
+	req.Header.Set("User-Agent", depotapi.Agent())
+
+	// Helper function allowing the HTTP method to change because some registries
+	// use GET rather than HEAD (according to an old comment).
+	return checkAuthKind(ctx, req)
+}
+
+func checkAuthKind(ctx context.Context, req *http.Request) (*auth.Challenge, error) {
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	_ = res.Body.Close()
+	switch res.StatusCode {
+	case http.StatusOK:
+		return nil, nil
+	case http.StatusUnauthorized:
+		challenges := auth.ParseAuthHeader(res.Header)
+		if len(challenges) == 0 {
+			return nil, fmt.Errorf("no auth challenges found")
+		}
+		return &challenges[0], nil
+	case http.StatusMethodNotAllowed:
+		// We have a callback here to allow us to retry the request with a `GET`if the registry doesn't support `HEAD`.
+		req.Method = http.MethodGet
+		return checkAuthKind(ctx, req)
+	}
+
+	return nil, fmt.Errorf("unexpected status code: %s", res.Status)
+}

pkg/cmd/push/blobs.go

@@ -0,0 +1,55 @@
+package push
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/opencontainers/go-digest"
+)
+
+type BlobRequest struct {
+	ParsedTag     *ParsedTag
+	RegistryToken *Token
+	BuildID       string
+	Digest        digest.Digest
+}
+
+// PushBlob requests a blob to be pushed from Depot to a destination registry.
+func PushBlob(ctx context.Context, depotToken string, req *BlobRequest) error {
+	pushRequest := struct {
+		RegistryHost        string `json:"registryHost"`
+		RepositoryNamespace string `json:"repositoryNamespace"`
+		RegistryToken       string `json:"registryToken"`
+		TokenScheme         string `json:"tokenScheme"`
+	}{
+		RegistryHost:        req.ParsedTag.Host,
+		RepositoryNamespace: req.ParsedTag.Path,
+		RegistryToken:       req.RegistryToken.Token,
+		TokenScheme:         req.RegistryToken.Scheme,
+	}
+	buf, _ := json.MarshalIndent(pushRequest, "", "  ")
+
+	url := fmt.Sprintf("https://blob.depot.dev/blobs/%s/%s", req.BuildID, req.Digest.String())
+	pushReq, err := http.NewRequestWithContext(ctx, "POST", url, strings.NewReader(string(buf)))
+	if err != nil {
+		return err
+	}
+	pushReq.Header.Add("Authorization", "Bearer "+depotToken)
+	pushReq.Header.Set("Content-Type", "application/json")
+	resp, err := http.DefaultClient.Do(pushReq)
+	if err != nil {
+		return err
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode/100 != 2 {
+		body, _ := io.ReadAll(resp.Body)
+		err := fmt.Errorf("unexpected status code: %d %s", resp.StatusCode, string(body))
+		return err
+	}
+
+	return nil
+}

pkg/cmd/push/manifests.go

@@ -0,0 +1,205 @@
+package push
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/bufbuild/connect-go"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/reference"
+	"github.com/containerd/containerd/remotes"
+	"github.com/containerd/containerd/remotes/docker"
+	depotapi "github.com/depot/cli/pkg/api"
+	cliv1 "github.com/depot/cli/pkg/proto/depot/cli/v1"
+	"github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"golang.org/x/sync/errgroup"
+)
+
+// PushManifest pushes a manifest to a registry.
+func PushManifest(ctx context.Context, registryToken string, refspec reference.Spec, tag string, manifest ocispecs.Descriptor, manifestBytes []byte) error {
+	// Reversing the refspec's path.Join behavior.
+	i := strings.Index(refspec.Locator, "/")
+	host, repository := refspec.Locator[:i], refspec.Locator[i+1:]
+	if host == "docker.io" {
+		host = "registry-1.docker.io"
+	}
+
+	url := fmt.Sprintf("https://%s/v2/%s/manifests/%s", host, repository, tag)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, bytes.NewReader(manifestBytes))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("User-Agent", depotapi.Agent())
+	req.Header.Set("Content-Type", manifest.MediaType)
+	// TODO:
+	req.Header.Set("Authorization", "Bearer "+registryToken)
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = res.Body.Close() }()
+	if res.StatusCode/100 == 2 {
+		return nil
+	}
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return err
+	}
+	return fmt.Errorf("unexpected status code: %s %s", res.Status, string(body))
+}
+
+type ImageDescriptors struct {
+	Indices   []ocispecs.Descriptor `json:"indices,omitempty"`
+	Manifests []ocispecs.Descriptor `json:"manifests,omitempty"`
+	Configs   []ocispecs.Descriptor `json:"configs,omitempty"`
+	Layers    []ocispecs.Descriptor `json:"layers,omitempty"`
+
+	IndexBytes    map[digest.Digest][]byte `json:"indexBytes,omitempty"`
+	ManifestBytes map[digest.Digest][]byte `json:"manifestBytes,omitempty"`
+}
+
+// GetImageDescriptors returns back all the descriptors for an image.
+func GetImageDescriptors(ctx context.Context, token, buildID string, logger StartLogDetailFunc) (*ImageDescriptors, error) {
+	// Download location and credentials of ephemeral image save.
+	client := depotapi.NewBuildClient()
+	req := &cliv1.GetPullInfoRequest{BuildId: buildID}
+	res, err := client.GetPullInfo(ctx, depotapi.WithAuthentication(connect.NewRequest(req), token))
+	if err != nil {
+		return nil, err
+	}
+
+	username, password, ref := res.Msg.Username, res.Msg.Password, res.Msg.Reference
+
+	authorizer := &Authorizer{Username: username, Password: password}
+	hosts := docker.ConfigureDefaultRegistries(docker.WithAuthorizer(authorizer))
+
+	headers := http.Header{}
+	headers.Set("User-Agent", depotapi.Agent())
+	resolver := docker.NewResolver(docker.ResolverOptions{
+		Hosts:   hosts,
+		Headers: headers,
+	})
+
+	fin := logger(fmt.Sprintf("Resolving %s", ref))
+	name, desc, err := resolver.Resolve(ctx, ref)
+	fin()
+	if err != nil {
+		return nil, err
+	}
+
+	mu := sync.Mutex{}
+	descs := ImageDescriptors{
+		IndexBytes:    map[digest.Digest][]byte{},
+		ManifestBytes: map[digest.Digest][]byte{},
+	}
+
+	// Recursively fetch all the image descriptors. If a descriptor contains
+	// other descriptors, an additional goroutine is spawned on the errgroup.
+	errgroup, ctx := errgroup.WithContext(ctx)
+	var fetchImageDescriptors func(ctx context.Context, desc ocispecs.Descriptor) error
+	fetchImageDescriptors = func(ctx context.Context, desc ocispecs.Descriptor) error {
+		fetcher, err := resolver.Fetcher(ctx, name)
+		if err != nil {
+			return err
+		}
+
+		fin := logger(fmt.Sprintf("Fetching manifest %s", desc.Digest.String()))
+		buf, err := fetch(ctx, fetcher, desc)
+		fin()
+		if err != nil {
+			return err
+		}
+
+		if images.IsIndexType(desc.MediaType) {
+			var index ocispecs.Index
+			if err := json.Unmarshal(buf, &index); err != nil {
+				return err
+			}
+
+			mu.Lock()
+			descs.Indices = append(descs.Indices, desc)
+			descs.IndexBytes[desc.Digest] = buf
+			mu.Unlock()
+
+			for _, m := range index.Manifests {
+				m := m
+				if m.Digest != "" {
+					// Only download unique descriptors.
+					completed := false
+					mu.Lock()
+					if _, ok := descs.IndexBytes[m.Digest]; ok {
+						completed = true
+					}
+					if _, ok := descs.ManifestBytes[m.Digest]; ok {
+						completed = true
+					}
+					mu.Unlock()
+
+					if !completed {
+						errgroup.Go(func() error {
+							return fetchImageDescriptors(ctx, m)
+						})
+					}
+				}
+			}
+		} else if images.IsManifestType(desc.MediaType) {
+			var manifest ocispecs.Manifest
+			if err := json.Unmarshal(buf, &manifest); err != nil {
+				return err
+			}
+
+			mu.Lock()
+			descs.Manifests = append(descs.Manifests, desc)
+			descs.ManifestBytes[desc.Digest] = buf
+			descs.Configs = append(descs.Configs, manifest.Config)
+			descs.Layers = append(descs.Layers, manifest.Layers...)
+			mu.Unlock()
+		}
+		return nil
+	}
+
+	errgroup.Go(func() error {
+		return fetchImageDescriptors(ctx, desc)
+	})
+
+	err = errgroup.Wait()
+	if err != nil {
+		return nil, err
+	}
+
+	return &descs, nil
+}
+
+func fetch(ctx context.Context, fetcher remotes.Fetcher, desc ocispecs.Descriptor) ([]byte, error) {
+	r, err := fetcher.Fetch(ctx, desc)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = r.Close() }()
+	return io.ReadAll(r)
+}
+
+// Authorizer is a static authorizer used to authenticate with the Depot ephemeral registry.
+type Authorizer struct {
+	Username string
+	Password string
+}
+
+func (a *Authorizer) Authorize(ctx context.Context, req *http.Request) error {
+	req.SetBasicAuth(a.Username, a.Password)
+	return nil
+}
+func (a *Authorizer) AddResponses(ctx context.Context, responses []*http.Response) error {
+	return errdefs.ErrNotImplemented
+}