diff --git a/go.mod b/go.mod index f9daf735b8..2cdbd1556e 100644 --- a/go.mod +++ b/go.mod @@ -66,6 +66,7 @@ require ( github.com/sirupsen/logrus v1.7.0 github.com/urfave/cli v1.22.2 google.golang.org/grpc v1.33.2 + gopkg.in/yaml.v2 v2.3.0 k8s.io/api v0.19.0 k8s.io/apimachinery v0.19.0 k8s.io/apiserver v0.19.0 diff --git a/pkg/bootstrap/bootstrap.go b/pkg/bootstrap/bootstrap.go index 36664ff3d4..a03fd701f6 100644 --- a/pkg/bootstrap/bootstrap.go +++ b/pkg/bootstrap/bootstrap.go @@ -114,8 +114,14 @@ func Stage(dataDir, privateRegistry string, resolver *images.Resolver) (string, // If we didn't find the requested image in a tarball, pull it from the remote registry. // Note that this will fail (potentially after a long delay) if the registry cannot be reached. if img == nil { - logrus.Infof("Pulling runtime image %s", ref) - img, err = remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + registries, err := getPrivateRegistries(privateRegistry) + if err != nil { + return "", errors.Wrapf(err, "failed to load private registry configuration from %s", privateRegistry) + } + multiKeychain := authn.NewMultiKeychain(registries, authn.DefaultKeychain) + + logrus.Infof("Pulling runtime image %s", ref.Name()) + img, err = remote.Image(ref, remote.WithAuthFromKeychain(multiKeychain), remote.WithTransport(registries)) if err != nil { return "", errors.Wrapf(err, "failed to get runtime image %s", ref) } @@ -129,7 +135,6 @@ func Stage(dataDir, privateRegistry string, resolver *images.Resolver) (string, if err := extractToDirs(img, dataDir, extractPaths); err != nil { return "", errors.Wrap(err, "failed to extract runtime image") } - // Ensure correct permissions on bin dir if err := os.Chmod(refBinDir, 0755); err != nil { return "", err @@ -228,7 +233,7 @@ func releaseRefDigest(ref name.Reference) (string, error) { } return parts[0], nil } - return "", fmt.Errorf("Runtime image %s is not a not a reference to a digest or version tag matching pattern %s", ref, releasePattern) + return "", fmt.Errorf("Runtime image %s is not a not a reference to a digest or version tag matching pattern %s", ref.Name(), releasePattern) } // extractToDirs extracts to targetDir all content from img, then moves the content into place using the directory map. @@ -347,19 +352,21 @@ func preloadBootstrapFromRuntime(dataDir string, resolver *images.Resolver) (v1. func preloadBootstrapImage(dataDir string, imageRef name.Reference) (v1.Image, error) { imageTag, ok := imageRef.(name.Tag) if !ok { - logrus.Debugf("No local image available for %s: reference is not a tag", imageRef) + logrus.Debugf("No local image available for %s: reference is not a tag", imageRef.Name()) return nil, nil } imagesDir := imagesDir(dataDir) if _, err := os.Stat(imagesDir); err != nil { if os.IsNotExist(err) { - logrus.Debugf("No local image available for %s: directory %s does not exist", imageTag, imagesDir) + logrus.Debugf("No local image available for %s: directory %s does not exist", imageTag.Name(), imagesDir) return nil, nil } return nil, err } + logrus.Infof("Checking local image archives in %s for %s", imagesDir, imageRef.Name()) + // Walk the images dir to get a list of tar files files := map[string]os.FileInfo{} if err := filepath.Walk(imagesDir, func(path string, info os.FileInfo, err error) error { @@ -378,14 +385,14 @@ func preloadBootstrapImage(dataDir string, imageRef name.Reference) (v1.Image, e for fileName := range files { img, err := preloadFile(imageTag, fileName) if img != nil { - logrus.Debugf("Found %s in %s", imageTag, fileName) + logrus.Debugf("Found %s in %s", imageTag.Name(), fileName) return img, nil } if err != nil { logrus.Infof("Failed to check %s: %v", fileName, err) } } - logrus.Debugf("No local image available for %s: not found in any file in %s", imageTag, imagesDir) + logrus.Debugf("No local image available for %s: not found in any file in %s", imageTag.Name(), imagesDir) return nil, nil } diff --git a/pkg/bootstrap/registries.go b/pkg/bootstrap/registries.go new file mode 100644 index 0000000000..1ae12f8cf8 --- /dev/null +++ b/pkg/bootstrap/registries.go @@ -0,0 +1,192 @@ +package bootstrap + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/pkg/errors" + "github.com/rancher/k3s/pkg/agent/templates" + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +// registry stores information necessary to configure authentication and +// connections to remote registries, including overriding registry endpoints +type registry struct { + r *templates.Registry + t map[string]*http.Transport +} + +// Explicit interface checks +var _ authn.Keychain = ®istry{} +var _ http.RoundTripper = ®istry{} + +// getPrivateRegistries loads private registry configuration from registries.yaml +func getPrivateRegistries(path string) (*registry, error) { + privRegistries := &templates.Registry{} + privRegistryFile, err := ioutil.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + logrus.Infof("Using registry config file at %s", path) + if err := yaml.Unmarshal(privRegistryFile, &privRegistries); err != nil { + return nil, err + } + return ®istry{ + r: privRegistries, + t: map[string]*http.Transport{}, + }, nil +} + +// Resolve returns an authenticator for the authn.Keychain interface. The authenticator +// provides credentials to a registry by looking up configuration from mirror endpoints. +func (r *registry) Resolve(target authn.Resource) (authn.Authenticator, error) { + endpointURL, err := r.getEndpointForHost(target.RegistryStr()) + if err != nil { + return nil, err + } + return r.getAuthenticatorForHost(endpointURL.Host) +} + +// RoundTrip round-trips a HTTP request for the http.RoundTripper interface. The round-tripper +// overrides the Host in the headers and URL based on mirror endpoint configuration. It also +// configures TLS based on the endpoint's TLS config, if any. +func (r *registry) RoundTrip(req *http.Request) (*http.Response, error) { + endpointURL, err := r.getEndpointForHost(req.URL.Host) + if err != nil { + return nil, err + } + + // override request host and scheme + req.Host = endpointURL.Host + req.URL.Host = endpointURL.Host + req.URL.Scheme = endpointURL.Scheme + + switch endpointURL.Scheme { + case "http": + return http.DefaultTransport.RoundTrip(req) + case "https": + // Create and cache transport if not found. + if _, ok := r.t[endpointURL.Host]; !ok { + tlsConfig, err := r.getTLSConfigForHost(endpointURL.Host) + if err != nil { + return nil, errors.Wrapf(err, "failed to get TLS config for endpoint %s", endpointURL.Host) + } + + r.t[endpointURL.Host] = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + TLSClientConfig: tlsConfig, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + } + return r.t[endpointURL.Host].RoundTrip(req) + } + return nil, fmt.Errorf("unsupported scheme %s in registry endpoint", endpointURL.Scheme) +} + +// getEndpointForHost gets endpoint configuration for a host. Because go-containerregistry's +// Keychain interface does not provide a good hook to check authentication for multiple endpoints, +// we only use the first endpoint from the mirror list. If no endpoint configuration is found, https +// is assumed. +func (r *registry) getEndpointForHost(host string) (*url.URL, error) { + keys := []string{host} + if host == name.DefaultRegistry { + keys = append(keys, "docker.io") + } + keys = append(keys, "*") + + for _, key := range keys { + if mirror, ok := r.r.Mirrors[key]; ok { + endpointCount := len(mirror.Endpoints) + switch { + case endpointCount > 1: + logrus.Warnf("Found more than one endpoint for %s; only the first entry will be used", host) + fallthrough + case endpointCount == 1: + return url.Parse(mirror.Endpoints[0]) + } + } + } + return url.Parse("https://" + host) +} + +// getAuthenticatorForHost returns an Authenticator for a given host. This should be the host from +// the mirror endpoint list, NOT the host from the request. If no configuration is present, +// Anonymous authentication is used. +func (r *registry) getAuthenticatorForHost(host string) (authn.Authenticator, error) { + if config, ok := r.r.Configs[host]; ok { + if config.Auth != nil { + return authn.FromConfig(authn.AuthConfig{ + Username: config.Auth.Username, + Password: config.Auth.Password, + Auth: config.Auth.Auth, + IdentityToken: config.Auth.IdentityToken, + }), nil + } + } + return authn.Anonymous, nil +} + +// getTLSConfigForHost returns TLS configuration for a given host. This should be the host from the +// mirror endpoint list, NOT the host from the request. This is cribbed from +// https://github.com/containerd/cri/blob/release/1.4/pkg/server/image_pull.go#L274 +func (r *registry) getTLSConfigForHost(host string) (*tls.Config, error) { + tlsConfig := &tls.Config{} + if config, ok := r.r.Configs[host]; ok { + if config.TLS != nil { + if config.TLS.CertFile != "" && config.TLS.KeyFile == "" { + return nil, errors.Errorf("cert file %q was specified, but no corresponding key file was specified", config.TLS.CertFile) + } + if config.TLS.CertFile == "" && config.TLS.KeyFile != "" { + return nil, errors.Errorf("key file %q was specified, but no corresponding cert file was specified", config.TLS.KeyFile) + } + if config.TLS.CertFile != "" && config.TLS.KeyFile != "" { + cert, err := tls.LoadX509KeyPair(config.TLS.CertFile, config.TLS.KeyFile) + if err != nil { + return nil, errors.Wrap(err, "failed to load cert file") + } + if len(cert.Certificate) != 0 { + tlsConfig.Certificates = []tls.Certificate{cert} + } + tlsConfig.BuildNameToCertificate() // nolint:staticcheck + } + + if config.TLS.CAFile != "" { + caCertPool, err := x509.SystemCertPool() + if err != nil { + return nil, errors.Wrap(err, "failed to get system cert pool") + } + caCert, err := ioutil.ReadFile(config.TLS.CAFile) + if err != nil { + return nil, errors.Wrap(err, "failed to load CA file") + } + caCertPool.AppendCertsFromPEM(caCert) + tlsConfig.RootCAs = caCertPool + } + + tlsConfig.InsecureSkipVerify = config.TLS.InsecureSkipVerify + } + } + + return tlsConfig, nil +} diff --git a/pkg/images/images.go b/pkg/images/images.go index 0ec510861d..f89fbb1abb 100644 --- a/pkg/images/images.go +++ b/pkg/images/images.go @@ -31,7 +31,7 @@ var ( ) // ResolverOpt is an option to modify image resolution behavior. -type ResolverOpt func(name.Reference) error +type ResolverOpt func(name.Reference) (name.Reference, error) // Resolver provides functionality to resolve an RKE2 image name to a reference. type Resolver struct { @@ -120,27 +120,33 @@ func (r *Resolver) SetOverride(i string, n name.Reference) { // otherwise the compile-time default is retrieved and default-registry settings applied. // Options can be passed to modify the reference before it is returned. func (r *Resolver) GetReference(i string, opts ...ResolverOpt) (name.Reference, error) { - // Return unmodified override if set - if ref, ok := r.overrides[i]; ok { - return ref, nil - } - - // No override; get compile-time default - ref, err := getDefaultImage(i) - if err != nil { - return nil, err - } + var ref name.Reference + if o, ok := r.overrides[i]; ok { + // Use override if set + ref = o + } else { + // No override; get compile-time default + d, err := getDefaultImage(i) + if err != nil { + return nil, err + } + ref = d - // Apply registry override - if err := setRegistry(ref, r.Registry); err != nil { - return nil, err + // Apply registry override + d, err = setRegistry(ref, r.Registry) + if err != nil { + return nil, err + } + ref = d } // Apply additional options for _, o := range opts { - if err := o(ref); err != nil { + r, err := o(ref) + if err != nil { return nil, err } + ref = r } return ref, nil } @@ -155,30 +161,30 @@ func (r *Resolver) MustGetReference(i string, opts ...ResolverOpt) name.Referenc // WithRegistry overrides the registry when resolving the reference to an image. func WithRegistry(s string) ResolverOpt { - return func(r name.Reference) error { + return func(r name.Reference) (name.Reference, error) { registry, err := name.NewRegistry(s) if err != nil { - return err + return nil, err } - err = setRegistry(r, registry) + s, err := setRegistry(r, registry) if err != nil { - return err + return nil, err } - return nil + return s, nil } } // setRegistry sets the registry on an image reference. This is necessary // because the Reference type doesn't expose the Registry field. -func setRegistry(ref name.Reference, registry name.Registry) error { +func setRegistry(ref name.Reference, registry name.Registry) (name.Reference, error) { if t, ok := ref.(name.Tag); ok { t.Registry = registry - return nil + return t, nil } else if d, ok := ref.(name.Digest); ok { d.Registry = registry - return nil + return d, nil } - return errors.Errorf("unhandled Reference type: %T", ref) + return ref, errors.Errorf("unhandled Reference type: %T", ref) } // getDefaultImage gets the compile-time default image for a given name.