diff --git a/controllers/kustomization_controller.go b/controllers/kustomization_controller.go index 548ccbd3..86f8ed74 100644 --- a/controllers/kustomization_controller.go +++ b/controllers/kustomization_controller.go @@ -50,12 +50,14 @@ import ( apiacl "github.com/fluxcd/pkg/apis/acl" "github.com/fluxcd/pkg/apis/meta" + "github.com/fluxcd/pkg/http/fetch" "github.com/fluxcd/pkg/runtime/acl" runtimeClient "github.com/fluxcd/pkg/runtime/client" "github.com/fluxcd/pkg/runtime/events" "github.com/fluxcd/pkg/runtime/metrics" "github.com/fluxcd/pkg/runtime/predicates" "github.com/fluxcd/pkg/ssa" + "github.com/fluxcd/pkg/tar" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2" @@ -74,7 +76,7 @@ import ( // KustomizationReconciler reconciles a Kustomization object type KustomizationReconciler struct { client.Client - artifactFetcher *ArtifactFetcher + artifactFetcher *fetch.ArchiveFetcher requeueDependency time.Duration Scheme *runtime.Scheme EventRecorder kuberecorder.EventRecorder @@ -124,7 +126,7 @@ func (r *KustomizationReconciler) SetupWithManager(mgr ctrl.Manager, opts Kustom r.requeueDependency = opts.DependencyRequeueInterval r.statusManager = fmt.Sprintf("gotk-%s", r.ControllerName) - r.artifactFetcher = NewArtifactFetcher(opts.HTTPRetry) + r.artifactFetcher = fetch.NewArchiveFetcher(opts.HTTPRetry, tar.UnlimitedUntarSize, os.Getenv("SOURCE_CONTROLLER_LOCALHOST")) return ctrl.NewControllerManagedBy(mgr). For(&kustomizev1.Kustomization{}, builder.WithPredicates( @@ -270,7 +272,7 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques reconciledKustomization, reconcileErr := r.reconcile(ctx, *kustomization.DeepCopy(), source) // requeue if the artifact is not found - if reconcileErr == ArtifactNotFoundError { + if reconcileErr == fetch.FileNotFoundError { msg := fmt.Sprintf("Source is not ready, artifact not found, retrying in %s", r.requeueDependency.String()) log.Info(msg) if err := r.patchStatus(ctx, req, kustomizev1.KustomizationProgressing(kustomization, msg).Status); err != nil { @@ -332,7 +334,7 @@ func (r *KustomizationReconciler) reconcile( defer os.RemoveAll(tmpDir) // download artifact and extract files - err = r.artifactFetcher.Fetch(source.GetArtifact(), tmpDir) + err = r.artifactFetcher.Fetch(source.GetArtifact().URL, source.GetArtifact().Checksum, tmpDir) if err != nil { return kustomizev1.KustomizationNotReady( kustomization, diff --git a/controllers/kustomization_decryptor_test.go b/controllers/kustomization_decryptor_test.go index b78576e5..530419ca 100644 --- a/controllers/kustomization_decryptor_test.go +++ b/controllers/kustomization_decryptor_test.go @@ -66,11 +66,11 @@ func TestKustomizationReconciler_Decryptor(t *testing.T) { g.Expect(err).NotTo(HaveOccurred(), "failed to create kubeconfig secret") artifactName := "sops-" + randStringRunes(5) - artifactChecksum, err := createArtifact(testServer, "testdata/sops", artifactName) + artifactChecksum, err := testServer.ArtifactFromDir("testdata/sops", artifactName) g.Expect(err).ToNot(HaveOccurred()) overlayArtifactName := "sops-" + randStringRunes(5) - overlayChecksum, err := createArtifact(testServer, "testdata/test-dotenv", overlayArtifactName) + overlayChecksum, err := testServer.ArtifactFromDir("testdata/test-dotenv", overlayArtifactName) g.Expect(err).ToNot(HaveOccurred()) repositoryName := types.NamespacedName{ diff --git a/controllers/kustomization_fetcher.go b/controllers/kustomization_fetcher.go deleted file mode 100644 index 04eb6630..00000000 --- a/controllers/kustomization_fetcher.go +++ /dev/null @@ -1,127 +0,0 @@ -/* -Copyright 2022 The Flux authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "bytes" - "crypto/sha1" - "crypto/sha256" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "os" - "time" - - "github.com/fluxcd/pkg/untar" - sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" - "github.com/hashicorp/go-retryablehttp" -) - -// ArtifactFetcher holds the HTTP client that reties with back off when -// the artifact server is offline. -type ArtifactFetcher struct { - httpClient *retryablehttp.Client -} - -// ArtifactNotFoundError is an error type used to signal 404 HTTP status code responses. -var ArtifactNotFoundError = errors.New("artifact not found") - -// NewArtifactFetcher configures the retryable http client used for fetching artifacts. -// By default, it retries 10 times within a 3.5 minutes window. -func NewArtifactFetcher(retries int) *ArtifactFetcher { - httpClient := retryablehttp.NewClient() - httpClient.RetryWaitMin = 5 * time.Second - httpClient.RetryWaitMax = 30 * time.Second - httpClient.RetryMax = retries - httpClient.Logger = nil - - return &ArtifactFetcher{httpClient: httpClient} -} - -// Fetch downloads, verifies and extracts the artifact content to the specified directory. -// If the artifact server responds with 5xx errors, the download operation is retried. -// If the artifact server responds with 404, the returned error is of type ArtifactNotFoundError. -// If the artifact server is unavailable for more than 3 minutes, the returned error contains the original status code. -func (r *ArtifactFetcher) Fetch(artifact *sourcev1.Artifact, dir string) error { - artifactURL := artifact.URL - if hostname := os.Getenv("SOURCE_CONTROLLER_LOCALHOST"); hostname != "" { - u, err := url.Parse(artifactURL) - if err != nil { - return err - } - u.Host = hostname - artifactURL = u.String() - } - - req, err := retryablehttp.NewRequest(http.MethodGet, artifactURL, nil) - if err != nil { - return fmt.Errorf("failed to create a new request: %w", err) - } - - resp, err := r.httpClient.Do(req) - if err != nil { - return fmt.Errorf("failed to download artifact, error: %w", err) - } - defer resp.Body.Close() - - if code := resp.StatusCode; code != http.StatusOK { - if code == http.StatusNotFound { - return ArtifactNotFoundError - } - return fmt.Errorf("failed to download artifact from %s, status: %s", artifactURL, resp.Status) - } - - var buf bytes.Buffer - - // verify checksum matches origin - if err := r.Verify(artifact, &buf, resp.Body); err != nil { - return err - } - - // extract - if _, err = untar.Untar(&buf, dir); err != nil { - return fmt.Errorf("failed to untar artifact, error: %w", err) - } - - return nil -} - -// Verify computes the checksum of the tarball and returns an error if the computed value -// does not match the artifact advertised checksum. -func (r *ArtifactFetcher) Verify(artifact *sourcev1.Artifact, buf *bytes.Buffer, reader io.Reader) error { - hasher := sha256.New() - - // for backwards compatibility with source-controller v0.17.2 and older - if len(artifact.Checksum) == 40 { - hasher = sha1.New() - } - - // compute checksum - mw := io.MultiWriter(hasher, buf) - if _, err := io.Copy(mw, reader); err != nil { - return err - } - - if checksum := fmt.Sprintf("%x", hasher.Sum(nil)); checksum != artifact.Checksum { - return fmt.Errorf("failed to verify artifact: computed checksum '%s' doesn't match advertised '%s'", - checksum, artifact.Checksum) - } - - return nil -} diff --git a/controllers/kustomization_transformer_test.go b/controllers/kustomization_transformer_test.go index d9c5542b..7670587e 100644 --- a/controllers/kustomization_transformer_test.go +++ b/controllers/kustomization_transformer_test.go @@ -50,7 +50,7 @@ func TestKustomizationReconciler_KustomizeTransformer(t *testing.T) { g.Expect(err).NotTo(HaveOccurred()) artifactFile := "patch-" + randStringRunes(5) - artifactChecksum, err := createArtifact(testServer, "testdata/transformers", artifactFile) + artifactChecksum, err := testServer.ArtifactFromDir("testdata/transformers", artifactFile) g.Expect(err).ToNot(HaveOccurred()) repositoryName := types.NamespacedName{ @@ -173,7 +173,7 @@ func TestKustomizationReconciler_KustomizeTransformerFiles(t *testing.T) { g.Expect(err).NotTo(HaveOccurred()) artifactFile := "patch-" + randStringRunes(5) - artifactChecksum, err := createArtifact(testServer, "testdata/file-transformer", artifactFile) + artifactChecksum, err := testServer.ArtifactFromDir("testdata/file-transformer", artifactFile) g.Expect(err).ToNot(HaveOccurred()) repositoryName := types.NamespacedName{ @@ -292,7 +292,7 @@ func TestKustomizationReconciler_FluxTransformers(t *testing.T) { g.Expect(err).NotTo(HaveOccurred(), "failed to create kubeconfig secret") artifactFile := "patch-" + randStringRunes(5) - artifactChecksum, err := createArtifact(testServer, "testdata/patch", artifactFile) + artifactChecksum, err := testServer.ArtifactFromDir("testdata/patch", artifactFile) g.Expect(err).ToNot(HaveOccurred()) repositoryName := types.NamespacedName{ diff --git a/controllers/kustomization_validation_test.go b/controllers/kustomization_validation_test.go index 05a6b30f..52898f92 100644 --- a/controllers/kustomization_validation_test.go +++ b/controllers/kustomization_validation_test.go @@ -43,11 +43,11 @@ func TestKustomizationReconciler_Validation(t *testing.T) { g.Expect(err).NotTo(HaveOccurred(), "failed to create kubeconfig secret") artifactName := "val-" + randStringRunes(5) - artifactChecksum, err := createArtifact(testServer, "testdata/invalid/plain", artifactName) + artifactChecksum, err := testServer.ArtifactFromDir("testdata/invalid/plain", artifactName) g.Expect(err).ToNot(HaveOccurred()) overlayArtifactName := "val-" + randStringRunes(5) - overlayChecksum, err := createArtifact(testServer, "testdata/invalid/overlay", overlayArtifactName) + overlayChecksum, err := testServer.ArtifactFromDir("testdata/invalid/overlay", overlayArtifactName) g.Expect(err).ToNot(HaveOccurred()) repositoryName := types.NamespacedName{ diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 0eab7285..05e4b19d 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -17,17 +17,12 @@ limitations under the License. package controllers import ( - "archive/tar" - "compress/gzip" "context" - "crypto/sha1" "crypto/sha256" "fmt" - "io" "math/rand" "os" "path/filepath" - "strings" "testing" "time" @@ -288,95 +283,6 @@ func applyGitRepository(objKey client.ObjectKey, artifactName string, revision s return nil } -func createArtifact(artifactServer *testserver.ArtifactServer, fixture, path string) (string, error) { - if f, err := os.Stat(fixture); os.IsNotExist(err) || !f.IsDir() { - return "", fmt.Errorf("invalid fixture path: %s", fixture) - } - f, err := os.Create(filepath.Join(artifactServer.Root(), path)) - if err != nil { - return "", err - } - defer func() { - if err != nil { - os.Remove(f.Name()) - } - }() - - h := sha1.New() - - mw := io.MultiWriter(h, f) - gw := gzip.NewWriter(mw) - tw := tar.NewWriter(gw) - - if err = filepath.Walk(fixture, func(p string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - - // Ignore anything that is not a file (directories, symlinks) - if !fi.Mode().IsRegular() { - return nil - } - - // Ignore dotfiles - if strings.HasPrefix(fi.Name(), ".") { - return nil - } - - header, err := tar.FileInfoHeader(fi, p) - if err != nil { - return err - } - // The name needs to be modified to maintain directory structure - // as tar.FileInfoHeader only has access to the base name of the file. - // Ref: https://golang.org/src/archive/tar/common.go?#L626 - relFilePath := p - if filepath.IsAbs(fixture) { - relFilePath, err = filepath.Rel(fixture, p) - if err != nil { - return err - } - } - header.Name = relFilePath - - if err := tw.WriteHeader(header); err != nil { - return err - } - - f, err := os.Open(p) - if err != nil { - f.Close() - return err - } - if _, err := io.Copy(tw, f); err != nil { - f.Close() - return err - } - return f.Close() - }); err != nil { - return "", err - } - - if err := tw.Close(); err != nil { - gw.Close() - f.Close() - return "", err - } - if err := gw.Close(); err != nil { - f.Close() - return "", err - } - if err := f.Close(); err != nil { - return "", err - } - - if err := os.Chmod(f.Name(), 0644); err != nil { - return "", err - } - - return fmt.Sprintf("%x", h.Sum(nil)), nil -} - func createVaultTestInstance() (*dockertest.Pool, *dockertest.Resource, error) { // uses a sensible default on windows (tcp/http) and linux/osx (socket) pool, err := dockertest.NewPool("") diff --git a/go.mod b/go.mod index d2544ce9..cd751209 100644 --- a/go.mod +++ b/go.mod @@ -24,15 +24,15 @@ require ( github.com/fluxcd/pkg/apis/acl v0.1.0 github.com/fluxcd/pkg/apis/kustomize v0.6.0 github.com/fluxcd/pkg/apis/meta v0.17.0 + github.com/fluxcd/pkg/http/fetch v0.1.0 github.com/fluxcd/pkg/kustomize v0.8.0 github.com/fluxcd/pkg/runtime v0.20.0 github.com/fluxcd/pkg/ssa v0.21.0 - github.com/fluxcd/pkg/testserver v0.3.0 - github.com/fluxcd/pkg/untar v0.2.0 + github.com/fluxcd/pkg/tar v0.1.0 + github.com/fluxcd/pkg/testserver v0.4.0 github.com/fluxcd/source-controller/api v0.30.0 - github.com/hashicorp/go-retryablehttp v0.7.1 github.com/hashicorp/vault/api v1.8.0 - github.com/onsi/gomega v1.20.2 + github.com/onsi/gomega v1.21.1 github.com/ory/dockertest v3.3.5+incompatible github.com/otiai10/copy v1.7.0 github.com/spf13/pflag v1.0.5 @@ -150,6 +150,7 @@ require ( github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-plugin v1.4.3 // indirect + github.com/hashicorp/go-retryablehttp v0.7.1 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-secure-stdlib/mlock v0.1.2 // indirect github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect diff --git a/go.sum b/go.sum index 5aa25dce..c4ec5a45 100644 --- a/go.sum +++ b/go.sum @@ -290,16 +290,18 @@ github.com/fluxcd/pkg/apis/kustomize v0.6.0 h1:Afxv3Uv+xiuettzqm3sP0ceWikDZTfHdH github.com/fluxcd/pkg/apis/kustomize v0.6.0/go.mod h1:iY0zSpK6eUiPfNt/yR6g0q/wQP+wH+Ax/L7KBOx5x2M= github.com/fluxcd/pkg/apis/meta v0.17.0 h1:Y2dfo1syHZDb9Mexjr2SWdcj1FnxnRXm015hEnhl6wU= github.com/fluxcd/pkg/apis/meta v0.17.0/go.mod h1:GrOVzWXiu22XjLNgLLe2EBYhQPqZetes5SIADb4bmHE= +github.com/fluxcd/pkg/http/fetch v0.1.0 h1:Ig/kZuM0+jHBJnwHn5UUseTKIYD5w8X4bInJyuyOZKI= +github.com/fluxcd/pkg/http/fetch v0.1.0/go.mod h1:1CjOSfn7aOeHf2ZRA2+GTKHg442zN6X/fSys3a0KLC0= github.com/fluxcd/pkg/kustomize v0.8.0 h1:8AdEvp6y38ISZzoi0H82Si5zkmLXClbeX10W7HevB00= github.com/fluxcd/pkg/kustomize v0.8.0/go.mod h1:zGtCZF6V3hMWcf46SqrQc10fS9yUlKzi2UcFUeabDAE= github.com/fluxcd/pkg/runtime v0.20.0 h1:F9q9wap0BhjQszboUroJrYOB1C831zkQwTAk2tlMIQc= github.com/fluxcd/pkg/runtime v0.20.0/go.mod h1:KVHNQMhccuLTjMDFVCr/SF+4Z554bcMH1LncC4sQf8o= github.com/fluxcd/pkg/ssa v0.21.0 h1:aeoTohPNf5x7jQjHidyLJAOHw3EyHOQoQN3mN2i+4cc= github.com/fluxcd/pkg/ssa v0.21.0/go.mod h1:jumyhUbEMDnduN7anSlKfxl2fEoyeyv+Ta5hWCbxI5Q= -github.com/fluxcd/pkg/testserver v0.3.0 h1:oyZW6YWHVZR7FRVNu7lN9F5H808TD2jCzBm8CenFoi0= -github.com/fluxcd/pkg/testserver v0.3.0/go.mod h1:gjOKX41okmrGYOa4oOF2fiLedDAfPo1XaG/EzrUUGBI= -github.com/fluxcd/pkg/untar v0.2.0 h1:sJXU+FbJcNUb2ffLJNjeR3hwt3X2loVpOMlCUjyFw6E= -github.com/fluxcd/pkg/untar v0.2.0/go.mod h1:33AyoWaPpjX/xXpczcfhQh2AkB63TFwiR2YwROtv23E= +github.com/fluxcd/pkg/tar v0.1.0 h1:ObyUml8NJtGQtz/cRgexd7HU2mQsTmgjz2dtX4xdnng= +github.com/fluxcd/pkg/tar v0.1.0/go.mod h1:w0/TOC7kwBJhnSJn7TCABkc/I7ib1f2Yz6vOsbLBnhw= +github.com/fluxcd/pkg/testserver v0.4.0 h1:pDZ3gistqYhwlf3sAjn1Q8NzN4Qe6I1BEmHMHi46lMg= +github.com/fluxcd/pkg/testserver v0.4.0/go.mod h1:gjOKX41okmrGYOa4oOF2fiLedDAfPo1XaG/EzrUUGBI= github.com/fluxcd/source-controller/api v0.30.0 h1:rPVPpwXcYG2n0DTRcRagfGDiccvCib5S09K5iMjlpRU= github.com/fluxcd/source-controller/api v0.30.0/go.mod h1:UkjAqQ6QAXNNesNQDTArTeiTp+UuhOUIA+JyFhGP/+Q= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -676,8 +678,8 @@ github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.20.2 h1:8uQq0zMgLEfa0vRrrBgaJF2gyW9Da9BmfGV+OyUzfkY= -github.com/onsi/gomega v1.20.2/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.21.1 h1:OB/euWYIExnPBohllTicTHmGTrMaqJ67nIu80j0/uEM= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=