diff --git a/examples/lazy/some-content/some-file.txt b/examples/lazy/some-content/some-file.txt new file mode 100644 index 00000000..e69de29b diff --git a/examples/lazy/vendir-lazy.yml b/examples/lazy/vendir-lazy.yml new file mode 100644 index 00000000..46d64abf --- /dev/null +++ b/examples/lazy/vendir-lazy.yml @@ -0,0 +1,9 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: +- path: vendor + contents: + - path: dir + lazy: true + directory: + path: ./some-content diff --git a/examples/lazy/vendir-nonlazy.yml b/examples/lazy/vendir-nonlazy.yml new file mode 100644 index 00000000..e8e0a528 --- /dev/null +++ b/examples/lazy/vendir-nonlazy.yml @@ -0,0 +1,8 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: +- path: vendor + contents: + - path: dir + directory: + path: ./some-content diff --git a/examples/lazy/vendir.lock.yml b/examples/lazy/vendir.lock.yml new file mode 100644 index 00000000..38c07d1c --- /dev/null +++ b/examples/lazy/vendir.lock.yml @@ -0,0 +1,7 @@ +apiVersion: vendir.k14s.io/v1alpha1 +directories: +- contents: + - directory: {} + path: dir + path: vendor +kind: LockConfig diff --git a/pkg/vendir/cmd/sync.go b/pkg/vendir/cmd/sync.go index c60882b9..4357ca30 100644 --- a/pkg/vendir/cmd/sync.go +++ b/pkg/vendir/cmd/sync.go @@ -30,6 +30,7 @@ type SyncOptions struct { Directories []string Locked bool + Lazy bool Chdir string AllowAllSymlinkDestinations bool @@ -50,6 +51,7 @@ func NewSyncCmd(o *SyncOptions) *cobra.Command { cmd.Flags().StringSliceVarP(&o.Directories, "directory", "d", nil, "Sync specific directory (format: dir/sub-dir[=local-dir])") cmd.Flags().BoolVarP(&o.Locked, "locked", "l", false, "Consult lock file to pull exact references (e.g. use git sha instead of branch name)") + cmd.Flags().BoolVar(&o.Lazy, "lazy", true, "Set to 'false' it ignores the 'lazy' flag in the directory content configuration") cmd.Flags().StringVar(&o.Chdir, "chdir", "", "Set current directory for process") cmd.Flags().BoolVar(&o.AllowAllSymlinkDestinations, "dangerous-allow-all-symlink-destinations", false, "Symlinks to all destinations are allowed") @@ -95,14 +97,14 @@ func (o *SyncOptions) Run() error { o.ui.PrintBlock(configBs) } + existingLockConfig, err := ctlconf.NewLockConfigFromFile(o.LockFile) + if err != nil && o.Locked { + return err + } + // If syncing against a lock file, apply lock information // on top of existing config if o.Locked { - existingLockConfig, err := ctlconf.NewLockConfigFromFile(o.LockFile) - if err != nil { - return err - } - err = conf.Lock(existingLockConfig) if err != nil { return err @@ -130,11 +132,14 @@ func (o *SyncOptions) Run() error { GithubAPIToken: os.Getenv("VENDIR_GITHUB_API_TOKEN"), HelmBinary: os.Getenv("VENDIR_HELM_BINARY"), Cache: cache, + Lazy: o.Lazy, } newLockConfig := ctlconf.NewLockConfig() for _, dirConf := range conf.Directories { - dirLockConf, err := ctldir.NewDirectory(dirConf, o.ui).Sync(syncOpts) + // error safe to ignore, since lock file might not exist + dirExistingLockConf, _ := existingLockConfig.FindDirectory(dirConf.Path) + dirLockConf, err := ctldir.NewDirectory(dirConf, dirExistingLockConf, o.ui).Sync(syncOpts) if err != nil { return fmt.Errorf("Syncing directory '%s': %s", dirConf.Path, err) } @@ -150,11 +155,6 @@ func (o *SyncOptions) Run() error { // Update only selected directories in lock file if len(dirs) > 0 { - existingLockConfig, err := ctlconf.NewLockConfigFromFile(o.LockFile) - if err != nil { - return err - } - err = existingLockConfig.Merge(newLockConfig) if err != nil { return err diff --git a/pkg/vendir/config/config.go b/pkg/vendir/config/config.go index 0ec3aef1..a5842324 100644 --- a/pkg/vendir/config/config.go +++ b/pkg/vendir/config/config.go @@ -183,6 +183,7 @@ func (c Config) UseDirectory(path, dirPath string) error { ExcludePaths: con.ExcludePaths, IgnorePaths: con.IgnorePaths, LegalPaths: con.LegalPaths, + Lazy: con.Lazy, } dir.Contents[j] = newCon c.Directories[i] = dir diff --git a/pkg/vendir/config/directory.go b/pkg/vendir/config/directory.go index a386cdd9..58ec782a 100644 --- a/pkg/vendir/config/directory.go +++ b/pkg/vendir/config/directory.go @@ -34,6 +34,7 @@ type Directory struct { type DirectoryContents struct { Path string `json:"path"` + Lazy bool `json:"lazy,omitempty"` Git *DirectoryContentsGit `json:"git,omitempty"` Hg *DirectoryContentsHg `json:"hg,omitempty"` diff --git a/pkg/vendir/config/lock_config.go b/pkg/vendir/config/lock_config.go index d2a0c2cb..c449ffff 100644 --- a/pkg/vendir/config/lock_config.go +++ b/pkg/vendir/config/lock_config.go @@ -28,6 +28,13 @@ func NewLockConfig() LockConfig { } } +func LockFileExists(path string) bool { + if _, err := os.Stat(path); err != nil { + return false + } + return true +} + func NewLockConfigFromFile(path string) (LockConfig, error) { bs, err := os.ReadFile(path) if err != nil { @@ -114,6 +121,16 @@ func (c LockConfig) FindContents(dirPath, conPath string) (LockDirectoryContents "Expected to find directory '%s' within lock config, but did not", dirPath) } +func (c LockConfig) FindDirectory(dirPath string) (LockDirectory, error) { + for _, dir := range c.Directories { + if dir.Path == dirPath { + return dir, nil + } + } + return LockDirectory{}, fmt.Errorf( + "Expected to find directory '%s' within lock config, but did not", dirPath) +} + func (c LockConfig) Merge(other LockConfig) error { for _, dir := range other.Directories { for _, con := range dir.Contents { diff --git a/pkg/vendir/config/lock_directory.go b/pkg/vendir/config/lock_directory.go index 41651e47..3e239f7c 100644 --- a/pkg/vendir/config/lock_directory.go +++ b/pkg/vendir/config/lock_directory.go @@ -3,13 +3,16 @@ package config +import "fmt" + type LockDirectory struct { Path string `json:"path"` Contents []LockDirectoryContents `json:"contents"` } type LockDirectoryContents struct { - Path string `json:"path"` + Path string `json:"path"` + ConfigDigest string `json:"configDigest,omitempty"` Git *LockDirectoryContentsGit `json:"git,omitempty"` Hg *LockDirectoryContentsHg `json:"hg,omitempty"` @@ -62,3 +65,13 @@ type LockDirectoryContentsManual struct{} type LockDirectoryContentsDirectory struct{} type LockDirectoryContentsInline struct{} + +func (d LockDirectory) FindContents(conPath string) (LockDirectoryContents, error) { + for _, con := range d.Contents { + if con.Path == conPath { + return con, nil + } + } + return LockDirectoryContents{}, fmt.Errorf("Expected to find contents '%s' "+ + "within directory '%s' in lock config, but did not", conPath, d.Path) +} diff --git a/pkg/vendir/directory/directory.go b/pkg/vendir/directory/directory.go index 8baf5b07..d0ba0bb1 100644 --- a/pkg/vendir/directory/directory.go +++ b/pkg/vendir/directory/directory.go @@ -4,9 +4,12 @@ package directory import ( + "crypto/sha256" + "encoding/hex" "fmt" "os" "path/filepath" + "sigs.k8s.io/yaml" "github.com/cppforlife/go-cli-ui/ui" dircopy "github.com/otiai10/copy" @@ -24,12 +27,13 @@ import ( ) type Directory struct { - opts ctlconf.Directory - ui ui.UI + opts ctlconf.Directory + lockDirectory ctlconf.LockDirectory + ui ui.UI } -func NewDirectory(opts ctlconf.Directory, ui ui.UI) *Directory { - return &Directory{opts, ui} +func NewDirectory(opts ctlconf.Directory, lockDirectory ctlconf.LockDirectory, ui ui.UI) *Directory { + return &Directory{opts, lockDirectory, ui} } type SyncOpts struct { @@ -37,6 +41,17 @@ type SyncOpts struct { GithubAPIToken string HelmBinary string Cache ctlcache.Cache + Lazy bool +} + +func createConfigDigest(contents ctlconf.DirectoryContents) (string, error) { + yaml, err := yaml.Marshal(contents) + if err != nil { + return "", fmt.Errorf("error during creating for config digest for path '%s': %s", contents.Path, err) + } + digest := sha256.Sum256(yaml) + digestStr := hex.EncodeToString(digest[:]) + return digestStr, nil } func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) { @@ -57,7 +72,25 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) { return lockConfig, err } - lockDirContents := ctlconf.LockDirectoryContents{Path: contents.Path} + // creates config digest for current content config + configDigest, err := createConfigDigest(contents) + if err != nil { + return lockConfig, err + } + + lockDirContents := ctlconf.LockDirectoryContents{ + Path: contents.Path, + } + + // error is safe to ignore, since it indicates that no lock file entry for the given path exists + oldLockContents, _ := d.lockDirectory.FindContents(contents.Path) + skipFetching, lazySyncAddConfigDigest := d.handleLazySync(oldLockContents.ConfigDigest, configDigest, syncOpts.Lazy, contents.Lazy) + + if skipFetching { + d.ui.PrintLinef("Skipping fetch: %s + %s (flagged as lazy, config has not changed since last sync)", d.opts.Path, contents.Path) + lockConfig.Contents = append(lockConfig.Contents, oldLockContents) + continue + } skipFileFilter := false skipNewRootPath := false @@ -72,7 +105,6 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) { if err != nil { return lockConfig, fmt.Errorf("Syncing directory '%s' with git contents: %s", contents.Path, err) } - lockDirContents.Git = &lock case contents.Hg != nil: @@ -147,7 +179,6 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) { if err != nil { return lockConfig, fmt.Errorf("Syncing directory '%s' with helm chart contents: %s", contents.Path, err) } - lockDirContents.HelmChart = &lock case contents.Manual != nil: @@ -215,6 +246,10 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) { return lockConfig, fmt.Errorf("chmod on '%s': %s", stagingDstPath, err) } + if lazySyncAddConfigDigest { + lockDirContents.ConfigDigest = configDigest + } + lockConfig.Contents = append(lockConfig.Contents, lockDirContents) } @@ -243,3 +278,17 @@ func maybeChmod(path string, potentialPerms ...*os.FileMode) error { return nil } + +func (d *Directory) handleLazySync(oldConfigDigest string, newConfigDigest string, fetchLazyGlobalOverride bool, fetchLazy bool) (bool, bool) { + skipFetching := false + addConfigDigest := false + // if lazy sync is enabled and config remains unchanged, skip fetching + if fetchLazyGlobalOverride && fetchLazy && oldConfigDigest == newConfigDigest { + skipFetching = true + } + // config digest is always added if lazy syncing is enabled locally and globally + if fetchLazy { + addConfigDigest = true + } + return skipFetching, addConfigDigest +} diff --git a/test/e2e/example_lazy_test.go b/test/e2e/example_lazy_test.go new file mode 100644 index 00000000..e460a74f --- /dev/null +++ b/test/e2e/example_lazy_test.go @@ -0,0 +1,56 @@ +// Copyright 2020 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "github.com/stretchr/testify/require" + ctlconf "github.com/vmware-tanzu/carvel-vendir/pkg/vendir/config" + "os" + "testing" +) + +func TestExampleLazy(t *testing.T) { + env := BuildEnv(t) + vendir := Vendir{t, env.BinaryPath, Logger{}} + + osEnv := []string{"VENDIR_HELM_BINARY=" + env.Helm2Binary} + + dir := "examples/lazy" + path := "../../" + dir + + // run lazy sync + _, err := vendir.RunWithOpts([]string{"sync", "-f=vendir-lazy.yml"}, RunOpts{Dir: path, Env: osEnv}) + require.NoError(t, err) + + // check that the lock file has config digest + lockConf, err := ctlconf.NewLockConfigFromFile(path + "/vendir.lock.yml") + require.NoError(t, err) + require.NotEmpty(t, lockConf.Directories[0].Contents[0].ConfigDigest, "Expected Config Digest in Lock File") + + // remove some directory + err = os.RemoveAll(path + "/vendor/dir") + require.NoError(t, err) + + // resync lazily, should not sync. Removed dir has not been reinstated + _, err = vendir.RunWithOpts([]string{"sync", "-f=vendir-lazy.yml"}, RunOpts{Dir: path, Env: osEnv}) + require.NoError(t, err) + require.NoDirExists(t, path+"/vendor/dir") + + // resync with lazy override, should not affect config digest + _, err = vendir.RunWithOpts([]string{"sync", "--lazy=false", "-f=vendir-lazy.yml"}, RunOpts{Dir: path, Env: osEnv}) + require.NoError(t, err) + require.DirExists(t, path+"/vendor/dir") + + // content digest is kept during lazy sync override + lockConf, err = ctlconf.NewLockConfigFromFile(path + "/vendir.lock.yml") + require.NoError(t, err) + require.NotEmpty(t, lockConf.Directories[0].Contents[0].ConfigDigest, "Expected Config Digest in Lock File") + + // if synced without lazy flag in vendir.yml, no config digest should be kept in lock file + _, err = vendir.RunWithOpts([]string{"sync", "-f=vendir-nonlazy.yml"}, RunOpts{Dir: path, Env: osEnv}) + require.NoError(t, err) + lockConf, err = ctlconf.NewLockConfigFromFile(path + "/vendir.lock.yml") + require.NoError(t, err) + require.Empty(t, lockConf.Directories[0].Contents[0].ConfigDigest, "Expected No Config Digest in Lock File") +}