Added lazy sync functionality

examples/lazy/vendir-lazy.yml

@@ -0,0 +1,9 @@
+apiVersion: vendir.k14s.io/v1alpha1
+kind: Config
+directories:
+- path: vendor
+  contents:
+  - path: dir
+    lazy: true
+    directory:
+      path: ./some-content

examples/lazy/vendir-nonlazy.yml

@@ -0,0 +1,8 @@
+apiVersion: vendir.k14s.io/v1alpha1
+kind: Config
+directories:
+- path: vendor
+  contents:
+  - path: dir
+    directory:
+      path: ./some-content

examples/lazy/vendir.lock.yml

@@ -0,0 +1,7 @@
+apiVersion: vendir.k14s.io/v1alpha1
+directories:
+- contents:
+  - directory: {}
+    path: dir
+  path: vendor
+kind: LockConfig

pkg/vendir/cmd/sync.go

@@ -30,6 +30,7 @@ type SyncOptions struct {
 
 	Directories []string
 	Locked      bool
+	Lazy        bool
 
 	Chdir                       string
 	AllowAllSymlinkDestinations bool
@@ -50,6 +51,7 @@ func NewSyncCmd(o *SyncOptions) *cobra.Command {
 
 	cmd.Flags().StringSliceVarP(&o.Directories, "directory", "d", nil, "Sync specific directory (format: dir/sub-dir[=local-dir])")
 	cmd.Flags().BoolVarP(&o.Locked, "locked", "l", false, "Consult lock file to pull exact references (e.g. use git sha instead of branch name)")
+	cmd.Flags().BoolVar(&o.Lazy, "lazy", true, "Set to 'false' it ignores the 'lazy' flag in the directory content configuration")
 
 	cmd.Flags().StringVar(&o.Chdir, "chdir", "", "Set current directory for process")
 	cmd.Flags().BoolVar(&o.AllowAllSymlinkDestinations, "dangerous-allow-all-symlink-destinations", false, "Symlinks to all destinations are allowed")
@@ -95,14 +97,14 @@ func (o *SyncOptions) Run() error {
 		o.ui.PrintBlock(configBs)
 	}
 
+	existingLockConfig, err := ctlconf.NewLockConfigFromFile(o.LockFile)
+	if err != nil && o.Locked {
+		return err
+	}
+
 	// If syncing against a lock file, apply lock information
 	// on top of existing config
 	if o.Locked {
-		existingLockConfig, err := ctlconf.NewLockConfigFromFile(o.LockFile)
-		if err != nil {
-			return err
-		}
-
 		err = conf.Lock(existingLockConfig)
 		if err != nil {
 			return err
@@ -130,11 +132,14 @@ func (o *SyncOptions) Run() error {
 		GithubAPIToken: os.Getenv("VENDIR_GITHUB_API_TOKEN"),
 		HelmBinary:     os.Getenv("VENDIR_HELM_BINARY"),
 		Cache:          cache,
+		Lazy:           o.Lazy,
 	}
 	newLockConfig := ctlconf.NewLockConfig()
 
 	for _, dirConf := range conf.Directories {
-		dirLockConf, err := ctldir.NewDirectory(dirConf, o.ui).Sync(syncOpts)
+		// error safe to ignore, since lock file might not exist
+		dirExistingLockConf, _ := existingLockConfig.FindDirectory(dirConf.Path)
+		dirLockConf, err := ctldir.NewDirectory(dirConf, dirExistingLockConf, o.ui).Sync(syncOpts)
 		if err != nil {
 			return fmt.Errorf("Syncing directory '%s': %s", dirConf.Path, err)
 		}
@@ -150,11 +155,6 @@ func (o *SyncOptions) Run() error {
 
 	// Update only selected directories in lock file
 	if len(dirs) > 0 {
-		existingLockConfig, err := ctlconf.NewLockConfigFromFile(o.LockFile)
-		if err != nil {
-			return err
-		}
-
 		err = existingLockConfig.Merge(newLockConfig)
 		if err != nil {
 			return err

pkg/vendir/config/config.go

@@ -183,6 +183,7 @@ func (c Config) UseDirectory(path, dirPath string) error {
 				ExcludePaths: con.ExcludePaths,
 				IgnorePaths:  con.IgnorePaths,
 				LegalPaths:   con.LegalPaths,
+				Lazy:         con.Lazy,
 			}
 			dir.Contents[j] = newCon
 			c.Directories[i] = dir

pkg/vendir/config/directory.go

@@ -34,6 +34,7 @@ type Directory struct {
 
 type DirectoryContents struct {
 	Path string `json:"path"`
+	Lazy bool   `json:"lazy,omitempty"`
 
 	Git           *DirectoryContentsGit           `json:"git,omitempty"`
 	Hg            *DirectoryContentsHg            `json:"hg,omitempty"`

pkg/vendir/config/lock_config.go

@@ -28,6 +28,13 @@ func NewLockConfig() LockConfig {
 	}
 }
 
+func LockFileExists(path string) bool {
+	if _, err := os.Stat(path); err != nil {
+		return false
+	}
+	return true
+}
+
 func NewLockConfigFromFile(path string) (LockConfig, error) {
 	bs, err := os.ReadFile(path)
 	if err != nil {
@@ -114,6 +121,16 @@ func (c LockConfig) FindContents(dirPath, conPath string) (LockDirectoryContents
 		"Expected to find directory '%s' within lock config, but did not", dirPath)
 }
 
+func (c LockConfig) FindDirectory(dirPath string) (LockDirectory, error) {
+	for _, dir := range c.Directories {
+		if dir.Path == dirPath {
+			return dir, nil
+		}
+	}
+	return LockDirectory{}, fmt.Errorf(
+		"Expected to find directory '%s' within lock config, but did not", dirPath)
+}
+
 func (c LockConfig) Merge(other LockConfig) error {
 	for _, dir := range other.Directories {
 		for _, con := range dir.Contents {

pkg/vendir/config/lock_directory.go

@@ -3,13 +3,16 @@
 
 package config
 
+import "fmt"
+
 type LockDirectory struct {
 	Path     string                  `json:"path"`
 	Contents []LockDirectoryContents `json:"contents"`
 }
 
 type LockDirectoryContents struct {
-	Path string `json:"path"`
+	Path         string `json:"path"`
+	ConfigDigest string `json:"configDigest,omitempty"`
 
 	Git           *LockDirectoryContentsGit           `json:"git,omitempty"`
 	Hg            *LockDirectoryContentsHg            `json:"hg,omitempty"`
@@ -62,3 +65,13 @@ type LockDirectoryContentsManual struct{}
 type LockDirectoryContentsDirectory struct{}
 
 type LockDirectoryContentsInline struct{}
+
+func (d LockDirectory) FindContents(conPath string) (LockDirectoryContents, error) {
+	for _, con := range d.Contents {
+		if con.Path == conPath {
+			return con, nil
+		}
+	}
+	return LockDirectoryContents{}, fmt.Errorf("Expected to find contents '%s' "+
+		"within directory '%s' in lock config, but did not", conPath, d.Path)
+}

pkg/vendir/directory/directory.go

@@ -4,9 +4,12 @@
 package directory
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"os"
 	"path/filepath"
+	"sigs.k8s.io/yaml"
 
 	"github.com/cppforlife/go-cli-ui/ui"
 	dircopy "github.com/otiai10/copy"
@@ -24,19 +27,31 @@ import (
 )
 
 type Directory struct {
-	opts ctlconf.Directory
-	ui   ui.UI
+	opts          ctlconf.Directory
+	lockDirectory ctlconf.LockDirectory
+	ui            ui.UI
 }
 
-func NewDirectory(opts ctlconf.Directory, ui ui.UI) *Directory {
-	return &Directory{opts, ui}
+func NewDirectory(opts ctlconf.Directory, lockDirectory ctlconf.LockDirectory, ui ui.UI) *Directory {
+	return &Directory{opts, lockDirectory, ui}
 }
 
 type SyncOpts struct {
 	RefFetcher     ctlfetch.RefFetcher
 	GithubAPIToken string
 	HelmBinary     string
 	Cache          ctlcache.Cache
+	Lazy           bool
+}
+
+func createConfigDigest(contents ctlconf.DirectoryContents) (string, error) {
+	yaml, err := yaml.Marshal(contents)
+	if err != nil {
+		return "", fmt.Errorf("error during creating for config digest for path '%s': %s", contents.Path, err)
+	}
+	digest := sha256.Sum256(yaml)
+	digestStr := hex.EncodeToString(digest[:])
+	return digestStr, nil
 }
 
 func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) {
@@ -57,7 +72,25 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) {
 			return lockConfig, err
 		}
 
-		lockDirContents := ctlconf.LockDirectoryContents{Path: contents.Path}
+		// creates config digest for current content config
+		configDigest, err := createConfigDigest(contents)
+		if err != nil {
+			return lockConfig, err
+		}
+
+		lockDirContents := ctlconf.LockDirectoryContents{
+			Path: contents.Path,
+		}
+
+		// error is safe to ignore, since it indicates that no lock file entry for the given path exists
+		oldLockContents, _ := d.lockDirectory.FindContents(contents.Path)
+		skipFetching, lazySyncAddConfigDigest := d.handleLazySync(oldLockContents.ConfigDigest, configDigest, syncOpts.Lazy, contents.Lazy)
+
+		if skipFetching {
+			d.ui.PrintLinef("Skipping fetch: %s + %s (flagged as lazy, config has not changed since last sync)", d.opts.Path, contents.Path)
+			lockConfig.Contents = append(lockConfig.Contents, oldLockContents)
+			continue
+		}
 
 		skipFileFilter := false
 		skipNewRootPath := false
@@ -72,7 +105,6 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) {
 			if err != nil {
 				return lockConfig, fmt.Errorf("Syncing directory '%s' with git contents: %s", contents.Path, err)
 			}
-
 			lockDirContents.Git = &lock
 
 		case contents.Hg != nil:
@@ -147,7 +179,6 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) {
 			if err != nil {
 				return lockConfig, fmt.Errorf("Syncing directory '%s' with helm chart contents: %s", contents.Path, err)
 			}
-
 			lockDirContents.HelmChart = &lock
 
 		case contents.Manual != nil:
@@ -215,6 +246,10 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) {
 			return lockConfig, fmt.Errorf("chmod on '%s': %s", stagingDstPath, err)
 		}
 
+		if lazySyncAddConfigDigest {
+			lockDirContents.ConfigDigest = configDigest
+		}
+
 		lockConfig.Contents = append(lockConfig.Contents, lockDirContents)
 	}
 
@@ -243,3 +278,17 @@ func maybeChmod(path string, potentialPerms ...*os.FileMode) error {
 
 	return nil
 }
+
+func (d *Directory) handleLazySync(oldConfigDigest string, newConfigDigest string, fetchLazyGlobalOverride bool, fetchLazy bool) (bool, bool) {
+	skipFetching := false
+	addConfigDigest := false
+	// if lazy sync is enabled and config remains unchanged, skip fetching
+	if fetchLazyGlobalOverride && fetchLazy && oldConfigDigest == newConfigDigest {
+		skipFetching = true
+	}
+	// config digest is always added if lazy syncing is enabled locally and globally
+	if fetchLazy {
+		addConfigDigest = true
+	}
+	return skipFetching, addConfigDigest
+}

test/e2e/example_lazy_test.go

@@ -0,0 +1,56 @@
+// Copyright 2020 VMware, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"github.com/stretchr/testify/require"
+	ctlconf "github.com/vmware-tanzu/carvel-vendir/pkg/vendir/config"
+	"os"
+	"testing"
+)
+
+func TestExampleLazy(t *testing.T) {
+	env := BuildEnv(t)
+	vendir := Vendir{t, env.BinaryPath, Logger{}}
+
+	osEnv := []string{"VENDIR_HELM_BINARY=" + env.Helm2Binary}
+
+	dir := "examples/lazy"
+	path := "../../" + dir
+
+	// run lazy sync
+	_, err := vendir.RunWithOpts([]string{"sync", "-f=vendir-lazy.yml"}, RunOpts{Dir: path, Env: osEnv})
+	require.NoError(t, err)
+
+	// check that the lock file has config digest
+	lockConf, err := ctlconf.NewLockConfigFromFile(path + "/vendir.lock.yml")
+	require.NoError(t, err)
+	require.NotEmpty(t, lockConf.Directories[0].Contents[0].ConfigDigest, "Expected Config Digest in Lock File")
+
+	// remove some directory
+	err = os.RemoveAll(path + "/vendor/dir")
+	require.NoError(t, err)
+
+	// resync lazily, should not sync. Removed dir has not been reinstated
+	_, err = vendir.RunWithOpts([]string{"sync", "-f=vendir-lazy.yml"}, RunOpts{Dir: path, Env: osEnv})
+	require.NoError(t, err)
+	require.NoDirExists(t, path+"/vendor/dir")
+
+	// resync with lazy override, should not affect config digest
+	_, err = vendir.RunWithOpts([]string{"sync", "--lazy=false", "-f=vendir-lazy.yml"}, RunOpts{Dir: path, Env: osEnv})
+	require.NoError(t, err)
+	require.DirExists(t, path+"/vendor/dir")
+
+	// content digest is kept during lazy sync override
+	lockConf, err = ctlconf.NewLockConfigFromFile(path + "/vendir.lock.yml")
+	require.NoError(t, err)
+	require.NotEmpty(t, lockConf.Directories[0].Contents[0].ConfigDigest, "Expected Config Digest in Lock File")
+
+	// if synced without lazy flag in vendir.yml, no config digest should be kept in lock file
+	_, err = vendir.RunWithOpts([]string{"sync", "-f=vendir-nonlazy.yml"}, RunOpts{Dir: path, Env: osEnv})
+	require.NoError(t, err)
+	lockConf, err = ctlconf.NewLockConfigFromFile(path + "/vendir.lock.yml")
+	require.NoError(t, err)
+	require.Empty(t, lockConf.Directories[0].Contents[0].ConfigDigest, "Expected No Config Digest in Lock File")
+}