fix: fixing error when running vendir sync with directory option whil…

…e corresponding entry is missing from vendir.lock.yaml

Signed-off-by: Fritz Duchardt <[email protected]>

examples/git-and-manual/local-dir-2/file.txt

@@ -0,0 +1 @@
+file-dir-2

examples/git-and-manual/local-dir-dev/file.txt

@@ -1 +1 @@
-local-dir2/file
+file-dir-dev

examples/git-and-manual/vendir.lock.yml

@@ -11,5 +11,7 @@ directories:
     path: github.com/GoogleCloudPlatform/metacontroller
   - directory: {}
     path: local-dir
+  - directory: {}
+    path: local-dir-2
   path: vendor
 kind: LockConfig

examples/git-and-manual/vendir.yml

@@ -17,3 +17,6 @@ directories:
   - path: local-dir
     directory:
       path: local-dir
+  - path: local-dir-2
+    directory:
+      path: local-dir-2

examples/git-and-manual/vendor/local-dir-2/file.txt

@@ -0,0 +1 @@
+file-dir-2

pkg/vendir/cmd/sync.go

@@ -36,6 +36,13 @@ type SyncOptions struct {
 	AllowAllSymlinkDestinations bool
 }
 
+func (o *SyncOptions) LockFileExists() bool {
+	if _, err := os.Stat(o.LockFile); err != nil {
+		return false
+	}
+	return true
+}
+
 func NewSyncOptions(ui ui.UI) *SyncOptions {
 	return &SyncOptions{ui: ui}
 }
@@ -133,13 +140,15 @@ func (o *SyncOptions) Run() error {
 		HelmBinary:     os.Getenv("VENDIR_HELM_BINARY"),
 		Cache:          cache,
 		Lazy:           o.Lazy,
+		Partial:        len(dirs) > 0,
 	}
 	newLockConfig := ctlconf.NewLockConfig()
 
 	for _, dirConf := range conf.Directories {
 		// error safe to ignore, since lock file might not exist
 		dirExistingLockConf, _ := existingLockConfig.FindDirectory(dirConf.Path)
-		dirLockConf, err := ctldir.NewDirectory(dirConf, dirExistingLockConf, o.ui).Sync(syncOpts)
+		directory := ctldir.NewDirectory(dirConf, dirExistingLockConf, o.ui)
+		dirLockConf, err := directory.Sync(syncOpts)
 		if err != nil {
 			return fmt.Errorf("Syncing directory '%s': %s", dirConf.Path, err)
 		}
@@ -155,12 +164,18 @@ func (o *SyncOptions) Run() error {
 
 	// Update only selected directories in lock file
 	if len(dirs) > 0 {
-		err = existingLockConfig.Merge(newLockConfig)
-		if err != nil {
-			return err
-		}
+		if o.LockFileExists() {
+			existingLockConfig, err := ctlconf.NewLockConfigFromFile(o.LockFile)
+			if err != nil {
+				return err
+			}
+			err = existingLockConfig.Merge(newLockConfig)
+			if err != nil {
+				return err
+			}
 
-		newLockConfig = existingLockConfig
+			newLockConfig = existingLockConfig
+		}
 	}
 
 	newLockConfigBs, err := newLockConfig.AsBytes()

pkg/vendir/config/config.go

@@ -209,22 +209,22 @@ func (c Config) Subset(paths []string) (Config, error) {
 
 	for _, dir := range c.Directories {
 		for _, con := range dir.Contents {
-			path := filepath.Join(dir.Path, con.Path)
+			entirePath := filepath.Join(dir.Path, con.Path)
 
-			seen, found := pathsToSeen[path]
+			seen, found := pathsToSeen[entirePath]
 			if !found {
 				continue
 			}
 			if seen {
-				return Config{}, fmt.Errorf("Expected to match path '%s' once, but matched multiple", path)
+				return Config{}, fmt.Errorf("Expected to match path '%s' once, but matched multiple", entirePath)
 			}
-			pathsToSeen[path] = true
+			pathsToSeen[entirePath] = true
 
 			newCon := con // copy (but not deep unfortunately)
-			newCon.Path = EntireDirPath
+			newCon.Path = con.Path
 
 			result.Directories = append(result.Directories, Directory{
-				Path:     path,
+				Path:     dir.Path,
 				Contents: []DirectoryContents{newCon},
 			})
 		}

pkg/vendir/config/lock_config.go

@@ -30,7 +30,10 @@ func NewLockConfig() LockConfig {
 
 func LockFileExists(path string) bool {
 	if _, err := os.Stat(path); err != nil {
-		return false
+		if errors.Is(err, fs.ErrNotExist) {
+			return false
+		}
+		panic(fmt.Errorf("can not stat file '%v': %v", path, err))
 	}
 	return true
 }
@@ -131,42 +134,48 @@ func (c LockConfig) FindDirectory(dirPath string) (LockDirectory, error) {
 		"Expected to find directory '%s' within lock config, but did not", dirPath)
 }
 
-func (c LockConfig) Merge(other LockConfig) error {
+func (c *LockConfig) Merge(other LockConfig) error {
 	for _, dir := range other.Directories {
+		replaced := false
 		for _, con := range dir.Contents {
-			err := c.MergeContents(filepath.Join(dir.Path, con.Path), con)
-			if err != nil {
-				return err
+			replaced = c.ReplaceContents(filepath.Join(dir.Path, con.Path), con)
+			if replaced {
+				continue
+			}
+			replaced = c.AppendContents(dir.Path, con)
+			if replaced {
+				continue
 			}
 		}
+		if !replaced {
+			c.Directories = append(c.Directories, dir)
+		}
 	}
 	return nil
 }
 
-func (c LockConfig) MergeContents(path string, replaceCon LockDirectoryContents) error {
-	var matched bool
+func (c *LockConfig) ReplaceContents(path string, replaceCon LockDirectoryContents) bool {
 
 	for i, dir := range c.Directories {
 		for j, con := range dir.Contents {
 			if filepath.Join(dir.Path, con.Path) != path {
 				continue
 			}
-
-			if matched {
-				return fmt.Errorf("Expected to match exactly one directory, but matched multiple")
-			}
-			matched = true
-
-			newCon := replaceCon
-			newCon.Path = con.Path
-
-			dir.Contents[j] = newCon
+			dir.Contents[j] = replaceCon
 			c.Directories[i] = dir
+			return true
 		}
 	}
 
-	if !matched {
-		return fmt.Errorf("Expected to match exactly one directory, but did not match any")
+	return false
+}
+
+func (c *LockConfig) AppendContents(path string, appendCon LockDirectoryContents) bool {
+	for i, dir := range c.Directories {
+		if dir.Path == path {
+			c.Directories[i].Contents = append(dir.Contents, appendCon)
+			return true
+		}
 	}
-	return nil
+	return false
 }

pkg/vendir/config/lock_config_test.go

@@ -148,3 +148,109 @@ func TestWriteToFile(t *testing.T) {
 
 	require.NoError(t, os.RemoveAll(tempDir))
 }
+
+func TestLockConfig_ReplaceContents(t *testing.T) {
+	lockConfig := config.LockConfig{
+		APIVersion: "vendir.k14s.io/v1alpha1",
+		Kind:       "LockConfig",
+		Directories: []config.LockDirectory{
+			{
+				Path: "dirpath",
+				Contents: []config.LockDirectoryContents{
+					{
+						Path: "dirpath",
+					},
+				},
+			},
+		},
+	}
+	lockContents := config.LockDirectoryContents{
+		Path: "dirpath",
+		HelmChart: &config.LockDirectoryContentsHelmChart{
+			Version: "test",
+		},
+	}
+	t.Run("replace contents", func(t *testing.T) {
+		appended := lockConfig.ReplaceContents("dirpath/dirpath", lockContents)
+		require.True(t, appended)
+		require.Equal(t, 1, len(lockConfig.Directories))
+		require.Equal(t, 1, len(lockConfig.Directories[0].Contents))
+		require.Equal(t, config.LockDirectoryContents{Path: "dirpath", HelmChart: &config.LockDirectoryContentsHelmChart{Version: "test"}}, lockConfig.Directories[0].Contents[0])
+	})
+
+	t.Run("fail to append content to config due to wrong path", func(t *testing.T) {
+		appended := lockConfig.AppendContents("wrong-path", lockContents)
+		require.False(t, appended)
+	})
+}
+
+func TestLockConfig_AppendContents(t *testing.T) {
+	lockConfig := config.LockConfig{
+		APIVersion: "vendir.k14s.io/v1alpha1",
+		Kind:       "LockConfig",
+		Directories: []config.LockDirectory{
+			{
+				Path: "dirpath",
+				Contents: []config.LockDirectoryContents{
+					{
+						Path: "dirpath",
+					},
+				},
+			},
+		},
+	}
+	lockContents := config.LockDirectoryContents{
+		Path: "gitpath",
+	}
+	t.Run("append contents to config", func(t *testing.T) {
+		appended := lockConfig.AppendContents("dirpath", lockContents)
+		require.True(t, appended)
+		require.Equal(t, 1, len(lockConfig.Directories))
+		require.Equal(t, 2, len(lockConfig.Directories[0].Contents))
+		require.Equal(t, config.LockDirectoryContents{Path: "gitpath"}, lockConfig.Directories[0].Contents[1])
+	})
+
+	t.Run("fail to append content to config due to wrong path", func(t *testing.T) {
+		appended := lockConfig.AppendContents("wrong-path", lockContents)
+		require.False(t, appended)
+	})
+}
+
+func TestLockConfig_Merge(t *testing.T) {
+	lockConfig := config.LockConfig{
+		APIVersion: "vendir.k14s.io/v1alpha1",
+		Kind:       "LockConfig",
+		Directories: []config.LockDirectory{
+			{
+				Path: "gitpath-1",
+				Contents: []config.LockDirectoryContents{
+					{
+						Path: "gitpath-1",
+					},
+				},
+			},
+		},
+	}
+	lockConfig2 := config.LockConfig{
+		APIVersion: "vendir.k14s.io/v1alpha1",
+		Kind:       "LockConfig",
+		Directories: []config.LockDirectory{
+			{
+				Path: "gitpath-2",
+				Contents: []config.LockDirectoryContents{
+					{
+						Path: "gitpath-2",
+					},
+				},
+			},
+		},
+	}
+	t.Run("append directory to config", func(t *testing.T) {
+		lockConfig.Merge(lockConfig2)
+		require.Equal(t, 2, len(lockConfig.Directories))
+		require.Equal(t, 1, len(lockConfig.Directories[0].Contents))
+		require.Equal(t, 1, len(lockConfig.Directories[1].Contents))
+		require.Equal(t, config.LockDirectoryContents{Path: "gitpath-1"}, lockConfig.Directories[0].Contents[0])
+		require.Equal(t, config.LockDirectoryContents{Path: "gitpath-2"}, lockConfig.Directories[1].Contents[0])
+	})
+}

pkg/vendir/directory/directory.go

@@ -42,6 +42,7 @@ type SyncOpts struct {
 	HelmBinary     string
 	Cache          ctlcache.Cache
 	Lazy           bool
+	Partial        bool
 }
 
 func createConfigDigest(contents ctlconf.DirectoryContents) (string, error) {
@@ -251,11 +252,20 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) {
 		}
 
 		lockConfig.Contents = append(lockConfig.Contents, lockDirContents)
+
+		if syncOpts.Partial {
+			err = stagingDir.PartialRepace(contents.Path, filepath.Join(d.opts.Path, contents.Path))
+			if err != nil {
+				return lockConfig, err
+			}
+		}
 	}
 
-	err = stagingDir.Replace(d.opts.Path)
-	if err != nil {
-		return lockConfig, err
+	if !syncOpts.Partial {
+		err = stagingDir.Replace(d.opts.Path)
+		if err != nil {
+			return lockConfig, err
+		}
 	}
 
 	// after everything else is done, ensure the outer dir's access perms are set

pkg/vendir/directory/staging_dir.go

@@ -130,25 +130,50 @@ func isPathIgnored(path string, ignorePaths []string) (bool, error) {
 	return false, nil
 }
 
+// Replaces entire final location directory with staging directory
 func (d StagingDir) Replace(path string) error {
-	err := os.RemoveAll(path)
+
+	err := d.prepareOutputDirectory(path)
 	if err != nil {
-		return fmt.Errorf("Deleting dir %s: %s", path, err)
+		return err
 	}
 
-	// Clean to avoid getting 'out/in/' from 'out/in/' instead of just 'out'
-	parentPath := filepath.Dir(filepath.Clean(path))
+	err = os.Rename(d.stagingDir, path)
+	if err != nil {
+		return fmt.Errorf("Moving staging directory '%s' to final location '%s': %s", d.stagingDir, path, err)
+	}
 
-	err = os.MkdirAll(parentPath, 0700)
+	return nil
+}
+
+// Replaces single directory of final location dir with single directory of staging dir
+func (d StagingDir) PartialRepace(contentPath string, directoryPath string) error {
+	err := d.prepareOutputDirectory(directoryPath)
 	if err != nil {
-		return fmt.Errorf("Creating final location parent dir %s: %s", parentPath, err)
+		return err
 	}
 
-	err = os.Rename(d.stagingDir, path)
+	err = os.Rename(filepath.Join(d.stagingDir, contentPath), directoryPath)
 	if err != nil {
-		return fmt.Errorf("Moving staging directory '%s' to final location '%s': %s", d.stagingDir, path, err)
+		return fmt.Errorf("Moving staging directory '%s' to final location '%s': %s", d.stagingDir, directoryPath, err)
+	}
+
+	return nil
+}
+
+func (d StagingDir) prepareOutputDirectory(directoryPath string) error {
+	err := os.RemoveAll(directoryPath)
+	if err != nil {
+		return fmt.Errorf("Deleting dir %s: %s", directoryPath, err)
 	}
 
+	// Clean to avoid getting 'out/in/' from 'out/in/' instead of just 'out'
+	parentPath := filepath.Dir(filepath.Clean(directoryPath))
+
+	err = os.MkdirAll(parentPath, 0700)
+	if err != nil {
+		return fmt.Errorf("Creating final location parent dir %s: %s", parentPath, err)
+	}
 	return nil
 }