Merge pull request #1365 from ps-gill/feature/745/allow-excluding-old…

…er-snapshots

Add refresh-exclude-older flag to only transfer new snapshots during instance/volume refresh

client/incus_instances.go

@@ -782,6 +782,10 @@ func (r *ProtocolIncus) CopyInstance(source InstanceServer, instance api.Instanc
 			}
 		}
 
+		if args.RefreshExcludeOlder && !source.HasExtension("custom_volume_refresh_exclude_older_snapshots") {
+			return nil, fmt.Errorf("The source server is missing the required \"custom_volume_refresh_exclude_older_snapshots\" API extension")
+		}
+
 		if args.AllowInconsistent {
 			if !r.HasExtension("instance_allow_inconsistent_copy") {
 				return nil, fmt.Errorf("The source server is missing the required \"instance_allow_inconsistent_copy\" API extension")
@@ -796,6 +800,7 @@ func (r *ProtocolIncus) CopyInstance(source InstanceServer, instance api.Instanc
 		req.Source.Live = args.Live
 		req.Source.InstanceOnly = args.InstanceOnly
 		req.Source.Refresh = args.Refresh
+		req.Source.RefreshExcludeOlder = args.RefreshExcludeOlder
 		req.Source.AllowInconsistent = args.AllowInconsistent
 	}
 
@@ -868,6 +873,7 @@ func (r *ProtocolIncus) CopyInstance(source InstanceServer, instance api.Instanc
 		req.Source.Type = "migration"
 		req.Source.Mode = "push"
 		req.Source.Refresh = args.Refresh
+		req.Source.RefreshExcludeOlder = args.RefreshExcludeOlder
 
 		op, err := r.CreateInstance(req)
 		if err != nil {

client/incus_storage_volumes.go

@@ -529,15 +529,20 @@ func (r *ProtocolIncus) CopyStoragePoolVolume(pool string, source InstanceServer
 		return nil, fmt.Errorf("The target server is missing the required \"custom_volume_refresh\" API extension")
 	}
 
+	if args != nil && args.RefreshExcludeOlder && !r.HasExtension("custom_volume_refresh_exclude_older_snapshots") {
+		return nil, fmt.Errorf("The target server is missing the required \"custom_volume_refresh_exclude_older_snapshots\" API extension")
+	}
+
 	req := api.StorageVolumesPost{
 		Name: args.Name,
 		Type: volume.Type,
 		Source: api.StorageVolumeSource{
-			Name:       volume.Name,
-			Type:       "copy",
-			Pool:       sourcePool,
-			VolumeOnly: args.VolumeOnly,
-			Refresh:    args.Refresh,
+			Name:                volume.Name,
+			Type:                "copy",
+			Pool:                sourcePool,
+			VolumeOnly:          args.VolumeOnly,
+			Refresh:             args.Refresh,
+			RefreshExcludeOlder: args.RefreshExcludeOlder,
 		},
 	}
 

client/interfaces.go

@@ -522,6 +522,9 @@ type StoragePoolVolumeCopyArgs struct {
 
 	// API extension: custom_volume_refresh
 	Refresh bool
+
+	// API extension: custom_volume_refresh_exclude_older_snapshots
+	RefreshExcludeOlder bool
 }
 
 // The StoragePoolVolumeMoveArgs struct is used to pass additional options
@@ -573,6 +576,9 @@ type InstanceCopyArgs struct {
 	// Perform an incremental copy
 	Refresh bool
 
+	// API extension: custom_volume_refresh_exclude_older_snapshots
+	RefreshExcludeOlder bool
+
 	// API extension: instance_allow_inconsistent_copy
 	AllowInconsistent bool
 }

cmd/incus/copy.go

@@ -17,19 +17,20 @@ import (
 type cmdCopy struct {
 	global *cmdGlobal
 
-	flagNoProfiles        bool
-	flagProfile           []string
-	flagConfig            []string
-	flagDevice            []string
-	flagEphemeral         bool
-	flagInstanceOnly      bool
-	flagMode              string
-	flagStateless         bool
-	flagStorage           string
-	flagTarget            string
-	flagTargetProject     string
-	flagRefresh           bool
-	flagAllowInconsistent bool
+	flagNoProfiles          bool
+	flagProfile             []string
+	flagConfig              []string
+	flagDevice              []string
+	flagEphemeral           bool
+	flagInstanceOnly        bool
+	flagMode                string
+	flagStateless           bool
+	flagStorage             string
+	flagTarget              string
+	flagTargetProject       string
+	flagRefresh             bool
+	flagRefreshExcludeOlder bool
+	flagAllowInconsistent   bool
 }
 
 func (c *cmdCopy) Command() *cobra.Command {
@@ -61,6 +62,7 @@ The pull transfer mode is the default as it is compatible with all server versio
 	cmd.Flags().StringVar(&c.flagTargetProject, "target-project", "", i18n.G("Copy to a project different from the source")+"``")
 	cmd.Flags().BoolVar(&c.flagNoProfiles, "no-profiles", false, i18n.G("Create the instance with no profiles applied"))
 	cmd.Flags().BoolVar(&c.flagRefresh, "refresh", false, i18n.G("Perform an incremental copy"))
+	cmd.Flags().BoolVar(&c.flagRefreshExcludeOlder, "refresh-exclude-older", false, i18n.G("During incremental copy, exclude source snapshots earlier than latest target snapshot"))
 	cmd.Flags().BoolVar(&c.flagAllowInconsistent, "allow-inconsistent", false, i18n.G("Ignore copy errors for volatile files"))
 
 	cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
@@ -256,12 +258,13 @@ func (c *cmdCopy) copyInstance(conf *config.Config, sourceResource string, destR
 	} else {
 		// Prepare the instance creation request
 		args := incus.InstanceCopyArgs{
-			Name:              destName,
-			Live:              stateful,
-			InstanceOnly:      instanceOnly,
-			Mode:              mode,
-			Refresh:           c.flagRefresh,
-			AllowInconsistent: c.flagAllowInconsistent,
+			Name:                destName,
+			Live:                stateful,
+			InstanceOnly:        instanceOnly,
+			Mode:                mode,
+			Refresh:             c.flagRefresh,
+			RefreshExcludeOlder: c.flagRefreshExcludeOlder,
+			AllowInconsistent:   c.flagAllowInconsistent,
 		}
 
 		// Copy of an instance into a new instance

cmd/incus/storage_volume.go

@@ -347,10 +347,11 @@ type cmdStorageVolumeCopy struct {
 	storage       *cmdStorage
 	storageVolume *cmdStorageVolume
 
-	flagMode          string
-	flagVolumeOnly    bool
-	flagTargetProject string
-	flagRefresh       bool
+	flagMode                string
+	flagVolumeOnly          bool
+	flagTargetProject       string
+	flagRefresh             bool
+	flagRefreshExcludeOlder bool
 }
 
 func (c *cmdStorageVolumeCopy) Command() *cobra.Command {
@@ -367,6 +368,7 @@ func (c *cmdStorageVolumeCopy) Command() *cobra.Command {
 	cmd.Flags().BoolVar(&c.flagVolumeOnly, "volume-only", false, i18n.G("Copy the volume without its snapshots"))
 	cmd.Flags().StringVar(&c.flagTargetProject, "target-project", "", i18n.G("Copy to a project different from the source")+"``")
 	cmd.Flags().BoolVar(&c.flagRefresh, "refresh", false, i18n.G("Refresh and update the existing storage volume copies"))
+	cmd.Flags().BoolVar(&c.flagRefreshExcludeOlder, "refresh-exclude-older", false, i18n.G("During refresh, exclude source snapshots earlier than latest target snapshot"))
 	cmd.RunE = c.Run
 
 	cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
@@ -514,6 +516,7 @@ func (c *cmdStorageVolumeCopy) Run(cmd *cobra.Command, args []string) error {
 		args.Mode = mode
 		args.VolumeOnly = c.flagVolumeOnly
 		args.Refresh = c.flagRefresh
+		args.RefreshExcludeOlder = c.flagRefreshExcludeOlder
 
 		if c.flagTargetProject != "" {
 			dstServer = dstServer.UseProject(c.flagTargetProject)

cmd/incusd/instance.go

@@ -290,6 +290,7 @@ type instanceCreateAsCopyOpts struct {
 	targetInstance       db.InstanceArgs   // Configuration for new instance.
 	instanceOnly         bool              // Only copy the instance and not it's snapshots.
 	refresh              bool              // Refresh an existing target instance.
+	refreshExcludeOlder  bool              // During refresh, exclude source snapshots earlier than latest target snapshot
 	applyTemplateTrigger bool              // Apply deferred TemplateTriggerCopy.
 	allowInconsistent    bool              // Ignore some copy errors
 }
@@ -372,7 +373,7 @@ func instanceCreateAsCopy(s *state.State, opts instanceCreateAsCopyOpts, op *ope
 				})
 			}
 
-			syncSourceSnapshotIndexes, deleteTargetSnapshotIndexes := storagePools.CompareSnapshots(sourceSnapshotComparable, targetSnapshotsComparable)
+			syncSourceSnapshotIndexes, deleteTargetSnapshotIndexes := storagePools.CompareSnapshots(sourceSnapshotComparable, targetSnapshotsComparable, opts.refreshExcludeOlder)
 
 			// Delete extra snapshots first.
 			for _, deleteTargetSnapIndex := range deleteTargetSnapshotIndexes {

cmd/incusd/instances_post.go

@@ -362,6 +362,7 @@ func createFromMigration(ctx context.Context, s *state.State, r *http.Request, p
 		InstanceOnly:          instanceOnly,
 		ClusterMoveSourceName: clusterMoveSourceName,
 		Refresh:               req.Source.Refresh,
+		RefreshExcludeOlder:   req.Source.RefreshExcludeOlder,
 	}
 
 	sink, err := newMigrationSink(&migrationArgs)
@@ -549,6 +550,7 @@ func createFromCopy(ctx context.Context, s *state.State, r *http.Request, projec
 			targetInstance:       args,
 			instanceOnly:         req.Source.InstanceOnly,
 			refresh:              req.Source.Refresh,
+			refreshExcludeOlder:  req.Source.RefreshExcludeOlder,
 			applyTemplateTrigger: true,
 			allowInconsistent:    req.Source.AllowInconsistent,
 		}, op)

cmd/incusd/migrate.go

@@ -185,6 +185,7 @@ type migrationSink struct {
 	push                  bool
 	clusterMoveSourceName string
 	refresh               bool
+	refreshExcludeOlder   bool
 }
 
 // MigrationSinkArgs arguments to configure migration sink.
@@ -201,6 +202,7 @@ type migrationSinkArgs struct {
 	Idmap                 *idmap.Set
 	Live                  bool
 	Refresh               bool
+	RefreshExcludeOlder   bool
 	ClusterMoveSourceName string
 	Snapshots             []*migration.Snapshot
 

cmd/incusd/migrate_instance.go

@@ -169,6 +169,7 @@ func newMigrationSink(args *migrationSinkArgs) (*migrationSink, error) {
 		clusterMoveSourceName: args.ClusterMoveSourceName,
 		push:                  args.Push,
 		refresh:               args.Refresh,
+		refreshExcludeOlder:   args.RefreshExcludeOlder,
 	}
 
 	secretNames := []string{api.SecretNameControl, api.SecretNameFilesystem}
@@ -275,8 +276,9 @@ func (c *migrationSink) Do(state *state.State, instOp *operationlock.InstanceOpe
 			},
 			ClusterMoveSourceName: c.clusterMoveSourceName,
 		},
-		InstanceOperation: instOp,
-		Refresh:           c.refresh,
+		InstanceOperation:   instOp,
+		Refresh:             c.refresh,
+		RefreshExcludeOlder: c.refreshExcludeOlder,
 	})
 	if err != nil {
 		l.Error("Failed migration on target", logger.Ctx{"err": err})

cmd/incusd/migrate_storage_volumes.go

@@ -208,9 +208,10 @@ func newStorageMigrationSink(args *migrationSinkArgs) (*migrationSink, error) {
 		migrationFields: migrationFields{
 			volumeOnly: args.VolumeOnly,
 		},
-		url:     args.URL,
-		push:    args.Push,
-		refresh: args.Refresh,
+		url:                 args.URL,
+		push:                args.Push,
+		refresh:             args.Refresh,
+		refreshExcludeOlder: args.RefreshExcludeOlder,
 	}
 
 	secretNames := []string{api.SecretNameControl, api.SecretNameFilesystem}
@@ -321,15 +322,16 @@ func (c *migrationSink) DoStorage(state *state.State, projectName string, poolNa
 	// with the new storage layer.
 	myTarget = func(conn io.ReadWriteCloser, op *operations.Operation, args migrationSinkArgs) error {
 		volTargetArgs := localMigration.VolumeTargetArgs{
-			IndexHeaderVersion: respHeader.GetIndexHeaderVersion(),
-			Name:               req.Name,
-			Config:             req.Config,
-			Description:        req.Description,
-			MigrationType:      respTypes[0],
-			TrackProgress:      true,
-			ContentType:        req.ContentType,
-			Refresh:            args.Refresh,
-			VolumeOnly:         args.VolumeOnly,
+			IndexHeaderVersion:  respHeader.GetIndexHeaderVersion(),
+			Name:                req.Name,
+			Config:              req.Config,
+			Description:         req.Description,
+			MigrationType:       respTypes[0],
+			TrackProgress:       true,
+			ContentType:         req.ContentType,
+			Refresh:             args.Refresh,
+			RefreshExcludeOlder: args.RefreshExcludeOlder,
+			VolumeOnly:          args.VolumeOnly,
 		}
 
 		// A zero length Snapshots slice indicates volume only migration in
@@ -373,7 +375,7 @@ func (c *migrationSink) DoStorage(state *state.State, projectName string, poolNa
 		}
 
 		// Compare the two sets.
-		syncSourceSnapshotIndexes, deleteTargetSnapshotIndexes := storagePools.CompareSnapshots(sourceSnapshotComparable, targetSnapshotsComparable)
+		syncSourceSnapshotIndexes, deleteTargetSnapshotIndexes := storagePools.CompareSnapshots(sourceSnapshotComparable, targetSnapshotsComparable, c.refreshExcludeOlder)
 
 		// Delete the extra local snapshots first.
 		for _, deleteTargetSnapshotIndex := range deleteTargetSnapshotIndexes {
@@ -419,10 +421,11 @@ func (c *migrationSink) DoStorage(state *state.State, projectName string, poolNa
 			// as part of MigrationSinkArgs below.
 			rsyncFeatures := respHeader.GetRsyncFeaturesSlice()
 			args := migrationSinkArgs{
-				RsyncFeatures: rsyncFeatures,
-				Snapshots:     respHeader.Snapshots,
-				VolumeOnly:    c.volumeOnly,
-				Refresh:       c.refresh,
+				RsyncFeatures:       rsyncFeatures,
+				Snapshots:           respHeader.Snapshots,
+				VolumeOnly:          c.volumeOnly,
+				Refresh:             c.refresh,
+				RefreshExcludeOlder: c.refreshExcludeOlder,
 			}
 
 			fsConn, err := c.conns[api.SecretNameFilesystem].WebsocketIO(state.ShutdownCtx)

cmd/incusd/storage_volumes.go

@@ -829,7 +829,7 @@ func doCustomVolumeRefresh(s *state.State, r *http.Request, requestProjectName s
 			return fmt.Errorf("No source volume name supplied")
 		}
 
-		err = pool.RefreshCustomVolume(projectName, srcProjectName, req.Name, req.Description, req.Config, req.Source.Pool, req.Source.Name, !req.Source.VolumeOnly, op)
+		err = pool.RefreshCustomVolume(projectName, srcProjectName, req.Name, req.Description, req.Config, req.Source.Pool, req.Source.Name, !req.Source.VolumeOnly, req.Source.RefreshExcludeOlder, op)
 		if err != nil {
 			return err
 		}
@@ -940,10 +940,11 @@ func doVolumeMigration(s *state.State, r *http.Request, requestProjectName strin
 			NetDialContext:   localtls.RFC3493Dialer,
 			HandshakeTimeout: time.Second * 5,
 		},
-		Secrets:    req.Source.Websockets,
-		Push:       push,
-		VolumeOnly: req.Source.VolumeOnly,
-		Refresh:    req.Source.Refresh,
+		Secrets:             req.Source.Websockets,
+		Push:                push,
+		VolumeOnly:          req.Source.VolumeOnly,
+		Refresh:             req.Source.Refresh,
+		RefreshExcludeOlder: req.Source.RefreshExcludeOlder,
 	}
 
 	sink, err := newStorageMigrationSink(&migrationArgs)

doc/api-extensions.md

@@ -2642,3 +2642,7 @@ As part of this, the following configuration options have been added:
 * `cluster.rebalance.cooldown`
 * `cluster.rebalance.interval`
 * `cluster.rebalance.threshold`
+
+## `custom_volume_refresh_exclude_older_snapshots`
+
+This adds support for excluding source snapshots earlier than latest target snapshot.

doc/rest-api.yaml

@@ -2111,6 +2111,11 @@ definitions:
                 example: false
                 type: boolean
                 x-go-name: Refresh
+            refresh_exclude_older:
+                description: Whether to exclude source snapshots earlier than latest target snapshot
+                example: false
+                type: boolean
+                x-go-name: RefreshExcludeOlder
             secret:
                 description: Remote server secret (for remote private images)
                 example: RANDOM-STRING
@@ -6504,6 +6509,11 @@ definitions:
                 example: false
                 type: boolean
                 x-go-name: Refresh
+            refresh_exclude_older:
+                description: Whether to exclude source snapshots earlier than latest target snapshot
+                example: false
+                type: boolean
+                x-go-name: RefreshExcludeOlder
             secrets:
                 additionalProperties:
                     type: string

internal/server/instance/drivers/driver_lxc.go

@@ -6206,7 +6206,7 @@ func (d *lxc) MigrateReceive(args instance.MigrateReceiveArgs) error {
 		}
 
 		// Compare the two sets.
-		syncSourceSnapshotIndexes, deleteTargetSnapshotIndexes := storagePools.CompareSnapshots(sourceSnapshotComparable, targetSnapshotsComparable)
+		syncSourceSnapshotIndexes, deleteTargetSnapshotIndexes := storagePools.CompareSnapshots(sourceSnapshotComparable, targetSnapshotsComparable, args.RefreshExcludeOlder)
 
 		// Delete the extra local snapshots first.
 		for _, deleteTargetSnapshotIndex := range deleteTargetSnapshotIndexes {

internal/server/instance/drivers/driver_qemu.go

@@ -7315,7 +7315,7 @@ func (d *qemu) MigrateReceive(args instance.MigrateReceiveArgs) error {
 		}
 
 		// Compare the two sets.
-		syncSourceSnapshotIndexes, deleteTargetSnapshotIndexes := storagePools.CompareSnapshots(sourceSnapshotComparable, targetSnapshotsComparable)
+		syncSourceSnapshotIndexes, deleteTargetSnapshotIndexes := storagePools.CompareSnapshots(sourceSnapshotComparable, targetSnapshotsComparable, args.RefreshExcludeOlder)
 
 		// Delete the extra local snapshots first.
 		for _, deleteTargetSnapshotIndex := range deleteTargetSnapshotIndexes {

internal/server/instance/instance_interface.go

@@ -238,6 +238,7 @@ type MigrateSendArgs struct {
 type MigrateReceiveArgs struct {
 	MigrateArgs
 
-	InstanceOperation *operationlock.InstanceOperation
-	Refresh           bool
+	InstanceOperation   *operationlock.InstanceOperation
+	Refresh             bool
+	RefreshExcludeOlder bool
 }

internal/server/migration/migration_volumes.go

@@ -73,6 +73,7 @@ type VolumeTargetArgs struct {
 	MigrationType         Type
 	TrackProgress         bool
 	Refresh               bool
+	RefreshExcludeOlder   bool
 	Live                  bool
 	VolumeSize            int64
 	ContentType           string