Storage: Add support for storage bucket backup (from Incus)

.github/workflows/tests.yml

@@ -10,7 +10,7 @@ on:
     - cron: '0 0 * * *'  # Test TICS daily
 
 env:
-  LXD_REQUIRED_TESTS: "storage_buckets,network_ovn"
+  LXD_REQUIRED_TESTS: "storage_buckets,storage_bucket_export,network_ovn"
   LXD_SKIP_TESTS: "clustering_upgrade clustering_upgrade_large"
   GOCOVERDIR: ${{ ( github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' ) && '/home/runner/work/lxd/lxd/coverage' || '' }}
 
@@ -131,7 +131,7 @@ jobs:
           strip -s /tmp/bin/*
 
           # bin/max (sizes are in MiB)
-          SIZES="lxc 15
+          SIZES="lxc 16
                  lxd-agent 13"
           MIB="$((1024 * 1024))"
 

client/interfaces.go

@@ -364,6 +364,12 @@ type InstanceServer interface {
 	UpdateStoragePoolBucketKey(poolName string, bucketName string, keyName string, key api.StorageBucketKeyPut, ETag string) (err error)
 	DeleteStoragePoolBucketKey(poolName string, bucketName string, keyName string) (err error)
 
+	// Storage bucket backup functions ("storage_bucket_backup" API extension)
+	CreateStoragePoolBucketBackup(poolName string, bucketName string, backup api.StorageBucketBackupsPost) (op Operation, err error)
+	DeleteStoragePoolBucketBackup(pool string, bucketName string, name string) (op Operation, err error)
+	GetStoragePoolBucketBackupFile(pool string, bucketName string, name string, req *BackupFileRequest) (resp *BackupFileResponse, err error)
+	CreateStoragePoolBucketFromBackup(pool string, args StoragePoolBucketBackupArgs) (op Operation, err error)
+
 	// List all volumes functions ("storage_volumes_all" API extension)
 	GetVolumesWithFilter(filters []string) (volumes []api.StorageVolume, err error)
 	GetVolumesWithFilterAllProjects(filters []string) (volumes []api.StorageVolume, err error)
@@ -754,3 +760,13 @@ type GetPermissionsArgs struct {
 	// level permissions will not be returned.
 	ProjectName string
 }
+
+// The StoragePoolBucketBackupArgs struct is used when creating a storage volume from a backup.
+// API extension: storage_bucket_backup.
+type StoragePoolBucketBackupArgs struct {
+	// The backup file
+	BackupFile io.Reader
+
+	// Name to import backup as
+	Name string
+}

client/lxd_storage_buckets.go

@@ -1,7 +1,16 @@
 package lxd
 
 import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+
 	"github.com/canonical/lxd/shared/api"
+	"github.com/canonical/lxd/shared/cancel"
+	"github.com/canonical/lxd/shared/ioprogress"
+	"github.com/canonical/lxd/shared/units"
 )
 
 // GetStoragePoolBucketNames returns a list of storage bucket names.
@@ -233,3 +242,151 @@ func (r *ProtocolLXD) DeleteStoragePoolBucketKey(poolName string, bucketName str
 
 	return nil
 }
+
+// CreateStoragePoolBucketBackup creates a new storage bucket backup.
+func (r *ProtocolLXD) CreateStoragePoolBucketBackup(poolName string, bucketName string, backup api.StorageBucketBackupsPost) (Operation, error) {
+	err := r.CheckExtension("storage_bucket_backup")
+	if err != nil {
+		return nil, err
+	}
+
+	op, _, err := r.queryOperation("POST", "/storage-pools/"+url.PathEscape(poolName)+"/buckets/"+url.PathEscape(bucketName)+"/backups", backup, "", true)
+	if err != nil {
+		return nil, err
+	}
+
+	return op, nil
+}
+
+// DeleteStoragePoolBucketBackup deletes an existing storage bucket backup.
+func (r *ProtocolLXD) DeleteStoragePoolBucketBackup(pool string, bucketName string, name string) (Operation, error) {
+	err := r.CheckExtension("storage_bucket_backup")
+	if err != nil {
+		return nil, err
+	}
+
+	op, _, err := r.queryOperation("DELETE", "/storage-pools/"+url.PathEscape(pool)+"/buckets/"+url.PathEscape(bucketName)+"/backups/"+url.PathEscape(name), nil, "", true)
+	if err != nil {
+		return nil, err
+	}
+
+	return op, nil
+}
+
+// GetStoragePoolBucketBackupFile returns the storage bucket file.
+func (r *ProtocolLXD) GetStoragePoolBucketBackupFile(pool string, bucketName string, name string, req *BackupFileRequest) (*BackupFileResponse, error) {
+	err := r.CheckExtension("storage_bucket_backup")
+	if err != nil {
+		return nil, err
+	}
+
+	// Build the URL
+	uri := r.httpBaseURL.String() + "/1.0/storage-pools/" + url.PathEscape(pool) + "/buckets/" + url.PathEscape(bucketName) + "/backups/" + url.PathEscape(name) + "/export"
+
+	if r.project != "" {
+		uri += "?project=" + url.QueryEscape(r.project)
+	}
+
+	// Prepare the download request
+	request, err := http.NewRequest("GET", uri, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if r.httpUserAgent != "" {
+		request.Header.Set("User-Agent", r.httpUserAgent)
+	}
+
+	// Start the request
+	response, doneCh, err := cancel.CancelableDownload(req.Canceler, r.DoHTTP, request)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() { _ = response.Body.Close() }()
+	defer close(doneCh)
+
+	if response.StatusCode != http.StatusOK {
+		_, _, err := lxdParseResponse(response)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Handle the data
+	body := response.Body
+	if req.ProgressHandler != nil {
+		body = &ioprogress.ProgressReader{
+			ReadCloser: response.Body,
+			Tracker: &ioprogress.ProgressTracker{
+				Length: response.ContentLength,
+				Handler: func(percent int64, speed int64) {
+					req.ProgressHandler(ioprogress.ProgressData{Text: strconv.FormatInt(percent, 10) + "% (" + units.GetByteSizeString(speed, 2) + "/s)"})
+				},
+			},
+		}
+	}
+
+	size, err := io.Copy(req.BackupFile, body)
+	if err != nil {
+		return nil, err
+	}
+
+	resp := BackupFileResponse{}
+	resp.Size = size
+
+	return &resp, nil
+}
+
+// CreateStoragePoolBucketFromBackup creates a storage pool bucket using a backup.
+func (r *ProtocolLXD) CreateStoragePoolBucketFromBackup(pool string, args StoragePoolBucketBackupArgs) (Operation, error) {
+	if !r.HasExtension("storage_bucket_backup") {
+		return nil, fmt.Errorf(`The server is missing the required "custom_volume_backup" API extension`)
+	}
+
+	path := "/storage-pools/" + url.PathEscape(pool) + "/buckets"
+
+	// Prepare the HTTP request.
+	reqURL, err := r.setQueryAttributes(r.httpBaseURL.String() + "/1.0" + path)
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequest("POST", reqURL, args.BackupFile)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/octet-stream")
+
+	if args.Name != "" {
+		req.Header.Set("X-LXD-name", args.Name)
+	}
+
+	// Send the request.
+	resp, err := r.DoHTTP(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() { _ = resp.Body.Close() }()
+
+	// Handle errors.
+	response, _, err := lxdParseResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	respOperation, err := response.MetadataAsOperation()
+	if err != nil {
+		return nil, err
+	}
+
+	op := operation{
+		Operation: *respOperation,
+		r:         r,
+		chActive:  make(chan bool),
+	}
+
+	return &op, nil
+}

doc/api-extensions.md

@@ -2599,3 +2599,18 @@ The following pool level configuration keys have been added:
 1. {config:option}`storage-pure-pool-conf:pure.api.token`
 1. {config:option}`storage-pure-pool-conf:pure.mode`
 1. {config:option}`storage-pure-pool-conf:pure.target`
+
+## `storage_bucket_backup`
+
+Add storage bucket backup support.
+
+This includes the following new endpoints (see [RESTful API](rest-api.md) for details):
+
+* `GET /1.0/storage-pools/<pool>/buckets/<bucket>/backups`
+* `POST /1.0/storage-pools/<pool>/buckets/<bucket>/backups`
+
+* `GET /1.0/storage-pools/<pool>/buckets/<bucket>/backups/<name>`
+* `POST /1.0/storage-pools/<pool>/buckets/<bucket>/backups/<name>`
+* `DELETE /1.0/storage-pools/<pool>/buckets/<bucket>/backups/<name>`
+
+* `GET /1.0/storage-pools/<pool>/buckets/<bucket>/backups/<name>/export`

doc/metadata.txt

@@ -6817,6 +6817,9 @@ using the `zfs` command in the container.
 `can_view`
 : Grants permission to view the storage bucket.
 
+`can_manage_backups`
+: Grants permission to create and delete backups of the storage bucket.
+
 
 <!-- entity group storage_bucket end -->
 <!-- entity group storage_pool start -->