Merge branch 'main' into immutable-backup-151024

docs/developer-guide.md

@@ -46,16 +46,6 @@ Take the following steps to get started in configuring and verifying the infrast
 
     Make a note of the name of the storage account in the script output - it's generated with a random suffix, and you'll need it in the following steps to initialise the terraform.
 
-1. Prepare Terraform Variables (Optional)
-
-    If you want to override the Terraform variables, make a copy of `tfvars.template` and amend any default settings as required.
-
-    In the next step add the following flag to the `terraform apply` command in order to use your variables:
-
-    ```pwsh
-    -var-file="<your-var-file>.tfvars
-    ```
-
 1. Initialise Terraform
 
     Change the working directory to `./infrastructure`.
@@ -66,6 +56,22 @@ Take the following steps to get started in configuring and verifying the infrast
     terraform init -backend=true -backend-config="resource_group_name=rg-nhsbackup" -backend-config="storage_account_name=<storage-account-name>" -backend-config="container_name=tfstate" -backend-config="key=terraform.tfstate"
     ````
 
+1. Prepare Terraform Variables
+
+    You need to specify the mandatory terraform variables as a minimum, and may want to specify a number of the optional variables.
+
+    You can specify the variables via the command line when executing `terraform apply`, or by preparing a tfvars file and specifying the path to that file.
+
+    Here are examples of each approach:
+
+    ```pwsh
+    terraform apply -var resource_group_name=<resource-group-name> -var backup_vault_name=<backup-vault-name> var tags={"tagOne" = "tagOneValue"} -var blob_storage_backups={"backup1" = { "backup_name" = "myblob", "retention_period" = "P7D", "backup_intervals" = ["R/2024-01-01T00:00:00+00:00/P1D"], "storage_account_id" = "id" }}
+    ```
+
+    ```pwsh
+    terraform apply -var-file="<your-var-file>.tfvars
+    ```
+
 1. Apply Terraform
 
     Apply the Terraform code to create the infrastructure.

docs/usage.md

@@ -32,14 +32,18 @@ module "my_backup" {
   }
   blob_storage_backups = {
     backup1 = {
-      backup_name        = "storage1"
-      retention_period   = "P7D"
-      storage_account_id = azurerm_storage_account.my_storage_account_1.id
+      backup_name                = "storage1"
+      retention_period           = "P7D"
+      backup_intervals           = ["R/2024-01-01T00:00:00+00:00/P1D"]
+      storage_account_id         = azurerm_storage_account.my_storage_account_1.id
+      storage_account_containers = ["container1", "container2"]
     }
     backup2 = {
-      backup_name        = "storage2"
-      retention_period   = "P30D"
-      storage_account_id = azurerm_storage_account.my_storage_account_2.id
+      backup_name                = "storage2"
+      retention_period           = "P30D"
+      backup_intervals           = ["R/2024-01-01T00:00:00+00:00/P2D"]
+      storage_account_id         = azurerm_storage_account.my_storage_account_2.id
+      storage_account_containers = ["container1", "container2"]
     }
   }
   managed_disk_backups = {
@@ -108,8 +112,10 @@ To deploy the module an Azure identity (typically an app registration with clien
 | `tags` | A map of tags which will be applied to the resource group and backup vault. When no tags are specified then no tags are added. | No | n/a |
 | `blob_storage_backups` | A map of blob storage backups that should be created. For each backup the following values should be provided: `storage_account_id`, `backup_name` and `retention_period`. When no value is provided then no backups are created. | No | n/a |
 | `blob_storage_backups.storage_account_id` | The id of the storage account that should be backed up. | Yes | n/a |
+| `blob_storage_backups.storage_account_containers` | A list of containers in the storage account that should be backed up. | Yes | n/a |
 | `blob_storage_backups.backup_name` | The name of the backup, which must be unique across blob storage backups. | Yes | n/a |
 | `blob_storage_backups.retention_period` | How long the backed up data will be retained for, which should be in `ISO 8601` duration format. [See the following link for the possible values](https://en.wikipedia.org/wiki/ISO_8601#Durations). | Yes | n/a |
+| `blob_storage_backups.backup_intervals` | A list of intervals at which backups should be taken, which should be in `ISO 8601` duration format. [See the following link for the possible values](https://en.wikipedia.org/wiki/ISO_8601#Time_intervals). | Yes | n/a |
 | `managed_disk_backups` | A map of managed disk backups that should be created. For each backup the following values should be provided: `managed_disk_id`, `backup_name` and `retention_period`. When no value is provided then no backups are created. | No | n/a |
 | `managed_disk_backups.managed_disk_id` | The id of the managed disk that should be backed up. | Yes | n/a |
 | `managed_disk_backups.backup_name` | The name of the backup, which must be unique across managed disk backups. | Yes | n/a |

infrastructure/backup_modules.tf

@@ -1,10 +1,12 @@
 module "blob_storage_backup" {
-  for_each           = var.blob_storage_backups
-  source             = "./modules/backup/blob_storage"
-  vault              = azurerm_data_protection_backup_vault.backup_vault
-  backup_name        = each.value.backup_name
-  retention_period   = each.value.retention_period
-  storage_account_id = each.value.storage_account_id
+  for_each                   = var.blob_storage_backups
+  source                     = "./modules/backup/blob_storage"
+  vault                      = azurerm_data_protection_backup_vault.backup_vault
+  backup_name                = each.value.backup_name
+  retention_period           = each.value.retention_period
+  backup_intervals           = each.value.backup_intervals
+  storage_account_id         = each.value.storage_account_id
+  storage_account_containers = each.value.storage_account_containers
 }
 
 module "managed_disk_backup" {

infrastructure/modules/backup/blob_storage/backup_instance.tf

@@ -5,11 +5,12 @@ resource "azurerm_role_assignment" "role_assignment" {
 }
 
 resource "azurerm_data_protection_backup_instance_blob_storage" "backup_instance" {
-  name               = "bkinst-blob-${var.backup_name}"
-  vault_id           = var.vault.id
-  location           = var.vault.location
-  storage_account_id = var.storage_account_id
-  backup_policy_id   = azurerm_data_protection_backup_policy_blob_storage.backup_policy.id
+  name                            = "bkinst-blob-${var.backup_name}"
+  vault_id                        = var.vault.id
+  location                        = var.vault.location
+  storage_account_id              = var.storage_account_id
+  backup_policy_id                = azurerm_data_protection_backup_policy_blob_storage.backup_policy.id
+  storage_account_container_names = var.storage_account_containers
 
   depends_on = [
     azurerm_role_assignment.role_assignment

infrastructure/modules/backup/blob_storage/backup_policy.tf

@@ -1,5 +1,6 @@
 resource "azurerm_data_protection_backup_policy_blob_storage" "backup_policy" {
-  name                                   = "bkpol-blob-${var.backup_name}"
-  vault_id                               = var.vault.id
-  operational_default_retention_duration = var.retention_period
+  name                             = "bkpol-blob-${var.backup_name}"
+  vault_id                         = var.vault.id
+  vault_default_retention_duration = var.retention_period
+  backup_repeating_time_intervals  = var.backup_intervals
 }

infrastructure/modules/backup/blob_storage/variables.tf

@@ -10,6 +10,14 @@ variable "retention_period" {
   type = string
 }
 
+variable "backup_intervals" {
+  type = list(string)
+}
+
 variable "storage_account_id" {
   type = string
 }
+
+variable "storage_account_containers" {
+  type = list(string)
+}

infrastructure/variables.tf

@@ -41,11 +41,24 @@ variable "tags" {
 variable "blob_storage_backups" {
   description = "A map of blob storage backups to create"
   type = map(object({
-    backup_name        = string
-    retention_period   = string
-    storage_account_id = string
+    backup_name                = string
+    retention_period           = string
+    backup_intervals           = list(string)
+    storage_account_id         = string
+    storage_account_containers = list(string)
   }))
+
   default = {}
+
+  validation {
+    condition     = length(var.blob_storage_backups) == 0 || alltrue([for k, v in var.blob_storage_backups : length(v.backup_intervals) > 0])
+    error_message = "At least one backup interval must be provided."
+  }
+
+  validation {
+    condition     = length(var.blob_storage_backups) == 0 || alltrue([for k, v in var.blob_storage_backups : length(v.storage_account_containers) > 0])
+    error_message = "At least one storage account container must be provided."
+  }
 }
 
 variable "managed_disk_backups" {
@@ -60,7 +73,13 @@ variable "managed_disk_backups" {
       name = string
     })
   }))
+
   default = {}
+
+  validation {
+    condition     = length(var.managed_disk_backups) == 0 || alltrue([for k, v in var.managed_disk_backups : length(v.backup_intervals) > 0])
+    error_message = "At least one backup interval must be provided."
+  }
 }
 
 variable "postgresql_flexible_server_backups" {
@@ -72,5 +91,11 @@ variable "postgresql_flexible_server_backups" {
     server_id                = string
     server_resource_group_id = string
   }))
+
   default = {}
+
+  validation {
+    condition     = length(var.postgresql_flexible_server_backups) == 0 || alltrue([for k, v in var.postgresql_flexible_server_backups : length(v.backup_intervals) > 0])
+    error_message = "At least one backup interval must be provided."
+  }
 }

tests/end-to-end-tests/blob_storage_backup_test.go

@@ -16,9 +16,11 @@ import (
 )
 
 type TestBlobStorageBackupExternalResources struct {
-	ResourceGroup     armresources.ResourceGroup
-	StorageAccountOne armstorage.Account
-	StorageAccountTwo armstorage.Account
+	ResourceGroup              armresources.ResourceGroup
+	StorageAccountOne          armstorage.Account
+	StorageAccountOneContainer armstorage.BlobContainer
+	StorageAccountTwo          armstorage.Account
+	StorageAccountTwoContainer armstorage.BlobContainer
 }
 
 /*
@@ -31,14 +33,18 @@ func setupExternalResourcesForBlobStorageBackupTest(t *testing.T, credential *az
 
 	storageAccountOneName := fmt.Sprintf("sa%sexternal1", strings.ToLower(uniqueId))
 	storageAccountOne := CreateStorageAccount(t, credential, subscriptionID, externalResourceGroupName, storageAccountOneName, resourceGroupLocation)
+	storageAccountOneContainer := CreateStorageAccountContainer(t, credential, subscriptionID, externalResourceGroupName, storageAccountOneName, "test-container")
 
 	storageAccountTwoName := fmt.Sprintf("sa%sexternal2", strings.ToLower(uniqueId))
 	storageAccountTwo := CreateStorageAccount(t, credential, subscriptionID, externalResourceGroupName, storageAccountTwoName, resourceGroupLocation)
+	storageAccountTwoContainer := CreateStorageAccountContainer(t, credential, subscriptionID, externalResourceGroupName, storageAccountTwoName, "test-container")
 
 	externalResources := &TestBlobStorageBackupExternalResources{
-		ResourceGroup:     resourceGroup,
-		StorageAccountOne: storageAccountOne,
-		StorageAccountTwo: storageAccountTwo,
+		ResourceGroup:              resourceGroup,
+		StorageAccountOne:          storageAccountOne,
+		StorageAccountOneContainer: storageAccountOneContainer,
+		StorageAccountTwo:          storageAccountTwo,
+		StorageAccountTwoContainer: storageAccountTwoContainer,
 	}
 
 	return externalResources
@@ -64,14 +70,18 @@ func TestBlobStorageBackup(t *testing.T) {
 	// policies have been created correctly
 	blobStorageBackups := map[string]map[string]interface{}{
 		"backup1": {
-			"backup_name":        "blob1",
-			"retention_period":   "P7D",
-			"storage_account_id": *externalResources.StorageAccountOne.ID,
+			"backup_name":                "blob1",
+			"retention_period":           "P7D",
+			"backup_intervals":           []string{"R/2024-01-01T00:00:00+00:00/P1D"},
+			"storage_account_id":         *externalResources.StorageAccountOne.ID,
+			"storage_account_containers": []string{*externalResources.StorageAccountOneContainer.Name},
 		},
 		"backup2": {
-			"backup_name":        "blob2",
-			"retention_period":   "P30D",
-			"storage_account_id": *externalResources.StorageAccountTwo.ID,
+			"backup_name":                "blob2",
+			"retention_period":           "P30D",
+			"backup_intervals":           []string{"R/2024-01-01T00:00:00+00:00/P2D"},
+			"storage_account_id":         *externalResources.StorageAccountTwo.ID,
+			"storage_account_containers": []string{*externalResources.StorageAccountTwoContainer.Name},
 		},
 	}
 
@@ -128,6 +138,7 @@ func TestBlobStorageBackup(t *testing.T) {
 		for _, backup := range blobStorageBackups {
 			backupName := backup["backup_name"].(string)
 			retentionPeriod := backup["retention_period"].(string)
+			backupIntervals := backup["backup_intervals"].([]string)
 			storageAccountId := backup["storage_account_id"].(string)
 
 			// Validate backup policy
@@ -141,6 +152,13 @@ func TestBlobStorageBackup(t *testing.T) {
 			deleteOption := retentionRule.Lifecycles[0].DeleteAfter.(*armdataprotection.AbsoluteDeleteOption)
 			assert.Equal(t, retentionPeriod, *deleteOption.Duration, "Expected the backup policy retention period to be %s", retentionPeriod)
 
+			// Validate backup intervals
+			backupRule := GetBackupPolicyRuleForName(backupPolicyProperties.PolicyRules, "BackupIntervals").(*armdataprotection.AzureBackupRule)
+			schedule := backupRule.Trigger.(*armdataprotection.ScheduleBasedTriggerContext).Schedule
+			for index, interval := range schedule.RepeatingTimeIntervals {
+				assert.Equal(t, backupIntervals[index], *interval, "Expected backup policy repeating interval %s to be %s", index, backupIntervals[index])
+			}
+
 			// Validate backup instance
 			backupInstanceName := fmt.Sprintf("bkinst-blob-%s", backupName)
 			backupInstance := GetBackupInstanceForName(backupInstances, backupInstanceName)

tests/end-to-end-tests/helpers.go

@@ -380,6 +380,29 @@ func CreateStorageAccount(t *testing.T, credential *azidentity.ClientSecretCrede
 	return resp.Account
 }
 
+/*
+ * Creates a storage account container that can be used for testing purposes.
+ */
+func CreateStorageAccountContainer(t *testing.T, credential *azidentity.ClientSecretCredential, subscriptionID string,
+	resourceGroupName string, storageAccountName string, containerName string) armstorage.BlobContainer {
+	containerClient, err := armstorage.NewBlobContainersClient(subscriptionID, credential, nil)
+	assert.NoError(t, err, "Failed to create container client: %v", err)
+
+	resp, err := containerClient.Create(
+		context.Background(),
+		resourceGroupName,
+		storageAccountName,
+		containerName,
+		armstorage.BlobContainer{},
+		nil,
+	)
+	assert.NoError(t, err, "Failed to create container: %v", err)
+
+	log.Printf("Container '%s' created successfully in storage account %s", containerName, storageAccountName)
+
+	return resp.BlobContainer
+}
+
 /*
  * Creates a managed disk that can be used for testing purposes.
  */