diff --git a/docs/developer-guide.md b/docs/developer-guide.md index 9551519..2070ad1 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -46,16 +46,6 @@ Take the following steps to get started in configuring and verifying the infrast Make a note of the name of the storage account in the script output - it's generated with a random suffix, and you'll need it in the following steps to initialise the terraform. -1. Prepare Terraform Variables (Optional) - - If you want to override the Terraform variables, make a copy of `tfvars.template` and amend any default settings as required. - - In the next step add the following flag to the `terraform apply` command in order to use your variables: - - ```pwsh - -var-file=".tfvars - ``` - 1. Initialise Terraform Change the working directory to `./infrastructure`. @@ -66,6 +56,22 @@ Take the following steps to get started in configuring and verifying the infrast terraform init -backend=true -backend-config="resource_group_name=rg-nhsbackup" -backend-config="storage_account_name=" -backend-config="container_name=tfstate" -backend-config="key=terraform.tfstate" ```` +1. Prepare Terraform Variables + + You need to specify the mandatory terraform variables as a minimum, and may want to specify a number of the optional variables. + + You can specify the variables via the command line when executing `terraform apply`, or by preparing a tfvars file and specifying the path to that file. + + Here are examples of each approach: + + ```pwsh + terraform apply -var resource_group_name= -var backup_vault_name= var tags={"tagOne" = "tagOneValue"} -var blob_storage_backups={"backup1" = { "backup_name" = "myblob", "retention_period" = "P7D", "backup_intervals" = ["R/2024-01-01T00:00:00+00:00/P1D"], "storage_account_id" = "id" }} + ``` + + ```pwsh + terraform apply -var-file=".tfvars + ``` + 1. Apply Terraform Apply the Terraform code to create the infrastructure. diff --git a/docs/usage.md b/docs/usage.md index 187f97e..75ff9c8 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -32,14 +32,18 @@ module "my_backup" { } blob_storage_backups = { backup1 = { - backup_name = "storage1" - retention_period = "P7D" - storage_account_id = azurerm_storage_account.my_storage_account_1.id + backup_name = "storage1" + retention_period = "P7D" + backup_intervals = ["R/2024-01-01T00:00:00+00:00/P1D"] + storage_account_id = azurerm_storage_account.my_storage_account_1.id + storage_account_containers = ["container1", "container2"] } backup2 = { - backup_name = "storage2" - retention_period = "P30D" - storage_account_id = azurerm_storage_account.my_storage_account_2.id + backup_name = "storage2" + retention_period = "P30D" + backup_intervals = ["R/2024-01-01T00:00:00+00:00/P2D"] + storage_account_id = azurerm_storage_account.my_storage_account_2.id + storage_account_containers = ["container1", "container2"] } } managed_disk_backups = { @@ -108,8 +112,10 @@ To deploy the module an Azure identity (typically an app registration with clien | `tags` | A map of tags which will be applied to the resource group and backup vault. When no tags are specified then no tags are added. | No | n/a | | `blob_storage_backups` | A map of blob storage backups that should be created. For each backup the following values should be provided: `storage_account_id`, `backup_name` and `retention_period`. When no value is provided then no backups are created. | No | n/a | | `blob_storage_backups.storage_account_id` | The id of the storage account that should be backed up. | Yes | n/a | +| `blob_storage_backups.storage_account_containers` | A list of containers in the storage account that should be backed up. | Yes | n/a | | `blob_storage_backups.backup_name` | The name of the backup, which must be unique across blob storage backups. | Yes | n/a | | `blob_storage_backups.retention_period` | How long the backed up data will be retained for, which should be in `ISO 8601` duration format. [See the following link for the possible values](https://en.wikipedia.org/wiki/ISO_8601#Durations). | Yes | n/a | +| `blob_storage_backups.backup_intervals` | A list of intervals at which backups should be taken, which should be in `ISO 8601` duration format. [See the following link for the possible values](https://en.wikipedia.org/wiki/ISO_8601#Time_intervals). | Yes | n/a | | `managed_disk_backups` | A map of managed disk backups that should be created. For each backup the following values should be provided: `managed_disk_id`, `backup_name` and `retention_period`. When no value is provided then no backups are created. | No | n/a | | `managed_disk_backups.managed_disk_id` | The id of the managed disk that should be backed up. | Yes | n/a | | `managed_disk_backups.backup_name` | The name of the backup, which must be unique across managed disk backups. | Yes | n/a | diff --git a/infrastructure/backup_modules.tf b/infrastructure/backup_modules.tf index 9c9f102..851aeee 100644 --- a/infrastructure/backup_modules.tf +++ b/infrastructure/backup_modules.tf @@ -1,10 +1,12 @@ module "blob_storage_backup" { - for_each = var.blob_storage_backups - source = "./modules/backup/blob_storage" - vault = azurerm_data_protection_backup_vault.backup_vault - backup_name = each.value.backup_name - retention_period = each.value.retention_period - storage_account_id = each.value.storage_account_id + for_each = var.blob_storage_backups + source = "./modules/backup/blob_storage" + vault = azurerm_data_protection_backup_vault.backup_vault + backup_name = each.value.backup_name + retention_period = each.value.retention_period + backup_intervals = each.value.backup_intervals + storage_account_id = each.value.storage_account_id + storage_account_containers = each.value.storage_account_containers } module "managed_disk_backup" { diff --git a/infrastructure/modules/backup/blob_storage/backup_instance.tf b/infrastructure/modules/backup/blob_storage/backup_instance.tf index c9605fc..c886b7c 100644 --- a/infrastructure/modules/backup/blob_storage/backup_instance.tf +++ b/infrastructure/modules/backup/blob_storage/backup_instance.tf @@ -5,11 +5,12 @@ resource "azurerm_role_assignment" "role_assignment" { } resource "azurerm_data_protection_backup_instance_blob_storage" "backup_instance" { - name = "bkinst-blob-${var.backup_name}" - vault_id = var.vault.id - location = var.vault.location - storage_account_id = var.storage_account_id - backup_policy_id = azurerm_data_protection_backup_policy_blob_storage.backup_policy.id + name = "bkinst-blob-${var.backup_name}" + vault_id = var.vault.id + location = var.vault.location + storage_account_id = var.storage_account_id + backup_policy_id = azurerm_data_protection_backup_policy_blob_storage.backup_policy.id + storage_account_container_names = var.storage_account_containers depends_on = [ azurerm_role_assignment.role_assignment diff --git a/infrastructure/modules/backup/blob_storage/backup_policy.tf b/infrastructure/modules/backup/blob_storage/backup_policy.tf index 12bc14a..815f5eb 100644 --- a/infrastructure/modules/backup/blob_storage/backup_policy.tf +++ b/infrastructure/modules/backup/blob_storage/backup_policy.tf @@ -1,5 +1,6 @@ resource "azurerm_data_protection_backup_policy_blob_storage" "backup_policy" { - name = "bkpol-blob-${var.backup_name}" - vault_id = var.vault.id - operational_default_retention_duration = var.retention_period + name = "bkpol-blob-${var.backup_name}" + vault_id = var.vault.id + vault_default_retention_duration = var.retention_period + backup_repeating_time_intervals = var.backup_intervals } diff --git a/infrastructure/modules/backup/blob_storage/variables.tf b/infrastructure/modules/backup/blob_storage/variables.tf index 053daf4..6a22a36 100644 --- a/infrastructure/modules/backup/blob_storage/variables.tf +++ b/infrastructure/modules/backup/blob_storage/variables.tf @@ -10,6 +10,14 @@ variable "retention_period" { type = string } +variable "backup_intervals" { + type = list(string) +} + variable "storage_account_id" { type = string } + +variable "storage_account_containers" { + type = list(string) +} diff --git a/infrastructure/variables.tf b/infrastructure/variables.tf index e0b1581..62bcf51 100644 --- a/infrastructure/variables.tf +++ b/infrastructure/variables.tf @@ -41,11 +41,24 @@ variable "tags" { variable "blob_storage_backups" { description = "A map of blob storage backups to create" type = map(object({ - backup_name = string - retention_period = string - storage_account_id = string + backup_name = string + retention_period = string + backup_intervals = list(string) + storage_account_id = string + storage_account_containers = list(string) })) + default = {} + + validation { + condition = length(var.blob_storage_backups) == 0 || alltrue([for k, v in var.blob_storage_backups : length(v.backup_intervals) > 0]) + error_message = "At least one backup interval must be provided." + } + + validation { + condition = length(var.blob_storage_backups) == 0 || alltrue([for k, v in var.blob_storage_backups : length(v.storage_account_containers) > 0]) + error_message = "At least one storage account container must be provided." + } } variable "managed_disk_backups" { @@ -60,7 +73,13 @@ variable "managed_disk_backups" { name = string }) })) + default = {} + + validation { + condition = length(var.managed_disk_backups) == 0 || alltrue([for k, v in var.managed_disk_backups : length(v.backup_intervals) > 0]) + error_message = "At least one backup interval must be provided." + } } variable "postgresql_flexible_server_backups" { @@ -72,5 +91,11 @@ variable "postgresql_flexible_server_backups" { server_id = string server_resource_group_id = string })) + default = {} + + validation { + condition = length(var.postgresql_flexible_server_backups) == 0 || alltrue([for k, v in var.postgresql_flexible_server_backups : length(v.backup_intervals) > 0]) + error_message = "At least one backup interval must be provided." + } } diff --git a/tests/end-to-end-tests/blob_storage_backup_test.go b/tests/end-to-end-tests/blob_storage_backup_test.go index 21920a3..c2879e3 100644 --- a/tests/end-to-end-tests/blob_storage_backup_test.go +++ b/tests/end-to-end-tests/blob_storage_backup_test.go @@ -16,9 +16,11 @@ import ( ) type TestBlobStorageBackupExternalResources struct { - ResourceGroup armresources.ResourceGroup - StorageAccountOne armstorage.Account - StorageAccountTwo armstorage.Account + ResourceGroup armresources.ResourceGroup + StorageAccountOne armstorage.Account + StorageAccountOneContainer armstorage.BlobContainer + StorageAccountTwo armstorage.Account + StorageAccountTwoContainer armstorage.BlobContainer } /* @@ -31,14 +33,18 @@ func setupExternalResourcesForBlobStorageBackupTest(t *testing.T, credential *az storageAccountOneName := fmt.Sprintf("sa%sexternal1", strings.ToLower(uniqueId)) storageAccountOne := CreateStorageAccount(t, credential, subscriptionID, externalResourceGroupName, storageAccountOneName, resourceGroupLocation) + storageAccountOneContainer := CreateStorageAccountContainer(t, credential, subscriptionID, externalResourceGroupName, storageAccountOneName, "test-container") storageAccountTwoName := fmt.Sprintf("sa%sexternal2", strings.ToLower(uniqueId)) storageAccountTwo := CreateStorageAccount(t, credential, subscriptionID, externalResourceGroupName, storageAccountTwoName, resourceGroupLocation) + storageAccountTwoContainer := CreateStorageAccountContainer(t, credential, subscriptionID, externalResourceGroupName, storageAccountTwoName, "test-container") externalResources := &TestBlobStorageBackupExternalResources{ - ResourceGroup: resourceGroup, - StorageAccountOne: storageAccountOne, - StorageAccountTwo: storageAccountTwo, + ResourceGroup: resourceGroup, + StorageAccountOne: storageAccountOne, + StorageAccountOneContainer: storageAccountOneContainer, + StorageAccountTwo: storageAccountTwo, + StorageAccountTwoContainer: storageAccountTwoContainer, } return externalResources @@ -64,14 +70,18 @@ func TestBlobStorageBackup(t *testing.T) { // policies have been created correctly blobStorageBackups := map[string]map[string]interface{}{ "backup1": { - "backup_name": "blob1", - "retention_period": "P7D", - "storage_account_id": *externalResources.StorageAccountOne.ID, + "backup_name": "blob1", + "retention_period": "P7D", + "backup_intervals": []string{"R/2024-01-01T00:00:00+00:00/P1D"}, + "storage_account_id": *externalResources.StorageAccountOne.ID, + "storage_account_containers": []string{*externalResources.StorageAccountOneContainer.Name}, }, "backup2": { - "backup_name": "blob2", - "retention_period": "P30D", - "storage_account_id": *externalResources.StorageAccountTwo.ID, + "backup_name": "blob2", + "retention_period": "P30D", + "backup_intervals": []string{"R/2024-01-01T00:00:00+00:00/P2D"}, + "storage_account_id": *externalResources.StorageAccountTwo.ID, + "storage_account_containers": []string{*externalResources.StorageAccountTwoContainer.Name}, }, } @@ -128,6 +138,7 @@ func TestBlobStorageBackup(t *testing.T) { for _, backup := range blobStorageBackups { backupName := backup["backup_name"].(string) retentionPeriod := backup["retention_period"].(string) + backupIntervals := backup["backup_intervals"].([]string) storageAccountId := backup["storage_account_id"].(string) // Validate backup policy @@ -141,6 +152,13 @@ func TestBlobStorageBackup(t *testing.T) { deleteOption := retentionRule.Lifecycles[0].DeleteAfter.(*armdataprotection.AbsoluteDeleteOption) assert.Equal(t, retentionPeriod, *deleteOption.Duration, "Expected the backup policy retention period to be %s", retentionPeriod) + // Validate backup intervals + backupRule := GetBackupPolicyRuleForName(backupPolicyProperties.PolicyRules, "BackupIntervals").(*armdataprotection.AzureBackupRule) + schedule := backupRule.Trigger.(*armdataprotection.ScheduleBasedTriggerContext).Schedule + for index, interval := range schedule.RepeatingTimeIntervals { + assert.Equal(t, backupIntervals[index], *interval, "Expected backup policy repeating interval %s to be %s", index, backupIntervals[index]) + } + // Validate backup instance backupInstanceName := fmt.Sprintf("bkinst-blob-%s", backupName) backupInstance := GetBackupInstanceForName(backupInstances, backupInstanceName) diff --git a/tests/end-to-end-tests/helpers.go b/tests/end-to-end-tests/helpers.go index cc725fd..f579ce7 100644 --- a/tests/end-to-end-tests/helpers.go +++ b/tests/end-to-end-tests/helpers.go @@ -380,6 +380,29 @@ func CreateStorageAccount(t *testing.T, credential *azidentity.ClientSecretCrede return resp.Account } +/* + * Creates a storage account container that can be used for testing purposes. + */ +func CreateStorageAccountContainer(t *testing.T, credential *azidentity.ClientSecretCredential, subscriptionID string, + resourceGroupName string, storageAccountName string, containerName string) armstorage.BlobContainer { + containerClient, err := armstorage.NewBlobContainersClient(subscriptionID, credential, nil) + assert.NoError(t, err, "Failed to create container client: %v", err) + + resp, err := containerClient.Create( + context.Background(), + resourceGroupName, + storageAccountName, + containerName, + armstorage.BlobContainer{}, + nil, + ) + assert.NoError(t, err, "Failed to create container: %v", err) + + log.Printf("Container '%s' created successfully in storage account %s", containerName, storageAccountName) + + return resp.BlobContainer +} + /* * Creates a managed disk that can be used for testing purposes. */ diff --git a/tests/integration-tests/backup_modules_blob_storage.tftest.hcl b/tests/integration-tests/backup_modules_blob_storage.tftest.hcl index 1bd6091..add8937 100644 --- a/tests/integration-tests/backup_modules_blob_storage.tftest.hcl +++ b/tests/integration-tests/backup_modules_blob_storage.tftest.hcl @@ -26,14 +26,18 @@ run "create_blob_storage_backup" { tags = run.setup_tests.tags blob_storage_backups = { backup1 = { - backup_name = "storage1" - retention_period = "P7D" - storage_account_id = "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.Storage/storageAccounts/sastorage1" + backup_name = "storage1" + retention_period = "P7D" + backup_intervals = ["R/2024-01-01T00:00:00+00:00/P1D"] + storage_account_id = "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.Storage/storageAccounts/sastorage1" + storage_account_containers = ["container1"] } backup2 = { - backup_name = "storage2" - retention_period = "P30D" - storage_account_id = "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.Storage/storageAccounts/sastorage2" + backup_name = "storage2" + retention_period = "P30D" + backup_intervals = ["R/2024-01-01T00:00:00+00:00/P2D"] + storage_account_id = "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.Storage/storageAccounts/sastorage2" + storage_account_containers = ["container2"] } } } @@ -59,10 +63,15 @@ run "create_blob_storage_backup" { } assert { - condition = module.blob_storage_backup["backup1"].backup_policy.operational_default_retention_duration == "P7D" + condition = module.blob_storage_backup["backup1"].backup_policy.vault_default_retention_duration == "P7D" error_message = "Blob storage backup policy retention period not as expected." } + assert { + condition = module.blob_storage_backup["backup1"].backup_policy.backup_repeating_time_intervals[0] == "R/2024-01-01T00:00:00+00:00/P1D" + error_message = "Blob storage backup policy backup intervals not as expected." + } + assert { condition = length(module.blob_storage_backup["backup1"].backup_instance.id) > 0 error_message = "Blob storage backup instance id not as expected." @@ -88,6 +97,11 @@ run "create_blob_storage_backup" { error_message = "Blob storage backup instance storage account id not as expected." } + assert { + condition = module.blob_storage_backup["backup1"].backup_instance.storage_account_container_names[0] == "container1" + error_message = "Blob storage backup instance storage account containers not as expected." + } + assert { condition = module.blob_storage_backup["backup1"].backup_instance.backup_policy_id == module.blob_storage_backup["backup1"].backup_policy.id error_message = "Blob storage backup instance backup policy id not as expected." @@ -109,10 +123,15 @@ run "create_blob_storage_backup" { } assert { - condition = module.blob_storage_backup["backup2"].backup_policy.operational_default_retention_duration == "P30D" + condition = module.blob_storage_backup["backup2"].backup_policy.vault_default_retention_duration == "P30D" error_message = "Blob storage backup policy retention period not as expected." } + assert { + condition = module.blob_storage_backup["backup2"].backup_policy.backup_repeating_time_intervals[0] == "R/2024-01-01T00:00:00+00:00/P2D" + error_message = "Blob storage backup policy backup intervals not as expected." + } + assert { condition = length(module.blob_storage_backup["backup2"].backup_instance.id) > 0 error_message = "Blob storage backup instance id not as expected." @@ -138,8 +157,69 @@ run "create_blob_storage_backup" { error_message = "Blob storage backup instance storage account id not as expected." } + assert { + condition = module.blob_storage_backup["backup2"].backup_instance.storage_account_container_names[0] == "container2" + error_message = "Blob storage backup instance storage account containers not as expected." + } + assert { condition = module.blob_storage_backup["backup2"].backup_instance.backup_policy_id == module.blob_storage_backup["backup2"].backup_policy.id error_message = "Blob storage backup instance backup policy id not as expected." } -} \ No newline at end of file +} + +run "validate_backup_intervals" { + command = plan + + module { + source = "../../infrastructure" + } + + variables { + resource_group_name = run.setup_tests.resource_group_name + resource_group_location = "uksouth" + backup_vault_name = run.setup_tests.backup_vault_name + tags = run.setup_tests.tags + blob_storage_backups = { + backup1 = { + backup_name = "storage1" + retention_period = "P30D" + backup_intervals = [] + storage_account_id = "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.Storage/storageAccounts/sastorage1" + storage_account_containers = ["container1"] + } + } + } + + expect_failures = [ + var.blob_storage_backups, + ] +} + +run "validate_storage_account_containers" { + command = plan + + module { + source = "../../infrastructure" + } + + variables { + resource_group_name = run.setup_tests.resource_group_name + resource_group_location = "uksouth" + backup_vault_name = run.setup_tests.backup_vault_name + tags = run.setup_tests.tags + blob_storage_backups = { + backup1 = { + backup_name = "storage1" + retention_period = "P30D" + backup_intervals = ["R/2024-01-01T00:00:00+00:00/P1D"] + storage_account_id = "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.Storage/storageAccounts/sastorage1" + storage_account_containers = [] + } + } + } + + expect_failures = [ + var.blob_storage_backups, + ] +} diff --git a/tests/integration-tests/backup_modules_managed_disk.tftest.hcl b/tests/integration-tests/backup_modules_managed_disk.tftest.hcl index e87255f..79f5ce1 100644 --- a/tests/integration-tests/backup_modules_managed_disk.tftest.hcl +++ b/tests/integration-tests/backup_modules_managed_disk.tftest.hcl @@ -172,4 +172,35 @@ run "create_managed_disk_backup" { condition = module.managed_disk_backup["backup2"].backup_instance.backup_policy_id == module.managed_disk_backup["backup2"].backup_policy.id error_message = "Managed disk backup instance backup policy id not as expected." } +} + +run "validate_backup_intervals" { + command = plan + + module { + source = "../../infrastructure" + } + + variables { + resource_group_name = run.setup_tests.resource_group_name + resource_group_location = "uksouth" + backup_vault_name = run.setup_tests.backup_vault_name + tags = run.setup_tests.tags + managed_disk_backups = { + backup1 = { + backup_name = "disk1" + retention_period = "P7D" + backup_intervals = [] + managed_disk_id = "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.Compute/disks/disk-1" + managed_disk_resource_group = { + id = "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group1" + name = "example-resource-group1" + } + } + } + } + + expect_failures = [ + var.managed_disk_backups, + ] } \ No newline at end of file diff --git a/tests/integration-tests/backup_modules_postgresql_flexible_server.tftest.hcl b/tests/integration-tests/backup_modules_postgresql_flexible_server.tftest.hcl index f985ca4..b192253 100644 --- a/tests/integration-tests/backup_modules_postgresql_flexible_server.tftest.hcl +++ b/tests/integration-tests/backup_modules_postgresql_flexible_server.tftest.hcl @@ -156,4 +156,32 @@ run "create_postgresql_flexible_server_backup" { condition = module.postgresql_flexible_server_backup["backup2"].backup_instance.backup_policy_id == module.postgresql_flexible_server_backup["backup2"].backup_policy.id error_message = "Postgresql flexible server backup instance backup policy id not as expected." } -} \ No newline at end of file +} + +run "validate_backup_intervals" { + command = plan + + module { + source = "../../infrastructure" + } + + variables { + resource_group_name = run.setup_tests.resource_group_name + resource_group_location = "uksouth" + backup_vault_name = run.setup_tests.backup_vault_name + tags = run.setup_tests.tags + postgresql_flexible_server_backups = { + backup1 = { + backup_name = "server1" + retention_period = "P7D" + backup_intervals = [] + server_id = "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.DBforPostgreSQL/flexibleServers/server-1" + server_resource_group_id = "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group1" + } + } + } + + expect_failures = [ + var.postgresql_flexible_server_backups, + ] +}