diff --git a/README.md b/README.md index 69f503a..9af3248 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ Users of Terragrunt can achieve similar results by using modules provided in the - [Basic example](https://github.com/terraform-aws-modules/terraform-aws-dynamodb-table/tree/master/examples/basic) - [Autoscaling example](https://github.com/terraform-aws-modules/terraform-aws-dynamodb-table/tree/master/examples/autoscaling) - [Global tables example](https://github.com/terraform-aws-modules/terraform-aws-dynamodb-table/tree/master/examples/global-tables) +- [S3 import examples](https://github.com/terraform-aws-modules/terraform-aws-dynamodb-table/tree/master/examples/s3-import) ## Requirements @@ -60,13 +61,13 @@ Users of Terragrunt can achieve similar results by using modules provided in the | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.59 | +| [aws](#requirement\_aws) | >= 5.21 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.59 | +| [aws](#provider\_aws) | >= 5.21 | ## Modules @@ -104,6 +105,7 @@ No modules. | [global\_secondary\_indexes](#input\_global\_secondary\_indexes) | Describe a GSI for the table; subject to the normal limits on the number of GSIs, projected attributes, etc. | `any` | `[]` | no | | [hash\_key](#input\_hash\_key) | The attribute to use as the hash (partition) key. Must also be defined as an attribute | `string` | `null` | no | | [ignore\_changes\_global\_secondary\_index](#input\_ignore\_changes\_global\_secondary\_index) | Whether to ignore changes lifecycle to global secondary indices, useful for provisioned tables with scaling | `bool` | `false` | no | +| [import\_table](#input\_import\_table) | Configurations for importing s3 data into a new table. | `any` | `{}` | no | | [local\_secondary\_indexes](#input\_local\_secondary\_indexes) | Describe an LSI on the table; these can only be allocated at creation so you cannot change this definition after you have created the resource. | `any` | `[]` | no | | [name](#input\_name) | Name of the DynamoDB table | `string` | `null` | no | | [point\_in\_time\_recovery\_enabled](#input\_point\_in\_time\_recovery\_enabled) | Whether to enable point-in-time recovery | `bool` | `false` | no | diff --git a/examples/autoscaling/README.md b/examples/autoscaling/README.md index 38b33bf..f19f35e 100644 --- a/examples/autoscaling/README.md +++ b/examples/autoscaling/README.md @@ -20,7 +20,7 @@ Note that this example may create resources which can cost money (AWS Elastic IP | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 3.69 | +| [aws](#requirement\_aws) | >= 5.21 | | [random](#requirement\_random) | >= 2.0 | ## Providers diff --git a/examples/autoscaling/versions.tf b/examples/autoscaling/versions.tf index 8d7f35e..d17ac73 100644 --- a/examples/autoscaling/versions.tf +++ b/examples/autoscaling/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.69" + version = ">= 5.21" } random = { source = "hashicorp/random" diff --git a/examples/basic/README.md b/examples/basic/README.md index caf8131..0c7c94c 100644 --- a/examples/basic/README.md +++ b/examples/basic/README.md @@ -20,7 +20,7 @@ Note that this example may create resources which can cost money (AWS Elastic IP | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 3.69 | +| [aws](#requirement\_aws) | >= 5.21 | | [random](#requirement\_random) | >= 2.0 | ## Providers diff --git a/examples/basic/versions.tf b/examples/basic/versions.tf index 8d7f35e..d17ac73 100644 --- a/examples/basic/versions.tf +++ b/examples/basic/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.69" + version = ">= 5.21" } random = { source = "hashicorp/random" diff --git a/examples/global-tables/README.md b/examples/global-tables/README.md index 94d7af6..9a3e348 100644 --- a/examples/global-tables/README.md +++ b/examples/global-tables/README.md @@ -20,15 +20,15 @@ Note that this example may create resources which can cost money (AWS Elastic IP | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.23 | +| [aws](#requirement\_aws) | >= 5.21 | | [random](#requirement\_random) | >= 2.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.23 | -| [aws.euwest2](#provider\_aws.euwest2) | >= 4.23 | +| [aws](#provider\_aws) | >= 5.21 | +| [aws.euwest2](#provider\_aws.euwest2) | >= 5.21 | | [random](#provider\_random) | >= 2.0 | ## Modules diff --git a/examples/global-tables/versions.tf b/examples/global-tables/versions.tf index d2bf143..d17ac73 100644 --- a/examples/global-tables/versions.tf +++ b/examples/global-tables/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.23" + version = ">= 5.21" } random = { source = "hashicorp/random" diff --git a/examples/s3-import/README.md b/examples/s3-import/README.md new file mode 100644 index 0000000..55f7562 --- /dev/null +++ b/examples/s3-import/README.md @@ -0,0 +1,64 @@ +# DynamoDB Table s3 import example + +Configuration in this directory creates an AWS DynamoDB table created from s3 imports (both json and csv examples). + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + +Note that this example may create resources which can cost money (AWS Elastic IP, for example). Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | >= 5.21 | +| [random](#requirement\_random) | >= 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [random](#provider\_random) | >= 2.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [import\_csv\_table](#module\_import\_csv\_table) | ../../ | n/a | +| [import\_json\_table](#module\_import\_json\_table) | ../../ | n/a | +| [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 3.15 | +| [s3\_import\_object\_csv](#module\_s3\_import\_object\_csv) | terraform-aws-modules/s3-bucket/aws//modules/object | ~> 3.15 | +| [s3\_import\_object\_json](#module\_s3\_import\_object\_json) | terraform-aws-modules/s3-bucket/aws//modules/object | ~> 3.15 | + +## Resources + +| Name | Type | +|------|------| +| [random_pet.this](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [import\_csv\_table\_arn](#output\_import\_csv\_table\_arn) | ARN of the DynamoDB table | +| [import\_csv\_table\_id](#output\_import\_csv\_table\_id) | ID of the DynamoDB table | +| [import\_csv\_table\_stream\_arn](#output\_import\_csv\_table\_stream\_arn) | The ARN of the Table Stream. Only available when var.stream\_enabled is true | +| [import\_csv\_table\_stream\_label](#output\_import\_csv\_table\_stream\_label) | A timestamp, in ISO 8601 format of the Table Stream. Only available when var.stream\_enabled is true | +| [import\_json\_table\_arn](#output\_import\_json\_table\_arn) | ARN of the DynamoDB table | +| [import\_json\_table\_id](#output\_import\_json\_table\_id) | ID of the DynamoDB table | +| [import\_json\_table\_stream\_arn](#output\_import\_json\_table\_stream\_arn) | The ARN of the Table Stream. Only available when var.stream\_enabled is true | +| [import\_json\_table\_stream\_label](#output\_import\_json\_table\_stream\_label) | A timestamp, in ISO 8601 format of the Table Stream. Only available when var.stream\_enabled is true | + diff --git a/examples/s3-import/files/sample.csv b/examples/s3-import/files/sample.csv new file mode 100644 index 0000000..822a6ec --- /dev/null +++ b/examples/s3-import/files/sample.csv @@ -0,0 +1,3 @@ +id;title +02;csv +03;csvtest diff --git a/examples/s3-import/files/sample.json b/examples/s3-import/files/sample.json new file mode 100644 index 0000000..9082164 --- /dev/null +++ b/examples/s3-import/files/sample.json @@ -0,0 +1,11 @@ +{"Item": +{ + "id":{"N":"01"}, + "title":{"S":"example1"} +}} + +{"Item": +{ + "id":{"N":"02"}, + "title":{"S":"example2"} +}} diff --git a/examples/s3-import/main.tf b/examples/s3-import/main.tf new file mode 100644 index 0000000..7228abc --- /dev/null +++ b/examples/s3-import/main.tf @@ -0,0 +1,107 @@ +provider "aws" { + region = "eu-west-1" +} + +resource "random_pet" "this" { + length = 2 +} + +module "import_json_table" { + source = "../../" + + name = "import-json-${random_pet.this.id}" + hash_key = "id" + range_key = "title" + table_class = "STANDARD" + deletion_protection_enabled = false + + attributes = [ + { + name = "id" + type = "N" + }, + { + name = "title" + type = "S" + }, + ] + + import_table = { + input_format = "DYNAMODB_JSON" + input_compression_type = "NONE" + bucket = module.s3_bucket.s3_bucket_id + key_prefix = "import-json-${random_pet.this.id}" + } + + tags = { + Terraform = "true" + Environment = "staging" + } +} + +module "import_csv_table" { + source = "../../" + + name = "import-csv-${random_pet.this.id}" + hash_key = "id" + range_key = "title" + table_class = "STANDARD" + deletion_protection_enabled = false + + attributes = [ + { + name = "id" + type = "N" + }, + { + name = "title" + type = "S" + }, + ] + + import_table = { + input_format = "CSV" + input_compression_type = "NONE" + bucket = module.s3_bucket.s3_bucket_id + key_prefix = "import-csv-${random_pet.this.id}" + input_format_options = { + csv = { + delimiter = ";" + } + } + } + + tags = { + Terraform = "true" + Environment = "staging" + } +} + +module "s3_bucket" { + source = "terraform-aws-modules/s3-bucket/aws" + version = "~> 3.15" + + bucket = "import-example-${random_pet.this.id}" + + force_destroy = true +} + +module "s3_import_object_json" { + source = "terraform-aws-modules/s3-bucket/aws//modules/object" + version = "~> 3.15" + + bucket = module.s3_bucket.s3_bucket_id + key = "import-json-${random_pet.this.id}/sample.json" + + content_base64 = filebase64("./files/sample.json") +} + +module "s3_import_object_csv" { + source = "terraform-aws-modules/s3-bucket/aws//modules/object" + version = "~> 3.15" + + bucket = module.s3_bucket.s3_bucket_id + key = "import-csv-${random_pet.this.id}/sample.csv" + + content_base64 = filebase64("./files/sample.csv") +} diff --git a/examples/s3-import/outputs.tf b/examples/s3-import/outputs.tf new file mode 100644 index 0000000..e826e9e --- /dev/null +++ b/examples/s3-import/outputs.tf @@ -0,0 +1,39 @@ +output "import_json_table_arn" { + description = "ARN of the DynamoDB table" + value = module.import_json_table.dynamodb_table_arn +} + +output "import_json_table_id" { + description = "ID of the DynamoDB table" + value = module.import_json_table.dynamodb_table_id +} + +output "import_json_table_stream_arn" { + description = "The ARN of the Table Stream. Only available when var.stream_enabled is true" + value = module.import_json_table.dynamodb_table_stream_arn +} + +output "import_json_table_stream_label" { + description = "A timestamp, in ISO 8601 format of the Table Stream. Only available when var.stream_enabled is true" + value = module.import_json_table.dynamodb_table_stream_label +} + +output "import_csv_table_arn" { + description = "ARN of the DynamoDB table" + value = module.import_csv_table.dynamodb_table_arn +} + +output "import_csv_table_id" { + description = "ID of the DynamoDB table" + value = module.import_csv_table.dynamodb_table_id +} + +output "import_csv_table_stream_arn" { + description = "The ARN of the Table Stream. Only available when var.stream_enabled is true" + value = module.import_csv_table.dynamodb_table_stream_arn +} + +output "import_csv_table_stream_label" { + description = "A timestamp, in ISO 8601 format of the Table Stream. Only available when var.stream_enabled is true" + value = module.import_csv_table.dynamodb_table_stream_label +} diff --git a/examples/s3-import/variables.tf b/examples/s3-import/variables.tf new file mode 100644 index 0000000..e69de29 diff --git a/examples/s3-import/versions.tf b/examples/s3-import/versions.tf new file mode 100644 index 0000000..d17ac73 --- /dev/null +++ b/examples/s3-import/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.21" + } + random = { + source = "hashicorp/random" + version = ">= 2.0" + } + } +} diff --git a/main.tf b/main.tf index 65a3f4e..8364a4f 100644 --- a/main.tf +++ b/main.tf @@ -71,6 +71,37 @@ resource "aws_dynamodb_table" "this" { kms_key_arn = var.server_side_encryption_kms_key_arn } + dynamic "import_table" { + for_each = length(var.import_table) > 0 ? [var.import_table] : [] + + content { + input_format = import_table.value.input_format + input_compression_type = try(import_table.value.input_compression_type, null) + + dynamic "input_format_options" { + for_each = try([import_table.value.input_format_options], []) + + content { + + dynamic "csv" { + for_each = try([input_format_options.value.csv], []) + + content { + delimiter = try(csv.value.delimiter, null) + header_list = try(csv.value.header_list, null) + } + } + } + } + + s3_bucket_source { + bucket = import_table.value.bucket + bucket_owner = try(import_table.value.bucket_owner, null) + key_prefix = try(import_table.value.key_prefix, null) + } + } + } + tags = merge( var.tags, { @@ -158,6 +189,37 @@ resource "aws_dynamodb_table" "autoscaled" { kms_key_arn = var.server_side_encryption_kms_key_arn } + dynamic "import_table" { + for_each = length(var.import_table) > 0 ? [var.import_table] : [] + + content { + input_format = import_table.value.input_format + input_compression_type = try(import_table.value.input_compression_type, null) + + dynamic "input_format_options" { + for_each = try([import_table.value.input_format_options], []) + + content { + + dynamic "csv" { + for_each = try([input_format_options.value.csv], []) + + content { + delimiter = try(csv.value.delimiter, null) + header_list = try(csv.value.header_list, null) + } + } + } + } + + s3_bucket_source { + bucket = import_table.value.bucket + bucket_owner = try(import_table.value.bucket_owner, null) + key_prefix = try(import_table.value.key_prefix, null) + } + } + } + tags = merge( var.tags, { diff --git a/variables.tf b/variables.tf index aa13fb1..577ace5 100644 --- a/variables.tf +++ b/variables.tf @@ -168,6 +168,12 @@ variable "deletion_protection_enabled" { default = null } +variable "import_table" { + description = "Configurations for importing s3 data into a new table." + type = any + default = {} +} + variable "ignore_changes_global_secondary_index" { description = "Whether to ignore changes lifecycle to global secondary indices, useful for provisioned tables with scaling" type = bool diff --git a/versions.tf b/versions.tf index 33eb30c..5cdb3d4 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.59" + version = ">= 5.21" } } } diff --git a/wrappers/main.tf b/wrappers/main.tf index 031be25..6ab374a 100644 --- a/wrappers/main.tf +++ b/wrappers/main.tf @@ -38,5 +38,6 @@ module "wrapper" { autoscaling_indexes = try(each.value.autoscaling_indexes, var.defaults.autoscaling_indexes, {}) table_class = try(each.value.table_class, var.defaults.table_class, null) deletion_protection_enabled = try(each.value.deletion_protection_enabled, var.defaults.deletion_protection_enabled, null) + import_table = try(each.value.import_table, var.defaults.import_table, {}) ignore_changes_global_secondary_index = try(each.value.ignore_changes_global_secondary_index, var.defaults.ignore_changes_global_secondary_index, false) }