diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..75fd15d --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.terraform* +terraform.* +test.tf \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index fac9c6f..c668efa 100644 --- a/docs/index.md +++ b/docs/index.md @@ -33,6 +33,26 @@ buckets, lifecycle rules and other features. 20588400 bytes downloaded Switched terraform to version "1.4.2" ``` + + + +??? success "Required roles and permissions" + + **On the project you want to deploy on:** + + - Broad roles that will work, but **not recommended** for service accounts or even people. + - `roles/owner` + - `roles/editor` + - Recommended roles to respect the least privilege principle. + - `roles/pubsub.admin` + - `roles/storage.admin` + - Granular permissions required to build a custom role specific for this deployment. + - `pubsub.topics.create` + - `pubsub.topics.delete` + - `pubsub.topics.setIamPolicy` + - `storage.buckets.create` + - `storage.buckets.setIamPolicy` + - `storage.buckets.update` ??? success "Log in to GCP with your default credentials" diff --git a/examples/standalone/main.tf b/examples/standalone/main.tf index c829a24..256eced 100644 --- a/examples/standalone/main.tf +++ b/examples/standalone/main.tf @@ -2,31 +2,41 @@ locals { project_id = "PROJECT_ID" # Replace this with your actual project id } -resource "random_string" "prefix" { - length = 4 - upper = false - special = false -} - provider "google" { user_project_override = true billing_project = local.project_id } +resource "random_string" "suffix" { + length = 4 + upper = false + special = false +} + module "datalake" { source = "artefactory/datalake/google" project_id = local.project_id - - # Naming convention naming_convention = { - "prefix": local.project_id - "suffix": random_string.prefix.result + "prefix" : local.project_id + "suffix" : random_string.suffix.result } - - # List of buckets to create - buckets = [ - "source-a", - "source-b" + buckets_config = [ + { + "bucket_name" : "sourcea", + "iam_rules" : [ + { "role" = "roles/storage.admin", "principals" = ["user:user@user.com"] } + ] + "autoclass" : false, + "lifecycle_rules" : [ + ], + "notification_topic" : "hello" + }, + { + "bucket_name" : "sourceb", + "iam_rules" : [{ "role" = "roles/storage.admin", "principals" = ["user:user@user.com"] }] + "autoclass" : false, + "lifecycle_rules" : [], + } ] } diff --git a/main.tf b/main.tf index fd503e7..f5cdd1a 100644 --- a/main.tf +++ b/main.tf @@ -1,15 +1,33 @@ +locals { + iam_list = distinct(flatten([ + for bucket_config in var.buckets_config : [ + for iam_rule in bucket_config.iam_rules : [ + for principal in iam_rule.principals : { + bucket_name = bucket_config.bucket_name + role = iam_rule.role + principal = principal + } + ] + ] + ])) + +} + resource "google_storage_bucket" "buckets" { - for_each = toset(var.buckets) + for_each = tomap({ for bucket_config in var.buckets_config : bucket_config.bucket_name => bucket_config }) - name = "${var.naming_convention.prefix}-${each.value}-${var.naming_convention.suffix}" + labels = var.labels + name = "${var.naming_convention.prefix}-${each.value.bucket_name}-${var.naming_convention.suffix}" location = var.location force_destroy = false project = var.project_id - storage_class = "STANDARD" + autoclass { + enabled = each.value.autoclass + } dynamic "lifecycle_rule" { - for_each = var.lifecycle_rules + for_each = each.value.lifecycle_rules content { condition { @@ -25,3 +43,39 @@ resource "google_storage_bucket" "buckets" { uniform_bucket_level_access = true public_access_prevention = "enforced" } + +resource "google_storage_bucket_iam_member" "member" { + for_each = { for entry in local.iam_list : "${entry.bucket_name}.${entry.role}.${entry.principal}" => entry } + bucket = "${var.naming_convention.prefix}-${each.value.bucket_name}-${var.naming_convention.suffix}" + role = each.value.role + member = each.value.principal + depends_on = [google_storage_bucket.buckets] +} + +resource "google_storage_notification" "notification" { + for_each = tomap({ for bucket_config in var.buckets_config : bucket_config.bucket_name => bucket_config if bucket_config.notification_topic != null }) + bucket = "${var.naming_convention.prefix}-${each.value.bucket_name}-${var.naming_convention.suffix}" + payload_format = "JSON_API_V1" + topic = each.value.notification_topic + event_types = ["OBJECT_FINALIZE", "OBJECT_DELETE", "OBJECT_ARCHIVE", "OBJECT_METADATA_UPDATE"] + depends_on = [google_pubsub_topic_iam_binding.bind_gcs_svc_acc, + google_storage_bucket.buckets] +} + +data "google_storage_project_service_account" "gcs_account" { + project = var.project_id +} + +resource "google_pubsub_topic_iam_binding" "bind_gcs_svc_acc" { + for_each = google_pubsub_topic.notification_topic + topic = each.value.id + role = "roles/pubsub.publisher" + members = ["serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"] +} + + +resource "google_pubsub_topic" "notification_topic" { + for_each = tomap({ for bucket_config in var.buckets_config : bucket_config.bucket_name => bucket_config if bucket_config.notification_topic != null }) + project = var.project_id + name = each.value.notification_topic +} \ No newline at end of file diff --git a/variables.tf b/variables.tf index 9559075..3c9a853 100644 --- a/variables.tf +++ b/variables.tf @@ -15,27 +15,34 @@ variable "labels" { default = {} } -variable "buckets" { - description = "Name of the buckets to create" - type = list(string) -} - -variable "lifecycle_rules" { - description = "Lifecycle rules to define for each bucket" +variable "buckets_config" { + description = "Data lake configuration per bucket" type = list( - object( - { - delay = number - storage_class = string - } - ) + object({ + bucket_name = string + autoclass = optional(bool, true) + lifecycle_rules = optional(list( + object({ + delay = number + storage_class = string + }) + ), []) + iam_rules = optional(list( + object({ + role = string + principals = list(string) + }) + ), []) + notification_topic = optional(string, null) + regex_validation = optional(string, ".*") + }) ) - default = [ - { - "delay" : 60, - "storage_class" : "ARCHIVE", - } - ] + validation { + condition = alltrue([ + for bucket_config in var.buckets_config : !(bucket_config.autoclass == true && length(bucket_config.lifecycle_rules) != 0) + ]) + error_message = "Autoclass cannot be true while lifecyle_rules are defined" + } } variable "naming_convention" {