draft new bucket config structure

add default value

set autoclass to optional (already in tf)

fix comment

wip

fix: fixed the lifecycle policy condition

wip: started the implementation of interface

help

wip: implement notificaitons

feat: add new datamodel for the bucket variable

fix: removed storage class standard as it is already set by default

.gitignore

@@ -0,0 +1,3 @@
+.terraform*
+terraform.*
+test.tf

docs/index.md

@@ -33,6 +33,26 @@ buckets, lifecycle rules and other features.
     20588400 bytes downloaded
     Switched terraform to version "1.4.2" 
     ```
+
+
+
+??? success "Required roles and permissions"
+
+  **On the project you want to deploy on:**
+
+  - Broad roles that will work, but **not recommended** for service accounts or even people.
+  - `roles/owner`
+  - `roles/editor`
+  - Recommended roles to respect the least privilege principle.
+  - `roles/pubsub.admin`
+  - `roles/storage.admin`
+  - Granular permissions required to build a custom role specific for this deployment.
+  - `pubsub.topics.create`
+  - `pubsub.topics.delete`
+  - `pubsub.topics.setIamPolicy`
+  - `storage.buckets.create`
+  - `storage.buckets.setIamPolicy`
+  - `storage.buckets.update`
 
 
 ??? success "Log in to GCP with your default credentials"

examples/standalone/main.tf

@@ -2,31 +2,41 @@ locals {
   project_id = "PROJECT_ID" # Replace this with your actual project id
 }
 
-resource "random_string" "prefix" {
-  length  = 4
-  upper   = false
-  special = false
-}
-
 provider "google" {
   user_project_override = true
   billing_project       = local.project_id
 }
 
+resource "random_string" "suffix" {
+  length  = 4
+  upper   = false
+  special = false
+}
+
 module "datalake" {
   source = "artefactory/datalake/google"
 
   project_id = local.project_id
-
-  # Naming convention
   naming_convention = {
-    "prefix": local.project_id
-    "suffix": random_string.prefix.result
+    "prefix" : local.project_id
+    "suffix" : random_string.suffix.result
   }
-
-  # List of buckets to create
-  buckets = [
-    "source-a",
-    "source-b"
+  buckets_config = [
+    {
+      "bucket_name" : "sourcea",
+      "iam_rules" : [
+        { "role" = "roles/storage.admin", "principals" = ["user:[email protected]"] }
+      ]
+      "autoclass" : false,
+      "lifecycle_rules" : [
+      ],
+      "notification_topic" : "hello"
+    },
+    {
+      "bucket_name" : "sourceb",
+      "iam_rules" : [{ "role" = "roles/storage.admin", "principals" = ["user:[email protected]"] }]
+      "autoclass" : false,
+      "lifecycle_rules" : [],
+    }
   ]
 }

main.tf

@@ -1,15 +1,33 @@
+locals {
+  iam_list = distinct(flatten([
+    for bucket_config in var.buckets_config : [
+      for iam_rule in bucket_config.iam_rules : [
+        for principal in iam_rule.principals : {
+          bucket_name = bucket_config.bucket_name
+          role        = iam_rule.role
+          principal   = principal
+        }
+      ]
+    ]
+  ]))
+
+}
+
 resource "google_storage_bucket" "buckets" {
-  for_each = toset(var.buckets)
+  for_each = tomap({ for bucket_config in var.buckets_config : bucket_config.bucket_name => bucket_config })
 
-  name     = "${var.naming_convention.prefix}-${each.value}-${var.naming_convention.suffix}"
+  labels   = var.labels
+  name     = "${var.naming_convention.prefix}-${each.value.bucket_name}-${var.naming_convention.suffix}"
   location = var.location
 
   force_destroy = false
   project       = var.project_id
-  storage_class = "STANDARD"
+  autoclass {
+    enabled = each.value.autoclass
+  }
 
   dynamic "lifecycle_rule" {
-    for_each = var.lifecycle_rules
+    for_each = each.value.lifecycle_rules
 
     content {
       condition {
@@ -25,3 +43,39 @@ resource "google_storage_bucket" "buckets" {
   uniform_bucket_level_access = true
   public_access_prevention    = "enforced"
 }
+
+resource "google_storage_bucket_iam_member" "member" {
+  for_each   = { for entry in local.iam_list : "${entry.bucket_name}.${entry.role}.${entry.principal}" => entry }
+  bucket     = "${var.naming_convention.prefix}-${each.value.bucket_name}-${var.naming_convention.suffix}"
+  role       = each.value.role
+  member     = each.value.principal
+  depends_on = [google_storage_bucket.buckets]
+}
+
+resource "google_storage_notification" "notification" {
+  for_each       = tomap({ for bucket_config in var.buckets_config : bucket_config.bucket_name => bucket_config if bucket_config.notification_topic != null })
+  bucket         = "${var.naming_convention.prefix}-${each.value.bucket_name}-${var.naming_convention.suffix}"
+  payload_format = "JSON_API_V1"
+  topic          = each.value.notification_topic
+  event_types    = ["OBJECT_FINALIZE", "OBJECT_DELETE", "OBJECT_ARCHIVE", "OBJECT_METADATA_UPDATE"]
+  depends_on = [google_pubsub_topic_iam_binding.bind_gcs_svc_acc,
+  google_storage_bucket.buckets]
+}
+
+data "google_storage_project_service_account" "gcs_account" {
+  project = var.project_id
+}
+
+resource "google_pubsub_topic_iam_binding" "bind_gcs_svc_acc" {
+  for_each = google_pubsub_topic.notification_topic
+  topic    = each.value.id
+  role     = "roles/pubsub.publisher"
+  members  = ["serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"]
+}
+
+
+resource "google_pubsub_topic" "notification_topic" {
+  for_each = tomap({ for bucket_config in var.buckets_config : bucket_config.bucket_name => bucket_config if bucket_config.notification_topic != null })
+  project  = var.project_id
+  name     = each.value.notification_topic
+}

variables.tf

@@ -15,27 +15,34 @@ variable "labels" {
   default     = {}
 }
 
-variable "buckets" {
-  description = "Name of the buckets to create"
-  type        = list(string)
-}
-
-variable "lifecycle_rules" {
-  description = "Lifecycle rules to define for each bucket"
+variable "buckets_config" {
+  description = "Data lake configuration per bucket"
   type = list(
-    object(
-      {
-        delay         = number
-        storage_class = string
-      }
-    )
+    object({
+      bucket_name = string
+      autoclass   = optional(bool, true)
+      lifecycle_rules = optional(list(
+        object({
+          delay         = number
+          storage_class = string
+        })
+      ), [])
+      iam_rules = optional(list(
+        object({
+          role       = string
+          principals = list(string)
+        })
+      ), [])
+      notification_topic = optional(string, null)
+      regex_validation   = optional(string, ".*")
+    })
   )
-  default = [
-    {
-      "delay" : 60,
-      "storage_class" : "ARCHIVE",
-    }
-  ]
+  validation {
+    condition = alltrue([
+      for bucket_config in var.buckets_config : !(bucket_config.autoclass == true && length(bucket_config.lifecycle_rules) != 0)
+    ])
+    error_message = "Autoclass cannot be true while lifecyle_rules are defined"
+  }
 }
 
 variable "naming_convention" {