👁️ Observability: Add Athena Data Source

terraform/environments/observability-platform/environment-configurations.tf

@@ -13,6 +13,7 @@ locals {
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = false
               xray_enabled                    = true
+              athena_enabled                  = false
             }
           }
         },
@@ -24,20 +25,23 @@ locals {
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = false
               xray_enabled                    = true
+              athena_enabled                  = false
             },
             "analytical-platform-compute-development" = {
               cloudwatch_enabled              = true
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = true
               amazon_prometheus_workspace_id  = "ws-bfdd5d7a-5571-4686-bfd4-43ab07cf8d54ba"
               xray_enabled                    = true
+              athena_enabled                  = false
             },
             "analytical-platform-compute-test" = {
               cloudwatch_enabled              = true
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = true
               amazon_prometheus_workspace_id  = "ws-a9d7f576-58b7-4748-b4c1-b02bbdc54a2922"
               xray_enabled                    = true
+              athena_enabled                  = false
             }
           }
         },
@@ -49,6 +53,7 @@ locals {
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = false
               xray_enabled                    = false
+              athena_enabled                  = false
             }
           }
         },
@@ -60,18 +65,21 @@ locals {
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = false
               xray_enabled                    = false
+              athena_enabled                  = false
             },
             "digital-prison-reporting-preproduction" = {
               cloudwatch_enabled              = true
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = false
               xray_enabled                    = false
+              athena_enabled                  = false
             },
             "digital-prison-reporting-test" = {
               cloudwatch_enabled              = true
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = false
               xray_enabled                    = false
+              athena_enabled                  = false
             }
           }
         },
@@ -83,12 +91,14 @@ locals {
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = false
               xray_enabled                    = false
+              athena_enabled                  = false
             }
             "oasys-test" = {
               cloudwatch_enabled              = true
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = false
               xray_enabled                    = false
+              athena_enabled                  = false
             }
           }
         }
@@ -107,6 +117,7 @@ locals {
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = false
               xray_enabled                    = true
+              athena_enabled                  = false
             }
           }
         },
@@ -118,20 +129,23 @@ locals {
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = false
               xray_enabled                    = true
+              athena_enabled                  = false
             },
             "analytical-platform-compute-production" = {
               cloudwatch_enabled              = true
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = true
               amazon_prometheus_workspace_id  = "ws-257796b7-4aa4-4c18-b906-6dd21e95d7b73e"
               xray_enabled                    = true
+              athena_enabled                  = false
             },
             "analytical-platform-production" = {
               cloudwatch_enabled              = true
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = true
               amazon_prometheus_workspace_id  = "ws-a7b353be-244a-47e7-8054-436b41c050d932"
               xray_enabled                    = false
+              athena_enabled                  = false
             }
           }
         },
@@ -143,6 +157,7 @@ locals {
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = false
               xray_enabled                    = false
+              athena_enabled                  = false
             }
           }
         },
@@ -155,30 +170,41 @@ locals {
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = false
               xray_enabled                    = false
+              athena_enabled                  = false
             },
             "core-logging-production" = {
               cloudwatch_enabled              = true
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = false
               xray_enabled                    = false
+              athena_enabled                  = true
+              athena_config = {
+                mod-platform-cur-reports = {
+                  database  = "data"
+                  workgroup = "mod-platform-cur-reports"
+                }
+              }
             },
             "core-security-production" = {
               cloudwatch_enabled              = true
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = false
               xray_enabled                    = false
+              athena_enabled                  = false
             },
             "core-shared-services-production" = {
               cloudwatch_enabled              = true
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = false
               xray_enabled                    = false
+              athena_enabled                  = false
             },
             "core-vpc-production" = {
               cloudwatch_enabled              = true
               prometheus_push_enabled         = false
               amazon_prometheus_query_enabled = false
               xray_enabled                    = false
+              athena_enabled                  = false
             }
           }
         }

terraform/environments/observability-platform/modules/grafana/athena-source/main.tf

@@ -0,0 +1,21 @@
+locals {
+  name = "${var.name}-athena"
+}
+
+data "grafana_data_source" "this" {
+  count = var.athena_enabled ? 1 : 0
+  name  = "${var.name}-athena"
+}
+
+resource "grafana_data_source" "this" {
+  type = "athena"
+  name = "${var.athena_workgroup}-${var.athena_database}"
+  json_data_encoded = jsonencode({
+    defaultRegion = "eu-west-2"
+    authType      = "ec2_iam_role"
+    assumeRoleArn = "arn:aws:iam::${var.account_id}:role/observability-platform"
+    externalId    = var.name
+    database      = var.athena_database
+    workgroup     = var.athena_workgroup
+  })
+}

terraform/environments/observability-platform/modules/grafana/athena-source/providers.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_providers {
+    grafana = {
+      source  = "grafana/grafana"
+      version = "~> 3.0"
+    }
+  }
+  required_version = "~> 1.0"
+}

terraform/environments/observability-platform/modules/grafana/athena-source/variables.tf

@@ -0,0 +1,20 @@
+variable "name" {
+  type = string
+}
+
+variable "account_id" {
+  type = string
+}
+
+variable "athena_enabled" {
+  type    = bool
+  default = false
+}
+
+variable "athena_workgroup" {
+  type = string
+}
+
+variable "athena_database" {
+  type = string
+}

terraform/environments/observability-platform/modules/grafana/team/main.tf

@@ -55,6 +55,27 @@ resource "grafana_data_source_permission" "xray" {
   }
 }
 
+data "grafana_data_source" "athena" {
+  for_each = {
+    for name, config in var.aws_accounts : name => config if config.athena_enabled
+  }
+
+  name = "${each.key}-athena"
+}
+
+resource "grafana_data_source_permission" "athena" {
+  for_each = {
+    for name, config in var.aws_accounts : name => config if config.athena_enabled
+  }
+
+  datasource_uid = trimprefix(data.grafana_data_source.athena[each.key].id, "1:")
+
+  permissions {
+    team_id    = grafana_team.this.id
+    permission = "Query"
+  }
+}
+
 data "grafana_data_source" "amazon_prometheus" {
   for_each = {
     for name, account in var.aws_accounts : name => account if account.amazon_prometheus_query_enabled

terraform/environments/observability-platform/modules/grafana/team/variables.tf

@@ -11,5 +11,8 @@ variable "aws_accounts" {
     cloudwatch_enabled              = optional(bool)
     xray_enabled                    = optional(bool)
     amazon_prometheus_query_enabled = optional(bool)
+    athena_enabled                  = optional(bool)
   }))
 }
+
+

terraform/environments/observability-platform/modules/observability-platform/tenant-configuration/main.tf

@@ -36,6 +36,32 @@ module "amazon_prometheus_query_source" {
   amazon_prometheus_workspace_id = each.value.amazon_prometheus_workspace_id
 }
 
+locals {
+  flattened_athena_configs = flatten([
+    for env_name, env_data in var.aws_accounts : [
+      for config_name, config_data in try(env_data.athena_config, {}) : {
+        key        = "${env_name}-${config_name}"
+        account_id = nonsensitive(var.environment_management.account_ids[env_name])
+        database   = config_data.database
+        workgroup  = config_data.workgroup
+      }
+    ] if env_data.athena_enabled == true
+  ])
+}
+
+module "athena_source" {
+  for_each = {
+    for config in local.flattened_athena_configs : config.key => config
+  }
+
+  source = "../../grafana/athena-source"
+
+  name             = each.key
+  account_id       = each.value.account_id
+  athena_workgroup = each.value.workgroup
+  athena_database  = each.value.database
+}
+
 module "prometheus_push" {
   for_each = {
     for name, account in var.aws_accounts : name => account if account.prometheus_push_enabled
@@ -61,6 +87,7 @@ module "team" {
   depends_on = [
     module.xray_source,
     module.cloudwatch_source,
-    module.amazon_prometheus_query_source
+    module.amazon_prometheus_query_source,
+    module.athena_source
   ]
 }

terraform/environments/observability-platform/modules/observability-platform/tenant-configuration/variables.tf

@@ -18,6 +18,11 @@ variable "aws_accounts" {
     amazon_prometheus_query_enabled = optional(bool)
     amazon_prometheus_workspace_id  = optional(string)
     xray_enabled                    = optional(bool)
+    athena_enabled                  = optional(bool)
+    athena_config = optional(map(object({
+      database  = string
+      workgroup = string
+    })))
   }))
   default = {
     default = {
@@ -26,6 +31,7 @@ variable "aws_accounts" {
       amazon_prometheus_query_enabled = false
       amazon_prometheus_workspace_id  = ""
       xray_enabled                    = false
+      athena_enabled                  = false
     }
   }
 }