From 6bfe015344d12588e6935a667d9517ef185ec5b3 Mon Sep 17 00:00:00 2001 From: Gary <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 22 Oct 2024 09:52:33 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=91=81=EF=B8=8F=20=20Observability:=20Add?= =?UTF-8?q?=20Athena=20Data=20Source=20(#8142)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * :wrench: Add Athena Data Source * :wrench: Modify for multiple Athena Data Sources * :recycle: Refactor --- .../environment-configurations.tf | 26 +++++++++++++++++ .../modules/grafana/athena-source/main.tf | 21 ++++++++++++++ .../grafana/athena-source/providers.tf | 9 ++++++ .../grafana/athena-source/variables.tf | 20 +++++++++++++ .../modules/grafana/team/main.tf | 21 ++++++++++++++ .../modules/grafana/team/variables.tf | 3 ++ .../tenant-configuration/main.tf | 29 ++++++++++++++++++- .../tenant-configuration/variables.tf | 6 ++++ 8 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 terraform/environments/observability-platform/modules/grafana/athena-source/main.tf create mode 100644 terraform/environments/observability-platform/modules/grafana/athena-source/providers.tf create mode 100644 terraform/environments/observability-platform/modules/grafana/athena-source/variables.tf diff --git a/terraform/environments/observability-platform/environment-configurations.tf b/terraform/environments/observability-platform/environment-configurations.tf index b3413d34e24..011bd6105bc 100644 --- a/terraform/environments/observability-platform/environment-configurations.tf +++ b/terraform/environments/observability-platform/environment-configurations.tf @@ -13,6 +13,7 @@ locals { prometheus_push_enabled = false amazon_prometheus_query_enabled = false xray_enabled = true + athena_enabled = false } } }, @@ -24,6 +25,7 @@ locals { prometheus_push_enabled = false amazon_prometheus_query_enabled = false xray_enabled = true + athena_enabled = false }, "analytical-platform-compute-development" = { cloudwatch_enabled = true @@ -31,6 +33,7 @@ locals { amazon_prometheus_query_enabled = true amazon_prometheus_workspace_id = "ws-bfdd5d7a-5571-4686-bfd4-43ab07cf8d54ba" xray_enabled = true + athena_enabled = false }, "analytical-platform-compute-test" = { cloudwatch_enabled = true @@ -38,6 +41,7 @@ locals { amazon_prometheus_query_enabled = true amazon_prometheus_workspace_id = "ws-a9d7f576-58b7-4748-b4c1-b02bbdc54a2922" xray_enabled = true + athena_enabled = false } } }, @@ -49,6 +53,7 @@ locals { prometheus_push_enabled = false amazon_prometheus_query_enabled = false xray_enabled = false + athena_enabled = false } } }, @@ -60,18 +65,21 @@ locals { prometheus_push_enabled = false amazon_prometheus_query_enabled = false xray_enabled = false + athena_enabled = false }, "digital-prison-reporting-preproduction" = { cloudwatch_enabled = true prometheus_push_enabled = false amazon_prometheus_query_enabled = false xray_enabled = false + athena_enabled = false }, "digital-prison-reporting-test" = { cloudwatch_enabled = true prometheus_push_enabled = false amazon_prometheus_query_enabled = false xray_enabled = false + athena_enabled = false } } }, @@ -83,12 +91,14 @@ locals { prometheus_push_enabled = false amazon_prometheus_query_enabled = false xray_enabled = false + athena_enabled = false } "oasys-test" = { cloudwatch_enabled = true prometheus_push_enabled = false amazon_prometheus_query_enabled = false xray_enabled = false + athena_enabled = false } } } @@ -107,6 +117,7 @@ locals { prometheus_push_enabled = false amazon_prometheus_query_enabled = false xray_enabled = true + athena_enabled = false } } }, @@ -118,6 +129,7 @@ locals { prometheus_push_enabled = false amazon_prometheus_query_enabled = false xray_enabled = true + athena_enabled = false }, "analytical-platform-compute-production" = { cloudwatch_enabled = true @@ -125,6 +137,7 @@ locals { amazon_prometheus_query_enabled = true amazon_prometheus_workspace_id = "ws-257796b7-4aa4-4c18-b906-6dd21e95d7b73e" xray_enabled = true + athena_enabled = false }, "analytical-platform-production" = { cloudwatch_enabled = true @@ -132,6 +145,7 @@ locals { amazon_prometheus_query_enabled = true amazon_prometheus_workspace_id = "ws-a7b353be-244a-47e7-8054-436b41c050d932" xray_enabled = false + athena_enabled = false } } }, @@ -143,6 +157,7 @@ locals { prometheus_push_enabled = false amazon_prometheus_query_enabled = false xray_enabled = false + athena_enabled = false } } }, @@ -155,30 +170,41 @@ locals { prometheus_push_enabled = false amazon_prometheus_query_enabled = false xray_enabled = false + athena_enabled = false }, "core-logging-production" = { cloudwatch_enabled = true prometheus_push_enabled = false amazon_prometheus_query_enabled = false xray_enabled = false + athena_enabled = true + athena_config = { + mod-platform-cur-reports = { + database = "data" + workgroup = "mod-platform-cur-reports" + } + } }, "core-security-production" = { cloudwatch_enabled = true prometheus_push_enabled = false amazon_prometheus_query_enabled = false xray_enabled = false + athena_enabled = false }, "core-shared-services-production" = { cloudwatch_enabled = true prometheus_push_enabled = false amazon_prometheus_query_enabled = false xray_enabled = false + athena_enabled = false }, "core-vpc-production" = { cloudwatch_enabled = true prometheus_push_enabled = false amazon_prometheus_query_enabled = false xray_enabled = false + athena_enabled = false } } } diff --git a/terraform/environments/observability-platform/modules/grafana/athena-source/main.tf b/terraform/environments/observability-platform/modules/grafana/athena-source/main.tf new file mode 100644 index 00000000000..178162769af --- /dev/null +++ b/terraform/environments/observability-platform/modules/grafana/athena-source/main.tf @@ -0,0 +1,21 @@ +locals { + name = "${var.name}-athena" +} + +data "grafana_data_source" "this" { + count = var.athena_enabled ? 1 : 0 + name = "${var.name}-athena" +} + +resource "grafana_data_source" "this" { + type = "athena" + name = "${var.athena_workgroup}-${var.athena_database}" + json_data_encoded = jsonencode({ + defaultRegion = "eu-west-2" + authType = "ec2_iam_role" + assumeRoleArn = "arn:aws:iam::${var.account_id}:role/observability-platform" + externalId = var.name + database = var.athena_database + workgroup = var.athena_workgroup + }) +} diff --git a/terraform/environments/observability-platform/modules/grafana/athena-source/providers.tf b/terraform/environments/observability-platform/modules/grafana/athena-source/providers.tf new file mode 100644 index 00000000000..516d493291f --- /dev/null +++ b/terraform/environments/observability-platform/modules/grafana/athena-source/providers.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + grafana = { + source = "grafana/grafana" + version = "~> 3.0" + } + } + required_version = "~> 1.0" +} diff --git a/terraform/environments/observability-platform/modules/grafana/athena-source/variables.tf b/terraform/environments/observability-platform/modules/grafana/athena-source/variables.tf new file mode 100644 index 00000000000..1fd8f3f8bc1 --- /dev/null +++ b/terraform/environments/observability-platform/modules/grafana/athena-source/variables.tf @@ -0,0 +1,20 @@ +variable "name" { + type = string +} + +variable "account_id" { + type = string +} + +variable "athena_enabled" { + type = bool + default = false +} + +variable "athena_workgroup" { + type = string +} + +variable "athena_database" { + type = string +} diff --git a/terraform/environments/observability-platform/modules/grafana/team/main.tf b/terraform/environments/observability-platform/modules/grafana/team/main.tf index 0bff958d733..ccdc31cc897 100644 --- a/terraform/environments/observability-platform/modules/grafana/team/main.tf +++ b/terraform/environments/observability-platform/modules/grafana/team/main.tf @@ -55,6 +55,27 @@ resource "grafana_data_source_permission" "xray" { } } +data "grafana_data_source" "athena" { + for_each = { + for name, config in var.aws_accounts : name => config if config.athena_enabled + } + + name = "${each.key}-athena" +} + +resource "grafana_data_source_permission" "athena" { + for_each = { + for name, config in var.aws_accounts : name => config if config.athena_enabled + } + + datasource_uid = trimprefix(data.grafana_data_source.athena[each.key].id, "1:") + + permissions { + team_id = grafana_team.this.id + permission = "Query" + } +} + data "grafana_data_source" "amazon_prometheus" { for_each = { for name, account in var.aws_accounts : name => account if account.amazon_prometheus_query_enabled diff --git a/terraform/environments/observability-platform/modules/grafana/team/variables.tf b/terraform/environments/observability-platform/modules/grafana/team/variables.tf index efed6445fcc..aeb472edd63 100644 --- a/terraform/environments/observability-platform/modules/grafana/team/variables.tf +++ b/terraform/environments/observability-platform/modules/grafana/team/variables.tf @@ -11,5 +11,8 @@ variable "aws_accounts" { cloudwatch_enabled = optional(bool) xray_enabled = optional(bool) amazon_prometheus_query_enabled = optional(bool) + athena_enabled = optional(bool) })) } + + diff --git a/terraform/environments/observability-platform/modules/observability-platform/tenant-configuration/main.tf b/terraform/environments/observability-platform/modules/observability-platform/tenant-configuration/main.tf index 637d7d604d5..b09155ebf1b 100644 --- a/terraform/environments/observability-platform/modules/observability-platform/tenant-configuration/main.tf +++ b/terraform/environments/observability-platform/modules/observability-platform/tenant-configuration/main.tf @@ -36,6 +36,32 @@ module "amazon_prometheus_query_source" { amazon_prometheus_workspace_id = each.value.amazon_prometheus_workspace_id } +locals { + flattened_athena_configs = flatten([ + for env_name, env_data in var.aws_accounts : [ + for config_name, config_data in try(env_data.athena_config, {}) : { + key = "${env_name}-${config_name}" + account_id = nonsensitive(var.environment_management.account_ids[env_name]) + database = config_data.database + workgroup = config_data.workgroup + } + ] if env_data.athena_enabled == true + ]) +} + +module "athena_source" { + for_each = { + for config in local.flattened_athena_configs : config.key => config + } + + source = "../../grafana/athena-source" + + name = each.key + account_id = each.value.account_id + athena_workgroup = each.value.workgroup + athena_database = each.value.database +} + module "prometheus_push" { for_each = { for name, account in var.aws_accounts : name => account if account.prometheus_push_enabled @@ -61,6 +87,7 @@ module "team" { depends_on = [ module.xray_source, module.cloudwatch_source, - module.amazon_prometheus_query_source + module.amazon_prometheus_query_source, + module.athena_source ] } diff --git a/terraform/environments/observability-platform/modules/observability-platform/tenant-configuration/variables.tf b/terraform/environments/observability-platform/modules/observability-platform/tenant-configuration/variables.tf index 6e55539a48b..0fe0ba87859 100644 --- a/terraform/environments/observability-platform/modules/observability-platform/tenant-configuration/variables.tf +++ b/terraform/environments/observability-platform/modules/observability-platform/tenant-configuration/variables.tf @@ -18,6 +18,11 @@ variable "aws_accounts" { amazon_prometheus_query_enabled = optional(bool) amazon_prometheus_workspace_id = optional(string) xray_enabled = optional(bool) + athena_enabled = optional(bool) + athena_config = optional(map(object({ + database = string + workgroup = string + }))) })) default = { default = { @@ -26,6 +31,7 @@ variable "aws_accounts" { amazon_prometheus_query_enabled = false amazon_prometheus_workspace_id = "" xray_enabled = false + athena_enabled = false } } }