Skip to content

Commit

Permalink
👁️ Observability: Add Athena Data Source (#8142)
Browse files Browse the repository at this point in the history 
* 🔧 Add Athena Data Source

* 🔧 Modify for multiple Athena Data Sources

* ♻️ Refactor
  • Loading branch information
@Gary-H9
Gary-H9 authored Oct 22, 2024
1 parent b70b626 commit 6bfe015
Show file tree
Hide file tree
Showing 8 changed files with 134 additions and 1 deletion.
26 changes: 26 additions & 0 deletions terraform/environments/observability-platform/environment-configurations.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ locals {
prometheus_push_enabled = false
amazon_prometheus_query_enabled = false
xray_enabled = true
athena_enabled = false
}
}
},
Expand All @@ -24,20 +25,23 @@ locals {
prometheus_push_enabled = false
amazon_prometheus_query_enabled = false
xray_enabled = true
athena_enabled = false
},
"analytical-platform-compute-development" = {
cloudwatch_enabled = true
prometheus_push_enabled = false
amazon_prometheus_query_enabled = true
amazon_prometheus_workspace_id = "ws-bfdd5d7a-5571-4686-bfd4-43ab07cf8d54ba"
xray_enabled = true
athena_enabled = false
},
"analytical-platform-compute-test" = {
cloudwatch_enabled = true
prometheus_push_enabled = false
amazon_prometheus_query_enabled = true
amazon_prometheus_workspace_id = "ws-a9d7f576-58b7-4748-b4c1-b02bbdc54a2922"
xray_enabled = true
athena_enabled = false
}
}
},
Expand All @@ -49,6 +53,7 @@ locals {
prometheus_push_enabled = false
amazon_prometheus_query_enabled = false
xray_enabled = false
athena_enabled = false
}
}
},
Expand All @@ -60,18 +65,21 @@ locals {
prometheus_push_enabled = false
amazon_prometheus_query_enabled = false
xray_enabled = false
athena_enabled = false
},
"digital-prison-reporting-preproduction" = {
cloudwatch_enabled = true
prometheus_push_enabled = false
amazon_prometheus_query_enabled = false
xray_enabled = false
athena_enabled = false
},
"digital-prison-reporting-test" = {
cloudwatch_enabled = true
prometheus_push_enabled = false
amazon_prometheus_query_enabled = false
xray_enabled = false
athena_enabled = false
}
}
},
Expand All @@ -83,12 +91,14 @@ locals {
prometheus_push_enabled = false
amazon_prometheus_query_enabled = false
xray_enabled = false
athena_enabled = false
}
"oasys-test" = {
cloudwatch_enabled = true
prometheus_push_enabled = false
amazon_prometheus_query_enabled = false
xray_enabled = false
athena_enabled = false
}
}
}
Expand All @@ -107,6 +117,7 @@ locals {
prometheus_push_enabled = false
amazon_prometheus_query_enabled = false
xray_enabled = true
athena_enabled = false
}
}
},
Expand All @@ -118,20 +129,23 @@ locals {
prometheus_push_enabled = false
amazon_prometheus_query_enabled = false
xray_enabled = true
athena_enabled = false
},
"analytical-platform-compute-production" = {
cloudwatch_enabled = true
prometheus_push_enabled = false
amazon_prometheus_query_enabled = true
amazon_prometheus_workspace_id = "ws-257796b7-4aa4-4c18-b906-6dd21e95d7b73e"
xray_enabled = true
athena_enabled = false
},
"analytical-platform-production" = {
cloudwatch_enabled = true
prometheus_push_enabled = false
amazon_prometheus_query_enabled = true
amazon_prometheus_workspace_id = "ws-a7b353be-244a-47e7-8054-436b41c050d932"
xray_enabled = false
athena_enabled = false
}
}
},
Expand All @@ -143,6 +157,7 @@ locals {
prometheus_push_enabled = false
amazon_prometheus_query_enabled = false
xray_enabled = false
athena_enabled = false
}
}
},
Expand All @@ -155,30 +170,41 @@ locals {
prometheus_push_enabled = false
amazon_prometheus_query_enabled = false
xray_enabled = false
athena_enabled = false
},
"core-logging-production" = {
cloudwatch_enabled = true
prometheus_push_enabled = false
amazon_prometheus_query_enabled = false
xray_enabled = false
athena_enabled = true
athena_config = {
mod-platform-cur-reports = {
database = "data"
workgroup = "mod-platform-cur-reports"
}
}
},
"core-security-production" = {
cloudwatch_enabled = true
prometheus_push_enabled = false
amazon_prometheus_query_enabled = false
xray_enabled = false
athena_enabled = false
},
"core-shared-services-production" = {
cloudwatch_enabled = true
prometheus_push_enabled = false
amazon_prometheus_query_enabled = false
xray_enabled = false
athena_enabled = false
},
"core-vpc-production" = {
cloudwatch_enabled = true
prometheus_push_enabled = false
amazon_prometheus_query_enabled = false
xray_enabled = false
athena_enabled = false
}
}
}
Expand Down
21 changes: 21 additions & 0 deletions terraform/environments/observability-platform/modules/grafana/athena-source/main.tf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
locals {
name = "${var.name}-athena"
}

data "grafana_data_source" "this" {
count = var.athena_enabled ? 1 : 0
name = "${var.name}-athena"
}

resource "grafana_data_source" "this" {
type = "athena"
name = "${var.athena_workgroup}-${var.athena_database}"
json_data_encoded = jsonencode({
defaultRegion = "eu-west-2"
authType = "ec2_iam_role"
assumeRoleArn = "arn:aws:iam::${var.account_id}:role/observability-platform"
externalId = var.name
database = var.athena_database
workgroup = var.athena_workgroup
})
}
9 changes: 9 additions & 0 deletions terraform/environments/observability-platform/modules/grafana/athena-source/providers.tf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
terraform {
required_providers {
grafana = {
source = "grafana/grafana"
version = "~> 3.0"
}
}
required_version = "~> 1.0"
}
20 changes: 20 additions & 0 deletions terraform/environments/observability-platform/modules/grafana/athena-source/variables.tf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
variable "name" {
type = string
}

variable "account_id" {
type = string
}

variable "athena_enabled" {
type = bool
default = false
}

variable "athena_workgroup" {
type = string
}

variable "athena_database" {
type = string
}
21 changes: 21 additions & 0 deletions terraform/environments/observability-platform/modules/grafana/team/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,27 @@ resource "grafana_data_source_permission" "xray" {
}
}

data "grafana_data_source" "athena" {
for_each = {
for name, config in var.aws_accounts : name => config if config.athena_enabled
}

name = "${each.key}-athena"
}

resource "grafana_data_source_permission" "athena" {
for_each = {
for name, config in var.aws_accounts : name => config if config.athena_enabled
}

datasource_uid = trimprefix(data.grafana_data_source.athena[each.key].id, "1:")

permissions {
team_id = grafana_team.this.id
permission = "Query"
}
}

data "grafana_data_source" "amazon_prometheus" {
for_each = {
for name, account in var.aws_accounts : name => account if account.amazon_prometheus_query_enabled
Expand Down
3 changes: 3 additions & 0 deletions terraform/environments/observability-platform/modules/grafana/team/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,8 @@ variable "aws_accounts" {
cloudwatch_enabled = optional(bool)
xray_enabled = optional(bool)
amazon_prometheus_query_enabled = optional(bool)
athena_enabled = optional(bool)
}))
}


29 changes: 28 additions & 1 deletion ...nments/observability-platform/modules/observability-platform/tenant-configuration/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,32 @@ module "amazon_prometheus_query_source" {
amazon_prometheus_workspace_id = each.value.amazon_prometheus_workspace_id
}

locals {
flattened_athena_configs = flatten([
for env_name, env_data in var.aws_accounts : [
for config_name, config_data in try(env_data.athena_config, {}) : {
key = "${env_name}-${config_name}"
account_id = nonsensitive(var.environment_management.account_ids[env_name])
database = config_data.database
workgroup = config_data.workgroup
}
] if env_data.athena_enabled == true
])
}

module "athena_source" {
for_each = {
for config in local.flattened_athena_configs : config.key => config
}

source = "../../grafana/athena-source"

name = each.key
account_id = each.value.account_id
athena_workgroup = each.value.workgroup
athena_database = each.value.database
}

module "prometheus_push" {
for_each = {
for name, account in var.aws_accounts : name => account if account.prometheus_push_enabled
Expand All @@ -61,6 +87,7 @@ module "team" {
depends_on = [
module.xray_source,
module.cloudwatch_source,
module.amazon_prometheus_query_source
module.amazon_prometheus_query_source,
module.athena_source
]
}
6 changes: 6 additions & 0 deletions ...s/observability-platform/modules/observability-platform/tenant-configuration/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ variable "aws_accounts" {
amazon_prometheus_query_enabled = optional(bool)
amazon_prometheus_workspace_id = optional(string)
xray_enabled = optional(bool)
athena_enabled = optional(bool)
athena_config = optional(map(object({
database = string
workgroup = string
})))
}))
default = {
default = {
Expand All @@ -26,6 +31,7 @@ variable "aws_accounts" {
amazon_prometheus_query_enabled = false
amazon_prometheus_workspace_id = ""
xray_enabled = false
athena_enabled = false
}
}
}

0 comments on commit 6bfe015

Please sign in to comment.