feat: allow disabling cloudwatch logs for es (#858)

* feat: allow disabling cloudwatch logs for es Adds a new configuration option to allow disabling cloudwatch for ES in case it's not required or the credentials being used don't have the appropriate permissions. * Update docs/config/deployer.md Co-authored-by: Alejandro García Montoro <[email protected]> --------- Co-authored-by: Alejandro García Montoro <[email protected]>
mattermost · Dec 3, 2024 · 5cf2fc2 · 5cf2fc2
1 parent 246292a
commit 5cf2fc2
Show file tree

Hide file tree

Showing 9 changed files with 47 additions and 18 deletions.
diff --git a/config/deployer.sample.json b/config/deployer.sample.json
@@ -32,7 +32,8 @@
     "RestoreTimeoutMinutes": 45,
     "ClusterTimeoutMinutes": 45,
     "ZoneAwarenessEnabled": false,
-    "ZoneAwarenessAZCount": 2
+    "ZoneAwarenessAZCount": 2,
+    "EnableCloudwatchLogs": true
   },
   "RedisSettings": {
     "Enabled": false,

diff --git a/config/deployer.sample.toml b/config/deployer.sample.toml
@@ -62,6 +62,7 @@ InstanceType = 'r6g.large.search'
 Version = 'Elasticsearch_7.10'
 ZoneAwarenessEnabled = false
 ZoneAwarenessAZCount = 2
+EnableCloudwatchLogs = true
 
 [ExternalBucketSettings]
 AmazonS3AccessKeyId = ''

diff --git a/deployment/config.go b/deployment/config.go
@@ -334,6 +334,8 @@ type ElasticSearchSettings struct {
 	ZoneAwarenessEnabled bool `default:"false"`
 	// ZoneAwarenessAZCount indicates the number of availability zones to use for zone awareness.
 	ZoneAwarenessAZCount int `default:"2" validate:"range:[1,3]"`
+	// EnableCloudwatchLogs indicates whether to enable Cloudwatch logs or not.
+	EnableCloudwatchLogs bool `default:"true"`
 }
 
 type RedisSettings struct {

diff --git a/deployment/terraform/assets/bindata.go b/deployment/terraform/assets/bindata.go
diff --git a/deployment/terraform/assets/elasticsearch.tf b/deployment/terraform/assets/elasticsearch.tf
@@ -67,9 +67,17 @@ resource "aws_iam_role_policy_attachment" "es_attach" {
    }
 */
 resource "aws_cloudwatch_log_group" "es_log_group" {
+  count = var.es_enable_cloudwatch_logs ? 1 : 0
   name = "${var.cluster_name}-log-group"
 }
 
+data "aws_subnets" "selected" {
+  filter {
+    name   = "vpc-id"
+    values = [var.cluster_vpc_id]
+  }
+}
+
 resource "aws_opensearch_domain" "es_server" {
   tags = {
     Name = "${var.cluster_name}-es_server"
@@ -79,7 +87,7 @@ resource "aws_opensearch_domain" "es_server" {
   engine_version = var.es_version
 
   vpc_options {
-    subnet_ids         = (length(var.cluster_subnet_ids.elasticsearch) > 0) ? tolist(var.cluster_subnet_ids.elasticsearch) : null
+    subnet_ids         = (length(var.cluster_subnet_ids.elasticsearch) > 0) ? tolist(var.cluster_subnet_ids.elasticsearch) : [element(tolist(data.aws_subnets.selected.ids), 0)]
     security_group_ids = [aws_security_group.elastic[0].id]
   }
 
@@ -116,9 +124,12 @@ resource "aws_opensearch_domain" "es_server" {
     aws_iam_service_linked_role.es,
   ]
 
-  log_publishing_options {
-    cloudwatch_log_group_arn = aws_cloudwatch_log_group.es_log_group.arn
-    log_type                 = "ES_APPLICATION_LOGS"
+  dynamic "log_publishing_options" {
+    for_each = var.es_enable_cloudwatch_logs ? [true] : []
+    content {
+      cloudwatch_log_group_arn = aws_cloudwatch_log_group.es_log_group.arn
+      log_type                 = "ES_APPLICATION_LOGS"
+    }
   }
 
   advanced_security_options {

diff --git a/deployment/terraform/assets/variables.tf b/deployment/terraform/assets/variables.tf
@@ -56,6 +56,9 @@ variable "es_zone_awareness_enabled" {
 variable "es_zone_awarness_availability_zone_count" {
 }
 
+variable "es_enable_cloudwatch_logs" {
+}
+
 # Proxy server
 
 variable "proxy_instance_count" {

diff --git a/deployment/terraform/create.go b/deployment/terraform/create.go
@@ -181,14 +181,16 @@ func (t *Terraform) Create(extAgent *ssh.ExtAgent, initData bool) error {
 	// policies: there can only be 10 such policies per region per account.
 	// Check the docs for more information:
 	// https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/cloudwatch_limits_cwl.html
-	if err = t.checkCloudWatchLogsPolicy(); err != nil {
-		if err != ErrNotFound {
-			return fmt.Errorf("failed to check CloudWatchLogs policy: %w", err)
-		}
+	if t.config.ElasticSearchSettings.EnableCloudwatchLogs {
+		if err = t.checkCloudWatchLogsPolicy(); err != nil {
+			if err != ErrNotFound {
+				return fmt.Errorf("failed to check CloudWatchLogs policy: %w", err)
+			}
 
-		mlog.Info("No CloudWatchLogs policy found, creating a new one")
-		if err := t.createCloudWatchLogsPolicy(); err != nil {
-			return fmt.Errorf("failed creating CloudWatchLogs policy")
+			mlog.Info("No CloudWatchLogs policy found, creating a new one")
+			if err := t.createCloudWatchLogsPolicy(); err != nil {
+				return fmt.Errorf("failed creating CloudWatchLogs policy")
+			}
 		}
 	}
 

diff --git a/deployment/terraform/utils.go b/deployment/terraform/utils.go
@@ -261,6 +261,7 @@ func (t *Terraform) getParams() []string {
 		"-var", fmt.Sprintf("es_snapshot_repository=%s", t.config.ElasticSearchSettings.SnapshotRepository),
 		"-var", fmt.Sprintf("es_zone_awareness_enabled=%t", t.config.ElasticSearchSettings.ZoneAwarenessEnabled),
 		"-var", fmt.Sprintf("es_zone_awarness_availability_zone_count=%d", t.config.ElasticSearchSettings.ZoneAwarenessAZCount),
+		"-var", fmt.Sprintf("es_enable_cloudwatch_logs=%t", t.config.ElasticSearchSettings.EnableCloudwatchLogs),
 		"-var", fmt.Sprintf("proxy_instance_count=%d", t.config.ProxyInstanceCount),
 		"-var", fmt.Sprintf("proxy_instance_type=%s", t.config.ProxyInstanceType),
 		"-var", fmt.Sprintf("ssh_public_key=%s", t.config.SSHPublicKey),

diff --git a/docs/config/deployer.md b/docs/config/deployer.md
@@ -211,6 +211,14 @@ The number of availability zones to use for the Elasticsearch cluster. This sett
 
 Check the [documentation](https://aws.amazon.com/blogs/big-data/increase-availability-for-amazon-opensearch-service-by-deploying-in-three-availability-zones/).
 
+### EnableCloudwatchLogs
+
+*bool* (Default: `true`)
+
+Whether to enable Cloudwatch logs for the Elasticsearch cluster.
+
+Check the [documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/AnalyzingLogData.html).
+
 ## JobServerSettings
 
 ### InstanceCount