MM-60286: Fix deletion of DB param group

We need to use the create_before_destroy
lifecycle configuration to allow Terraform
to properly change settings of the parameter
group: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/db_parameter_group#create_before_destroy-lifecycle-configuration.

Additionally, we induce a sleep of 30 seconds
to delay the destruction of param group
after the destruction of db cluster. This
ensures that everything happens in the correct
order.

While here, we also remove installation
of inux-tools-aws-lts-22.04.
More context on that: https://community.mattermost.com/core/pl/a7krecfgjpyoty8q7edy61k4ao

Bonus fix: we create the redis security
group conditionally as well. Just one of those
small oversights.

https://mattermost.atlassian.net/browse/MM-60286

deployment/terraform/assets/cluster.tf

@@ -318,6 +318,7 @@ resource "aws_rds_cluster" "db_cluster" {
   vpc_security_group_ids = [aws_security_group.db[0].id]
 }
 
+# This resource will create (potentially immediately) after aws_db_parameter_group.db_params_group
 resource "aws_rds_cluster_instance" "cluster_instances" {
   tags = {
     Name = "${var.cluster_name}-db-${count.index}"
@@ -331,13 +332,16 @@ resource "aws_rds_cluster_instance" "cluster_instances" {
   apply_immediately            = true
   auto_minor_version_upgrade   = false
   performance_insights_enabled = var.db_enable_performance_insights
-  db_parameter_group_name      = length(var.db_parameters) > 0 ? "${var.cluster_name}-db-pg" : ""
+  db_parameter_group_name      = length(var.db_parameters) > 0 ? aws_db_parameter_group.db_params_group.name : ""
   availability_zone            = var.aws_az
   db_subnet_group_name         = var.app_instance_count > 0 && var.db_instance_count > 0 && length(var.cluster_subnet_ids.database) > 1 ? aws_db_subnet_group.db[0].name : ""
+
+  depends_on = [time_sleep.wait_30_seconds]
 }
 
+# This resource will destroy (at least) 30 seconds after aws_rds_cluster_instance.cluster_instances
 resource "aws_db_parameter_group" "db_params_group" {
-  name   = "${var.cluster_name}-db-pg"
+  name_prefix   = "${var.cluster_name}-db-pg"
   family = var.db_instance_engine == "aurora-mysql" ? "aurora-mysql8.0" : "aurora-postgresql14"
   dynamic "parameter" {
     for_each = var.db_parameters
@@ -347,6 +351,16 @@ resource "aws_db_parameter_group" "db_params_group" {
       apply_method = parameter.value["apply_method"]
     }
   }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "time_sleep" "wait_30_seconds" {
+  depends_on = [aws_db_parameter_group.db_params_group]
+
+  destroy_duration = "30s"
 }
 
 resource "aws_instance" "loadtest_agent" {
@@ -629,7 +643,7 @@ resource "aws_security_group" "redis" {
     security_groups = [aws_security_group.app[0].id, aws_security_group.metrics[0].id]
   }
 
-  count = 1
+  count      = var.redis_enabled ? 1 : 0
 }
 
 resource "aws_security_group" "elastic" {

deployment/terraform/assets/provisioners/agent.sh

@@ -13,7 +13,7 @@ do
         echo "Attempt ${n}"
         sudo apt-get -y update && \
         sudo apt-get install -y prometheus-node-exporter && \
-        sudo apt-get install -y numactl linux-tools-aws linux-tools-aws-lts-22.04 && \
+        sudo apt-get install -y numactl linux-tools-aws && \
         # Install OpenTelemetry collector, using ubuntu user to avoid permission issues
         wget https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.110.0/otelcol-contrib_0.110.0_linux_amd64.deb && \
         sudo dpkg -i otelcol-contrib_0.110.0_linux_amd64.deb && \

deployment/terraform/assets/provisioners/app.sh

@@ -19,7 +19,7 @@ do
       sudo apt-get install -y mysql-client-8.0 && \
       sudo apt-get install -y postgresql-client-14 && \
       sudo apt-get install -y prometheus-node-exporter && \
-      sudo apt-get install -y numactl linux-tools-aws linux-tools-aws-lts-22.04 && \
+      sudo apt-get install -y numactl linux-tools-aws && \
       wget https://github.com/streamer45/netpeek/releases/download/v0.1.4/netpeek-v0.1.4 && \
       sudo mv netpeek-v* /usr/local/bin/netpeek && sudo chmod +x /usr/local/bin/netpeek && \
       # Install OpenTelemetry collector, using ubuntu user to avoid permission issues