Merge pull request #53 from Discngine/1.9.3

1.9.3

CHANGELOG.md

@@ -4,6 +4,30 @@ Dates are ISO8601 / YYYY-MM-DD
 Version equals the version of the equivalent 3decision helm chart release
 Add a `-0` with incrementing numbers in case of a terraform / cloudformation change without equivalent helm changes 
 
+## [3.0.7] - 2024-07-12
+### Cloudformation
+#### Added
+- Added a permission to the user to update the description of IAM roles @JonathanManass
+
+#### Changed
+- Nothing
+
+#### Removed
+- Nothing
+
+### Terraform
+#### Added
+- Added the reloader annotation to the sqlcl container @JonathanManass
+- Added a reprocessing timestamp for the transfer from redis to oracle @aphilippejolivel
+
+#### Changed
+- Updated 3decision helm chart default value to 3.0.7 @aphilippejolivel
+- Updated the time at which secrets update from every 30 days to every first sunday of the month at 2 AM @JonathanManass
+
+#### Removed
+- Removed the Redis bucket and its references @JonathanManass
+---
+
 ## [3.0.5] - 2024-05-02
 ### Cloudformation
 #### Added

templates/discngine-3decision-existing-vpc.template.yaml

@@ -625,6 +625,7 @@ Resources:
                   - iam:DeletePolicyVersion
                   - iam:CreateServiceLinkedRole
                   - iam:UpdateAssumeRolePolicy
+                  - iam:UpdateRoleDescription
                 Resource:
                   - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:oidc-provider/*
                   - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/3decision-*
@@ -870,6 +871,8 @@ Resources:
             terraform init -backend-config=backend.conf -reconfigure -upgrade
 
             sudo yum install jq -y
+
+            terraform state rm module.storage[\"redis\"].aws_s3_bucket.bucket
             terraform state pull > terraform.tfstate
             final_snapshot_value=$(jq -r '.resources[] | select(.name == "public_data") | .instances[0].attributes.final_snapshot' terraform.tfstate)
 

terraform/main.tf

@@ -180,43 +180,12 @@ module "volumes" {
 }
 
 locals {
-  buckets = toset(["redis", "alphafold"])
+  buckets = toset(["alphafold"])
   allowed_service_accounts = {
-    "redis"     = ["system:serviceaccount:*:redis-s3-upload", "system:serviceaccount:*:sentinel-redis"]
     "alphafold" = ["system:serviceaccount:tdecision:*"]
   }
 }
 
-moved {
-  from = module.storage.aws_iam_role.redis_role
-  to   = module.storage["redis"].aws_iam_role.role
-}
-
-moved {
-  from = module.storage.aws_iam_role_policy_attachment.secret_rotator_lambda_role_policy_attachment
-  to   = module.storage["redis"].aws_iam_role_policy_attachment.policy_attachment
-}
-
-moved {
-  from = module.storage.aws_iam_policy.redis_policy
-  to   = module.storage["redis"].aws_iam_policy.policy
-}
-
-moved {
-  from = module.storage.aws_s3_bucket.bucket
-  to   = module.storage["redis"].aws_s3_bucket.bucket
-}
-
-moved {
-  from = module.storage.aws_s3_bucket_ownership_controls.bucket_ownership
-  to   = module.storage["redis"].aws_s3_bucket_ownership_controls.bucket_ownership
-}
-
-moved {
-  from = module.storage.aws_s3_bucket_public_access_block.public_access_block
-  to   = module.storage["redis"].aws_s3_bucket_public_access_block.public_access_block
-}
-
 module "storage" {
   for_each = local.buckets
   source   = "./modules/storage"
@@ -271,9 +240,7 @@ module "kubernetes" {
   jwt_ssh_private         = module.secrets.jwt_private_key
   jwt_ssh_public          = module.secrets.jwt_public_key
   secrets_access_role_arn = module.secrets.secrets_access_role_arn
-  redis_bucket_name       = module.storage["redis"].bucket_name
   alphafold_bucket_name   = module.storage["alphafold"].bucket_name
-  redis_s3_role_arn       = module.storage["redis"].s3_role_arn
   alphafold_s3_role_arn   = module.storage["alphafold"].s3_role_arn
   public_volume_id        = module.volumes.public_volume_id
   private_volume_id       = module.volumes.private_volume_id

terraform/modules/kubernetes/main.tf

@@ -140,6 +140,8 @@ metadata:
   labels:
     role: help
     app: sqlcl
+  annotations:
+    reloader.stakater.com/auto: "true"
 spec:
   replicas: 1
   selector:
@@ -258,27 +260,6 @@ resource "kubernetes_secret" "nest_authentication_secrets" {
   depends_on = [kubernetes_namespace.tdecision_namespace, kubernetes_config_map_v1.aws_auth]
 }
 
-resource "kubectl_manifest" "sentinel_configmap_redis" {
-  for_each = toset([var.tdecision_chart.namespace, "redis-cluster"])
-
-  yaml_body = <<YAML
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: sentinel-backup-env-cm
-  namespace: ${each.key}
-data:
-  BUCKET_NAME: ${var.redis_bucket_name}
-  PROVIDER: aws
-YAML
-  depends_on = [
-    kubernetes_namespace.redis_namespace,
-    kubernetes_namespace.tdecision_namespace,
-    kubernetes_config_map_v1.aws_auth
-  ]
-}
-
 resource "kubernetes_job_v1" "af_bucket_files_push" {
 
   metadata {
@@ -372,11 +353,6 @@ locals {
   values_config = <<YAML
 global:
   storageClass: gp2-encrypted
-serviceAccount:
-  create: true
-  name: sentinel-redis
-  annotations:
-    eks.amazonaws.com/role-arn: ${var.redis_s3_role_arn}  
 commonConfiguration: |-
   # Enable AOF https://redis.io/topics/persistence#append-only-file
   appendonly no
@@ -393,31 +369,11 @@ master:
     ports:
       redis: 6380
 replica:
+  replicaCount: 1
   resources:
     requests:
       cpu: 1000m
       memory: 2Gi
-  extraVolumes:
-  - name: secret-key
-    secret:
-      secretName: ssh-key-secret
-      optional: true
-  initContainers:
-    - name: redis-pull-container
-      envFrom:
-      - configMapRef:
-          name: sentinel-backup-env-cm
-          optional: true
-      image: fra.ocir.io/discngine1/3decision_kube/redis-backup:0.0.1
-      command: ["./entrypoint.sh"]
-      args: ["pull"]
-      imagePullPolicy: Always
-      volumeMounts:
-      - mountPath: /root/.ssh/
-        name: secret-key
-        readOnly: true
-      - mountPath: /data
-        name: redis-data
 global:
   redis:
     password: lapin80
@@ -467,7 +423,6 @@ resource "helm_release" "sentinel_release" {
   timeout          = 1200
   values           = [local.values_config]
   depends_on = [
-    kubectl_manifest.sentinel_configmap_redis,
     kubernetes_storage_class_v1.encrypted_storage_class,
     kubernetes_config_map_v1.aws_auth,
     terraform_data.delete_sentinel_statefulsets
@@ -518,23 +473,22 @@ resource "time_static" "tdecision_version_timestamp" {
   }
 }
 
-resource "time_static" "redis_timestamp" {}
 
 locals {
   # Update this list for any version of the 3decision helm chart needing reprocessing
   public_interaction_registration_reprocessing_version_list = ["2.3.3"]
   private_structure_reprocessing_version_list               = ["2.3.4"]
   missing_structure_registration_reprocessing_version_list  = ["2.3.7"]
   alphafold_structure_registration_version_list             = ["3.0.1"]
+  redis_to_oracle_transfer_version_list                     = ["3.0.7"]
 
-  reprocessing_timestamp       = timeadd(time_static.tdecision_version_timestamp.rfc3339, "24h")
-  redis_reprocessing_timestamp = timeadd(time_static.redis_timestamp.rfc3339, "4h")
-  redis_configmap_timestamp    = timeadd(local.redis_reprocessing_timestamp, "24h")
+  reprocessing_timestamp = timeadd(time_static.tdecision_version_timestamp.rfc3339, "24h")
 
   launch_public_interaction_registration_reprocessing = contains(local.public_interaction_registration_reprocessing_version_list, var.tdecision_chart.version)
   launch_private_structure_reprocessing               = contains(local.private_structure_reprocessing_version_list, var.tdecision_chart.version)
   launch_missing_structure_registration_reprocessing  = contains(local.missing_structure_registration_reprocessing_version_list, var.tdecision_chart.version)
   launch_alphafold_structure_registration             = contains(local.alphafold_structure_registration_version_list, var.tdecision_chart.version)
+  launch_redis_to_oracle_transfer                     = contains(local.redis_to_oracle_transfer_version_list, var.tdecision_chart.version)
 }
 
 locals {
@@ -578,15 +532,14 @@ nest:
       value: ${local.launch_public_interaction_registration_reprocessing ? local.reprocessing_timestamp : "2000-01-01T00:00:00"}
     rcsb_str_reg_repro_timestamp:
       value: ${local.launch_missing_structure_registration_reprocessing ? local.reprocessing_timestamp : "2000-01-01T00:00:00"}
-    redis_synchro_timestamp:
-      value: ${local.redis_reprocessing_timestamp}
-    private_structures_reprocessing_event_types:
-      value: rcsbStructureRegistration,sequenceMappingAnalysis,pocketDetectionAnalysis,ligandCavityOverlapAnalysis,pocketFeaturesAnalysis,interactionRegistration
     private_structure_reprocessing_timestamp:
       value: ${local.launch_private_structure_reprocessing ? local.reprocessing_timestamp : "2000-01-01T00:00:00"}
     alphafold_structure_registration_timestamp:
       name: ALPHAFOLD_STRUCTURE_REGISTRATION_TIMESTAMP
       value: ${local.launch_alphafold_structure_registration ? local.reprocessing_timestamp : "2000-01-01T00:00:00"}
+    event_log_data_transfer_timestamp:
+      name: "EVENT_LOG_DATA_TRANSFER_TIMESTAMP"
+      value: ${local.launch_redis_to_oracle_transfer ? local.reprocessing_timestamp : "2000-01-01T00:00:00"}
   env:
     okta_client_id:
       name: OKTA_CLIENT_ID
@@ -624,10 +577,6 @@ rbac:
       serviceAccountName: s3-access
       annotations:
         eks.amazonaws.com/role-arn: ${var.alphafold_s3_role_arn}
-  cluster:
-    redisBackup:
-      annotations:
-        eks.amazonaws.com/role-arn: ${var.redis_s3_role_arn}
 redis:
   nodeSelector: null
 pocket_features:
@@ -750,111 +699,6 @@ rm clean_choral.yaml
   depends_on = [kubectl_manifest.ClusterExternalSecret]
 }
 
-# This keeps the CONFORMATION_DEPENDENT_ANALYSIS_EVENT_TTL value low in a seperate 3decision configmap until a day after redis reprocessing
-# If this is not done the reprocessing will cache for too long and break the app
-# The patch is only done once since patching the configmap restarts most pods, so it has to be rerun if the chart is updated since that will reset the value
-resource "terraform_data" "redis_synchro_configmap_change" {
-  triggers_replace = [local.redis_configmap_timestamp, helm_release.tdecision_chart.metadata.0.revision]
-  provisioner "local-exec" {
-    interpreter = ["/bin/bash", "-c"]
-    command     = <<EOF
-target_time=$(date -d "${local.redis_configmap_timestamp}" +"%s")
-current_time=$(date +"%s")
-time_diff=$(($${target_time} - $${current_time}))
-if [ $${time_diff} -lt 0 ]; then
-  echo "redis synchro already passed... not launching pod."
-  exit 0
-fi
-
-aws eks update-kubeconfig --name EKS-tdecision --kubeconfig $HOME/.kube/config
-export KUBECONFIG=$HOME/.kube/config
-cat > redis_synchro.yaml << YAML
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: synchro-redis
----
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: synchro-redis
-rules:
-  - apiGroups:
-    - ""
-    resourceNames:
-    - nest-env-configmap
-    resources:
-    - configmaps
-    verbs:
-    - patch
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: synchro-redis
-subjects:
-- kind: ServiceAccount
-  name: synchro-redis
-roleRef:
-  kind: Role
-  name: synchro-redis
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: synchro-redis
-spec:
-  serviceAccountName: synchro-redis
-  restartPolicy: Never
-  containers:
-    - name: synchro-redis
-      image: alpine/curl:8.4.0
-      command: ["/bin/sh", "-c"]
-      args:
-        - |
-          target_time=\$(date -d \$(echo "${local.redis_configmap_timestamp}" | tr -d "TZ") +"%s")
-          current_time=\$(date +"%s")
-          time_diff=\$((\$${target_time} - \$${current_time}))
-          if [ \$${time_diff} -gt 0 ]; then
-              sec=/var/run/secrets/kubernetes.io/serviceaccount
-              curl -sS \
-                -H "Authorization: Bearer \$(cat \$${sec}/token)" \
-                -H "Content-Type: application/strategic-merge-patch+json" \
-                --cacert \$${sec}/ca.crt \
-                --request PATCH \
-                --data '{"data":{"CONFORMATION_DEPENDENT_ANALYSIS_EVENT_TTL":"600"}}' \
-                https://"\$${KUBERNETES_SERVICE_HOST}"/api/v1/namespaces/${var.tdecision_chart.namespace}/configmaps/nest-env-configmap
-
-              echo "Sleeping for \$${time_diff} seconds until ${local.redis_configmap_timestamp}"
-
-              sleep \$${time_diff}
-
-              sec=/var/run/secrets/kubernetes.io/serviceaccount
-              curl -sS \
-                -H "Authorization: Bearer \$(cat \$${sec}/token)" \
-                -H "Content-Type: application/strategic-merge-patch+json" \
-                --cacert \$${sec}/ca.crt \
-                --request PATCH \
-                --data '{"data":{"CONFORMATION_DEPENDENT_ANALYSIS_EVENT_TTL":"7890000"}}' \
-                https://"\$${KUBERNETES_SERVICE_HOST}"/api/v1/namespaces/${var.tdecision_chart.namespace}/configmaps/nest-env-configmap
-              echo "Woke up at \$(date)"
-          else
-              echo "The target time has already passed."
-          fi
-YAML
-kubectl delete -f redis_synchro.yaml -n ${var.tdecision_chart.namespace}
-kubectl apply -f redis_synchro.yaml -n ${var.tdecision_chart.namespace}
-rm -f redis_synchro.yaml
-    EOF
-  }
-  lifecycle {
-    ignore_changes = all
-  }
-  depends_on = [helm_release.tdecision_chart]
-}
-
 resource "helm_release" "tdecision_chart" {
   name       = var.tdecision_chart.name
   repository = var.tdecision_chart.repository

terraform/modules/kubernetes/variables.tf

@@ -28,9 +28,7 @@ variable "additional_eks_roles_arn" {}
 variable "additional_eks_users_arn" {}
 variable "custom_ami" {}
 variable "secrets_access_role_arn" {}
-variable "redis_bucket_name" {}
 variable "alphafold_bucket_name" {}
-variable "redis_s3_role_arn" {}
 variable "alphafold_s3_role_arn" {}
 variable "public_volume_id" {}
 variable "private_volume_id" {}

terraform/modules/secrets/main.tf

@@ -231,7 +231,8 @@ resource "aws_secretsmanager_secret_rotation" "db_master_password_rotation" {
   rotation_lambda_arn = aws_lambda_function.secret_rotator_lambda.arn
 
   rotation_rules {
-    automatically_after_days = 30
+    # Run on the first sunday of the month at 2 AM
+    schedule_expression = "cron(0 2 ? * SUN#1 *)"
   }
   depends_on = [aws_secretsmanager_secret_version.db_passwords_version]
 }

terraform/modules/storage/main.tf

@@ -49,7 +49,7 @@ resource "aws_iam_role" "role" {
     ]
   })
 
-  description = "Role designed to access the Redis access point inside EKS pods"
+  description = "Role designed to access the access point inside EKS pods"
 }
 
 

terraform/variables.tf

@@ -210,7 +210,7 @@ variable "tdecision_chart" {
     repository       = optional(string, "oci://fra.ocir.io/discngine1/3decision_kube")
     chart            = optional(string, "tdecision")
     namespace        = optional(string, "tdecision")
-    version          = optional(string, "3.0.5")
+    version          = optional(string, "3.0.7")
     create_namespace = optional(bool, true)
   })
   default = {}