From aaa462c009d76e488f5f2d1f536a75e90ed85cae Mon Sep 17 00:00:00 2001 From: Yannick Evers Date: Wed, 3 Apr 2024 09:03:58 +0000 Subject: [PATCH 1/9] Use generated config/credentials on dev --- .../templates/configmap.yml.j2 | 4 ++++ .../templates/deployment.yml.j2 | 13 ++++++++++++- .../templates/job_init_db.yml.j2 | 13 ++++++++++++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/ansible/roles/schulcloud-calendar-core/templates/configmap.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/configmap.yml.j2 index 5f0fc88..1033ba9 100644 --- a/ansible/roles/schulcloud-calendar-core/templates/configmap.yml.j2 +++ b/ansible/roles/schulcloud-calendar-core/templates/configmap.yml.j2 @@ -16,3 +16,7 @@ data: {% if CAL_IS_MIGRATION is defined %} IS_MIGRATION: "{{ CAL_IS_MIGRATION }}" {% endif %} +{% if WITH_BRANCH_POSTGRES_DB_MANAGEMENT is defined and WITH_BRANCH_POSTGRES_DB_MANAGEMENT|bool %} + DB_HOST: {{ POSTGRES_HOST }} + DB_SSL: true +{% endif %} \ No newline at end of file diff --git a/ansible/roles/schulcloud-calendar-core/templates/deployment.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/deployment.yml.j2 index 551e356..bf7401f 100644 --- a/ansible/roles/schulcloud-calendar-core/templates/deployment.yml.j2 +++ b/ansible/roles/schulcloud-calendar-core/templates/deployment.yml.j2 @@ -43,10 +43,21 @@ spec: ports: - containerPort: 3000 envFrom: + - secretRef: + name: calendar-secret - configMapRef: name: calendar-configmap +{% if WITH_BRANCH_POSTGRES_DB_MANAGEMENT is defined and WITH_BRANCH_POSTGRES_DB_MANAGEMENT|bool %} - secretRef: - name: calendar-secret + name: pg-calendar-secret + env: + - name: DB_PASSWORD + value: "${DB_USER_PASSWORD}" + - name: DB_USERNAME + value: "${DB_USER}" + - name: DB_DATABASE + value: "${DB_NAME}" +{% endif %} livenessProbe: failureThreshold: 3 httpGet: diff --git a/ansible/roles/schulcloud-calendar-init/templates/job_init_db.yml.j2 b/ansible/roles/schulcloud-calendar-init/templates/job_init_db.yml.j2 index 11be6e0..a0c61f0 100644 --- a/ansible/roles/schulcloud-calendar-init/templates/job_init_db.yml.j2 +++ b/ansible/roles/schulcloud-calendar-init/templates/job_init_db.yml.j2 @@ -11,10 +11,21 @@ spec: - name: calendar-db-init image: schulcloud/infra-tools:latest envFrom: + - secretRef: + name: calendar-secret - configMapRef: name: calendar-configmap +{% if WITH_BRANCH_POSTGRES_DB_MANAGEMENT is defined and WITH_BRANCH_POSTGRES_DB_MANAGEMENT|bool %} - secretRef: - name: calendar-secret + name: pg-calendar-secret + env: + - name: DB_PASSWORD + value: "${DB_USER_PASSWORD}" + - name: DB_USERNAME + value: "${DB_USER}" + - name: DB_DATABASE + value: "${DB_NAME}" +{% endif %} volumeMounts: - name: script mountPath: /update.sh From 25187a7a572387cb5f5315128babdb6884dbd3b0 Mon Sep 17 00:00:00 2001 From: Yannick Evers Date: Thu, 4 Apr 2024 12:07:41 +0000 Subject: [PATCH 2/9] Try adding quotes --- .../roles/schulcloud-calendar-core/templates/configmap.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/schulcloud-calendar-core/templates/configmap.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/configmap.yml.j2 index 1033ba9..e35e4a6 100644 --- a/ansible/roles/schulcloud-calendar-core/templates/configmap.yml.j2 +++ b/ansible/roles/schulcloud-calendar-core/templates/configmap.yml.j2 @@ -17,6 +17,6 @@ data: IS_MIGRATION: "{{ CAL_IS_MIGRATION }}" {% endif %} {% if WITH_BRANCH_POSTGRES_DB_MANAGEMENT is defined and WITH_BRANCH_POSTGRES_DB_MANAGEMENT|bool %} - DB_HOST: {{ POSTGRES_HOST }} - DB_SSL: true + DB_HOST: "{{ POSTGRES_HOST }}" + DB_SSL: "true" {% endif %} \ No newline at end of file From 17afe22af9048c77f326251ebf63b789868d4ccc Mon Sep 17 00:00:00 2001 From: Yannick Evers Date: Thu, 4 Apr 2024 14:34:07 +0000 Subject: [PATCH 3/9] Fix parentheses --- .../schulcloud-calendar-core/templates/deployment.yml.j2 | 6 +++--- .../schulcloud-calendar-init/templates/job_init_db.yml.j2 | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ansible/roles/schulcloud-calendar-core/templates/deployment.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/deployment.yml.j2 index bf7401f..c1a02bc 100644 --- a/ansible/roles/schulcloud-calendar-core/templates/deployment.yml.j2 +++ b/ansible/roles/schulcloud-calendar-core/templates/deployment.yml.j2 @@ -52,11 +52,11 @@ spec: name: pg-calendar-secret env: - name: DB_PASSWORD - value: "${DB_USER_PASSWORD}" + value: "$(DB_USER_PASSWORD)" - name: DB_USERNAME - value: "${DB_USER}" + value: "$(DB_USER)" - name: DB_DATABASE - value: "${DB_NAME}" + value: "$(DB_NAME)" {% endif %} livenessProbe: failureThreshold: 3 diff --git a/ansible/roles/schulcloud-calendar-init/templates/job_init_db.yml.j2 b/ansible/roles/schulcloud-calendar-init/templates/job_init_db.yml.j2 index a0c61f0..1619dd9 100644 --- a/ansible/roles/schulcloud-calendar-init/templates/job_init_db.yml.j2 +++ b/ansible/roles/schulcloud-calendar-init/templates/job_init_db.yml.j2 @@ -20,11 +20,11 @@ spec: name: pg-calendar-secret env: - name: DB_PASSWORD - value: "${DB_USER_PASSWORD}" + value: "$(DB_USER_PASSWORD)" - name: DB_USERNAME - value: "${DB_USER}" + value: "$(DB_USER)" - name: DB_DATABASE - value: "${DB_NAME}" + value: "$(DB_NAME)" {% endif %} volumeMounts: - name: script From ceb096ba35daa1449ec3e89cebcb676342eb0f96 Mon Sep 17 00:00:00 2001 From: Yannick Evers Date: Fri, 5 Apr 2024 13:09:09 +0000 Subject: [PATCH 4/9] Add check if Job already exists (can't be patched) --- .../roles/schulcloud-calendar-init/tasks/main.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ansible/roles/schulcloud-calendar-init/tasks/main.yml b/ansible/roles/schulcloud-calendar-init/tasks/main.yml index bfa6d41..c2574dd 100644 --- a/ansible/roles/schulcloud-calendar-init/tasks/main.yml +++ b/ansible/roles/schulcloud-calendar-init/tasks/main.yml @@ -15,12 +15,22 @@ name: calendar-db-init-file when: not WITH_CALENDAR_INIT + - name: Test if init job exits + kubernetes.core.k8s_info: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + api_version: batch/v1 + kind: Job + name: calendar-db-init-job + register: calendar_init_job_present + when: WITH_CALENDAR_INIT + - name: Calendar db init job kubernetes.core.k8s: kubeconfig: ~/.kube/config namespace: "{{ NAMESPACE }}" template: job_init_db.yml.j2 - when: WITH_CALENDAR_INIT + when: WITH_CALENDAR_INIT and calendar_init_job_present.resources|length == 0 - name: Calendar db init job kubernetes.core.k8s: From d19e57cc7d49c1cec1db84a11fba9d4217b31df9 Mon Sep 17 00:00:00 2001 From: Yannick Evers Date: Fri, 5 Apr 2024 13:37:28 +0000 Subject: [PATCH 5/9] Fix typo --- ansible/roles/schulcloud-calendar-init/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/schulcloud-calendar-init/tasks/main.yml b/ansible/roles/schulcloud-calendar-init/tasks/main.yml index c2574dd..1c842b2 100644 --- a/ansible/roles/schulcloud-calendar-init/tasks/main.yml +++ b/ansible/roles/schulcloud-calendar-init/tasks/main.yml @@ -15,7 +15,7 @@ name: calendar-db-init-file when: not WITH_CALENDAR_INIT - - name: Test if init job exits + - name: Test if init job exists kubernetes.core.k8s_info: kubeconfig: ~/.kube/config namespace: "{{ NAMESPACE }}" From 85b5d943117c0e1cedb5f68885479b029e925c87 Mon Sep 17 00:00:00 2001 From: Yannick Evers Date: Thu, 11 Apr 2024 13:56:42 +0000 Subject: [PATCH 6/9] Add database creation task --- ansible/roles/schulcloud-calendar-core/tasks/main.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ansible/roles/schulcloud-calendar-core/tasks/main.yml b/ansible/roles/schulcloud-calendar-core/tasks/main.yml index 806482c..00c90d9 100644 --- a/ansible/roles/schulcloud-calendar-core/tasks/main.yml +++ b/ansible/roles/schulcloud-calendar-core/tasks/main.yml @@ -1,3 +1,10 @@ + - name: Create database + include_role: + name: dof_postgresql_management + vars: + database_name: calendar + when: WITH_BRANCH_POSTGRES_DB_MANAGEMENT is defined and WITH_BRANCH_POSTGRES_DB_MANAGEMENT + - name: Service kubernetes.core.k8s: kubeconfig: ~/.kube/config From d9156531560a3a7196858792e93fb0cf5c90ff04 Mon Sep 17 00:00:00 2001 From: Yannick Evers Date: Mon, 22 Apr 2024 09:13:51 +0000 Subject: [PATCH 7/9] Integrate/move postgres role into this one --- .../schulcloud-calendar-core/tasks/main.yml | 3 +- .../tasks/postgres_management.yml | 48 +++++++++++++++++ .../configmap-database-deletion.yml.j2 | 19 +++++++ .../configmap-database-init.yml.j2 | 19 +++++++ .../job-database-deletion.yml.j2 | 51 ++++++++++++++++++ .../job-database-init.yml.j2 | 53 +++++++++++++++++++ .../onepassword-pg-cluster.yml.j2 | 9 ++++ .../secret-database.yml.j2 | 12 +++++ 8 files changed, 212 insertions(+), 2 deletions(-) create mode 100644 ansible/roles/schulcloud-calendar-core/tasks/postgres_management.yml create mode 100644 ansible/roles/schulcloud-calendar-core/templates/postgres_management/configmap-database-deletion.yml.j2 create mode 100644 ansible/roles/schulcloud-calendar-core/templates/postgres_management/configmap-database-init.yml.j2 create mode 100644 ansible/roles/schulcloud-calendar-core/templates/postgres_management/job-database-deletion.yml.j2 create mode 100644 ansible/roles/schulcloud-calendar-core/templates/postgres_management/job-database-init.yml.j2 create mode 100644 ansible/roles/schulcloud-calendar-core/templates/postgres_management/onepassword-pg-cluster.yml.j2 create mode 100644 ansible/roles/schulcloud-calendar-core/templates/postgres_management/secret-database.yml.j2 diff --git a/ansible/roles/schulcloud-calendar-core/tasks/main.yml b/ansible/roles/schulcloud-calendar-core/tasks/main.yml index 00c90d9..37f44ce 100644 --- a/ansible/roles/schulcloud-calendar-core/tasks/main.yml +++ b/ansible/roles/schulcloud-calendar-core/tasks/main.yml @@ -1,6 +1,5 @@ - name: Create database - include_role: - name: dof_postgresql_management + include_tasks: postgres_management.yml vars: database_name: calendar when: WITH_BRANCH_POSTGRES_DB_MANAGEMENT is defined and WITH_BRANCH_POSTGRES_DB_MANAGEMENT diff --git a/ansible/roles/schulcloud-calendar-core/tasks/postgres_management.yml b/ansible/roles/schulcloud-calendar-core/tasks/postgres_management.yml new file mode 100644 index 0000000..0fe21b6 --- /dev/null +++ b/ansible/roles/schulcloud-calendar-core/tasks/postgres_management.yml @@ -0,0 +1,48 @@ +- name: Add or Update Postgres Cluster Secret by 1Password + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: postgres_management/onepassword-pg-cluster.yml.j2 + when: ONEPASSWORD_OPERATOR is defined and ONEPASSWORD_OPERATOR|bool + +- name: Check if secret with database credentials already exists + kubernetes.core.k8s_info: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + kind: Secret + name: "pg-{{ database_name }}-secret" + register: db_secret_present + +- name: Create Secret for the database (if not existing) + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: postgres_management/secret-database.yml.j2 + when: db_secret_present.resources|length == 0 + +- name: Create ConfigMap with Script + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: postgres_management/configmap-database-init.yml.j2 + apply: yes + +- name: Create/execute database configuration script + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: postgres_management/job-database-init.yml.j2 + +- name: Create ConfigMap with Script for database deletion + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: postgres_management/configmap-database-deletion.yml.j2 + apply: yes + +- name: Create suspended Job for database deletion + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: postgres_management/job-database-deletion.yml.j2 + apply: yes diff --git a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/configmap-database-deletion.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/postgres_management/configmap-database-deletion.yml.j2 new file mode 100644 index 0000000..3338fbf --- /dev/null +++ b/ansible/roles/schulcloud-calendar-core/templates/postgres_management/configmap-database-deletion.yml.j2 @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: pg-configmap-deletion + namespace: {{ NAMESPACE }} + labels: + app: postgres +data: + config_script.sh: | + #!/bin/bash + DB_PREFIX="{{ POSTGRES_PREFIX }}" + if [[ {{ '${#DB_PREFIX}' }} -le 5 ]]; then + echo "Postgres prefix \"{{ POSTGRES_PREFIX }}\" seems too short. Dropping all matching databases could be dangerous. Aborting." + exit 1 + fi + echo "Delete databases starting with {{ POSTGRES_PREFIX }}" + echo "SELECT 'DROP DATABASE ' || quote_ident(datname) || ' WITH (FORCE);' FROM pg_database WHERE datname LIKE '{{ POSTGRES_PREFIX | replace('_','#_')}}%' ESCAPE '#' \gexec" | psql -d postgres -w + echo "Delete users starting with {{ POSTGRES_PREFIX }}" + echo "SELECT 'DROP USER ' || quote_ident(usename) || ';' FROM pg_catalog.pg_user WHERE usename LIKE '{{ POSTGRES_PREFIX | replace('_','#_')}}%' ESCAPE '#' \gexec" | psql -d postgres -w \ No newline at end of file diff --git a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/configmap-database-init.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/postgres_management/configmap-database-init.yml.j2 new file mode 100644 index 0000000..8e34ca2 --- /dev/null +++ b/ansible/roles/schulcloud-calendar-core/templates/postgres_management/configmap-database-init.yml.j2 @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: pg-{{ database_name }}-configmap-init + namespace: {{ NAMESPACE }} + labels: + app: postgres +data: + config_script.sh: | + #!/bin/bash + echo "Create owner of the DB" + echo "SELECT 'CREATE USER $DB_USER' WHERE NOT EXISTS (SELECT FROM pg_user WHERE usename = '$DB_USER')\gexec" | psql -d postgres -w + echo "GRANT $DB_USER TO $PGUSER;" | psql -d postgres -w + echo "Set/update password for user $DB_USER" + echo "ALTER USER $DB_USER WITH ENCRYPTED PASSWORD '$DB_USER_PASSWORD';" | psql -d postgres -w + echo "Create database" + echo "SELECT 'CREATE DATABASE $DB_NAME OWNER $DB_USER' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '$DB_NAME')\gexec" | psql -d postgres -w + echo "Revoke permissions for public role" + echo "REVOKE ALL ON DATABASE $DB_NAME FROM PUBLIC;" | psql -d postgres -w diff --git a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/job-database-deletion.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/postgres_management/job-database-deletion.yml.j2 new file mode 100644 index 0000000..6529bbe --- /dev/null +++ b/ansible/roles/schulcloud-calendar-core/templates/postgres_management/job-database-deletion.yml.j2 @@ -0,0 +1,51 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: pg-deletion-job + namespace: {{ NAMESPACE }} +spec: + template: + metadata: + labels: + app: postgres + spec: + volumes: + - name: config-script + configMap: + name: pg-configmap-deletion + # 711 in decimal is 457 + defaultMode: 457 + containers: + - name: psql-config + image: {{ POSTGRES_JOB_IMAGE }} + command: + - /bin/bash + - -c + args: + - /scripts/config_script.sh + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - name: config-script + mountPath: /scripts/ + env: + - name: PGHOST + value: {{ POSTGRES_HOST }} + - name: PGUSER + valueFrom: + secretKeyRef: + name: pg-cluster-secret + key: username + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: pg-cluster-secret + key: password + restartPolicy: Never + suspend: true + ttlSecondsAfterFinished: 0 \ No newline at end of file diff --git a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/job-database-init.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/postgres_management/job-database-init.yml.j2 new file mode 100644 index 0000000..f8d730d --- /dev/null +++ b/ansible/roles/schulcloud-calendar-core/templates/postgres_management/job-database-init.yml.j2 @@ -0,0 +1,53 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: pg-{{ database_name }}-init-job-{{ 1000000 | random | hash('md5') }} + namespace: {{ NAMESPACE }} +spec: + template: + metadata: + labels: + app: postgres + spec: + volumes: + - name: config-script + configMap: + name: pg-{{ database_name }}-configmap-init + # 711 in decimal is 457 + defaultMode: 457 + containers: + - name: psql-config + image: {{ POSTGRES_JOB_IMAGE }} + command: + - /bin/bash + - -c + args: + - /scripts/config_script.sh + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - name: config-script + mountPath: /scripts/ + envFrom: + - secretRef: + name: pg-{{ database_name }}-secret + env: + - name: PGHOST + value: {{ POSTGRES_HOST }} + - name: PGUSER + valueFrom: + secretKeyRef: + name: pg-cluster-secret + key: username + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: pg-cluster-secret + key: password + restartPolicy: Never + ttlSecondsAfterFinished: 86400 \ No newline at end of file diff --git a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/onepassword-pg-cluster.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/postgres_management/onepassword-pg-cluster.yml.j2 new file mode 100644 index 0000000..e23e932 --- /dev/null +++ b/ansible/roles/schulcloud-calendar-core/templates/postgres_management/onepassword-pg-cluster.yml.j2 @@ -0,0 +1,9 @@ +apiVersion: onepassword.com/v1 +kind: OnePasswordItem +metadata: + name: pg-cluster-secret + namespace: {{ NAMESPACE }} + labels: + app: postgres +spec: + itemPath: "vaults/{{ ONEPASSWORD_OPERATOR_VAULT }}/items/pg-cluster-schulcloud" \ No newline at end of file diff --git a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/secret-database.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/postgres_management/secret-database.yml.j2 new file mode 100644 index 0000000..3dc45f5 --- /dev/null +++ b/ansible/roles/schulcloud-calendar-core/templates/postgres_management/secret-database.yml.j2 @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: pg-{{ database_name }}-secret + namespace: {{ NAMESPACE }} + labels: + app: postgres +type: Opaque +data: + DB_USER: "{{ (POSTGRES_PREFIX + database_name) | b64encode }}" + DB_USER_PASSWORD: "{{ lookup('ansible.builtin.password', '/dev/null') | b64encode }}" + DB_NAME: "{{ (POSTGRES_PREFIX + database_name) | b64encode }}" \ No newline at end of file From f47476bc11a502b8db43dbfba9fbe2047add6c41 Mon Sep 17 00:00:00 2001 From: Yannick Evers Date: Tue, 23 Apr 2024 10:19:14 +0000 Subject: [PATCH 8/9] Move cluster secret and db deletion back to general role --- .../schulcloud-calendar-core/tasks/main.yml | 35 +++++++++++-- .../tasks/postgres_management.yml | 48 ----------------- .../configmap-database-init.yml.j2 | 4 +- .../templates/configmap.yml.j2 | 2 +- .../job-database-init.yml.j2 | 16 +++--- .../configmap-database-deletion.yml.j2 | 19 ------- .../job-database-deletion.yml.j2 | 51 ------------------- .../onepassword-pg-cluster.yml.j2 | 9 ---- .../secret-database.yml.j2 | 12 ----- .../templates/secret-database.yml.j2 | 12 +++++ 10 files changed, 53 insertions(+), 155 deletions(-) delete mode 100644 ansible/roles/schulcloud-calendar-core/tasks/postgres_management.yml rename ansible/roles/schulcloud-calendar-core/templates/{postgres_management => }/configmap-database-init.yml.j2 (92%) rename ansible/roles/schulcloud-calendar-core/templates/{postgres_management => }/job-database-init.yml.j2 (74%) delete mode 100644 ansible/roles/schulcloud-calendar-core/templates/postgres_management/configmap-database-deletion.yml.j2 delete mode 100644 ansible/roles/schulcloud-calendar-core/templates/postgres_management/job-database-deletion.yml.j2 delete mode 100644 ansible/roles/schulcloud-calendar-core/templates/postgres_management/onepassword-pg-cluster.yml.j2 delete mode 100644 ansible/roles/schulcloud-calendar-core/templates/postgres_management/secret-database.yml.j2 create mode 100644 ansible/roles/schulcloud-calendar-core/templates/secret-database.yml.j2 diff --git a/ansible/roles/schulcloud-calendar-core/tasks/main.yml b/ansible/roles/schulcloud-calendar-core/tasks/main.yml index 37f44ce..91c14ba 100644 --- a/ansible/roles/schulcloud-calendar-core/tasks/main.yml +++ b/ansible/roles/schulcloud-calendar-core/tasks/main.yml @@ -1,8 +1,33 @@ - - name: Create database - include_tasks: postgres_management.yml - vars: - database_name: calendar - when: WITH_BRANCH_POSTGRES_DB_MANAGEMENT is defined and WITH_BRANCH_POSTGRES_DB_MANAGEMENT + - name: Check if secret with database credentials already exists + kubernetes.core.k8s_info: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + kind: Secret + name: "pg-calendar-secret" + register: db_secret_present + when: WITH_BRANCH_POSTGRES_DB_MANAGEMENT + + - name: Create Secret for the database (if not existing) + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: secret-database.yml.j2 + when: WITH_BRANCH_POSTGRES_DB_MANAGEMENT and db_secret_present.resources|length == 0 + + - name: Create ConfigMap with database configuration script + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: configmap-database-init.yml.j2 + apply: yes + when: WITH_BRANCH_POSTGRES_DB_MANAGEMENT + + - name: Create/execute database configuration script + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: job-database-init.yml.j2 + when: WITH_BRANCH_POSTGRES_DB_MANAGEMENT - name: Service kubernetes.core.k8s: diff --git a/ansible/roles/schulcloud-calendar-core/tasks/postgres_management.yml b/ansible/roles/schulcloud-calendar-core/tasks/postgres_management.yml deleted file mode 100644 index 0fe21b6..0000000 --- a/ansible/roles/schulcloud-calendar-core/tasks/postgres_management.yml +++ /dev/null @@ -1,48 +0,0 @@ -- name: Add or Update Postgres Cluster Secret by 1Password - kubernetes.core.k8s: - kubeconfig: ~/.kube/config - namespace: "{{ NAMESPACE }}" - template: postgres_management/onepassword-pg-cluster.yml.j2 - when: ONEPASSWORD_OPERATOR is defined and ONEPASSWORD_OPERATOR|bool - -- name: Check if secret with database credentials already exists - kubernetes.core.k8s_info: - kubeconfig: ~/.kube/config - namespace: "{{ NAMESPACE }}" - kind: Secret - name: "pg-{{ database_name }}-secret" - register: db_secret_present - -- name: Create Secret for the database (if not existing) - kubernetes.core.k8s: - kubeconfig: ~/.kube/config - namespace: "{{ NAMESPACE }}" - template: postgres_management/secret-database.yml.j2 - when: db_secret_present.resources|length == 0 - -- name: Create ConfigMap with Script - kubernetes.core.k8s: - kubeconfig: ~/.kube/config - namespace: "{{ NAMESPACE }}" - template: postgres_management/configmap-database-init.yml.j2 - apply: yes - -- name: Create/execute database configuration script - kubernetes.core.k8s: - kubeconfig: ~/.kube/config - namespace: "{{ NAMESPACE }}" - template: postgres_management/job-database-init.yml.j2 - -- name: Create ConfigMap with Script for database deletion - kubernetes.core.k8s: - kubeconfig: ~/.kube/config - namespace: "{{ NAMESPACE }}" - template: postgres_management/configmap-database-deletion.yml.j2 - apply: yes - -- name: Create suspended Job for database deletion - kubernetes.core.k8s: - kubeconfig: ~/.kube/config - namespace: "{{ NAMESPACE }}" - template: postgres_management/job-database-deletion.yml.j2 - apply: yes diff --git a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/configmap-database-init.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/configmap-database-init.yml.j2 similarity index 92% rename from ansible/roles/schulcloud-calendar-core/templates/postgres_management/configmap-database-init.yml.j2 rename to ansible/roles/schulcloud-calendar-core/templates/configmap-database-init.yml.j2 index 8e34ca2..833c27a 100644 --- a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/configmap-database-init.yml.j2 +++ b/ansible/roles/schulcloud-calendar-core/templates/configmap-database-init.yml.j2 @@ -1,10 +1,10 @@ apiVersion: v1 kind: ConfigMap metadata: - name: pg-{{ database_name }}-configmap-init + name: pg-calendar-configmap-init namespace: {{ NAMESPACE }} labels: - app: postgres + app: calendar-postgres-init data: config_script.sh: | #!/bin/bash diff --git a/ansible/roles/schulcloud-calendar-core/templates/configmap.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/configmap.yml.j2 index e35e4a6..10b7795 100644 --- a/ansible/roles/schulcloud-calendar-core/templates/configmap.yml.j2 +++ b/ansible/roles/schulcloud-calendar-core/templates/configmap.yml.j2 @@ -17,6 +17,6 @@ data: IS_MIGRATION: "{{ CAL_IS_MIGRATION }}" {% endif %} {% if WITH_BRANCH_POSTGRES_DB_MANAGEMENT is defined and WITH_BRANCH_POSTGRES_DB_MANAGEMENT|bool %} - DB_HOST: "{{ POSTGRES_HOST }}" + DB_HOST: "{{ POSTGRES_MANAGEMENT_HOST }}" DB_SSL: "true" {% endif %} \ No newline at end of file diff --git a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/job-database-init.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/job-database-init.yml.j2 similarity index 74% rename from ansible/roles/schulcloud-calendar-core/templates/postgres_management/job-database-init.yml.j2 rename to ansible/roles/schulcloud-calendar-core/templates/job-database-init.yml.j2 index f8d730d..ad780d6 100644 --- a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/job-database-init.yml.j2 +++ b/ansible/roles/schulcloud-calendar-core/templates/job-database-init.yml.j2 @@ -1,23 +1,23 @@ apiVersion: batch/v1 kind: Job metadata: - name: pg-{{ database_name }}-init-job-{{ 1000000 | random | hash('md5') }} + name: pg-calendar-init-job-{{ 1000000 | random | hash('md5') }} namespace: {{ NAMESPACE }} spec: template: metadata: labels: - app: postgres + app: calendar-postgres-init spec: volumes: - name: config-script configMap: - name: pg-{{ database_name }}-configmap-init + name: pg-calendar-configmap-init # 711 in decimal is 457 defaultMode: 457 containers: - - name: psql-config - image: {{ POSTGRES_JOB_IMAGE }} + - name: psql-calendar-config + image: {{ POSTGRES_MANAGEMENT_JOB_IMAGE }} command: - /bin/bash - -c @@ -35,10 +35,10 @@ spec: mountPath: /scripts/ envFrom: - secretRef: - name: pg-{{ database_name }}-secret + name: pg-calendar-secret env: - name: PGHOST - value: {{ POSTGRES_HOST }} + value: {{ POSTGRES_MANAGEMENT_HOST }} - name: PGUSER valueFrom: secretKeyRef: @@ -50,4 +50,4 @@ spec: name: pg-cluster-secret key: password restartPolicy: Never - ttlSecondsAfterFinished: 86400 \ No newline at end of file + ttlSecondsAfterFinished: 1800 \ No newline at end of file diff --git a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/configmap-database-deletion.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/postgres_management/configmap-database-deletion.yml.j2 deleted file mode 100644 index 3338fbf..0000000 --- a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/configmap-database-deletion.yml.j2 +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: pg-configmap-deletion - namespace: {{ NAMESPACE }} - labels: - app: postgres -data: - config_script.sh: | - #!/bin/bash - DB_PREFIX="{{ POSTGRES_PREFIX }}" - if [[ {{ '${#DB_PREFIX}' }} -le 5 ]]; then - echo "Postgres prefix \"{{ POSTGRES_PREFIX }}\" seems too short. Dropping all matching databases could be dangerous. Aborting." - exit 1 - fi - echo "Delete databases starting with {{ POSTGRES_PREFIX }}" - echo "SELECT 'DROP DATABASE ' || quote_ident(datname) || ' WITH (FORCE);' FROM pg_database WHERE datname LIKE '{{ POSTGRES_PREFIX | replace('_','#_')}}%' ESCAPE '#' \gexec" | psql -d postgres -w - echo "Delete users starting with {{ POSTGRES_PREFIX }}" - echo "SELECT 'DROP USER ' || quote_ident(usename) || ';' FROM pg_catalog.pg_user WHERE usename LIKE '{{ POSTGRES_PREFIX | replace('_','#_')}}%' ESCAPE '#' \gexec" | psql -d postgres -w \ No newline at end of file diff --git a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/job-database-deletion.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/postgres_management/job-database-deletion.yml.j2 deleted file mode 100644 index 6529bbe..0000000 --- a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/job-database-deletion.yml.j2 +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: pg-deletion-job - namespace: {{ NAMESPACE }} -spec: - template: - metadata: - labels: - app: postgres - spec: - volumes: - - name: config-script - configMap: - name: pg-configmap-deletion - # 711 in decimal is 457 - defaultMode: 457 - containers: - - name: psql-config - image: {{ POSTGRES_JOB_IMAGE }} - command: - - /bin/bash - - -c - args: - - /scripts/config_script.sh - resources: - limits: - cpu: 1000m - memory: 1Gi - requests: - cpu: 100m - memory: 200Mi - volumeMounts: - - name: config-script - mountPath: /scripts/ - env: - - name: PGHOST - value: {{ POSTGRES_HOST }} - - name: PGUSER - valueFrom: - secretKeyRef: - name: pg-cluster-secret - key: username - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: pg-cluster-secret - key: password - restartPolicy: Never - suspend: true - ttlSecondsAfterFinished: 0 \ No newline at end of file diff --git a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/onepassword-pg-cluster.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/postgres_management/onepassword-pg-cluster.yml.j2 deleted file mode 100644 index e23e932..0000000 --- a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/onepassword-pg-cluster.yml.j2 +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: onepassword.com/v1 -kind: OnePasswordItem -metadata: - name: pg-cluster-secret - namespace: {{ NAMESPACE }} - labels: - app: postgres -spec: - itemPath: "vaults/{{ ONEPASSWORD_OPERATOR_VAULT }}/items/pg-cluster-schulcloud" \ No newline at end of file diff --git a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/secret-database.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/postgres_management/secret-database.yml.j2 deleted file mode 100644 index 3dc45f5..0000000 --- a/ansible/roles/schulcloud-calendar-core/templates/postgres_management/secret-database.yml.j2 +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: pg-{{ database_name }}-secret - namespace: {{ NAMESPACE }} - labels: - app: postgres -type: Opaque -data: - DB_USER: "{{ (POSTGRES_PREFIX + database_name) | b64encode }}" - DB_USER_PASSWORD: "{{ lookup('ansible.builtin.password', '/dev/null') | b64encode }}" - DB_NAME: "{{ (POSTGRES_PREFIX + database_name) | b64encode }}" \ No newline at end of file diff --git a/ansible/roles/schulcloud-calendar-core/templates/secret-database.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/secret-database.yml.j2 new file mode 100644 index 0000000..cd593f9 --- /dev/null +++ b/ansible/roles/schulcloud-calendar-core/templates/secret-database.yml.j2 @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: pg-calendar-secret + namespace: {{ NAMESPACE }} + labels: + app: calendar-postgres-init +type: Opaque +data: + DB_USER: "{{ (POSTGRES_MANAGEMENT_PREFIX + 'calendar') | b64encode }}" + DB_USER_PASSWORD: "{{ lookup('ansible.builtin.password', '/dev/null') | b64encode }}" + DB_NAME: "{{ (POSTGRES_MANAGEMENT_PREFIX + 'calendar') | b64encode }}" \ No newline at end of file From 48b7804ff08f37da0e7480f60e5b756c0b11052e Mon Sep 17 00:00:00 2001 From: Yannick Evers Date: Tue, 23 Apr 2024 11:12:20 +0000 Subject: [PATCH 9/9] Add more labels for job (based on the ones of the deployment) --- .../templates/job-database-init.yml.j2 | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ansible/roles/schulcloud-calendar-core/templates/job-database-init.yml.j2 b/ansible/roles/schulcloud-calendar-core/templates/job-database-init.yml.j2 index ad780d6..6b480f3 100644 --- a/ansible/roles/schulcloud-calendar-core/templates/job-database-init.yml.j2 +++ b/ansible/roles/schulcloud-calendar-core/templates/job-database-init.yml.j2 @@ -3,11 +3,23 @@ kind: Job metadata: name: pg-calendar-init-job-{{ 1000000 | random | hash('md5') }} namespace: {{ NAMESPACE }} + labels: + app: calendar-postgres-init + app.kubernetes.io/part-of: schulcloud-verbund + app.kubernetes.io/name: calendar-postgres-init + app.kubernetes.io/component: calendar + app.kubernetes.io/managed-by: ansible + git.repo: {{ SCHULCLOUD_CALENDAR_REPO_NAME }} spec: template: metadata: labels: app: calendar-postgres-init + app.kubernetes.io/part-of: schulcloud-verbund + app.kubernetes.io/name: calendar-postgres-init + app.kubernetes.io/component: calendar + app.kubernetes.io/managed-by: ansible + git.repo: {{ SCHULCLOUD_CALENDAR_REPO_NAME }} spec: volumes: - name: config-script