Merge branch 'master' into feat/outline-healthcheck

README.md

@@ -46,7 +46,7 @@ server to become Sticky's production server.
 The code in this repository depends on the following software:
 
 - [nix]
-- A Discord webhook, which should be put in `ansible/.discord-webhook`
+- Two Discord webhooks, which should be put in `ansible/.env`
 
 Furthermore, the Ansible playbooks assume a **vanilla Ubuntu 20.04 host** to be
 deployed on.
@@ -157,16 +157,14 @@ performed, which are explained in detail in [this guide][deployment-new-producti
 ##### On your local terminal:
 1. Install the Nix package manager via the steps on this page: https://nixos.org/download.html
 
-1. Download the repository and enter the folder.
+2. Download the repository and enter the folder.
 `$ git clone https://github.com/svsticky/sadserver`
 `$ cd sadserver/ansible`
 
-1. Create a file `.discord-webhook` containing the webhook to be used for Discord notifications. Put the value of the
-[`slack_notifications_webhook_url` secret]
-(https://vault.bitwarden.com/#/vault?search=slack&itemId=c02392e6-728e-4bce-ae4e-ae900153afc9&cipherId=c02392e6-728e-4bce-ae4e-ae900153afc9)
-in that file. You will need to login to bitwarden as `[email protected]` to read this secret.
-Yes, the secret is still called `slack_notifications_webhook_url` because of legacy reasons, but you
-should not just change the name because it is used in the ansible code.
+3. Copy `sample.env` to `.env` and fill in the missing discord webhooks.
+You will need to login to bitwarden as `[email protected]` to read this secret.
+(If you find the `slack_notifications_webhook_url`, do _not_ change the name of
+the secret for legacy reasons. Ansible's code is dependent on the name.)
 
 To install all required dependencies, run the following command to enter a nix shell.
 `$ nix-shell`
@@ -209,9 +207,6 @@ Godspeed!
   [ansible/group_vars/all/websites.yml]:ansible/group_vars_example/all/websites.yml
   [Ansible Vault]:http://docs.ansible.com/ansible/playbooks_vault.html
   [inventory]:https://docs.ansible.com/ansible/intro_inventory.html
-  [slacktee]:https://github.com/course-hero/slacktee
-  [ansible]:https://github.com/ansible/ansible
-  [Bitwarden CLI]:https://help.bitwarden.com/article/cli/#download--install
   [deployment-new-production]:docs/deployment-new-production.md
   [IT Crowd]:mailto:[email protected]
   [deployment-guide]:#setting-up-the-staging-and-production-environment

ansible/.gitignore

@@ -24,3 +24,5 @@ tramp
 #-- SublimeText
 *.sublime-workspace
 #*.sublime-project
+
+.env

ansible/default.nix

@@ -13,6 +13,7 @@ let
     types-requests
     pyaml
     types-pyyaml
+    python-dotenv
   ]);
 
   linuxOnlyTools = if pkgs.stdenv.isLinux then [ pkgs.ansible-lint ] else [];

ansible/deploy.py

@@ -9,10 +9,24 @@
 import yaml
 
 from git.repo import Repo
-from typing import Optional
+from typing import Optional, List
+from dotenv import load_dotenv
 
 import scripts.bitwarden as bitwarden
 
+# Import .env file
+load_dotenv()
+discord_webhook_staging_deployments = os.getenv("DISCORD_WEBHOOK_STAGING_DEPLOYMENTS")
+discord_webhook_production_deployments = os.getenv(
+    "DISCORD_WEBHOOK_PRODUCTION_DEPLOYMENTS"
+)
+
+if (
+    discord_webhook_production_deployments is None
+    or discord_webhook_staging_deployments is None
+):
+    print("At least one Discord webhook is missing. Please fill in the .env file")
+
 
 @click.command()
 @click.option(
@@ -90,11 +104,7 @@ def deploy(
     if check:
         arguments.append("--check")
 
-    if not tags is None:
-        arguments.append("--tags")
-        arguments.append(tags)
-
-    # From-until logic:
+    # First determine all roles defined by from-until logic
     with open("main.yml", "r") as yaml_file:
         data = yaml.safe_load(yaml_file)
 
@@ -114,27 +124,68 @@ def deploy(
         roles = roles[: roles.index(until_playbook) + 1]
         from_until = True
 
-    final_roles = ",".join(roles)
     if from_until:
+        from_until_roles = roles  # Filtered by the from_until logic
+    else:
+        from_until_roles = []
+
+    # Then add all roles specified manually
+    if tags is not None:
+        manual_roles = [role.strip() for role in tags.split(",")]
+    else:
+        manual_roles = []
+
+    # Finally, pass down all roles specified, if any
+    specified_roles = from_until_roles + manual_roles
+    if specified_roles:
         arguments.append("--tags")
-        arguments.append(final_roles)
+        arguments.append(",".join(specified_roles))
 
     arguments.append(playbook)
 
+    if host == "production":
+        discord_deployment_webhook = str(discord_webhook_production_deployments)
+    else:
+        discord_deployment_webhook = str(discord_webhook_staging_deployments)
+
     if not check:
-        notify_deploy_start(playbook, host, user, branch, revision)
+        notify_deploy_start(
+            playbook,
+            host,
+            user,
+            branch,
+            revision,
+            specified_roles,
+            discord_deployment_webhook,
+        )
 
     print("Running the following playbook:")
     print(" ".join(arguments))
 
     try:
         subprocess.run(arguments, check=True, env=env)
         if not check:
-            notify_deploy_succes(playbook, host, branch, revision)
+            notify_deploy_succes(
+                playbook,
+                host,
+                user,
+                branch,
+                revision,
+                specified_roles,
+                discord_deployment_webhook,
+            )
 
     except subprocess.CalledProcessError:
         if not check:
-            notify_deploy_failure(playbook, host, branch, revision)
+            notify_deploy_failure(
+                playbook,
+                host,
+                user,
+                branch,
+                revision,
+                specified_roles,
+                discord_deployment_webhook,
+            )
 
 
 def current_branch_name() -> str:
@@ -148,41 +199,66 @@ def current_git_revision() -> str:
 
 
 def notify_deploy_start(
-    playbook: str, host: str, user: str, git_branch: str, git_revision: str
+    playbook: str,
+    host: str,
+    user: str,
+    git_branch: str,
+    git_revision: str,
+    roles: List[str],
+    discord_webhook: str,
 ) -> None:
+    roles_str = ", ".join(roles)
     discord_notify(
-        f"*Deployment of playbook {playbook} in {host} environment started by {user}*\n"
-        + f'_(branch: {git_branch} - revision "{git_revision}")_',
+        f"**Deployment of playbook {playbook} to {host} started by {user}**\n"
+        + f'Branch: {git_branch} - revision "{git_revision}"\n'
+        + f"Roles: {roles_str}",
         ":construction:",
         "#46c4ff",
+        discord_webhook,
     )
 
 
 def notify_deploy_succes(
-    playbook: str, host: str, git_branch: str, git_revision: str
+    playbook: str,
+    host: str,
+    user: str,
+    git_branch: str,
+    git_revision: str,
+    roles: List[str],
+    discord_webhook: str,
 ) -> None:
+    roles_str = ", ".join(roles)
     discord_notify(
-        f"*Deployment of playbook {playbook} in {host} environment succesfully completed*\n"
-        + f'_(branch: {git_branch} - revision "{git_revision}")_',
+        f"**Deployment of playbook {playbook} to {host}, started by {user}, succesfully completed**\n"
+        + f'Branch: {git_branch} - revision "{git_revision}"\n'
+        + f"Roles: {roles_str}",
         ":construction:",
         "good",
+        discord_webhook,
     )
 
 
 def notify_deploy_failure(
-    playbook: str, host: str, git_branch: str, git_revision: str
+    playbook: str,
+    host: str,
+    user: str,
+    git_branch: str,
+    git_revision: str,
+    roles: List[str],
+    discord_webhook: str,
 ) -> None:
+    roles_str = ", ".join(roles)
     discord_notify(
-        f"*Deployment of playbook {playbook} in {host} environment FAILED!*\n"
-        + f'_(branch: {git_branch} - revision "{git_revision}")_',
+        f"**Deployment of playbook {playbook} to {host}, started by {user}, FAILED!**\n"
+        + f'Branch: {git_branch} - revision "{git_revision}"\n'
+        + f"Roles: {roles_str}",
         ":exclamation:",
         "danger",
+        discord_webhook,
     )
 
 
-def discord_notify(message: str, icon: str, color: str) -> None:
-    url = get_discord_webhook().strip()
-
+def discord_notify(message: str, icon: str, color: str, webhook_url: str) -> None:
     data = {
         "username": "Ansible",
         "attachments": [
@@ -199,7 +275,7 @@ def discord_notify(message: str, icon: str, color: str) -> None:
     }
 
     r = requests.post(
-        url,
+        webhook_url,
         json=data,
     )
 
@@ -232,18 +308,5 @@ def verify_on_latest_master(host: str) -> None:
         click.ClickException("There is uncommited in your working tree or staging area")
 
 
-def get_discord_webhook() -> str:
-    webhook_filename = ".discord-webhook"
-
-    if not os.path.exists(webhook_filename):
-        raise click.ClickException(
-            "Please create .discord-webhook with a webhook URL for deploy notifications"
-        )
-    else:
-        # maybe should be a try-catch block here
-        with open(webhook_filename) as f:
-            return f.read()
-
-
 if __name__ == "__main__":
     deploy()

ansible/group_vars/all/websites.yml

@@ -25,6 +25,14 @@ websites:
     alternative_names: []
     oauth2_callback_url: "/api/v1/login"
 
+  - name: "voeljeveilig.{{ canonical_hostname }}"
+    user: "voeljeveilig"
+    authenticated: true
+    custom_config: true
+    state: "present"
+    alternative_names: []
+    oauth2_callback_url: "/api/oauth/callback"
+
   - name: "doorgeefluik.{{ canonical_hostname }}"
     user: "doorgeefluik"
     custom_config: true

ansible/group_vars/production/vars.yml

@@ -30,7 +30,7 @@ secret_oauth2_proxy:
 koala_env:
   environment: "production"
   oidc_signing_key_location: "/var/www/koala/signing-key.pem"
-  git_ref: "v2.23.1"
+  git_ref: "v2.26.0"
 
 secret_koala:
   # To change, generate new token using 'rake secret', and recompile + deploy
@@ -156,11 +156,14 @@ secret_grafana:
   loki_id: "{{ vault_secret_grafana.loki_id }}"
 
 chroma:
-  git_tag: "latest"
+  git_tag: "0.1.26"
 
 secret_chroma:
   s3_access_key: "{{ vault_secret_chroma.s3_access_key }}"
   s3_secret_access_key: "{{ vault_secret_chroma.s3_secret_access_key }}"
   s3_region: "{{ vault_secret_chroma.s3_region }}"
   s3_bucket: "{{ vault_secret_chroma.s3_bucket }}"
   service_tokens: "{{ vault_secret_chroma.service_tokens }}"
+
+fallacious_rooster:
+  git_tag: "0.1.4"

ansible/group_vars/staging/vars.yml

@@ -155,3 +155,6 @@ secret_chroma:
   s3_region: "{{ vault_secret_chroma.s3_region }}"
   s3_bucket: "{{ vault_secret_chroma.s3_bucket }}"
   service_tokens: "{{ vault_secret_chroma.service_tokens }}"
+
+fallacious_rooster:
+  git_tag: "0.1.4"

ansible/main.yml

@@ -85,3 +85,5 @@
     # exists. Otherwise fail2ban would miss the log files it has to monitor.
     - role: "fail2ban"
       tags: "always"
+    - role: "rooster"
+      tags: "rooster"

ansible/roles/chroma/tasks/main.yaml

@@ -39,7 +39,7 @@
 
 - name: "Download and extract chroma frontend"
   ansible.builtin.unarchive:
-    src: https://github.com/svsticky/chroma/releases/{{ chroma.git_tag }}/download/frontend.tar.gz
+    src: https://github.com/svsticky/chroma/releases/download/{{ chroma.git_tag }}/frontend.tar.gz
     dest: "/var/www/chroma/frontend"
     remote_src: true
     owner: "chroma"
@@ -50,7 +50,7 @@
 
 - name: "Download and extract chroma docs"
   ansible.builtin.unarchive:
-    src: https://github.com/svsticky/chroma/releases/{{ chroma.git_tag }}/download/docs.tar.gz
+    src: https://github.com/svsticky/chroma/releases/download/{{ chroma.git_tag }}/docs.tar.gz
     dest: "/var/www/chroma/docs"
     remote_src: true
     owner: "chroma"
@@ -61,7 +61,7 @@
 
 - name: "Download and extract chroma server"
   ansible.builtin.get_url:
-    url: https://github.com/svsticky/chroma/releases/{{ chroma.git_tag }}/download/server-x86_64-unknown-linux-musl
+    url: https://github.com/svsticky/chroma/releases/download/{{ chroma.git_tag }}/server-x86_64-unknown-linux-musl
     dest: "/var/www/chroma/server"
     owner: "chroma"
     group: "chroma"

ansible/roles/rooster/handlers/main.yaml

@@ -0,0 +1,5 @@
+---
+- name: "restart fallacious-rooster"
+  ansible.builtin.service:
+    name: "fallacious-rooster"
+    state: "restarted"