WPB-14388 stackit env

stackIT/README.md

@@ -0,0 +1,102 @@
+# StackIT Deployment and Configuration Guide
+
+This guide outlines the steps to set up and deploy the StackIT environment, including DNS configuration, Minikube cluster creation, Docker container setup, and Helm chart deployment. Each task and its associated commands are provided for clarity and customization.
+
+---
+
+## Steps to Deploy StackIT
+
+### 1. Run the Ansible Playbook
+- Prepare DNS records, STackIT public IP and set up Cert Manager to start before next step.
+- Use the following command to set up the VM:
+  ```bash
+  ansible-playbook -i host stackIT/stackit-vm-setup.yml --private-key ~/.ssh/stackit_private_key
+  ```
+
+- **Optional Skips:**
+  Use the following variables to skip tasks:
+  ```bash
+  -e skip_install=true
+  -e skip_minikube=true
+  -e skip_docker=true
+  -e skip_ssh=true
+  -e skip_inventory=true
+  ```
+
+- **Artifacts and Tasks:**
+  - Minikube Kubernetes cluster and four Docker containers to support node requirements.
+  - Generate `hosts.ini` based on the IPs of above containers for further ansible operations on node
+  - Download wire-server-deploy artifacts based on the specified hash 
+  - Configure iptables rules for DNAT to Coturn and k8s Nginx Controller (used by Wire applications).
+
+---
+
+### 2. Commands to Run on the StackIT Nodes
+
+1. **Load the environment:**
+   ```bash
+   source stackIT/offline-env.sh
+   ```
+
+2. **Access the environment:**
+   ```bash
+   d bash
+   ```
+
+3. **Generate secrets:**
+   ```bash
+   bash bin/offline-secrets.sh
+   ```
+
+4. **Set up and configure the environment:**
+   Run the following to set up the AssetHost, loading containers for k8s cluster, sync time, cassandra, elasticsearch and minio:
+   ```bash
+   bash offline-cluster.sh
+   ```
+
+5. **Deploy Helm charts:**
+   Use the following script to set up Helm chart values and deploy them:
+   ```bash
+   bash stackIT/setting-values.sh
+   ```
+
+---
+
+## To-Dos
+
+1. **Modify `bin/offline-env.sh`:**
+   - Add definitions for Kubernetes config for kubectl for non-kubespray environments like in stackIT
+   - When Copying ssh env from the host drop or re-initialize the known_hosts to avoid ip change clashes
+
+2. **Update `bin/offline-cluster.sh`:**
+   - Remove references to `restund.yml`.
+   - Introduce a check for Kubespray to avoid execution if Minikube is already running.
+
+3. **Enhance Helm charts:**
+   - Ensure pods reload when there are changes in:
+     - ConfigMaps.
+     - Environment variables.
+     - Public IPs parsed at pod startup.
+   - Introduce hashing to track changes and trigger restarts as needed.
+   - Current upgrades don't restart the pods for example, sftd and coturn
+
+---
+
+### File Structure Highlights
+
+- **Ansible Playbook Files:**
+  - `stackIT/stackit-vm-setup.yml`
+
+- **Environment Scripts:**
+  - `stackIT/offline-env.sh`
+  - `../bin/offline-secrets.sh`
+
+- **Cluster and Helm Setup:**
+  - `../bin/offline-cluster.sh`
+  - `stackIT/setting-values.sh`
+
+---
+
+## Notes
+-  Read all the files involved before executing them to understand defaults.
+

stackIT/host

@@ -0,0 +1 @@
+StackIT-public-ip

stackIT/offline-env.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+ZAUTH_CONTAINER=$(sudo docker load -i $SCRIPT_DIR/../containers-adminhost/quay.io_wire_zauth_*.tar | awk '{print $3}')
+export ZAUTH_CONTAINER
+
+WSD_CONTAINER=$(sudo docker load -i $SCRIPT_DIR/../containers-adminhost/container-wire-server-deploy.tgz | awk '{print $3}')
+
+alias d="sudo docker run -it --network=host -v ${SSH_AUTH_SOCK:-nonexistent}:/ssh-agent -e SSH_AUTH_SOCK=/ssh-agent -v $HOME/.ssh:/root/.ssh -v $PWD:/wire-server-deploy -v /home
+/ubuntu/.kube:/root/.kube -v /home/ubuntu/.minikube:/home/ubuntu/.minikube -e KUBECONFIG=/root/.kube/config $WSD_CONTAINER"

stackIT/setting-values.sh

@@ -0,0 +1,191 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2087
+set -Eeuo pipefail
+
+BASE_DIR="/wire-server-deploy"
+TARGET_SYSTEM="kittensonfire.com"
+HOST_IP=$(wget -qO- https://api.ipify.org)
+SFT_NODE="minikube"
+COTURN_NODE="minikube-m03"
+COTURN_NODE_IP=$(kubectl get node $COTURN_NODE -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')
+CHART_URL="https://charts.jetstack.io/charts/cert-manager-v1.13.2.tgz"
+NGINX_K8S_NODE="minikube-m02"
+
+# it create the values.yaml from prod-values.example.yaml/example.yaml to values.yaml
+process_charts() {  
+
+  # values for cassandra-external, elasticsearch-external, minio-external are created from offline-cluster.sh - helm_external.yml
+  # List of Helm charts to process values are here:
+  charts=(
+    fake-aws demo-smtp
+    rabbitmq databases-ephemeral reaper wire-server webapp account-pages
+    team-settings smallstep-accomp cert-manager-ns
+    nginx-ingress-services sftd coturn
+  )
+
+  for chart in "${charts[@]}"; do
+    chart_dir="$BASE_DIR/values/$chart"
+
+    if [[ -d "$chart_dir" ]]; then
+      if [[ -f "$chart_dir/prod-values.example.yaml" ]]; then
+        if [[ ! -f "$chart_dir/values.yaml" ]]; then
+          cp "$chart_dir/prod-values.example.yaml" "$chart_dir/values.yaml"
+          echo "Used template prod-values.example.yaml to create $chart_dir/values.yaml"
+        fi
+      fi
+    fi
+
+  done
+
+  # some manual secrets
+  if [[ ! -f "$BASE_DIR/values/rabbitmq/secrets.yaml" ]]; then
+  cp "$BASE_DIR/values/rabbitmq/prod-secrets.example.yaml" "$BASE_DIR/values/rabbitmq/secrets.yaml"
+  echo "Used template prod-secrets.example.yaml to create $BASE_DIR/values/rabbitmq/secrets.yaml"
+  fi
+  if [[ ! -f "$BASE_DIR/values/team-settings/secrets.yaml" ]]; then
+  cp "$BASE_DIR/values/team-settings/prod-secrets.example.yaml" "$BASE_DIR/values/team-settings/secrets.yaml"
+  echo "Used template prod-secrets.example.yaml to create $BASE_DIR/values/team-settings/secrets.yaml"
+  fi
+  if [[ ! -f "$BASE_DIR/values/ingress-nginx-controller/values.yaml" ]]; then
+  cp "$BASE_DIR/values/ingress-nginx-controller/hetzner-ci.example.yaml" "$BASE_DIR/values/ingress-nginx-controller/values.yaml"
+  echo "Used template hetzner-ci.example.yaml to create $BASE_DIR/values/ingress-nginx-controller/values.yaml"
+  fi
+}
+
+process_values() {
+  TEMP_DIR=$(mktemp -d)
+  trap "rm -rf $TEMP_DIR" EXIT
+
+  # Fixing the hosts with TARGET_SYSTEM and setting the turn server
+  sed -e "s/example.com/$TARGET_SYSTEM/g" \
+      -e "s/# - \"turn:<IP of restund1>:80\"/- \"turn:$HOST_IP:3478\"/g" \
+      -e "s/# - \"turn:<IP of restund1>:80?transport=tcp\"/- \"turn:$HOST_IP:3478?transport=tcp\"/g" \
+      "$BASE_DIR/values/wire-server/values.yaml" > "$TEMP_DIR/wire-server-values.yaml"
+
+  # Fixing the hosts in webapp team-settings and account-pages charts
+  for chart in webapp team-settings account-pages; do
+    sed "s/example.com/$TARGET_SYSTEM/g" "$BASE_DIR/values/$chart/values.yaml" > "$TEMP_DIR/$chart-values.yaml"
+  done
+
+  # Setting certManager and DNS records
+  sed -e 's/useCertManager: false/useCertManager: true/g' \
+    -e "/certmasterEmail:$/s/certmasterEmail:/certmasterEmail: [email protected]/" \
+    -e "s/example.com/$TARGET_SYSTEM/" \
+    "$BASE_DIR/values/nginx-ingress-services/values.yaml" > "$TEMP_DIR/nginx-ingress-services-values.yaml"
+
+  # adding nodeSelector for ingress controller as it should run as Deployment in the k8s cluster i.e. lack of external load balancer
+  sed -e 's/kind: DaemonSet/kind: Deployment/' \
+      "$BASE_DIR/values/ingress-nginx-controller/values.yaml" > "$TEMP_DIR/ingress-nginx-controller-values.yaml"
+  if ! grep -q "kubernetes.io/hostname: $NGINX_K8S_NODE" "$TEMP_DIR/ingress-nginx-controller-values.yaml"; then
+    echo -e "    nodeSelector:\n      kubernetes.io/hostname: $NGINX_K8S_NODE" >> "$TEMP_DIR/ingress-nginx-controller-values.yaml"
+  fi 
+
+  # Fixing SFTD hosts and setting the cert-manager to http01 and setting the replicaCount to 1
+  sed -e "s/webapp.example.com/webapp.$TARGET_SYSTEM/" \
+      -e "s/sftd.example.com/sftd.$TARGET_SYSTEM/" \
+      -e 's/name: letsencrypt-prod/name: letsencrypt-http01/' \
+      -e "s/replicaCount: 3/replicaCount: 1/" \
+      "$BASE_DIR/values/sftd/values.yaml" > "$TEMP_DIR/sftd-values.yaml"
+
+  # Creating coturn values and secrets
+  ZREST_SECRET=$(grep -A1 turn "$BASE_DIR/values/wire-server/secrets.yaml" | grep secret | tr -d '"' | awk '{print $NF}')
+  cat >"$TEMP_DIR/coturn-secrets.yaml"<<EOF
+secrets:
+  zrestSecrets:
+    - "$ZREST_SECRET"
+EOF
+
+  cat >"$TEMP_DIR/coturn-values.yaml"<<EOF
+nodeSelector:
+  wire.com/role: coturn
+
+coturnTurnListenIP: "$COTURN_NODE_IP"
+coturnTurnRelayIP: "$COTURN_NODE_IP"
+coturnTurnExternalIP: '$HOST_IP'
+EOF
+
+  # Compare and copy files if different
+  for file in wire-server-values.yaml webapp-values.yaml team-settings-values.yaml account-pages-values.yaml \
+              nginx-ingress-services-values.yaml ingress-nginx-controller-values.yaml sftd-values.yaml; do
+    if ! cmp -s "$TEMP_DIR/$file" "$BASE_DIR/values/${file%-values.yaml}/values.yaml"; then
+      cp "$TEMP_DIR/$file" "$BASE_DIR/values/${file%-values.yaml}/values.yaml"
+      echo "Updating  $BASE_DIR/values/${file%-values.yaml}/values.yaml"
+    fi
+  done
+
+  if ! cmp -s "$TEMP_DIR/coturn-secrets.yaml" "$BASE_DIR/values/coturn/secrets.yaml"; then
+    cp "$TEMP_DIR/coturn-secrets.yaml" "$BASE_DIR/values/coturn/secrets.yaml"
+    echo "Updating $BASE_DIR/values/coturn/secrets.yaml"
+  fi
+
+  if ! cmp -s "$TEMP_DIR/coturn-values.yaml" "$BASE_DIR/values/coturn/values.yaml"; then
+    cp "$TEMP_DIR/coturn-values.yaml" "$BASE_DIR/values/coturn/values.yaml"
+    echo "Updating $BASE_DIR/values/coturn/values.yaml"
+  fi
+}
+
+
+deploy_charts() {
+  echo "Deploying cassandra, elasticsearch-external, minio-external, fake-aws, demo-smtp, rabbitmq, databases-ephemeral, reaper"
+
+  helm upgrade --install --wait cassandra-external $BASE_DIR/charts/cassandra-external --values $BASE_DIR/values/cassandra-external/values.yaml
+  helm upgrade --install --wait elasticsearch-external $BASE_DIR/charts/elasticsearch-external --values $BASE_DIR/values/elasticsearch-external/values.yaml
+  helm upgrade --install --wait minio-external $BASE_DIR/charts/minio-external --values $BASE_DIR/values/minio-external/values.yaml
+  helm upgrade --install --wait fake-aws $BASE_DIR/charts/fake-aws --values $BASE_DIR/values/fake-aws/values.yaml
+  helm upgrade --install --wait demo-smtp $BASE_DIR/charts/demo-smtp --values $BASE_DIR/values/demo-smtp/values.yaml
+  helm upgrade --install --wait rabbitmq $BASE_DIR/charts/rabbitmq --values $BASE_DIR/values/rabbitmq/values.yaml --values $BASE_DIR/values/rabbitmq/secrets.yaml
+  helm upgrade --install --wait databases-ephemeral $BASE_DIR/charts/databases-ephemeral --values $BASE_DIR/values/databases-ephemeral/values.yaml
+  helm upgrade --install --wait reaper $BASE_DIR/charts/reaper
+
+  echo "Printing current pods status:"
+  kubectl get pods --sort-by=.metadata.creationTimestamp
+
+
+  echo "Deploying wire-server, webapp, account-pages, team-settings, smallstep-accomp, ingress-nginx-controller"
+
+  helm upgrade --install --wait --timeout=15m0s wire-server $BASE_DIR/charts/wire-server --values $BASE_DIR/values/wire-server/values.yaml --values $BASE_DIR/values/wire-server/secrets.yaml
+  if [ -d "$BASE_DIR/charts/webapp" ]; then
+      helm upgrade --install --wait --timeout=15m0s webapp $BASE_DIR/charts/webapp --values $BASE_DIR/values/webapp/values.yaml
+  fi
+  if [ -d "$BASE_DIR/charts/account-pages" ]; then
+      helm upgrade --install --wait --timeout=15m0s account-pages $BASE_DIR/charts/account-pages --values $BASE_DIR/values/account-pages/values.yaml
+  fi
+  if [ -d "$BASE_DIR/charts/team-settings" ]; then
+      helm upgrade --install --wait --timeout=15m0s team-settings $BASE_DIR/charts/team-settings --values $BASE_DIR/values/team-settings/values.yaml --values $BASE_DIR/values/team-settings/secrets.yaml
+  fi
+
+  helm upgrade --install --wait --timeout=15m0s smallstep-accomp $BASE_DIR/charts/smallstep-accomp --values $BASE_DIR/values/smallstep-accomp/values.yaml
+  helm upgrade --install --wait --timeout=15m0s ingress-nginx-controller $BASE_DIR/charts/ingress-nginx-controller --values $BASE_DIR/values/ingress-nginx-controller/values.yaml
+
+  echo "Printing current pods status:"
+  kubectl get pods --sort-by=.metadata.creationTimestamp
+
+  echo "Deploying cert-manager-ns, nginx-ingress-services, sftd, coturn"
+
+  # downloading the chart if not present
+  if [[ ! -d "$BASE_DIR/charts/cert-manager" ]]; then
+    wget -qO- "$CHART_URL" | tar -xz -C "$BASE_DIR/charts"
+  fi
+
+  kubectl get namespace cert-manager-ns || kubectl create namespace cert-manager-ns
+  helm upgrade --install -n cert-manager-ns --set 'installCRDs=true' cert-manager  $BASE_DIR/charts/cert-manager
+
+  helm upgrade --install nginx-ingress-services charts/nginx-ingress-services -f  $BASE_DIR/values/nginx-ingress-services/values.yaml
+  kubectl get certificate
+
+  # select the node to deploy sftd
+  kubectl label node $SFT_NODE wire.com/role=sftd
+  helm upgrade --install sftd $BASE_DIR/charts/sftd --set 'nodeSelector.wire\.com/role=sftd' --set 'node_annotations="{'wire\.com/external-ip': '"$HOST_IP"'}"' --values  $BASE_DIR/values/sftd/values.yaml
+
+  kubectl label node $COTURN_NODE wire.com/role=coturn
+  kubectl annotate node $COTURN_NODE wire.com/external-ip="$HOST_IP" --overwrite
+  helm upgrade --install coturn ./charts/coturn --values  $BASE_DIR/values/coturn/values.yaml --values  $BASE_DIR/values/coturn/secrets.yaml
+
+  kubectl get pods --sort-by=.metadata.creationTimestamp
+  kubectl get pods --sort-by=.metadata.creationTimestamp -n cert-manager-ns
+
+}
+
+process_charts
+process_values
+deploy_charts