refactored to separate agent and server runtimes + fixes + review com…

…ments

ansible/bootstrap.yml

@@ -310,6 +310,30 @@
   become: yes
   tags: k3s
   tasks:
-    - ansible.builtin.include_role:
+    - name: Install k3s
+      when: "'builder' in group_names"
+      ansible.builtin.include_role:
+        name: k3s
+        tasks_from: install.yml
+
+- hosts: k3s_server
+  become: yes
+  tags: k3s
+  tasks:
+    - name: Start k3s server
+      when: "'builder' not in group_names"
+      ansible.builtin.include_role:
+        name: k3s
+        tasks_from: server-runtime.yml
+
+- hosts: k3s_agent
+  become: yes
+  tags: k3s
+  tasks:
+    - name: Start k3s agents
+      when: "'builder' not in group_names"
+      vars: # set outside of role to allow compute init to define own value
+        k3s_bootstrap_token: "{{ hostvars[groups['k3s_server'] | first]._k3s_token_output.stdout | default('') }}"
+      ansible.builtin.include_role:
         name: k3s
-        tasks_from: "{{ 'install.yml' if 'builder' in group_names else 'runtime.yml' }}"
+        tasks_from: agent-runtime.yml

ansible/roles/k3s/defaults/main.yml

@@ -3,4 +3,6 @@ k3s_version: "v1.31.0+k3s1"
 k3s_selinux_release: v1.6.latest.1
 k3s_selinux_rpm_version: 1.6-1
 k3s_helm_version: v3.11.0
-k3s_bootstrap_token_expiry: 20m
+k3s_bootstrap_token: "{{ None }}" # ansible managed
+k3s_bootstrap_token_expiry: 10m
+k3s_server_name: "{{ None }}" # ansible managed

ansible/roles/k3s/tasks/agent-runtime.yml

@@ -0,0 +1,32 @@
+---
+
+- name: Template k3s agent env file
+  when: k3s_bootstrap_token != ""
+  ansible.builtin.template:
+    dest: /etc/systemd/system/k3s-agent.service.env
+    src: k3s-agent.service.env.j2
+  register: _k3s_agent_token_result
+
+- name: Ensure password directory exists
+  ansible.builtin.file: 
+    path: "/etc/rancher/node"
+    state: directory
+    owner: root
+    group: root
+    mode: 0640
+
+- name: Write node password
+  ansible.builtin.copy:
+    dest: /etc/rancher/node/password
+    content: "{{ vault_k3s_node_password }}"
+    owner: root
+    group: root
+    mode: 0640 # normal k3s install is 644 but that doesn't feel right
+
+- name: Start/restart k3s agent
+  when: _k3s_agent_token_result.changed
+  ansible.builtin.systemd:
+    name: k3s-agent
+    daemon_reload: true
+    state: restarted
+    enabled: true

ansible/roles/k3s/tasks/server-runtime.yml

@@ -0,0 +1,31 @@
+---
+
+- name: Template k3s env file
+  ansible.builtin.template:
+    dest: /etc/systemd/system/k3s.service.env
+    src: k3s.service.env.j2
+  register: _k3s_env_file_status
+
+- name: Start k3s server
+  ansible.builtin.systemd:
+    name: k3s
+    daemon_reload: "{{ _k3s_env_file_status.changed }}"
+    state: started
+    enabled: true
+
+# Possible race here as there is a delay between agents disconnecting and being registered as down, probably won't be hit in general use though
+- name: Check if k3s agents are connected
+  ignore_errors: true
+  ansible.builtin.shell:
+    cmd: kubectl get nodes --no-headers | grep -w Ready
+  register: _k3s_connected_nodes
+  retries: 5 # there may be a delay before the server reconnects to itself
+  delay: 10
+  until: not _k3s_connected_nodes.failed
+
+- name: Generate new bootstrap token
+  no_log: true
+  when: _k3s_connected_nodes.stdout_lines | length != groups['k3s'] | length
+  shell:
+    cmd: "k3s token create --ttl {{ k3s_bootstrap_token_expiry }}"
+  register: _k3s_token_output

ansible/roles/k3s/templates/k3s-agent.service.env.j2

@@ -1,3 +1,3 @@
-K3S_NODE_IP={{ access_ip }}
-K3S_TOKEN={{ hostvars[groups['control'] | first]._token_output.stdout }}
+K3S_NODE_IP={{ ansible_host }}
+K3S_TOKEN={{ k3s_bootstrap_token }}
 K3S_URL=https://{{ k3s_server_name }}:6443

ansible/roles/k3s/templates/k3s.service.env.j2

@@ -1 +1 @@
-K3S_NODE_IP={{ access_ip }}
+K3S_NODE_IP={{ ansible_host }}

environments/common/inventory/group_vars/all/defaults.yml

@@ -21,6 +21,7 @@ opensearch_address: "127.0.0.1"
 prometheus_address: "{{ hostvars[groups['prometheus'].0].api_address }}"
 openondemand_address: "{{ hostvars[groups['openondemand'].0].api_address if groups['openondemand'] | count > 0 else '' }}"
 grafana_address: "{{ hostvars[groups['grafana'].0].api_address }}"
+k3s_server_name: "{{ hostvars[groups['k3s_server'] | first].ansible_host }}"
 
 ############################# bootstrap: local user configuration #########################
 

environments/skeleton/{{cookiecutter.environment}}/tofu/data.tf

@@ -1,11 +1,3 @@
-data "external" "inventory_secrets" {
-  program = ["${path.module}/read-inventory-secrets.py"]
-
-  query = {
-    path = var.inventory_secrets_path == "" ? "${path.module}/../inventory/group_vars/all/secrets.yml" : var.inventory_secrets_path
-  }
-}
-
 data "external" "baremetal_nodes" {
   # returns an empty map if cannot list baremetal nodes
   program = ["${path.module}/baremetal-node-list.py"]

environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf

@@ -183,9 +183,3 @@ variable "root_volume_size" {
     type = number
     default = 40
 }
-
-variable "inventory_secrets_path" {
-  description = "Path to inventory secrets.yml file. Default is standard cookiecutter location."
-  type = string
-  default = ""
-}