Update/upstream 0.15.0

.github/workflows/build_test_images.yaml

@@ -87,25 +87,25 @@ jobs:
       - name: mount qcow2 file
         run: sudo guestmount -a ${{ steps.publish-image.outputs.image-name }}.qcow2 -i --ro -o allow_other './${{ steps.publish-image.outputs.image-name }}'
 
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/[email protected]
-        with:
-          scan-type: fs
-          scan-ref: "./${{ steps.publish-image.outputs.image-name }}"
-          scanners: "vuln"
-          format: sarif
-          output: "${{ steps.publish-image.outputs.image-name }}.sarif"
-          # limit to medium and higher as we get too many results for GH security otherwise
-          limit-severities-for-sarif: 'true'
-          severity: 'MEDIUM,HIGH,CRITICAL'
-        env:
-          TRIVY_DB_REPOSITORY: ghcr.io/azimuth-cloud/trivy-db:2
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: "${{ steps.publish-image.outputs.image-name }}.sarif"
-          category: "${{ matrix.name }}"
+#      - name: Run Trivy vulnerability scanner
+#        uses: aquasecurity/[email protected]
+#        with:
+#          scan-type: fs
+#          scan-ref: "./${{ steps.publish-image.outputs.image-name }}"
+#          scanners: "vuln"
+#          format: sarif
+#          output: "${{ steps.publish-image.outputs.image-name }}.sarif"
+#          # limit to high and critical as we get too many results for GH security otherwise
+#          limit-severities-for-sarif: 'true'
+#          severity: 'HIGH,CRITICAL'
+#        env:
+#          TRIVY_DB_REPOSITORY: ghcr.io/azimuth-cloud/trivy-db:2
+
+#      - name: Upload Trivy scan results to GitHub Security tab
+#        uses: github/codeql-action/upload-sarif@v3
+#        with:
+#          sarif_file: "${{ steps.publish-image.outputs.image-name }}.sarif"
+#          category: "${{ matrix.name }}"
 
       - name: Fail if scan has CRITICAL vulnerabilities
         uses: aquasecurity/[email protected]

ansible/linux-common.yml

@@ -0,0 +1,4 @@
+---
+- hosts: all
+  roles:
+    - role: linux-common

ansible/roles/linux-broadcom/tasks/main.yml

@@ -9,6 +9,7 @@
         - libasan8
         - libtsan2
         - dkms
+        - xz-utils
       state: latest    
 
   - name: Install the broadcom driver for NetXtreme-E Ethernet

ansible/roles/linux-common/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+kernel_modules:
+  - ib_ipoib

ansible/roles/linux-common/tasks/main.yml

@@ -1,24 +1,16 @@
 ---
-
-- name: Update apt cache
+# ACL Required for become to an unprivileged user to work
+# Ensure up-to-date CA certificates
+# Use linux-generic to track linux-modules-extra (IB)
+- name: Install common packages
   apt:
     update_cache: true
-  when: ansible_os_family == "Debian"
-
-- name: Ensure up-to-date CA certificates
-  package:
-    name: ca-certificates
+    pkg:
+    - acl
+    - ca-certificates
+      #- linux-generic
     state: latest
 
-# Required for become to an unprivileged user to work
-# Using the apt module seems to work more reliably than package :-/
-- name: Install ACL package
-  apt:
-    update_cache: true
-    name: acl
-    state: present
-  when: ansible_os_family == "Debian"
-
 # Enables the ansible-init system, but does not install any default playbooks
 - include_role:
     name: azimuth_cloud.image_utils.linux_ansible_init
@@ -30,3 +22,28 @@
 # Install the Broadcom network driver for NetXtreme-E Ethernet
 - include_role:
     name: linux-broadcom
+
+- name: Load relevant kernel modules
+  template:
+    src: modules.conf.j2
+    dest: /etc/modules-load.d/modules.conf
+
+# The cloud-init netplan renderer is default for ubuntu
+# but does not support InfiniBand devices
+- name: Switch cloud-init to alternative network renderer
+  lineinfile:
+    path: /etc/cloud/cloud.cfg
+    regexp: "^    renderers:.*"
+    line: "    renderers: ['networkd', 'network-manager']"
+    state: present
+
+- name: Switch cloud-init to alternative network activator
+  lineinfile:
+    path: /etc/cloud/cloud.cfg
+    regexp: "^    activators:.*"
+    line: "    activators: ['networkd', 'eni', 'network-manager']"
+    state: present
+
+- name: Clean up after ourselves
+  ansible.builtin.apt:
+    clean: yes

ansible/roles/linux-common/templates/modules.conf.j2

@@ -0,0 +1,3 @@
+{% for module in kernel_modules %}
+{{ module }}
+{% endfor %}

ansible/roles/linux-guacamole/defaults/main.yml

@@ -14,7 +14,7 @@ guacamole_client_image: "{{ guacamole_image_prefix }}/{{ guacamole_client_image_
 
 guacamole_nginx_image_prefix: docker.io
 guacamole_nginx_image_name: nginx
-guacamole_nginx_image_tag: 1.27.0
+guacamole_nginx_image_tag: 1.27.2
 guacamole_nginx_image: "{{ guacamole_nginx_image_prefix }}/{{ guacamole_nginx_image_name }}:{{ guacamole_nginx_image_tag }}"
 
 guacamole_mitm_log_level: notice

ansible/roles/linux-monitoring/defaults/main.yml

@@ -9,6 +9,6 @@ grafana_dashboards:
     home: true
 
 prometheus_container_image: "prom/prometheus"
-prometheus_version: "v2.54.0"
+prometheus_version: "v2.55.0"
 
 podman_service_user: "podman"

ansible/roles/linux-webconsole/tasks/main.yml

@@ -51,6 +51,9 @@
         owner: azimuth
         group: azimuth
 
+- include_role:
+    name: linux-zenith-ssh
+
 - include_role:
     name: stackhpc.os-manila-mount
     tasks_from: install.yml

ansible/roles/linux-zenith-client/defaults/main.yml

@@ -2,11 +2,17 @@
 
 zenith_client_image_prefix: ghcr.io/azimuth-cloud
 zenith_client_image_name: zenith-client
-zenith_client_image_tag: 0.12.0
+# TODO(mkjpryor) update when SSH protocol is in a release
+zenith_client_image_tag: 0.13.0
 zenith_client_image: "{{ zenith_client_image_prefix }}/{{ zenith_client_image_name }}:{{ zenith_client_image_tag }}"
 
 zenith_client_wants: []
 
+# The default host for forwarding is the pod that the client is in
 zenith_forward_to_host: 127.0.0.1
 
+# Any extra configuration for the Zenith client
+zenith_client_config_extra: {}
+
+# The user to run systemd services as
 podman_service_user: podman

ansible/roles/linux-zenith-client/tasks/main.yml

@@ -9,6 +9,7 @@
     - zenith_client_pod
     - zenith_registrar_token_metadata_key
     - zenith_client_playbook_priority
+    - zenith_forward_to_port
 
 - name: Ensure Zenith config directory exists
   file:
@@ -56,7 +57,7 @@
       zenith_forward_to_host: "{{ zenith_forward_to_host }}"
       zenith_forward_to_port: "{{ zenith_forward_to_port }}"
       zenith_registrar_token_metadata_key: "{{ zenith_registrar_token_metadata_key }}"
-
+      zenith_client_config_extra: "{{ zenith_client_config_extra }}"
 
 - name: Install ansible-init playbook for Zenith client
   template:

ansible/roles/linux-zenith-client/templates/zenith-playbook.yml

@@ -9,14 +9,17 @@
   vars:
     os_metadata: "{{ lookup('url', 'http://169.254.169.254/openstack/latest/meta_data.json') | from_json }}"
     os_user_metadata: "{{ os_metadata.get('meta', {}) }}"
-    os_project_id: "{{ os_metadata.project_id }}"
   tasks:
-    - name: Ensure Zenith SSH directory exists
-      file:
-        path: /etc/zenith/ssh/{{ zenith_client_name }}
-        state: directory
-        owner: podman
-        group: podman
+    # We end the play rather than failing if the required metadata is not available
+    # From the outside, this has the same effect as failing, i.e. the service never comes up
+    # However subsequent playbooks are able to proceed, meaning Zenith services can be optional
+    - name: End play if required metadata items are not available
+      meta: end_play
+      when: >-
+        "zenith_registrar_url" not in os_user_metadata or
+        "zenith_sshd_host" not in os_user_metadata or
+        "zenith_sshd_port" not in os_user_metadata or
+        zenith_registrar_token_metadata_key not in os_user_metadata
 
     - name: Set Zenith facts
       set_fact:
@@ -25,7 +28,13 @@
         zenith_client_registrar_verify_ssl: false
         zenith_client_sshd_host: "{{ os_user_metadata.zenith_sshd_host }}"
         zenith_client_sshd_port: "{{ os_user_metadata.zenith_sshd_port }}"
-        azimuth_cloud_name: "{{ os_user_metadata.get('azimuth_cloud_name', '') }}"
+
+    - name: Ensure Zenith SSH directory exists
+      file:
+        path: /etc/zenith/ssh/{{ zenith_client_name }}
+        state: directory
+        owner: podman
+        group: podman
 
     - name: Write the Zenith client configuration
       copy:
@@ -43,9 +52,9 @@
           forward_to_host: {{ ansible_default_ipv4.address if zenith_forward_to_host == 'HOST_IP' else zenith_forward_to_host }}
           forward_to_port: {{ zenith_forward_to_port }}
 
-          auth_params:
-            cloud-name: {{ azimuth_cloud_name }}
-            tenancy-id: {{ os_project_id }}
+          {% if zenith_client_config_extra -%}
+          {{ zenith_client_config_extra | to_nice_yaml }}
+          {%- endif %}
         dest: /etc/zenith/{{ zenith_client_name }}/client.yaml
 
     # The podman_container module doesn't re-run the container every time

ansible/roles/linux-zenith-ssh/defaults/main.yml

@@ -0,0 +1,10 @@
+---
+
+# The name of the SSH service
+zenith_ssh_service_name: zenith-ssh
+
+# The name of the SSH service in metadata items
+zenith_ssh_metadata_name: ssh
+
+# The SSH playbook must happen after the data volumes are set up, which have priority "10"
+zenith_ssh_playbook_priority: "11"

ansible/roles/linux-zenith-ssh/tasks/main.yml

@@ -0,0 +1,25 @@
+---
+
+- name: Install systemd unit for SSH pod
+  include_role: 
+    name: linux-podman
+    tasks_from: systemd-unit.yml
+  vars:
+    podman_service_name: "{{ zenith_ssh_service_name }}"
+    podman_service_type: pod
+
+- name: Configure SSH Zenith client
+  include_role:
+    name: linux-zenith-client
+  vars:
+    zenith_client_name: "{{ zenith_ssh_service_name }}-client"
+    zenith_client_pod: "{{ zenith_ssh_service_name }}"
+    zenith_forward_to_host: HOST_IP
+    zenith_forward_to_port: "22"
+    zenith_registrar_token_metadata_key: "zenith_registrar_token_{{ zenith_ssh_metadata_name }}"
+    zenith_client_playbook_priority: "{{ zenith_ssh_playbook_priority }}"
+    # Use an internal tunnel with the SSH protocol
+    zenith_client_config_extra:
+      backend_protocol: ssh
+      internal: true
+      skip_auth: true

packer/jupyter-repo2docker.pkr.hcl

@@ -30,7 +30,7 @@ variable "volume_type" {
 
 variable "volume_size" {
   type = number
-  default = 5
+  default = 10
 }
 
 variable "disk_format" {

packer/linux-desktop.pkr.hcl

@@ -30,7 +30,7 @@ variable "volume_type" {
 
 variable "volume_size" {
   type = number
-  default = 10
+  default = 15
 }
 
 variable "disk_format" {

requirements.yml

@@ -1,19 +1,19 @@
 ---
 collections:
   - name: ansible.posix
-    version: 1.5.4
+    version: 1.6.2
   # - name: ansible.windows
   #   version: 1.10.0
   # - name: chocolatey.chocolatey
   #   version: 1.2.0
   - name: community.general
-    version: 9.3.0
+    version: 9.5.0
   # - name: community.windows
   #   version: 1.10.0
   - name: containers.podman
-    version: 1.15.4
+    version: 1.16.1
   - name: prometheus.prometheus
-    version: 0.19.0
+    version: 0.20.1
   - name: https://github.com/azimuth-cloud/ansible-collection-image-utils
     type: git
     version: 0.2.0

vars/base/kubernetes_1_28.json

@@ -1,6 +1,6 @@
 {
-    "kubernetes_deb_version": "1.28.14-*",
-    "kubernetes_rpm_version": "1.28.14",
-    "kubernetes_semver": "v1.28.14",
+    "kubernetes_deb_version": "1.28.15-*",
+    "kubernetes_rpm_version": "1.28.15",
+    "kubernetes_semver": "v1.28.15",
     "kubernetes_series": "v1.28"
 }

vars/base/kubernetes_1_29.json

@@ -1,6 +1,6 @@
 {
-    "kubernetes_deb_version": "1.29.9-*",
-    "kubernetes_rpm_version": "1.29.9",
-    "kubernetes_semver": "v1.29.9",
+    "kubernetes_deb_version": "1.29.10-*",
+    "kubernetes_rpm_version": "1.29.10",
+    "kubernetes_semver": "v1.29.10",
     "kubernetes_series": "v1.29"
 }

vars/base/kubernetes_1_30.json

@@ -1,6 +1,6 @@
 {
-    "kubernetes_deb_version": "1.30.5-*",
-    "kubernetes_rpm_version": "1.30.5",
-    "kubernetes_semver": "v1.30.5",
+    "kubernetes_deb_version": "1.30.6-*",
+    "kubernetes_rpm_version": "1.30.6",
+    "kubernetes_semver": "v1.30.6",
     "kubernetes_series": "v1.30"
 }

vars/base/kubernetes_1_31.json

@@ -1,6 +1,6 @@
 {
-    "kubernetes_deb_version": "1.31.1-*",
-    "kubernetes_rpm_version": "1.31.1",
-    "kubernetes_semver": "v1.31.1",
+    "kubernetes_deb_version": "1.31.2-*",
+    "kubernetes_rpm_version": "1.31.2",
+    "kubernetes_semver": "v1.31.2",
     "kubernetes_series": "v1.31"
 }