Add CI staging job testing Ubuntu 24.04 (noble)

.github/workflows/staging.yml

@@ -9,9 +9,13 @@ on:
 
 jobs:
   staging:
+    strategy:
+      matrix:
+        ubuntu_version: ["focal", "noble"]
     runs-on: ubuntu-latest
     env:
       GOOGLE_CREDENTIALS: ${{ secrets.GOOGLE_CREDENTIALS }}
+      UBUNTU_VERSION: ${{ matrix.ubuntu_version }}
     steps:
       - uses: actions/checkout@v4
       - name: Run staging tests on GCE

Makefile

@@ -248,7 +248,7 @@ demo-landing-page: ## Serve the landing page for the SecureDrop demo
 .PHONY: staging
 staging:  ## Create a local staging environment in virtual machines (Focal)
 	@echo "███ Creating staging environment on Ubuntu Focal..."
-	@$(SDROOT)/devops/scripts/create-staging-env focal
+	@$(SDROOT)/devops/scripts/create-staging-env
 	@echo
 
 .PHONY: testinfra

devops/gce-nested/ci-env.sh

@@ -13,14 +13,15 @@ TOPLEVEL="$(git rev-parse --show-toplevel)"
 export TOPLEVEL
 GCE_CREDS_FILE="${TOPLEVEL}/.gce.creds"
 export GCE_CREDS_FILE
+export UBUNTU_VERSION="${UBUNTU_VERSION:-focal}"
 export BUILD_NUM="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
 export PROJECT_ID="securedrop-ci"
 export JOB_NAME="sd-ci-nested"
 export GCLOUD_MACHINE_TYPE="c2-standard-8"
 export GCLOUD_CONTAINER_VER
 export CLOUDSDK_COMPUTE_ZONE="us-west1-c"
 export EPHEMERAL_DIRECTORY="/tmp/gce-nested"
-export FULL_JOB_ID="${JOB_NAME}-${BUILD_NUM}"
+export FULL_JOB_ID="${JOB_NAME}-${UBUNTU_VERSION}-${BUILD_NUM}"
 export SSH_USER_NAME=sdci
 export SSH_PRIVKEY="${EPHEMERAL_DIRECTORY}/gce"
 export SSH_PUBKEY="${SSH_PRIVKEY}.pub"

devops/gce-nested/ci-go.sh

@@ -12,8 +12,6 @@ set -e
 set -u
 set -o pipefail
 
-export BASE_OS="${BASE_OS:-focal}"
-
 ./devops/gce-nested/gce-start.sh
 ./devops/gce-nested/gce-runner.sh
 ./devops/gce-nested/gce-stop.sh

devops/gce-nested/gce-runner.sh

@@ -4,15 +4,15 @@
 # for storage as artifacts on the build, so devs can review via web.
 set -e
 set -u
-BASE_OS="${BASE_OS:-focal}"
+UBUNTU_VERSION="${UBUNTU_VERSION:-focal}"
 
 
 TOPLEVEL="$(git rev-parse --show-toplevel)"
 # shellcheck source=devops/gce-nested/ci-env.sh
 . "${TOPLEVEL}/devops/gce-nested/ci-env.sh"
 
 REMOTE_IP="$(gcloud_call compute instances describe \
-            "${JOB_NAME}-${BUILD_NUM}" \
+            "${FULL_JOB_ID}" \
             --format="value(networkInterfaces[0].accessConfigs.natIP)")"
 SSH_TARGET="${SSH_USER_NAME}@${REMOTE_IP}"
 SSH_OPTS=(-i "$SSH_PRIVKEY" -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null")
@@ -56,6 +56,6 @@ copy_securedrop_repo
 # so register a trap to ensure the fetch always runs.
 trap fetch_junit_test_results EXIT
 
-ssh_gce "make build-debs-notest"
-ssh_gce "make build-debs-ossec-notest"
-ssh_gce "make staging"
+ssh_gce "UBUNTU_VERSION=\"${UBUNTU_VERSION}\" make build-debs-notest"
+ssh_gce "UBUNTU_VERSION=\"${UBUNTU_VERSION}\" make build-debs-ossec-notest"
+ssh_gce "UBUNTU_VERSION=\"${UBUNTU_VERSION}\" make staging"

devops/gce-nested/gce-start.sh

@@ -25,7 +25,7 @@ function find_latest_ci_image() {
     #    --filter="family:fpf-securedrop AND name ~ ^ci-nested-virt" \
     #    --sort-by=~Name --limit=1 --format="value(Name)"
     # Return hardcoded image id to prevent newer builds from breaking CI
-    echo "ci-nested-virt-bullseye-1651694527"
+    echo "ci-nested-virt-bullseye-1732663778"
 }
 
 # Call out to GCE API and start a new instance, designating

devops/gce-nested/gce-stop.sh

@@ -11,4 +11,4 @@ TOPLEVEL="$(git rev-parse --show-toplevel)"
 . "${TOPLEVEL}/devops/gce-nested/ci-env.sh"
 
 # Destroy remote instance
-gcloud_call compute instances delete "${JOB_NAME}-${BUILD_NUM}"
+gcloud_call compute instances delete "${FULL_JOB_ID}"

devops/scripts/create-staging-env

@@ -14,7 +14,9 @@ set -o pipefail
 
 . ./devops/scripts/boot-strap-venv.sh
 
-securedrop_staging_scenario="$(./devops/scripts/select-staging-env "${1}")"
+export PLATFORM="${UBUNTU_VERSION:-focal}"
+
+securedrop_staging_scenario="$(./devops/scripts/select-staging-env "${PLATFORM}")"
 
 if [ -z "$TEST_DATA_FILE" ]
 then

install_files/ansible-base/roles/common/tasks/apt_sources.yml

@@ -24,7 +24,7 @@
 # this is an empty file with a comment pointing to ubuntu.sources.
 # On upgrade from focal, we're deleting this file.
 - name: Remove obsolete sources.list (noble)
-  template:
+  file:
     dest: /etc/apt/sources.list
     state: absent
   when: ansible_distribution_release != "focal"

molecule/libvirt-staging-noble/ansible-override-vars.yml

@@ -0,0 +1,10 @@
+---
+# Permit direct access via SSH
+ssh_net_in_override: 0.0.0.0/0
+
+# In libvirt, we want to connect over eth0, not eth1 which is used for
+# inter-VM communication for OSSEC.
+ssh_ip: "{{ ansible_default_ipv4.address }}"
+
+# Make sure correct packages are used during installation
+securedrop_target_distribution: "noble"

molecule/libvirt-staging-noble/create.yml

@@ -0,0 +1,56 @@
+---
+- name: Create
+  hosts: localhost
+  connection: local
+  vars:
+    molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}"
+    molecule_instance_config: "{{ lookup('env', 'MOLECULE_INSTANCE_CONFIG') }}"
+    molecule_yml: "{{ lookup('file', molecule_file) | from_yaml }}"
+  tasks:
+
+    - name: Create molecule instance(s)
+      vagrant:
+        instance_name: "{{ item.name }}"
+        instance_interfaces: "{{ item.interfaces | default(omit) }}"
+        instance_raw_config_args: "{{ item.instance_raw_config_args | default(omit) }}"
+
+        platform_box: "{{ item.box }}"
+        platform_box_version: "{{ item.box_version | default(omit) }}"
+        platform_box_url: "{{ item.box_url | default(omit) }}"
+
+        provider_name: "{{ molecule_yml.driver.provider.name }}"
+        provider_memory: "{{ item.memory | default(omit) }}"
+        provider_cpus: "{{ item.cpus | default(omit) }}"
+        provider_raw_config_args: "{{ item.raw_config_args | default(omit) }}"
+        force_stop: yes
+
+        state: up
+      register: server
+      loop: "{{ molecule_yml.platforms | flatten(levels=1) }}"
+
+    # Mandatory configuration for Molecule to function.
+
+    - name: Populate instance config dict
+      set_fact:
+        instance_conf_dict: {
+          'instance': "{{ item.Host }}",
+          'address': "{{ item.HostName }}",
+          'user': "{{ item.User }}",
+          'port': "{{ item.Port }}",
+          'identity_file': "{{ item.IdentityFile }}", }
+      loop: "{{ server.results | flatten(levels=1) }}"
+      register: instance_config_dict
+      when: server.changed | bool
+
+    - name: Convert instance config dict to a list
+      set_fact:
+        instance_conf: "{{ instance_config_dict.results | map(attribute='ansible_facts.instance_conf_dict') | list }}"
+      when: server.changed | bool
+
+    - name: Dump instance config
+      copy:
+        content: |
+          # Molecule managed
+          {{ instance_conf | to_json | from_json }}
+        dest: "{{ molecule_instance_config }}"
+      when: server.changed | bool

molecule/libvirt-staging-noble/destroy.yml

@@ -0,0 +1,35 @@
+---
+
+- name: Destroy
+  hosts: localhost
+  connection: local
+  vars:
+    molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}"
+    molecule_instance_config: "{{ lookup('env',' MOLECULE_INSTANCE_CONFIG') }}"
+    molecule_yml: "{{ lookup('file', molecule_file) | from_yaml }}"
+    molecule_ephemeral_directory: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}"
+  tasks:
+    - name: Destroy molecule instance(s)
+      vagrant:
+        instance_name: "{{ item.name }}"
+        platform_box: "{{ item.box }}"
+        provider_name: "{{ molecule_yml.driver.provider.name }}"
+        force_stop: "{{ item.force_stop | default(True) }}"
+
+        state: destroy
+      register: server
+      loop: "{{ molecule_yml.platforms | flatten(levels=1) }}"
+
+    # Mandatory configuration for Molecule to function.
+
+    - name: Populate instance config
+      set_fact:
+        instance_conf: {}
+
+    - name: Dump instance config
+      copy:
+        content: |
+          # Molecule managed
+          {{ instance_conf | to_json | from_json | to_yaml }}
+        dest: "{{ molecule_instance_config }}"
+      when: server.changed | bool

molecule/libvirt-staging-noble/molecule.yml

@@ -0,0 +1,78 @@
+---
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+lint: |
+  yamllint
+
+platforms:
+  - name: app-staging
+    box: bento/ubuntu-24.04
+    raw_config_args:
+      - "cpu_mode = 'host-passthrough'"
+      - "video_type = 'virtio'"
+      - "machine_virtual_size = 10"
+    instance_raw_config_args:
+      - "vm.synced_folder './', '/vagrant', disabled: true"
+      - "vm.network 'private_network', ip: '10.0.1.2'"
+      - "ssh.insert_key = false"
+    memory: 1024
+    private_ip: 10.0.1.2
+    groups:
+      - securedrop_application_server
+      - securedrop
+      - staging
+
+  - name: mon-staging
+    box: bento/ubuntu-24.04
+    raw_config_args:
+      - "cpu_mode = 'host-passthrough'"
+      - "video_type = 'virtio'"
+      - "machine_virtual_size = 10"
+    instance_raw_config_args:
+      - "vm.synced_folder './', '/vagrant', disabled: true"
+      - "vm.network 'private_network', ip: '10.0.1.3'"
+      - "ssh.insert_key = false"
+    memory: 1024
+    private_ip: 10.0.1.3
+    groups:
+      - securedrop_monitor_server
+      - securedrop
+      - staging
+
+provisioner:
+  name: ansible
+  lint: |
+    ansible-lint
+  config_options:
+    defaults:
+      interpreter_python: auto
+  options:
+    e: "@molecule/libvirt-staging-noble/ansible-override-vars.yml"
+  playbooks:
+    converge: ../../install_files/ansible-base/securedrop-staging.yml
+    create: create.yml
+    destroy: destroy.yml
+    prepare: prepare.yml
+  env:
+    ANSIBLE_CONFIG: ../../install_files/ansible-base/ansible.cfg
+
+scenario:
+  name: libvirt-staging-noble
+  test_sequence:
+    - destroy
+    - create
+    - converge
+    - verify
+verifier:
+  name: testinfra
+  lint: |
+    flake8
+  directory: ../testinfra
+  options:
+    n: auto
+    v: 2
+    junit-xml: junit/testinfra-results.xml
+  env:
+    SECUREDROP_TARGET_DISTRIBUTION: noble

molecule/libvirt-staging-noble/prepare.yml

@@ -0,0 +1,9 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: False
+  tasks:
+    - name: Install python for Ansible
+      raw: test -e /usr/bin/python3 || (apt -y update && apt install -y python3-minimal)
+      become: True
+      changed_when: False