Build overcloud host and IPA images using packages from Ark

.github/workflows/stackhpc-container-image-build.yml

@@ -171,17 +171,6 @@ jobs:
           localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3
           EOF
 
-      # See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details.
-      # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the
-      # runner and dind containers.
-      - name: Deploy an authenticating package repository mirror proxy
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-
       - name: Create build logs output directory
         run: mkdir image-build-logs
 
@@ -192,7 +181,9 @@ jobs:
           args="${{ inputs.regexes }}"
           args="$args -e kolla_base_distro=${{ matrix.distro }}"
           args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
-          args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
+          # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the
+          # runner and dind containers.
+          args="$args -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy"
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe overcloud container image build $args

etc/kayobe/environments/ci-builder/hooks/overcloud-container-image-build/pre.d/10-pulp-auth-proxy.yml

@@ -0,0 +1 @@
+../../../../../ansible/pulp-auth-proxy.yml

etc/kayobe/environments/ci-builder/hooks/overcloud-deployment-image-build/pre.d/10-pulp-auth-proxy.yml

@@ -0,0 +1 @@
+../../../../../ansible/pulp-auth-proxy.yml

etc/kayobe/environments/ci-builder/hooks/overcloud-host-image-build/pre.d/10-pulp-auth-proxy.yml

@@ -0,0 +1 @@
+../../../../../ansible/pulp-auth-proxy.yml

etc/kayobe/environments/ci-builder/inventory/group_vars/seed/dev-tools

@@ -0,0 +1,3 @@
+# Ensure we're using modern docker-buildx instead of legacy docker-build
+dev_tools_packages_extra:
+  - "{% if os_distribution == 'ubuntu' %}docker-buildx-plugin{% endif %}"

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -36,6 +36,7 @@ kolla_build_neutron_ovs: true
 # Host and port of a package repository mirror.
 # Build against the development Pulp service repositories.
 # Use Ark's package repositories to install packages.
+stackhpc_repo_mirror_auth_proxy_enabled: true
 stackhpc_repo_mirror_url: "{{ stackhpc_repo_mirror_auth_proxy_url if stackhpc_repo_mirror_auth_proxy_enabled | bool else stackhpc_release_pulp_url }}"
 stackhpc_repo_mirror_username: "skc-ci-aio"
 stackhpc_repo_mirror_password: !vault |

etc/kayobe/ipa.yml

@@ -33,6 +33,7 @@
 ipa_build_dib_elements_extra:
   - extra-hardware
   - mellanox
+  - purge-command-not-found
 
 # List of Diskimage Builder (DIB) elements to use when building IPA images.
 # Default is combination of ipa_build_dib_elements_default and
@@ -58,7 +59,7 @@ ipa_build_dib_elements_extra:
 
 # List of additional git repositories containing Diskimage Builder (DIB)
 # elements. See stackhpc.openstack.os_images role for usage. Default is empty.
-#ipa_build_dib_git_elements_extra:
+ipa_build_dib_git_elements_extra: "{{ stackhpc_dib_image_elements_repos }}"
 
 # List of git repositories containing Diskimage Builder (DIB) elements. See
 # stackhpc.openstack.os_images role for usage. Default is a combination of

etc/kayobe/overcloud-dib.yml

@@ -64,11 +64,7 @@ overcloud_dib_host_packages_extra:
 
 # List of additional git repositories containing Diskimage Builder (DIB)
 # elements. See stackhpc.openstack.os_images role for usage. Default is empty.
-overcloud_dib_git_elements_extra:
-  - repo: "https://github.com/stackhpc/stackhpc-image-elements"
-    local: "{{ source_checkout_path }}/stackhpc-image-elements"
-    version: "v1.6.3"
-    elements_path: "elements"
+overcloud_dib_git_elements_extra: "{{ stackhpc_dib_image_elements_repos }}"
 
 # List of git repositories containing Diskimage Builder (DIB) elements. See
 # stackhpc.openstack.os_images role for usage. Default is a combination of

etc/kayobe/stackhpc-overcloud-dib.yml

@@ -22,6 +22,7 @@ stackhpc_overcloud_dib_name: "deployment_image"
 # StackHPC overcloud DIB image elements.
 stackhpc_overcloud_dib_elements:
   - "{{ os_distribution }}-{% if os_distribution == 'rocky' %}container-stackhpc{% else %}minimal{% endif %}"
+  - "{% if os_distribution == 'ubuntu' and stackhpc_repo_mirror_auth_proxy_enabled %}apt-no-verify-peer{% endif %}"
   - "cloud-init-datasources"
   - "{% if os_distribution == 'rocky' %}selinux-permissive{% endif %}"
   - "enable-serial-console"
@@ -41,18 +42,16 @@ stackhpc_overcloud_dib_env_vars:
   DIB_CONTAINERFILE_RUNTIME: "docker"
   DIB_CONTAINERFILE_NETWORK_DRIVER: "host"
   DIB_CONTAINERFILE_DOCKERFILE: "/opt/kayobe/src/stackhpc-image-elements/elements/rocky-container-stackhpc/containerfiles/9-stackhpc"
-  # NOTE: Not currently syncing Ubuntu packages, since the on_demand mirror in
-  # Ark does not work if the upstream mirror pulls packages (which it does
-  # sometimes).
-  # DIB_DISTRIBUTION_MIRROR: "{{ stackhpc_repo_ubuntu_focal_url if os_distribution == 'ubuntu' else '' }}"
+  DIB_CONTAINERFILE_BUILDOPTS: >-
+    --build-arg=ROCKY_USE_MIRRORS=true
+    --build-arg=ROCKY_MIRROR_URLS={{ [stackhpc_repo_rocky_9_baseos_url, stackhpc_repo_rocky_9_appstream_url] | join(',') }}
+  DIB_DISTRIBUTION_MIRROR: "{{ stackhpc_repo_ubuntu_jammy_url if os_distribution == 'ubuntu' else '' }}"
   DIB_DRACUT_ENABLED_MODULES_DEFAULT_CONFIG: "{{ stackhpc_overcloud_dib_dracut_enabled_modules_default_config }}"
   DIB_RELEASE: "{{ overcloud_dib_os_release }}"
   DIB_SUDOERS_FILENAME: "no-fqdn"
   # Avoid DNS queries during sudo commands, since we might not always have working DNS.
   DIB_SUDOERS_CONFIG: |
     Defaults  !fqdn
-  # FIXME: Support templating repo files.
-  # DIB_YUM_MINIMAL_BOOTSTRAP_REPOS: /path/to/dir/containing/dib-mirror-*.repo
   YUM: dnf
   # Workaround for stack user home ownership bug
   DIB_IMAGE_CACHE: "/tmp/yum"

etc/kayobe/stackhpc.yml

@@ -166,3 +166,11 @@ stackhpc_docker_registry_password: "{{ pulp_password }}"
 
 # Whether or not to run CIS benchmark hardening playbooks. Default is false.
 #stackhpc_enable_cis_benchmark_hardening_hook:
+
+###############################################################################
+# diskimage-builder elements repos
+stackhpc_dib_image_elements_repos:
+  - repo: "https://github.com/stackhpc/stackhpc-image-elements"
+    local: "{{ source_checkout_path }}/stackhpc-image-elements"
+    version: "rocky-container-pulp-2"
+    elements_path: "elements"