Hybrid deployments

README.md

@@ -137,6 +137,7 @@ Make sure to set/review the following vars:
 | `lab_cloud` | the cloud within the lab environment for Red Hat Performance labs (Example: `cloud42`)
 | `cluster_type` | either `mno`, or `sno` for the respective cluster layout
 | `worker_node_count` | applies to mno cluster type for the desired worker count, ideal for leaving left over inventory hosts for other purposes
+| `hybrid_worker_count` | applies to mno cluster type for the desired virtual worker count, HV nodes and VMs are required to be setup.
 | `bastion_lab_interface` | set to the bastion machine's lab accessible interface
 | `bastion_controlplane_interface` | set to the interface in which the bastion will be networked to the deployed ocp cluster
 | `controlplane_lab_interface` | applies to mno cluster type and should map to the nodes interface in which the lab provides dhcp to and also required for public routable vlan based sno deployment(to disable this interface)

ansible/mno-deploy.yml

@@ -23,7 +23,7 @@
   - role: boot-iso
     vars:
       inventory_group: worker
-      index: "{{ worker_node_count }}"
+      index: "{{ worker_node_count|int + (hybrid_worker_count | default(0)) }}"
   - wait-hosts-discovered
   - configure-local-storage
   - install-cluster

ansible/roles/boot-iso/tasks/libvirt.yml

@@ -0,0 +1,85 @@
+---
+# Libvirt tasks for booting an iso
+# Couldn't use ansible redfish_command it requires username and password to be used.
+# URLs modeled from http://docs.openstack.org/sushy-tools/latest/user/dynamic-emulator.html
+
+- name: Libvirt - Power down machine prior to booting iso
+  uri:
+    url: "http://{{ hostvars[item]['ansible_host'] }}:9000/redfish/v1/Systems/{{ hostvars[item]['domain_uuid'] }}/Actions/ComputerSystem.Reset"
+    method: POST
+    headers:
+      content-type: application/json
+      Accept: application/json
+    body: {"ResetType":"ForceOff"}
+    body_format: json
+    validate_certs: no
+    status_code: 204
+    return_content: yes
+  register: redfish_forceoff
+
+- name: Libvirt - Set OneTimeBoot VirtualCD
+  uri:
+    url: "http://{{ hostvars[item]['ansible_host'] }}:9000/redfish/v1/Systems/{{ hostvars[item]['domain_uuid'] }}"
+    method: PATCH
+    headers:
+      content-type: application/json
+      Accept: application/json
+    body: { "Boot": { "BootSourceOverrideTarget": "Cd", "BootSourceOverrideMode": "UEFI", "BootSourceOverrideEnabled": "Continuous" } }
+    body_format: json
+    validate_certs: no
+    status_code: 204
+    return_content: yes
+
+- name: Libvirt - Check for Virtual Media
+  uri:
+    url: "http://{{ hostvars[item]['ansible_host'] }}:9000/redfish/v1/Systems/{{ hostvars[item]['domain_uuid'] }}/VirtualMedia/Cd"
+    method: Get
+    headers:
+      content-type: application/json
+      Accept: application/json
+    body: {}
+    body_format: json
+    validate_certs: no
+    status_code: 200
+    return_content: yes
+  register: check_virtual_media
+
+- name: Libvirt - Eject any CD Virtual Media
+  uri:
+    url: "http://{{ hostvars[item]['ansible_host'] }}:9000/redfish/v1/Systems/{{ hostvars[item]['domain_uuid'] }}/VirtualMedia/Cd/Actions/VirtualMedia.EjectMedia"
+    method: POST
+    headers:
+      content-type: application/json
+      Accept: application/json
+    body: {}
+    body_format: json
+    validate_certs: no
+    status_code: 204
+    return_content: yes
+  when: check_virtual_media.json.Image
+
+- name: Libvirt - Insert virtual media
+  uri:
+    url: "http://{{ hostvars[item]['ansible_host'] }}:9000/redfish/v1/Systems/{{ hostvars[item]['domain_uuid'] }}/VirtualMedia/Cd/Actions/VirtualMedia.InsertMedia"
+    method: POST
+    headers:
+      content-type: application/json
+      Accept: application/json
+    body: {"Image":"http://{{ http_store_host }}:{{ http_store_port }}/{{ hostvars[item]['boot_iso'] }}", "Inserted": true}
+    body_format: json
+    validate_certs: no
+    status_code: 204
+    return_content: yes
+
+- name: Libvirt - Power on
+  uri:
+    url: "http://{{ hostvars[item]['ansible_host'] }}:9000/redfish/v1/Systems/{{ hostvars[item]['domain_uuid'] }}/Actions/ComputerSystem.Reset"
+    method: POST
+    headers:
+      content-type: application/json
+      Accept: application/json
+    body: {"ResetType":"On"}
+    body_format: json
+    validate_certs: no
+    status_code: 204
+    return_content: yes

ansible/roles/boot-iso/tasks/main.yml

@@ -24,3 +24,9 @@
   with_items:
     - "{{ groups[inventory_group][:index|int] }}"
   when: hostvars[item]['vendor'] == 'Lenovo'
+
+- name: Boot iso on libvirt vm
+  include_tasks: libvirt.yml
+  with_items:
+    - "{{ groups[inventory_group][:index|int] }}"
+  when: hostvars[item]['vendor'] == 'Libvirt'

ansible/roles/create-ai-cluster/defaults/main/main.yml

@@ -18,3 +18,7 @@ override_kni_infra_haproxy: false
 # Manually override the openshift-kni-haproxy max connections via a machineconfig that replaces the
 # haproxy templatized config file with one with greater max connections (Original default 20000)
 openshift_kni_infra_haproxy_maxconn: 40000
+
+# If HV is setup, how many workers are VMs
+# This runs hosts from hv_vm through the normal workers workflow for deployment
+hybrid_worker_count: 0

ansible/roles/create-ai-cluster/tasks/main.yml

@@ -32,9 +32,6 @@
   - cluster_type == "mno"
   loop: "{{ groups['worker'] }}"
 
-# - debug:
-#     msg: "{{ static_network_config }}"
-
 - name: Create cluster
   uri:
     url: "http://{{ assisted_installer_host }}:{{ assisted_installer_port }}/api/assisted-install/v2/clusters"

ansible/roles/create-ai-cluster/templates/mac_interface_map.json.j2

@@ -2,9 +2,11 @@
     {
         "mac_address": "{{ hostvars[item]['mac_address'] }}",
         "logical_nic_name": "{{ hostvars[item]['network_interface'] }}"
+    {% if 'lab_mac' in hostvars[item] %}
     },
     {
         "mac_address": "{{ hostvars[item]['lab_mac'] }}",
         "logical_nic_name": "{{ hostvars[item]['lab_interface'] }}"
+    {% endif %}
     }
 ]

ansible/roles/create-inventory/defaults/main/main.yml

@@ -33,3 +33,7 @@ hv_vm_bandwidth_burst: 11750
 
 # Path for VMs on disk2
 disk2_mount_path: /mnt/disk2
+
+# If HV is setup, how many workers are VMs
+# This adds vm workers to the workers section of the inventory
+hybrid_worker_count: 0

ansible/roles/create-inventory/templates/inventory-mno.j2

@@ -45,6 +45,26 @@ dns2={{ labs[lab]['dns'][1] | default('') }}
 {% for worker in ocpinventory_worker_nodes %}
 {{ worker.pm_addr.split('.')[0] | replace('mgmt-','') }} bmc_address={{ worker.pm_addr }} mac_address={{ worker.mac[controlplane_network_interface_idx|int] }} lab_mac={{ ( (mno_foreman_data.results| selectattr('json.name', 'eq', worker.pm_addr | replace('mgmt-',''))|first).json.interfaces | selectattr('primary', 'eq', True)|first).mac }} ip={{ controlplane_network | ansible.utils.nthhost(loop.index + mno_worker_node_offset) }} vendor={{ hw_vendor[(worker.pm_addr.split('.')[0]).split('-')[-1]] }} install_disk={{ worker_install_disk }}
 {% endfor %}
+{% if hybrid_worker_count > 0 %}
+{% set ctr = namespace(vm=1) %}
+{% for hv in ocpinventory_hv_nodes %}
+{% set hv_loop = loop %}
+{% for vm in range(hw_vm_counts[lab][(hv.pm_addr.split('.')[0]).split('-')[-1]]['default']) %}
+{% if ctr.vm <= hybrid_worker_count %}
+{{ hv_vm_prefix }}{{ '%05d' % ctr.vm }} bmc_address={{ hv.pm_addr | replace('mgmt-','') }} ip={{ controlplane_network | ansible.utils.nthhost(hv_vm_ip_offset + ctr.vm - 1) }} mac_address={{ (90520730730496 + ctr.vm) | ansible.utils.hwaddr('linux') }} domain_uuid={{ ctr.vm | to_uuid }} vendor=Libvirt install_disk=/dev/sda
+{% endif %}
+{% set ctr.vm = ctr.vm + 1 %}
+{% endfor %}
+{% if hv.disk2_enable %}
+{% for vm in range(hw_vm_counts[lab][(hv.pm_addr.split('.')[0]).split('-')[-1]][hv.disk2_device]) %}
+{% if ctr.vm <= hybrid_worker_count %}
+{{ hv_vm_prefix }}{{ '%05d' % ctr.vm }} bmc_address={{ hv.pm_addr | replace('mgmt-','') }} ip={{ controlplane_network | ansible.utils.nthhost(hv_vm_ip_offset + ctr.vm - 1) }} mac_address={{ (90520730730496 + ctr.vm) | ansible.utils.hwaddr('linux') }} domain_uuid={{ ctr.vm | to_uuid }} vendor=Libvirt install_disk=/dev/sda
+{% endif %}
+{% set ctr.vm = ctr.vm + 1 %}
+{% endfor %}
+{% endif %}
+{% endfor %}
+{% endif %}
 
 [worker:vars]
 role=worker
@@ -106,6 +126,7 @@ machine_network={{ controlplane_network }}
 network_prefix={{ controlplane_network_prefix }}
 gateway={{ controlplane_network_gateway }}
 bw_limit={{ hv_vm_bandwidth_limit }}
+
 {% else %}
 [hv]
 # Set `hv_inventory: true` to populate

ansible/roles/mno-post-cluster-install/defaults/main/main.yml

@@ -45,3 +45,7 @@ minio_pv_storageclass: localstorage2-sc
 # Performance-addon-operator vars_files
 install_performance_addon_operator: false
 ocp_channel: "{{ openshift_version }}"
+
+# If HV is setup, how many workers are VMs
+# This runs hosts from hv_vm through the normal workers workflow for deployment
+hybrid_worker_count: 0