Tkgm refactoring (#14)

add pattern for deploying TKGM without nesting
fix outstanding bugs in tkgm automation

---------

Co-authored-by: Matt Proud <[email protected]>

.github/workflows/yamllint.yml

@@ -1,7 +1,6 @@
 ---
 name: 'Yamllint GitHub Actions'
 on:
-  - push
   - pull_request
 jobs:
   yamllint:

.gitignore

@@ -2,6 +2,7 @@
 temp/*
 settings.json
 tmp/
+ignored/
 
 # Tanzu Multi-cloud outputs
 cluster-config.yml

deploy.yml

@@ -26,20 +26,28 @@
     # Base vsphere section
     - role: vsphere/vsphere-deploy-vc-and-hosts
       tags: ["vsphere"]
+      when: 'nested_vcenter is defined and nested_hosts is defined'
     - role: vsphere/vsphere-datacenter
       tags: ["vsphere"]
+      when: 'nested_vcenter is defined'
     - role: vsphere/vsphere-clusters
       tags: ["vsphere"]
+      when: 'nested_clusters is defined'
     - role: vsphere/vsphere-vswitch0-port-groups
       tags: ["vsphere"]
+      when: 'nested_hosts is defined'
     - role: vsphere/vsphere-local-datastores
       tags: ["vsphere"]
+      when: 'nested_clusters is defined'
     - role: vsphere/vsphere-nfs-datastores
       tags: ["vsphere"]
+      when: 'nested_clusters is defined'
     - role: vsphere/vsphere-enable-cluster-services
       tags: ["vsphere"]
+      when: 'nested_clusters is defined'
     - role: vsphere/vsphere-resource-pools
       tags: ["vsphere"]
+      when: 'nested_clusters is defined'
     - role: vsphere/vsphere-distributed-switches
       when: 'distributed_switches is defined'
       tags: ["vsphere"]
@@ -94,17 +102,17 @@
     # NSX-ALB (Avi) Section
     - role: nsx-alb/deploy-controller
       when: 'nsx_alb is defined'
-      tags: ["nsx-alb"]
+      tags: ["nsx-alb", "alb-controller"]
     - role: nsx-alb/configure-cloud
       when: 'nsx_alb is defined'
       tags: ["nsx-alb", "alb-cloud"]
     - role: nsx-alb/dummy-service
       when: 'nsx_alb.dummy_service is defined'
-      tags: ["nsx-alb"]
+      tags: ["nsx-alb", "alb-dummy"]
 
     # Tanzu Multi-Cloud Secton
     - role: tanzu/multi-cloud-generate-config
-      when: 'tanzu_multi_cloud.generated_config_file_name is defined'
+      when: 'tanzu_multi_cloud.generated_config_file is defined'
       tags: ["tkgm-config"]
 
     # vSphere with Kubernetes Section

roles/display-configuration/tasks/main.yml

@@ -8,11 +8,14 @@
   ansible.builtin.set_fact:
     build_outputs: >-
       {
+        {% if nested_vcenter is defined %}
         "vcenter": {
           "ip": "{{ nested_vcenter.ip }}",
           "username": "{{ nested_vcenter.username }}",
           "password": "{{ nested_vcenter.password }}"
         },
+        {% endif %}
+        {% if nested_hosts is defined %}
         "esxi_hosts": [
           {% for host in nested_hosts %}
           {
@@ -22,6 +25,7 @@
           },
           {% endfor %}
         ],
+        {% endif %}
         {% if nsxt is defined %}
         "nsxt": {
           "manager_ip": "{{ nsxt.manager.ip }}",
@@ -69,9 +73,10 @@
         },
         {% endif %}
 
-        {% if tanzu_multi_cloud.generated_config_file_name is defined %}
+        {% if tanzu_multi_cloud.generated_config_file is defined %}
         "tanzu_multi_cloud": {
-          "config_file": "{{ playbook_dir }}/{{ tanzu_multi_cloud.generated_config_file_name }}"
+          "config_file": "{{ tanzu_multi_cloud.generated_config_file }}",
+          "mgmt_create_command" : "tanzu management-cluster create --file {{ tanzu_multi_cloud.generated_config_file }}"
         },
         {% endif %}
 

roles/nsx-alb/configure-cloud/tasks/main.yml

@@ -3,14 +3,13 @@
 - name: Create a VMWare cloud with write access mode
   vmware.alb.avi_cloud:
     avi_credentials: "{{ avi_credentials }}"
-    # apic_mode: false
     dhcp_enabled: true
     enable_vip_static_routes: false
-    name: Default-Cloud
+    name: "{{ cloud_name }}"
     prefer_static_routes: false
     vcenter_configuration:
       datacenter: "{{ cloud_vcenter_datacenter }}"
-      management_network: /api/vimgrnwruntime/?name={{ se_management_port_group }}
+      management_network: "/api/vimgrnwruntime/?name={{ se_management_port_group }}"
       password: "{{ cloud_vcenter_password }}"
       privilege: WRITE_ACCESS
       username: "{{ cloud_vcenter_username }}"
@@ -22,7 +21,7 @@
 - name: Set Management Network Default Gateway
   vmware.alb.avi_vrfcontext:
     avi_credentials: "{{ avi_credentials }}"
-    cloud_ref: "/api/cloud?name=Default-Cloud"
+    cloud_ref: "/api/cloud?name={{ cloud_name }}"
     name: "management"
     static_routes:
       - route_id: "1"
@@ -47,7 +46,7 @@
     http_method: get
     path: cloudruntime
     params:
-      name: Default-Cloud
+      name: "{{ cloud_name }}"
   register: cloud_status
 
 - name: Abort on sync error
@@ -60,7 +59,7 @@
 - name: Set VIP Network Default Gateway
   vmware.alb.avi_vrfcontext:
     avi_credentials: "{{ avi_credentials }}"
-    cloud_ref: "/api/cloud?name=Default-Cloud"
+    cloud_ref: "/api/cloud?name={{ cloud_name }}"
     name: "global"
     static_routes:
       - route_id: "1"
@@ -79,6 +78,8 @@
     avi_credentials: "{{ avi_credentials }}"
     name: "{{ se_management_port_group }}"
     dhcp_enabled: false
+    exclude_discovered_subnets: true
+    cloud_ref: "/api/cloud?name={{ cloud_name }}"
     configured_subnets:
       - prefix:
           ip_addr:
@@ -93,14 +94,67 @@
               end:
                 addr: "{{ se_management_network_range.split('-')[1] }}"
                 type: "V4"
-  register: management_network_result
+  when: "se_management_network_range is defined and '__omit' not in se_management_network_range"
+
+- name: Gather all DV portgroup info
+  community.vmware.vmware_dvs_portgroup_info:
+    hostname: "{{ cloud_vcenter_url }}"
+    username: "{{ cloud_vcenter_username }}"
+    password: "{{ cloud_vcenter_password }}"
+    validate_certs: false
+    datacenter: "{{ cloud_vcenter_datacenter }}"
+  register: result_dv_port_groups
+  when: cloud_exclude_other_networks
+
+- name: Gather portgroup info about all ESXi Host in given Cluster
+  community.vmware.vmware_portgroup_info:
+    hostname: "{{ cloud_vcenter_url }}"
+    username: "{{ cloud_vcenter_username }}"
+    password: "{{ cloud_vcenter_password }}"
+    validate_certs: false
+    cluster_name: "{{ se_vcenter_cluster }}"
+  register: result_port_groups
+  when: cloud_exclude_other_networks
+
+- name: Set list of available DV port groups
+  ansible.builtin.set_fact:
+    exclude_dv_pgs: >-
+      [
+        {% for vds_pg_list in result_dv_port_groups.dvs_portgroup_info.values() %}
+          {% for pg_details in vds_pg_list %}
+            {% if pg_details.portgroup_name != se_management_port_group and pg_details.portgroup_name != se_vip_port_group %}
+              "{{ pg_details.portgroup_name }}",
+            {% endif %}
+          {% endfor %}
+        {% endfor %}
+
+        {% for host_pg_list in result_port_groups.hosts_portgroup_info.values() %}
+          {% for pg_details in host_pg_list %}
+            {% if pg_details.portgroup != se_management_port_group and pg_details.portgroup != se_vip_port_group and
+               pg_details.portgroup != 'Management Network' %}
+              "{{ pg_details.portgroup }}",
+            {% endif %}
+          {% endfor %}
+        {% endfor %}
+      ]
+  when: cloud_exclude_other_networks
+
+- name: Exclude networks
+  vmware.alb.avi_network:
+    avi_credentials: "{{ avi_credentials }}"
+    name: "{{ item }}"
+    cloud_ref: "/api/cloud?name={{ cloud_name }}"
+    dhcp_enabled: false
+    exclude_discovered_subnets: true
+  loop: "{{ exclude_dv_pgs }}"
+  when: cloud_exclude_other_networks
 
 - name: Create IPAM provider
   vmware.alb.avi_ipamdnsproviderprofile:
     avi_credentials: "{{ avi_credentials }}"
     internal_profile:
       usable_networks:
-        - nw_ref: /api/network/?name={{ se_vip_port_group }}
+        - nw_ref: "{{ se_vip_port_group }}"
       ttl: 30
     name: tkg-ipam-profile
     type: IPAMDNS_TYPE_INTERNAL
@@ -109,8 +163,8 @@
 - name: Configure SE Group
   vmware.alb.avi_serviceenginegroup:
     avi_credentials: "{{ avi_credentials }}"
-    name: "Default-Group"
-    cloud_ref: "/api/cloud?name=Default-Cloud"
+    name: "{{ se_group_name }}"
+    cloud_ref: "/api/cloud?name={{ cloud_name }}"
     se_name_prefix: "{{ seg_prefix | default('Avi') }}"
     vcpus_per_se: "{{ se_vcpu | default(1) }}"
     ha_mode: "{{ ha_mode | default('HA_MODE_LEGACY_ACTIVE_STANDBY') }}"  # Defaults to for Essential
@@ -137,11 +191,11 @@
     dhcp_enabled: false
     enable_vip_static_routes: false
     ipam_provider_ref: /api/ipamdnsproviderprofile/?name=tkg-ipam-profile
-    name: Default-Cloud
+    name: "{{ cloud_name }}"
     prefer_static_routes: false
     vcenter_configuration:
-      datacenter: Lab
-      management_network: /api/vimgrnwruntime/?name={{ se_management_port_group }}
+      datacenter: "{{ cloud_vcenter_datacenter }}"
+      management_network: "/api/vimgrnwruntime/?name={{ se_management_port_group }}"
       password: "{{ cloud_vcenter_password }}"
       privilege: WRITE_ACCESS
       username: "{{ cloud_vcenter_username }}"
@@ -165,7 +219,9 @@
   vmware.alb.avi_network:
     avi_credentials: "{{ avi_credentials }}"
     name: "{{ se_vip_port_group }}"
+    cloud_ref: "/api/cloud?name={{ cloud_name }}"
     dhcp_enabled: false
+    exclude_discovered_subnets: true
     configured_subnets:
       - prefix:
           ip_addr:
@@ -180,4 +236,5 @@
               end:
                 addr: "{{ se_vip_network_range.split('-')[1] }}"
                 type: "V4"
+            type: "STATIC_IPS_FOR_VIP_AND_SE"
   register: vip_network_result

roles/nsx-alb/configure-cloud/vars/main.yml

@@ -2,6 +2,7 @@
 controller_ip: "{{ nsx_alb.controller_ip }}"
 controller_username: "{{ nsx_alb.controller_username }}"
 controller_password: "{{ nsx_alb.controller_password }}"
+controller_vcenter_cluster: "{{ nsx_alb.controller_vcenter_cluster }}"
 api_version: "{{ nsx_alb.api_version }}"
 # api_version: 22.1.3
 
@@ -12,17 +13,20 @@ avi_credentials:
   api_version: "{{ api_version }}"
 
 # The cloud vCenter will host the service engines and Tanzu clusters
+cloud_name: "{{ nsx_alb.cloud_name }}"
 cloud_vcenter_username: "{{ nsx_alb.cloud_vcenter_username }}"
 cloud_vcenter_password: "{{ nsx_alb.cloud_vcenter_password }}"
 cloud_vcenter_url: "{{ nsx_alb.cloud_vcenter_url }}"
 cloud_vcenter_datacenter: "{{ nsx_alb.cloud_vcenter_datacenter }}"
+cloud_exclude_other_networks: "{{ nsx_alb.cloud_exclude_other_networks }}"
 
+se_group_name: "{{ nsx_alb.se_group_name }}"
 se_vcenter_cluster: "{{ nsx_alb.se_vcenter_cluster }}"
 # The management network will host he service engine management interface
 se_management_port_group: "{{ nsx_alb.se_management_port_group }}"
 se_management_network_cidr: "{{ nsx_alb.se_management_network_cidr }}"
-se_management_network_range: "{{ nsx_alb.se_management_network_range }}"
-se_management_network_gateway: "{{ nsx_alb.se_management_network_gateway }}"
+se_management_network_range: "{{ nsx_alb.se_management_network_range | default(omit) }}"
+# se_management_network_gateway: "{{ nsx_alb.se_management_network_gateway | default(omit) }}"
 # The vip network will contain the virtual servers created by Avi
 se_vip_port_group: "{{ nsx_alb.se_vip_port_group }}"
 se_vip_network_cidr: "{{ nsx_alb.se_vip_network_cidr }}"

roles/nsx-alb/deploy-controller/tasks/main.yml

@@ -38,8 +38,8 @@
     status_code: 200,302,301,503
   register: result_controller_check
   until: result_controller_check.status == 200
-  retries: 120
-  delay: 60
+  retries: 480
+  delay: 15
 
 - name: Check Cluster Status
   ansible.builtin.uri:
@@ -49,16 +49,16 @@
     status_code: 200,302,301,503
   register: cluster_status
   until: cluster_status.json.cluster_state.progress == 100
-  retries: 120
-  delay: 60
+  retries: 480
+  delay: 15
 
 - name: Wait to all services be ready
   ansible.builtin.pause:
     echo: false
     seconds: 60
   when: controller_results.changed
 
-- name: Get main Cluster Information using avi_api_session
+- name: Check to see if password needs to be reset
   vmware.alb.avi_api_session:
     avi_credentials: "{{ avi_credentials }}"
     http_method: get
@@ -82,7 +82,9 @@
     password: "{{ controller_password }}"
     api_version: "{{ api_version }}"
     old_password: "{{ controller_default_password }}"
-  when: "access_test.module_stderr is defined and 'Invalid credentials' in access_test.module_stderr"
+  when: >
+    "access_test.module_stderr is defined and
+    'Invalid credentials' in access_test.module_stderr"
   register: useraccount
 
 - name: Wait to for pw change
@@ -106,7 +108,7 @@
         - "{{ controller_ip }}"
   register: controller_cert_result
 
-- name: Basic Controller Config
+- name: SettingbBasic Controller config
   vmware.alb.avi_systemconfiguration:
     avi_credentials: "{{ avi_credentials }}"
     welcome_workflow_complete: true
@@ -137,11 +139,9 @@
     seconds: 15
   when: system_config_result.changed
 
-# Backup configuration currently broken
 - name: Create BackupConfiguration object
   vmware.alb.avi_backupconfiguration:
     avi_credentials: "{{ avi_credentials }}"
-    # state: present
     name: Backup-Configuration
     backup_passphrase: VMware1!
     upload_to_remote_host: false

roles/tanzu/multi-cloud-generate-config/tasks/main.yml

@@ -1,7 +1,31 @@
 ---
 
-- name: Create YAML template file for Tanzu Management Cluster
-  ansible.builtin.template:
-    src: management-cluster.yml
-    dest: "{{ playbook_dir }}/{{ tanzu_multi_cloud.generated_config_file_name }}"
-    mode: '660'
+- name: Load management cluster config
+  ansible.builtin.set_fact:
+    mgmt_cluster_config: "{{ lookup('template', 'management-cluster.yml') | from_yaml }}"
+
+# Allow insertion of variable if set
+# Workaround for j2 tempolate not removing keys when omit used
+- name: Set VSPHERE_CONTROL_PLANE_ENDPOINT if exists
+  ansible.builtin.set_fact:
+    mgmt_cluster_config: |
+      {{
+        mgmt_cluster_config | combine
+        ({ 'VSPHERE_CONTROL_PLANE_ENDPOINT': tanzu_multi_cloud.vsphere_control_plane_endpoint })
+      }}
+  when: tanzu_multi_cloud.vsphere_control_plane_endpoint is defined
+
+# Workaround for j2 always templating strings instead of booleans
+- name: Set AVI_CONTROL_PLANE_HA_PROVIDER
+  ansible.builtin.set_fact:
+    mgmt_cluster_config: |
+      {{
+        mgmt_cluster_config | combine
+        ({ 'AVI_CONTROL_PLANE_HA_PROVIDER': tanzu_multi_cloud.avi_control_plane_ha_provider })
+      }}
+
+- name: Write management cluster config
+  ansible.builtin.copy:
+    content: "{{ mgmt_cluster_config | to_nice_yaml }}"
+    dest: "{{ tanzu_multi_cloud.generated_config_file }}"
+    mode: '0644'