From aaa16daee8169e7d3fc2972490428e582e3a4209 Mon Sep 17 00:00:00 2001 From: Tom Weininger Date: Wed, 13 Nov 2024 12:09:05 +0100 Subject: [PATCH] Octavia adoption docs Updates for adoption of octavia. Split Octavia adoption steps into control plane adoption and data plane adoption. --- .../assembly_adopting-the-data-plane.adoc | 2 + ...c_adopting-the-loadbalancer-dataplane.adoc | 41 +++ .../proc_adopting-the-identity-service.adoc | 2 +- ...roc_adopting-the-loadbalancer-service.adoc | 249 ++++++++++++++++-- ...lling-back-the-control-plane-adoption.adoc | 8 +- .../proc_stopping-openstack-services.adoc | 6 + .../control_plane_rollback/tasks/main.yaml | 8 +- .../stop_openstack_services/tasks/main.yaml | 6 + 8 files changed, 301 insertions(+), 21 deletions(-) create mode 100644 docs_user/assemblies/proc_adopting-the-loadbalancer-dataplane.adoc diff --git a/docs_user/assemblies/assembly_adopting-the-data-plane.adoc b/docs_user/assemblies/assembly_adopting-the-data-plane.adoc index 39c1494d2..4ed424f75 100644 --- a/docs_user/assemblies/assembly_adopting-the-data-plane.adoc +++ b/docs_user/assemblies/assembly_adopting-the-data-plane.adoc @@ -24,5 +24,7 @@ include::../modules/proc_performing-a-fast-forward-upgrade-on-compute-services.a include::../modules/proc_adopting-networker-services-to-the-data-plane.adoc[leveloffset=+1] +include::../modules/proc_adopting-the-loadbalancer-dataplane.adoc[leveloffset=+1] + ifdef::parent-context[:context: {parent-context}] ifndef::parent-context[:!context:] diff --git a/docs_user/assemblies/proc_adopting-the-loadbalancer-dataplane.adoc b/docs_user/assemblies/proc_adopting-the-loadbalancer-dataplane.adoc new file mode 100644 index 000000000..b9c4c6597 --- /dev/null +++ b/docs_user/assemblies/proc_adopting-the-loadbalancer-dataplane.adoc @@ -0,0 +1,41 @@ +[id="adopting-the-loadbalancer-dataplane_{context}"] + += {loadbalancer_service} data plane adoption +At this point all services on the new control plane should be running including +the {loadbalancer_service} services. + +== Failover load balancers in ERROR state + +As the management network configuration and the controller IP has changed +in the new control plane the amphora load balancers will show up in ERROR +state at this point. + +[source,bash] +---- +$ openstack loadbalancer list ++--------------------------------------+------+----------------------------------+--------------+---------------------+------------------+----------+ +| id | name | project_id | vip_address | provisioning_status | operating_status | provider | ++--------------------------------------+------+----------------------------------+--------------+---------------------+------------------+----------+ +| 5dabaa20-d9df-42a7-b74e-8533ac99b152 | lb1 | c388c1dbd3ff40009a523726e009b4cf | 172.24.4.250 | ERROR | ONLINE | amphora | ++--------------------------------------+------+----------------------------------+--------------+---------------------+------------------+----------+ +---- + +A failover is necessary to configure the loadbalancers for the new environment and to +upgrade them to use the new amphora image. + +[source,bash] +---- +$ openstack loadbalancer failover --wait 5dabaa20-d9df-42a7-b74e-8533ac99b152 +---- + +This can be automated if necessary. The following script does this by +failing over all loadbalancers in error, but only five at a time. + +// TODO(tweining): test + +[source,bash] +---- +# TODO(tweining): verify it is provisioning status, not operating status +$ openstack loadbalancer list --provisioning-status ERROR -f value -c id | \ + xargs -P 5 -I {} openstack loadbalancer failover --wait {} +---- diff --git a/docs_user/modules/proc_adopting-the-identity-service.adoc b/docs_user/modules/proc_adopting-the-identity-service.adoc index 039723a1a..822caf8ee 100644 --- a/docs_user/modules/proc_adopting-the-identity-service.adoc +++ b/docs_user/modules/proc_adopting-the-identity-service.adoc @@ -63,7 +63,7 @@ control plane, excluding the {identity_service} and its endpoints: ---- $ openstack endpoint list | grep keystone | awk '/admin/{ print $2; }' | xargs ${BASH_ALIASES[openstack]} endpoint delete || true -for service in aodh heat heat-cfn barbican cinderv3 glance gnocchi manila manilav2 neutron nova placement swift ironic-inspector ironic; do +for service in aodh heat heat-cfn barbican cinderv3 glance gnocchi manila manilav2 neutron nova placement swift ironic-inspector ironic octavia; do openstack service list | awk "/ $service /{ print \$2; }" | xargs -r ${BASH_ALIASES[openstack]} service delete || true done ---- diff --git a/docs_user/modules/proc_adopting-the-loadbalancer-service.adoc b/docs_user/modules/proc_adopting-the-loadbalancer-service.adoc index 483519253..9dd9d300c 100644 --- a/docs_user/modules/proc_adopting-the-loadbalancer-service.adoc +++ b/docs_user/modules/proc_adopting-the-loadbalancer-service.adoc @@ -5,39 +5,252 @@ During the adoption process the {loadbalancer_first_ref} service must stay disabled in the new control plane. -== Certificates -Before running the script below the shell variables `CONTROLLER1_SSH` and -`CONTROLLER1_SCP` must be set to contain the command to log into one of the -controllers using `ssh` and `scp` respectively as root user as shown below. +== Control plane adoption steps +. Create an alias for the `openstack` command ++ [source,bash] ---- -$ CONTROLLER1_SSH="ssh -i root@192.168.122.100" -$ CONTROLLER1_SCP="scp -i root@192.168.122.100" +$ alias openstack="oc exec -t openstackclient -- openstack" ---- - -Make sure to replace `` with the correct path to the ssh -key for connecting to the controller. - +. Certificates ++ +Before running the script below set the +`CONTROLLER1_SCP` shell variable based on the value of the +existing `CONTROLLER1_SSH` variable: ++ +[source,bash] +---- +$ CONTROLLER1_SCP=$(echo "$CONTROLLER1_SSH" | sed 's/^ssh/scp/g') +---- ++ +Then run the following set of commands in order to regenerate the +keys and certificates and install the data in OpenShift. These +commands convert the existing single CA configuration into a +dual CA configuration. ++ [source,bash] ---- include::../../tests/roles/dataplane_adoption/tasks/octavia_certs.yaml[lines="7..83",indent=0] ---- -These commands convert the existing single CA configuration into a dual CA configuration. - -// == TODO: Other things -// TODO +. Add the octavia interfaces to each NodeNetworkConfigurationPolicy. ++ +The following command adds the network interface that is being +used as the VLAN base interface for the management network +(in this example `enp6s0`) for network isolation on the OpenShift nodes. +Please replace `enp6s0` with the name of the network interface in your +OpenShift setup. ++ +[source,bash] +---- +oc get -n openstack --no-headers nncp | cut -f 1 -d ' ' | while read ; do + oc patch -n openstack nncp $REPLY --type=merge --patch ' + spec: + desiredState: + interfaces: + - description: Octavia vlan host interface + name: enp6s0.24 + state: up + type: vlan + vlan: + base-iface: enp6s0 + id: 24 + - bridge: + options: + stp: + enabled: false + port: + - name: enp6s0.24 + description: Configuring bridge octbr + mtu: 1500 + name: octbr + state: up + type: linux-bridge + ' +done +---- -== Enabling the {loadbalancer_service} in OpenShift -Run the following command in order to enable the {loadbalancer_service} CR. +. Configure the network attachment definition ++ +The `octavia` network attachment is needed to connect pods +that manage amphorae +and the OpenvSwitch pods (managed by the OVN operator): ++ +[source,bash] +---- +$ cat >> octavia-nad.yaml << EOF_CAT +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + labels: + osp/net: octavia + name: octavia + namespace: openstack +spec: + config: | + { + "cniVersion": "0.3.1", + "name": "octavia", + "type": "bridge", + "bridge": "octbr", + "ipam": { + "type": "whereabouts", + "range": "172.23.0.0/24", + "range_start": "172.23.0.30", + "range_end": "172.23.0.70", + "routes": [ + { + "dst": "172.24.0.0/16", + "gw" : "172.23.0.150" + } + ] + } + } +EOF_CAT +$ oc apply -n openstack -f octavia-nad.yaml +---- +. Enable the {loadbalancer_service} service in OpenShift ++ [source,bash] ---- -$ oc patch openstackcontrolplane openstack --type=merge --patch ' +$ oc patch openstackcontrolplane openstack -n openstack --type=merge --patch ' spec: + ovn: + template: + ovnController: + nicMappings: + octavia: octbr octavia: enabled: true - template: {} + template: + apacheContainerImage: registry.redhat.io/rhel8/httpd-24:latest + amphoraImageContainerImage: quay.io/gthiemonge/octavia-amphora-image + databaseInstance: openstack + octaviaHousekeeping: + networkAttachments: + - octavia + octaviaHealthManager: + networkAttachments: + - octavia + octaviaWorker: + networkAttachments: + - octavia ' ---- + +. Wait for the {loadbalancer_service} control plane services CRs to become ready ++ +[source,bash] +---- +$ oc wait --for condition=Ready --timeout=300s octavia.octavia.openstack.org octavia +---- ++ + +. Ensure the {loadbalancer_service} service is registered in the {identity_service}: ++ +[source,bash] +---- +$ openstack service list | grep load-balancer +| bd078ca6f90c4b86a48801f45eb6f0d7 | octavia | load-balancer | +$ openstack endpoint list | grep load-balancer +| f1ae7756b6164baf9cb82a1a670067a2 | regionOne | octavia | load-balancer | True | public | https://octavia-public-openstack.apps-crc.testing | +| ff3222b4621843669e89843395213049 | regionOne | octavia | load-balancer | True | internal | http://octavia-internal.openstack.svc:9876 | +---- + +== Clean up old resources + +. Delete old flavors that have been migrated ++ +[source,bash] +---- +$ openstack flavor list --all | grep octavia +| 484c351a-57ca-4a4b-8e6e-93d31596fec5 | octavia-amphora-4vcpus | 4096 | 3 | 0 | 4 | False | +| 65 | octavia_65 | 1024 | 3 | 0 | 1 | False | +| amphora-mvcpu-ha | octavia_amphora-mvcpu-ha | 4096 | 3 | 0 | 4 | False | +| cf9d1d80-5680-4ed8-a051-e8ec4c5871e0 | octavia-amphora | 1024 | 3 | 0 | 1 | False | +$ openstack flavor delete octavia_65 +$ openstack flavor delete octavia_amphora-mvcpu-ha +$ openstack flavor list --all | grep octavia +| 484c351a-57ca-4a4b-8e6e-93d31596fec5 | octavia-amphora-4vcpus | 4096 | 3 | 0 | 4 | False | +| cf9d1d80-5680-4ed8-a051-e8ec4c5871e0 | octavia-amphora | 1024 | 3 | 0 | 1 | False | +---- + +. Delete old octavia flavors that have been migrated ++ +[source,bash] +---- +$ openstack loadbalancer flavor list ++--------------------------------------+--------------------------+--------------------------------------+---------+ +| id | name | flavor_profile_id | enabled | ++--------------------------------------+--------------------------+--------------------------------------+---------+ +| 5db54d9b-ba08-4b51-a859-0a81533604aa | octavia_amphora-mvcpu-ha | 4fa6a793-4c20-4480-be4f-806912840511 | True | +| 6d649fd5-6322-4265-b5f3-c3277fc29ec8 | amphora-4vcpus | d9764a80-99f5-4f22-bbe0-3ddbdc5c485c | True | +| 93f34308-24a7-42de-9065-959a3b36e7f6 | amphora | e75e50c8-7786-4623-abcf-bccbea59d213 | True | ++--------------------------------------+--------------------------+--------------------------------------+---------+ +$ openstack loadbalancer flavor delete octavia_amphora-mvcpu-ha +$ openstack loadbalancer flavor list ++--------------------------------------+----------------+--------------------------------------+---------+ +| id | name | flavor_profile_id | enabled | ++--------------------------------------+----------------+--------------------------------------+---------+ +| 6d649fd5-6322-4265-b5f3-c3277fc29ec8 | amphora-4vcpus | d9764a80-99f5-4f22-bbe0-3ddbdc5c485c | True | +| 93f34308-24a7-42de-9065-959a3b36e7f6 | amphora | e75e50c8-7786-4623-abcf-bccbea59d213 | True | ++--------------------------------------+----------------+--------------------------------------+---------+ +---- + +. Delete old flavorprofile that have been migrated ++ +[source,bash] +---- +$ openstack loadbalancer flavorprofile list ++--------------------------------------+----------------------------------+---------------+ +| id | name | provider_name | ++--------------------------------------+----------------------------------+---------------+ +| 4fa6a793-4c20-4480-be4f-806912840511 | octavia_amphora-mvcpu-ha_profile | amphora | +| d9764a80-99f5-4f22-bbe0-3ddbdc5c485c | amphora-4vcpus | amphora | +| e75e50c8-7786-4623-abcf-bccbea59d213 | amphora | amphora | ++--------------------------------------+----------------------------------+---------------+ +$ openstack loadbalancer flavorprofile delete octavia_amphora-mvcpu-ha_profile +$ openstack loadbalancer flavorprofile list ++--------------------------------------+----------------+---------------+ +| id | name | provider_name | ++--------------------------------------+----------------+---------------+ +| d9764a80-99f5-4f22-bbe0-3ddbdc5c485c | amphora-4vcpus | amphora | +| e75e50c8-7786-4623-abcf-bccbea59d213 | amphora | amphora | ++--------------------------------------+----------------+---------------+ +---- + +. Delete the old management network ports ++ +The first command stores the network ID of the old management network in the +variable WALLABY_LB_MGMT_NET_ID for later use. Then all ports that are used in the network get deleted. ++ +[source,bash] +---- +HM_PORTID="$(openstack port list --device-owner Octavia:health-mgr -f value -c ID)" +openstack port delete "$HM_PORTID" + +# TODO(tweining): remove once command above has been verified: +$ for net_id in $(openstack network list -f value -c ID --name lb-mgmt-net); do desc=$(openstack network show "$net_id" -f value -c description); [ -z "$desc" ] && WALLABY_LB_MGMT_NET_ID="$net_id" ; done +$ echo $WALLABY_LB_MGMT_NET_ID +1e21f9c1-7485-4104-a2f3-eed098ab9cad +$ for id in $(openstack port list --network "$WALLABY_LB_MGMT_NET_ID" -f value -c ID) ; do openstack port delete "$id" ; done +---- + +. Delete the old management network ++ +[source,bash] +---- +$ openstack network delete "$WALLABY_LB_MGMT_NET_ID" +---- + +. Verify only one lb-mgmt-net and one lb-mgmt-subnet exists ++ +[source,bash] +---- +$ openstack network list | grep lb-mgmt-net +| fe470c29-0482-4809-9996-6d636e3feea3 | lb-mgmt-net | 6a881091-097d-441c-937b-5a23f4f243b7 | +$ openstack subnet list | grep lb-mgmt-subnet +| 6a881091-097d-441c-937b-5a23f4f243b7 | lb-mgmt-subnet | fe470c29-0482-4809-9996-6d636e3feea3 | 172.24.0.0/16 | +---- diff --git a/docs_user/modules/proc_rolling-back-the-control-plane-adoption.adoc b/docs_user/modules/proc_rolling-back-the-control-plane-adoption.adoc index a3802a022..bd9e293c7 100644 --- a/docs_user/modules/proc_rolling-back-the-control-plane-adoption.adoc +++ b/docs_user/modules/proc_rolling-back-the-control-plane-adoption.adoc @@ -56,7 +56,13 @@ ServicesToStart=("tripleo_horizon.service" "tripleo_ceilometer_agent_notification.service" "tripleo_ovn_cluster_north_db_server.service" "tripleo_ovn_cluster_south_db_server.service" - "tripleo_ovn_cluster_northd.service") + "tripleo_ovn_cluster_northd.service" + "tripleo_octavia_api.service" + "tripleo_octavia_health_manager.service" + "tripleo_octavia_rsyslog.service" + "tripleo_octavia_driver_agent.service" + "tripleo_octavia_housekeeping.service" + "tripleo_octavia_worker.service") PacemakerResourcesToStart=("galera-bundle" "haproxy-bundle" diff --git a/docs_user/modules/proc_stopping-openstack-services.adoc b/docs_user/modules/proc_stopping-openstack-services.adoc index 5ad49efc4..2cc018cb0 100644 --- a/docs_user/modules/proc_stopping-openstack-services.adoc +++ b/docs_user/modules/proc_stopping-openstack-services.adoc @@ -64,6 +64,12 @@ ServicesToStop=("tripleo_aodh_api.service" "tripleo_aodh_notifier.service" "tripleo_ceilometer_agent_central.service" "tripleo_ceilometer_agent_notification.service" + "tripleo_octavia_api.service" + "tripleo_octavia_health_manager.service" + "tripleo_octavia_rsyslog.service" + "tripleo_octavia_driver_agent.service" + "tripleo_octavia_housekeeping.service" + "tripleo_octavia_worker.service" "tripleo_horizon.service" "tripleo_keystone.service" "tripleo_barbican_api.service" diff --git a/tests/roles/control_plane_rollback/tasks/main.yaml b/tests/roles/control_plane_rollback/tasks/main.yaml index 11ce151f7..8e918baa6 100644 --- a/tests/roles/control_plane_rollback/tasks/main.yaml +++ b/tests/roles/control_plane_rollback/tasks/main.yaml @@ -44,7 +44,13 @@ "tripleo_ceilometer_agent_notification.service" "tripleo_ovn_cluster_north_db_server.service" "tripleo_ovn_cluster_south_db_server.service" - "tripleo_ovn_cluster_northd.service") + "tripleo_ovn_cluster_northd.service" + "tripleo_octavia_api.service" + "tripleo_octavia_health_manager.service" + "tripleo_octavia_rsyslog.service" + "tripleo_octavia_driver_agent.service" + "tripleo_octavia_housekeeping.service" + "tripleo_octavia_worker.service") PacemakerResourcesToStart=("galera-bundle" "haproxy-bundle" diff --git a/tests/roles/stop_openstack_services/tasks/main.yaml b/tests/roles/stop_openstack_services/tasks/main.yaml index 1e95a5f7b..01514df19 100644 --- a/tests/roles/stop_openstack_services/tasks/main.yaml +++ b/tests/roles/stop_openstack_services/tasks/main.yaml @@ -20,6 +20,12 @@ "tripleo_aodh_notifier.service" "tripleo_ceilometer_agent_central.service" "tripleo_ceilometer_agent_notification.service" + "tripleo_octavia_api.service" + "tripleo_octavia_health_manager.service" + "tripleo_octavia_rsyslog.service" + "tripleo_octavia_driver_agent.service" + "tripleo_octavia_housekeeping.service" + "tripleo_octavia_worker.service" "tripleo_horizon.service" "tripleo_keystone.service" "tripleo_barbican_api.service"