From 1845dec7cebc4b08fc32db705d776afac801612c Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Tue, 5 Nov 2024 16:29:55 +0100 Subject: [PATCH 1/7] Fix openstack_deploy_prep target In the `openstack_deploy_prep` target we are setting `CTLPLANE_IP_ADDRESS_PREFIX` to `NNCP_CTLPLANE_IPV6_ADDRESS_PREFIX`, which isn't correct because we are assigning an IPv6 prefix to an IPv4 prefix variable. This patch changes it to use the right prefix `NNCP_CTLPLANE_IP_ADDRESS_PREFIX`. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ca292c6..194a72c 100644 --- a/Makefile +++ b/Makefile @@ -749,7 +749,7 @@ openstack_deploy_prep: export KIND=OpenStackControlPlane openstack_deploy_prep: export OVN_NICMAPPING=${OVNCONTROLLER_NMAP} openstack_deploy_prep: export NEUTRON_CUSTOM_CONF=${DEPLOY_DIR}/neutron-custom-conf.patch openstack_deploy_prep: export BRIDGE_NAME=${NNCP_BRIDGE} -openstack_deploy_prep: export CTLPLANE_IP_ADDRESS_PREFIX=${NNCP_CTLPLANE_IPV6_ADDRESS_PREFIX} +openstack_deploy_prep: export CTLPLANE_IP_ADDRESS_PREFIX=${NNCP_CTLPLANE_IP_ADDRESS_PREFIX} ifeq ($(NETWORK_ISOLATION_IPV4), true) openstack_deploy_prep: export IPV4_ENABLED=true endif From 547f194fea81556c411926b9828ee46171e7632d Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Wed, 6 Nov 2024 20:16:10 +0100 Subject: [PATCH 2/7] Support changing dataplane scenario directory Current `edpm_deploy_prep` target only allows to change the dataplane kustomize scenario within those provided by the openstack operator, but we may want to provide our own. This patch introduces a way to change the samples dir with a custom one using the `DATAPLANE_SAMPLES_DIR` variable, which would still use the `DATAPLANE_KUSTOMIZE_SCENARIO` variable to form the full location. It also introduces a way to set the full path to the scenario ignoring the scenario variable using variable `DATAPLANE_KUSTOMIZE_DIR`. --- Makefile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 194a72c..01ce628 100644 --- a/Makefile +++ b/Makefile @@ -379,6 +379,7 @@ BAREMETAL_OS_CONTAINER_IMG ?= # Dataplane Operator DATAPLANE_TIMEOUT ?= 30m +DATAPLANE_SAMPLES_DIR ?= ${OPERATOR_BASE_DIR}/openstack-operator/config/samples/dataplane ifeq ($(NETWORK_BGP), true) ifeq ($(BGP_OVN_ROUTING), true) DATAPLANE_KUSTOMIZE_SCENARIO ?= bgp_ovn_cluster @@ -401,6 +402,7 @@ DATAPLANE_NETWORKER_IP ?=172.16.1.200 DATAPLANE_SSHD_ALLOWED_RANGES ?=['172.16.1.0/24'] DATAPLANE_DEFAULT_GW ?= 172.16.1.1 endif +DATAPLANE_KUSTOMIZE_DIR ?= ${DATAPLANE_SAMPLES_DIR}/${DATAPLANE_KUSTOMIZE_SCENARIO} DATAPLANE_TOTAL_NODES ?=1 DATAPLANE_GROWVOLS_ARGS ?=/=8GB /tmp=1GB /home=1GB /var=100% DATAPLANE_TOTAL_NETWORKER_NODES ?=1 @@ -842,7 +844,7 @@ edpm_deploy_prep: edpm_deploy_cleanup openstack_repo ## prepares the CR to insta mkdir -p ${DEPLOY_DIR} cp ${DATAPLANE_EXTRA_NOVA_CONFIG_FILE} ${EDPM_EXTRA_NOVA_CONFIG_FILE} oc apply -f devsetup/edpm/config/ansible-ee-env.yaml - oc kustomize --load-restrictor LoadRestrictionsNone ${OPERATOR_BASE_DIR}/openstack-operator/config/samples/dataplane/${DATAPLANE_KUSTOMIZE_SCENARIO} > ${DEPLOY_DIR}/dataplane.yaml + oc kustomize --load-restrictor LoadRestrictionsNone ${DATAPLANE_KUSTOMIZE_DIR} > ${DEPLOY_DIR}/dataplane.yaml bash scripts/gen-edpm-kustomize.sh ifeq ($(GENERATE_SSH_KEYS), true) make edpm_deploy_generate_keys @@ -882,7 +884,7 @@ edpm_deploy_baremetal_prep: export EDPM_GROWVOLS_ARGS=${DATAPLANE_GROWVOLS_ARGS} edpm_deploy_baremetal_prep: export REPO=${OPENSTACK_REPO} edpm_deploy_baremetal_prep: export BRANCH=${OPENSTACK_BRANCH} edpm_deploy_baremetal_prep: export HASH=${OPENSTACK_COMMIT_HASH} -edpm_deploy_baremetal_prep: export DATAPLANE_KUSTOMIZE_SCENARIO=baremetal +edpm_deploy_baremetal_prep: export DATAPLANE_KUSTOMIZE_DIR=${DATAPLANE_SAMPLES_DIR}/baremetal edpm_deploy_baremetal_prep: export EDPM_ROOT_PASSWORD=${BM_ROOT_PASSWORD} edpm_deploy_baremetal_prep: export EDPM_EXTRA_NOVA_CONFIG_FILE=${DEPLOY_DIR}/25-nova-extra.conf edpm_deploy_baremetal_prep: export EDPM_SERVER_ROLE=compute @@ -891,7 +893,7 @@ edpm_deploy_baremetal_prep: edpm_deploy_cleanup openstack_repo ## prepares the C mkdir -p ${DEPLOY_DIR} cp ${DATAPLANE_EXTRA_NOVA_CONFIG_FILE} ${EDPM_EXTRA_NOVA_CONFIG_FILE} oc apply -f devsetup/edpm/config/ansible-ee-env.yaml - oc kustomize --load-restrictor LoadRestrictionsNone ${OPERATOR_BASE_DIR}/openstack-operator/config/samples/dataplane/${DATAPLANE_KUSTOMIZE_SCENARIO} > ${DEPLOY_DIR}/dataplane.yaml + oc kustomize --load-restrictor LoadRestrictionsNone ${DATAPLANE_KUSTOMIZE_DIR} > ${DEPLOY_DIR}/dataplane.yaml bash scripts/gen-edpm-baremetal-kustomize.sh ifeq ($(GENERATE_SSH_KEYS), true) make edpm_deploy_generate_keys @@ -951,7 +953,7 @@ edpm_deploy_networker_prep: export EDPM_SERVER_ROLE=networker edpm_deploy_networker_prep: export REPO=${OPENSTACK_REPO} edpm_deploy_networker_prep: export BRANCH=${OPENSTACK_BRANCH} edpm_deploy_networker_prep: export HASH=${OPENSTACK_COMMIT_HASH} -edpm_deploy_networker_prep: export DATAPLANE_KUSTOMIZE_SCENARIO=networker +edpm_deploy_networker_prep: export DATAPLANE_KUSTOMIZE_DIR=${DATAPLANE_SAMPLES_DIR}/networker ifeq ($(NETWORK_BGP), true) ifeq ($(BGP_OVN_ROUTING), true) edpm_deploy_networker_prep: export BGP=ovn @@ -964,7 +966,7 @@ edpm_deploy_networker_prep: edpm_deploy_networker_cleanup openstack_repo ## prep $(eval $(call vars,$@,dataplane)) mkdir -p ${DEPLOY_DIR_EDPM_NETWORKER} oc apply -f devsetup/edpm/config/ansible-ee-env.yaml - oc kustomize --load-restrictor LoadRestrictionsNone ${OPERATOR_BASE_DIR}/openstack-operator/config/samples/dataplane/${DATAPLANE_KUSTOMIZE_SCENARIO} > ${DEPLOY_DIR_EDPM_NETWORKER}/dataplane.yaml + oc kustomize --load-restrictor LoadRestrictionsNone ${DATAPLANE_KUSTOMIZE_DIR} > ${DEPLOY_DIR_EDPM_NETWORKER}/dataplane.yaml bash scripts/gen-edpm-kustomize.sh ifeq ($(GENERATE_SSH_KEYS), true) make edpm_deploy_generate_keys From f1375736358901e67262c93cee52cc431d186d7d Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Fri, 8 Nov 2024 11:25:45 +0100 Subject: [PATCH 3/7] Devsetup: Remove dhcp entry even without VM When using the `attach_default_interface_cleanup` target of devsetup on a system where the CRC VM is no longer present it will fail to remove the static DHCP entry from the libvirt network. This patch makes the `interfaces-setup-cleanup.sh` script more robust as it will now remove the static entry even when the VM is gone (using the static IP address assigned). --- devsetup/Makefile | 1 + devsetup/scripts/interfaces-setup-cleanup.sh | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/devsetup/Makefile b/devsetup/Makefile index f62aa10..ee2b492 100644 --- a/devsetup/Makefile +++ b/devsetup/Makefile @@ -341,6 +341,7 @@ attach_default_interface_cleanup: export BGP_NIC_2_MAC=${CRC_BGP_NIC_2_MAC} endif attach_default_interface_cleanup: export INSTANCE_NAME=${NETWORK_ISOLATION_INSTANCE_NAME} attach_default_interface_cleanup: export NETWORK_NAME=${NETWORK_ISOLATION_NET_NAME} +attach_default_interface_cleanup: export IP_ADDRESS=${NETWORK_ISOLATION_IP_ADDRESS} attach_default_interface_cleanup: ## Detach default libvirt network from CRC bash scripts/interfaces-setup-cleanup.sh diff --git a/devsetup/scripts/interfaces-setup-cleanup.sh b/devsetup/scripts/interfaces-setup-cleanup.sh index e69e719..0d74800 100644 --- a/devsetup/scripts/interfaces-setup-cleanup.sh +++ b/devsetup/scripts/interfaces-setup-cleanup.sh @@ -6,16 +6,24 @@ if [ "$EUID" -eq 0 ]; then exit fi -MAC_ADDRESS=$(virsh --connect=qemu:///system dumpxml $INSTANCE_NAME | xmllint --xpath "string(/domain/devices/interface/source[@network=\"$NETWORK_NAME\"]/../mac/@address)" -) +MAC_ADDRESS=$(virsh --connect=qemu:///system dumpxml $INSTANCE_NAME | xmllint --xpath "string(/domain/devices/interface/source[@network=\"$NETWORK_NAME\"]/../mac/@address)" - 2>/dev/null) if [ -n "${MAC_ADDRESS}" ]; then virsh --connect=qemu:///system detach-interface $INSTANCE_NAME network --mac $MAC_ADDRESS # First try to remove the DHCP static IP entry by MAC, if it fails try by hostname - if ! virsh --connect=qemu:///system net-update $NETWORK_NAME delete ip-dhcp-host "" --config --live; then - virsh --connect=qemu:///system net-update $NETWORK_NAME delete ip-dhcp-host "" --config --live + if virsh --connect=qemu:///system net-update $NETWORK_NAME delete ip-dhcp-host "" --config --live 2>/dev/null; then + DHCP_REMOVED=true fi - sleep 5 fi +# Without MAC we try to remove it using the host name or IP address +if [ -z "${DHCP_REMOVED}" ]; then + if ! virsh --connect=qemu:///system net-update $NETWORK_NAME delete ip-dhcp-host "" --config --live 2>/dev/null; then + virsh --connect=qemu:///system net-update $NETWORK_NAME delete ip-dhcp-host "" --config --live 2>/dev/null + fi +fi + +sleep 5 + if [ -n "$BGP" ]; then # We don't destroy the PCI devices here but before adding them, to avoid having to restart the CRC VM twice From 3c536f709cf4c9da503aa9e73c1936f7b0744fb1 Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Thu, 14 Nov 2024 17:13:28 +0100 Subject: [PATCH 4/7] Support post gen edpm kustomize scripts This patch introduces a mechanism to run an arbitrary script on the `edpm_deploy_prep` target after the `kustomization.yaml` file has been created by the `gen-edpm-kustomize.sh` script. This provides a flexible mechanism to modify things in the kustomization that are not currently possible. An example is to use a different ansible host for deployment than the IP that will be used as the `ctlplane` network IP. The variable used to define the location of the script is `EDPM_POST_GEN_SCRIPT`. Example of usage: ``` $ make edpm_deploy \ DATAPLANE_POST_GEN_SCRIPT="$(realpath ./scripts/change-ansible-hosts.sh)" ``` And the contents of `change-ansible-hosts.sh` are: ``` sed -i "s/value: 192.168.122.100/value: 192.168.1.13/" kustomization.yaml ``` --- Makefile | 1 + scripts/gen-edpm-kustomize.sh | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 01ce628..3a68d61 100644 --- a/Makefile +++ b/Makefile @@ -832,6 +832,7 @@ edpm_deploy_prep: export BRANCH=${OPENSTACK_BRANCH} edpm_deploy_prep: export HASH=${OPENSTACK_COMMIT_HASH} edpm_deploy_prep: export EDPM_TLS_ENABLED=${DATAPLANE_TLS_ENABLED} edpm_deploy_prep: export EDPM_NOVA_NFS_PATH=${DATAPLANE_NOVA_NFS_PATH} +edpm_deploy_prep: export EDPM_POST_GEN_SCRIPT=${DATAPLANE_POST_GEN_SCRIPT} ifeq ($(NETWORK_BGP), true) ifeq ($(BGP_OVN_ROUTING), true) edpm_deploy_prep: export BGP=ovn diff --git a/scripts/gen-edpm-kustomize.sh b/scripts/gen-edpm-kustomize.sh index 3089e29..5f3b0d2 100644 --- a/scripts/gen-edpm-kustomize.sh +++ b/scripts/gen-edpm-kustomize.sh @@ -222,4 +222,8 @@ fi kustomization_add_resources +if [ -n "${EDPM_POST_GEN_SCRIPT}" ]; then + . "${EDPM_POST_GEN_SCRIPT}" +fi + popd From bf41a5a863d8e7c05a87edc0fac1fd35699b6781 Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Fri, 29 Nov 2024 23:28:32 +0100 Subject: [PATCH 5/7] NNCP Set MTU for all VLANs If we don't set the VLANs MTUs in the NNCP we will always get a default value and not a value based on the bridge like it should. With this patch we allow setting the VLAN MTU specifically for the OCP cluster. This is necessary when the underlying interface is a tunnel and not a physical NIC. By default it will be set to the same value as the network mtu. --- Makefile | 3 +++ scripts/gen-nncp.sh | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3a68d61..8126a88 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,8 @@ NETWORK_ISOLATION_USE_DEFAULT_NETWORK ?= true NETWORK_ISOLATION_IPV4 ?= true NETWORK_ISOLATION_IPV6 ?= false NETWORK_MTU ?= 1500 +# VLAN MTU needs to be 4 bytes less than the NETWORK_MTU when tunneling +NETWORK_VLAN_MTU ?= NETWORK_VLAN_START ?= 20 NETWORK_VLAN_STEP ?= 1 NETWORK_ISOLATION_IPV4_ADDRESS ?= 172.16.1.1/24 @@ -2304,6 +2306,7 @@ nncp: export CTLPLANE_IP_ADDRESS_SUFFIX=${NNCP_CTLPLANE_IP_ADDRESS_SUFFIX} nncp: export DNS_SERVER=${NNCP_DNS_SERVER} endif nncp: export INTERFACE_MTU=${NETWORK_MTU} +nncp: export VLAN_MTU=${NETWORK_VLAN_MTU} nncp: export VLAN_START=${NETWORK_VLAN_START} nncp: export VLAN_STEP=${NETWORK_VLAN_STEP} nncp: export STORAGE_MACVLAN=${NETWORK_STORAGE_MACVLAN} diff --git a/scripts/gen-nncp.sh b/scripts/gen-nncp.sh index 52a9995..98db5ac 100644 --- a/scripts/gen-nncp.sh +++ b/scripts/gen-nncp.sh @@ -46,6 +46,9 @@ if [ -n "$BGP" ]; then check_var_set BGP_2_IP_ADDRESS fi +# Default VLANs MTU to the interface MTU if not defined or empty +[ -n "${VLAN_MTU}" ] || VLAN_MTU="${INTERFACE_MTU}" + echo DEPLOY_DIR ${DEPLOY_DIR} echo WORKERS ${WORKERS} echo INTERFACE ${INTERFACE} @@ -53,6 +56,7 @@ echo BRIDGE_NAME ${BRIDGE_NAME} echo INTERFACE_BGP_1 ${INTERFACE_BGP_1} echo INTERFACE_BGP_2 ${INTERFACE_BGP_2} echo INTERFACE_MTU ${INTERFACE_MTU} +echo VLAN_MTU ${VLAN_MTU} echo VLAN_START ${VLAN_START} echo VLAN_STEP ${VLAN_STEP} echo STORAGE_MACVLAN ${STORAGE_MACVLAN} @@ -187,6 +191,7 @@ EOF_CAT cat >> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> "${DEPLOY_DIR}/${WORKER}_nncp.yaml" <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml < Date: Wed, 11 Dec 2024 19:49:59 +0100 Subject: [PATCH 6/7] Allow skipping RDO repo setup Currently we always try to setup the RDO repo, but if we run the script on a RHEL repo it will fail to find the URL. With this patch we allow not setting up the repository. --- Makefile | 2 ++ scripts/gen-edpm-kustomize.sh | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/Makefile b/Makefile index 8126a88..f72dc7a 100644 --- a/Makefile +++ b/Makefile @@ -421,6 +421,7 @@ DATAPLANE_EXTRA_NOVA_CONFIG_FILE ?= /dev/null DATAPLANE_SERVER_ROLE ?= compute DATAPLANE_TLS_ENABLED ?= true DATAPLANE_NOVA_NFS_PATH ?= +DATAPLANE_SKIP_REPO_SETUP ?= # Manila MANILA_IMG ?= quay.io/openstack-k8s-operators/manila-operator-index:${OPENSTACK_K8S_TAG} @@ -835,6 +836,7 @@ edpm_deploy_prep: export HASH=${OPENSTACK_COMMIT_HASH} edpm_deploy_prep: export EDPM_TLS_ENABLED=${DATAPLANE_TLS_ENABLED} edpm_deploy_prep: export EDPM_NOVA_NFS_PATH=${DATAPLANE_NOVA_NFS_PATH} edpm_deploy_prep: export EDPM_POST_GEN_SCRIPT=${DATAPLANE_POST_GEN_SCRIPT} +edpm_deploy_prep: export EDPM_SKIP_REPO_SETUP=${DATAPLANE_SKIP_REPO_SETUP} ifeq ($(NETWORK_BGP), true) ifeq ($(BGP_OVN_ROUTING), true) edpm_deploy_prep: export BGP=ovn diff --git a/scripts/gen-edpm-kustomize.sh b/scripts/gen-edpm-kustomize.sh index 5f3b0d2..c82db18 100644 --- a/scripts/gen-edpm-kustomize.sh +++ b/scripts/gen-edpm-kustomize.sh @@ -95,10 +95,15 @@ cat <>kustomization.yaml EOF fi +if [ -z "$EDPM_SKIP_REPO_SETUP" ]; then cat <>kustomization.yaml - op: add path: /spec/services/0 value: repo-setup +EOF +fi + +cat <>kustomization.yaml - op: replace path: /spec/nodeTemplate/ansible/ansibleVars/timesync_ntp_servers value: From 1d22cd195241aad80922622dcdbb23fbb08e8e60 Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Wed, 11 Dec 2024 19:53:19 +0100 Subject: [PATCH 7/7] Allow using default images values for EDPM With this patch we'll be able to allow using the default values for `registry_url`, `image_prefix` and `image_tag` when the values for their corresponding variables are empty. --- scripts/gen-edpm-kustomize.sh | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/scripts/gen-edpm-kustomize.sh b/scripts/gen-edpm-kustomize.sh index c82db18..a8d606c 100644 --- a/scripts/gen-edpm-kustomize.sh +++ b/scripts/gen-edpm-kustomize.sh @@ -103,23 +103,38 @@ cat <>kustomization.yaml EOF fi +if [ -n "$EDPM_REGISTRY_URL" ]; then cat <>kustomization.yaml - - op: replace - path: /spec/nodeTemplate/ansible/ansibleVars/timesync_ntp_servers - value: - - {hostname: ${EDPM_NTP_SERVER}} - - op: replace - path: /spec/nodeTemplate/ansible/ansibleVars/neutron_public_interface_name - value: ${EDPM_NETWORK_INTERFACE_NAME} - op: replace path: /spec/nodeTemplate/ansible/ansibleVars/registry_url value: ${EDPM_REGISTRY_URL} +EOF +fi + +if [ -n "$EDPM_CONTAINER_PREFIX" ]; then +cat <>kustomization.yaml - op: replace path: /spec/nodeTemplate/ansible/ansibleVars/image_prefix value: ${EDPM_CONTAINER_PREFIX} +EOF +fi + +if [ -n "$EDPM_CONTAINER_TAG" ]; then +cat <>kustomization.yaml - op: replace path: /spec/nodeTemplate/ansible/ansibleVars/image_tag value: ${EDPM_CONTAINER_TAG} +EOF +fi + +cat <>kustomization.yaml + - op: replace + path: /spec/nodeTemplate/ansible/ansibleVars/timesync_ntp_servers + value: + - {hostname: ${EDPM_NTP_SERVER}} + - op: replace + path: /spec/nodeTemplate/ansible/ansibleVars/neutron_public_interface_name + value: ${EDPM_NETWORK_INTERFACE_NAME} - op: replace path: /spec/nodeTemplate/ansible/ansibleVars/edpm_sshd_allowed_ranges value: ${EDPM_SSHD_ALLOWED_RANGES}