Add CAPI+CAPM3 wf to multi-conductor experiment

Support/Multitenancy/Multiple-Ironic-conductors/.gitignore

@@ -9,3 +9,4 @@ macaddrs
 uuids
 sushy-tools-conf/*
 bmc-*.yaml
+ironic.env

Support/Multitenancy/Multiple-Ironic-conductors/Init-environment-v3.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+trap 'trap - SIGTERM && kill -- -'$$'' SIGINT SIGTERM EXIT
+__dir__=$(realpath "$(dirname "$0")")
+# shellcheck disable=SC1091
+. ./config.sh
+# This is temporarily required since https://review.opendev.org/c/openstack/sushy-tools/+/875366 has not been merged.
+./build-sushy-tools-image.sh
+sudo ./vm-setup.sh
+./configure-minikube.sh
+sudo ./handle-images.sh
+./generate_unique_nodes.sh
+./start_containers.sh
+./start-minikube.sh
+./build-api-server-container-image.sh
+
+./install-ironic.sh
+./install-bmo.sh
+python create_nodes_v3.py
+
+export CLUSTER_TOPOLOGY=true
+clusterctl init --infrastructure=metal3
+kubectl apply -f capim-modified.yaml
+./generate-certificates.sh
+# Wait for apiserver pod to exists
+sleep 60
+
+./create-cluster.sh

Support/Multitenancy/Multiple-Ironic-conductors/README.md

@@ -135,6 +135,19 @@ Now, if you open another terminal and run `kubectl -n metal3 get BMH --watch`, y
 
 Just like before, all of the steps can be ran at once by running the `./Init-environment-v2.sh` script. This script also respects configuration in `config.sh`.
 
+# Multiple ironics - full setup
+
+With BMO already working, we can now proceed to making the multiple ironic conductor and fake ipa work with CAPI and CAPM3, i.e. we will aim to "create" clusters with these fake nodes. Since we do not have any nodes to install the k8s apiserver onto, we will attempt to install the apiserver directly on top of the management cluster, using the great research and experiment that was done by our colleague Lennart Jern, which can be read in full [here](https://github.com/metal3-io/metal3-io.github.io/blob/0592e636bb10b1659437790b38f85cc49c552239/_posts/2023-05-17-Scaling_part_2.md)
+
+In short, for this story to work, you will need to install `kubeadm` and `clustctl` on your system. To simulate the `etcd` server, we added the script `start_fake_etcd.sh` into the equation.
+
+All the setup steps can be run at once with the script `Init-environment-v3.sh`. After that, each time we run the script `create-cluster.sh`, a new BMH man ifest will be applied, and a new 1-node cluster will be created (the 1 node is, of course, coming with 1 kcp object, 1 `Machine` object, and 1 `Metal3Machine` object as usual).
+
+Compared to Lennart's setup, ours has a couple of differences and notes:
+- Our BMO doesn't run in test mode. Instead, we use `fake-ipa` to "trick" `ironic` to think that it is talking with real nodes.
+- We don't expose the apiservers using the domain `test-kube-apiserver.NAMESPACE.svc.cluster.local` (in fact, we still do, but it doesn't seem to expose anything). Instead, we use the ClusterIP ip of the apiserver service.
+- We also bump into the issue of lacking resources due to apiservers taking up too much, so the number of nodes/clusters we can simulate will not be too high. (So far, we have not been able to try running these apiservers on external VMs yet.) Another way to solve this issue might be to come up with some sort of apiserver simulation, the kind of things we already did with `fake-ipa`.
+
 # Requirements
 
 This study was conducted on a VM with the following specs:

Support/Multitenancy/Multiple-Ironic-conductors/build-api-server-container-image.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+IMAGE_NAME="172.22.0.1:5000/localimages/capim"
+
+if [[ $(sudo podman images | grep ${IMAGE_NAME}) != "" ]]; then
+    exit 0
+fi
+CAPI_DIR="/tmp/cluster-api"
+rm -rf "${CAPI_DIR}"
+git clone https://github.com/kubernetes-sigs/cluster-api.git "${CAPI_DIR}"
+
+INMEMORY_DIR="${CAPI_DIR}/test/infrastructure/inmemory"
+
+cp main.go "${INMEMORY_DIR}/main.go"
+
+cd "${INMEMORY_DIR}" || exit
+
+sudo podman build --build-arg=builder_image=docker.io/library/golang:1.20.8 --build-arg=goproxy=https://proxy.golang.org,direct --build-arg=ARCH=amd64 --build-arg=ldflags="-X 'sigs.k8s.io/cluster-api/version.buildDate=2023-10-10T11:47:30Z'  -X 'sigs.k8s.io/cluster-api/version.gitCommit=8ba3f47b053da8bbf63cf407c930a2ee10bfd754' -X 'sigs.k8s.io/cluster-api/version.gitTreeState=dirty' -X 'sigs.k8s.io/cluster-api/version.gitMajor=1' -X 'sigs.k8s.io/cluster-api/version.gitMinor=0' -X 'sigs.k8s.io/cluster-api/version.gitVersion=v1.0.0-4041-8ba3f47b053da8-dirty' -X 'sigs.k8s.io/cluster-api/version.gitReleaseCommit=e09ed61cc9ba8bd37b0760291c833b4da744a985'" ../../.. -t "${IMAGE_NAME}" --file Dockerfile
+
+sudo podman push --tls-verify=false "${IMAGE_NAME}"

Support/Multitenancy/Multiple-Ironic-conductors/build-sushy-tools-image.sh

@@ -1,10 +1,14 @@
 #!/bin/bash
 #
-SUSHYTOOLS_DIR="$HOME/sushy-tools"
+IMAGE_NAME="127.0.0.1:5000/localimages/sushy-tools"
+if [[ $(sudo podman images | grep ${IMAGE_NAME}) != "" ]]; then
+    exit 0
+fi
+SUSHYTOOLS_DIR="/tmp/sushy-tools"
 rm -rf "$SUSHYTOOLS_DIR"
 git clone https://opendev.org/openstack/sushy-tools.git "$SUSHYTOOLS_DIR"
 cd "$SUSHYTOOLS_DIR" || exit
-git fetch https://review.opendev.org/openstack/sushy-tools refs/changes/66/875366/35 && git cherry-pick FETCH_HEAD
+git fetch https://review.opendev.org/openstack/sushy-tools refs/changes/66/875366/36 && git cherry-pick FETCH_HEAD
 
 pip3 install build
 python3 -m build
@@ -43,5 +47,4 @@ RUN mkdir -p /root/sushy
 CMD ["sushy-emulator", "-i", "::", "--config", "/root/sushy/conf.py"]
 EOF
 
-sudo podman build -t 127.0.0.1:5000/localimages/sushy-tools .
-rm -rf "$SUSHYTOOLS_DIR"
+sudo podman build -t "${IMAGE_NAME}" .

Support/Multitenancy/Multiple-Ironic-conductors/capim-modified.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: apiserver
+  labels:
+    app: manager
+spec:
+  containers:
+    - image: 172.22.0.1:5000/localimages/capim
+      imagePullPolicy: Always
+      name: capim
+      env:
+      - name: POD_IP
+        valueFrom:
+          fieldRef:
+            fieldPath: status.podIP
+  restartPolicy: Always

Support/Multitenancy/Multiple-Ironic-conductors/clean.sh

@@ -31,14 +31,18 @@ minikube stop
 minikube delete --all --purge
 
 # Stop and delete containers
-containers=("ironic-ipa-downloader" "ironic" "keepalived" "registry" "ironic-client" "fake-ipa" "openstack-client" "httpd-infra")
-for i in $(seq 1 "$N_SUSHY"); do
-    containers+=("sushy-tools-$i")
-done
-for container in "${containers[@]}"; do
-    echo "Deleting the container: $container"
-    sudo podman stop "$container" &>/dev/null
-    sudo podman rm "$container" &>/dev/null
+declare -a running_containers=($(sudo podman ps --all --format json | jq -r '.[].Names[0]'))
+echo ${running_containers[0]}
+declare -a containers=("ipa-downloader" "ironic" "keepalived" "registry" "ironic-client" "fake-ipa" "openstack-client" "httpd-infra")
+
+for container in "${running_containers[@]}"; do
+    if [[ "${containers[@]}" =~  "${container}" || "${container}" =~ "sushy-tools-"* ]]; then
+        echo "Deleting the container: ${container}"
+        sudo podman stop "$container" &>/dev/null
+        sudo podman rm "$container" &>/dev/null
+    fi
 done
 
-rm -rf macaddrs uuids node.json nodes.json batch.json
+rm -rf bmc-*.yaml
+
+rm -rf macaddrs uuids node.json nodes.json batch.json in-memory-development.yaml sushy-tools-conf ironic.env

Support/Multitenancy/Multiple-Ironic-conductors/config.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-export N_NODES=1000
-export N_SUSHY=30
+export N_NODES=500
+export N_SUSHY=10
 # Put the endpoints of different ironics, separated by spaces
 export IRONIC_ENDPOINTS="172.22.0.2 172.22.0.3 172.22.0.4 172.22.0.5"

Support/Multitenancy/Multiple-Ironic-conductors/configure-minikube.sh

@@ -5,7 +5,7 @@ minikube config set memory 4096
 sudo usermod --append --groups libvirt "$(whoami)"
 while /bin/true; do
   minikube_error=0
-  minikube start  --insecure-registry 172.22.0.1:5000 || minikube_error=1
+  minikube start --insecure-registry 172.22.0.1:5000 --memory 8192 --cpus 8 || minikube_error=1
   if [[ $minikube_error -eq 0 ]]; then
     break
   fi

Support/Multitenancy/Multiple-Ironic-conductors/create-cluster.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+#
+
+source ./config.sh
+CLUSTER_TEMPLATE=manifests/cluster-template.yaml
+export CLUSTER_APIENDPOINT_PORT="6443"
+export IMAGE_CHECKSUM="97830b21ed272a3d854615beb54cf004"
+export IMAGE_CHECKSUM_TYPE="md5"
+export IMAGE_FORMAT="raw"
+export IMAGE_URL="http://192.168.111.1:9999/images/rhcos-ootpa-latest.qcow2"
+export KUBERNETES_VERSION="v1.26.0"
+export WORKERS_KUBEADM_EXTRA_CONFIG=""
+export WORKER_MACHINE_COUNT="0"
+export NODE_DRAIN_TIMEOUT="60s"
+export CTLPLANE_KUBEADM_EXTRA_CONFIG=""
+
+
+create_cluster() {
+  cluster="${1:-test-1}"
+  namespace="${2:-${cluster}}"
+  nodename=${3:-"test1"}
+
+
+  kubectl port-forward pod/apiserver 3333:3333 2>/dev/null&
+
+  echo "Creating cluster ${cluster} in namespace ${namespace}"
+  kubectl create namespace "${namespace}"
+  kubectl -n "${namespace}" apply -f bmc-${nodename}.yaml
+
+  caKeyEncoded=$(cat /tmp/ca.key | base64 -w 0)
+  caCertEncoded=$(cat /tmp/ca.crt | base64 -w 0)
+  etcdKeyEncoded=$(cat /tmp/etcd.key | base64 -w 0)
+  etcdCertEncoded=$(cat /tmp/etcd.crt | base64 -w 0)
+
+  cluster_endpoints=$(curl "localhost:3333/register?resource=${namespace}/${cluster}&caKey=${caKeyEncoded}&caCert=${caCertEncoded}&etcdKey=${etcdKeyEncoded}&etcdCert=${etcdCertEncoded}")
+  host=$(echo ${cluster_endpoints} | jq -r ".Host")
+  port=$(echo ${cluster_endpoints} | jq -r ".Port")
+
+  cat <<EOF > "/tmp/${cluster}-ca-secrets.yaml"
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ${cluster}
+  name: ${cluster}-ca
+  namespace: ${namespace}
+type: kubernetes.io/tls
+data:
+  tls.crt: ${caCertEncoded}
+  tls.key: ${caKeyEncoded}
+EOF
+
+  kubectl -n ${namespace} apply -f /tmp/${cluster}-ca-secrets.yaml
+
+  cat <<EOF > "/tmp/${cluster}-etcd-secrets.yaml"
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ${cluster}
+  name: ${cluster}-etcd
+  namespace: ${namespace}
+type: kubernetes.io/tls
+data:
+  tls.crt: ${etcdCertEncoded}
+  tls.key: ${etcdKeyEncoded}
+EOF
+
+  kubectl -n ${namespace} apply -f /tmp/${cluster}-etcd-secrets.yaml
+
+  # Generate metal3 cluster
+  export CLUSTER_APIENDPOINT_HOST="${host}"
+  export CLUSTER_APIENDPOINT_PORT="${port}"
+  echo "Generating cluster ${cluster} with clusterctl"
+  clusterctl generate cluster "${cluster}" \
+    --from "${CLUSTER_TEMPLATE}" \
+    --target-namespace "${namespace}" > /tmp/${cluster}-cluster.yaml
+  kubectl apply -f /tmp/${cluster}-cluster.yaml
+
+  sleep 10
+
+  wait_for_resource() {
+    resource=$1
+    jsonpath=${2:-"{.items[0].metadata.name}"}
+    while true; do
+      kubectl -n "${namespace}" get "${resource}" -o jsonpath="${jsonpath}" 2> /dev/null
+      if [ $? -eq 0 ]; then
+        return
+      fi
+      sleep 2
+    done
+  }
+
+  bmh_name=$(wait_for_resource "bmh")
+  metal3machine=$(wait_for_resource "m3m")
+  machine=$(wait_for_resource "machine")
+
+  providerID="metal3://$namespace/$bmh_name/$metal3machine"
+  curl "localhost:3333/updateNode?resource=${namespace}/${cluster}&nodeName=${machine}&providerID=${providerID}"
+  echo "Done generating cluster ${cluster} with clusterctl"
+}
+
+for i in $(seq 1 $N_NODES); do
+  namespace="test${i}"
+  if [[ $(kubectl get ns | grep "${namespace}") != "" ]]; then
+    echo "ERROR: Namespace ${namespace} exists. Skip creating cluster"
+    continue
+  fi
+  create_cluster "test${i}" "test${i}" "test${i}"
+done
+
+# Wait for all BMHs to be available. Clusters should be more or less ready by then.
+desired_states=("available" "provisioning" "provisioned")
+for i in $(seq 1 $N_NODES); do
+  namespace="test${i}"
+  bmh_name="$(kubectl -n ${namespace} get bmh -o jsonpath='{.items[0].metadata.name}')"
+  echo "Waiting for BMH ${bmh_name} to become available."
+  while true; do
+    bmh_state="$(kubectl -n ${namespace} get bmh -o jsonpath='{.items[0].status.provisioning.state}')"
+    if [[ ! "${desired_states[@]}" =~ "${bmh_state}" ]]; then
+      break
+    fi
+    sleep 3
+  done
+done
+
+# Run describe for all clusters
+for i in $(seq 1 $N_NODES); do
+  clusterctl -n "test${i}" describe cluster "test${i}"
+done

Support/Multitenancy/Multiple-Ironic-conductors/create_nodes_v3.py

@@ -0,0 +1,60 @@
+import json
+import subprocess
+import time
+import random
+import os
+from multiprocessing import Pool
+
+with open("nodes.json") as f:
+    nodes = json.load(f)
+
+def generate_random_mac():
+    # Generate a random MAC address
+    mac = [random.randint(0x00, 0xff) for _ in range(6)]
+    # Set the locally administered address bit (2nd least significant bit of the 1st byte) to 1
+    mac[0] |= 0x02
+    # Format the MAC address
+    mac_address = ':'.join('%02x' % b for b in mac)
+    return mac_address
+
+def create_node(node):
+    uuid = node["uuid"]
+    name = node["name"]
+    port = 8001 + (int(name.strip("test")) - 1) % int(os.environ.get("N_SUSHY", 10))
+    # subprocess.run(["baremetal", "node", "create", "--driver", "redfish", "--driver-info",
+    # f"redfish_address=http://192.168.111.1:{port}", "--driver-info",
+    # f"redfish_system_id=/redfish/v1/Systems/{uuid}", "--driver-info",
+    # "redfish_username=admin", "--driver-info", "redfish_password=password",
+    #                 "--uuid", uuid, "--name", name], stdout=subprocess.DEVNULL)
+    random_mac = generate_random_mac()
+    manifest = f"""---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {name}-bmc-secret
+  labels:
+      environment.metal3.io: baremetal
+type: Opaque
+data:
+  username: YWRtaW4=
+  password: cGFzc3dvcmQ=
+---
+apiVersion: metal3.io/v1alpha1
+kind: BareMetalHost
+metadata:
+  name: {name}
+spec:
+  online: true
+  bmc:
+    address: redfish+http://192.168.111.1:{port}/redfish/v1/Systems/{uuid}
+    credentialsName: {name}-bmc-secret
+  bootMACAddress: {random_mac}
+  bootMode: legacy
+"""
+    with open(f"bmc-{name}.yaml", "w") as f:
+        f.write(manifest)
+    print(f"Created manifests for node {name}")
+
+if __name__ == "__main__":
+    with Pool(100) as p:
+        conductors = p.map(create_node, nodes)

Support/Multitenancy/Multiple-Ironic-conductors/generate-certificates.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+#
+
+openssl req -x509 -subj "/CN=Kubernetes API" -new -newkey rsa:2048 -nodes -keyout "/tmp/ca.key" -sha256 -days 3650 -out "/tmp/ca.crt"
+
+openssl req -x509 -subj "/CN=ETCD CA" -new -newkey rsa:2048 -nodes -keyout "/tmp/etcd.key" -sha256 -days 3650 -out "/tmp/etcd.crt"

Support/Multitenancy/Multiple-Ironic-conductors/generate_unique_nodes.sh

@@ -23,7 +23,7 @@ echo '[]' > nodes.json
 for i in $(seq 1 "${N_NODES:-100}"); do
   uuid=$(generate_unique uuidgen uuids)
   macaddr=$(generate_unique macgen macaddrs)
-  name="fake${i}" 
+  name="test${i}" 
   jq --arg node_name "${name}" \
     --arg uuid "${uuid}" \
     --arg macaddr "${macaddr}" \

Support/Multitenancy/Multiple-Ironic-conductors/get_ironic_logs.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+log_dir="ironic_logs"
+rm -rf ${log_dir}
+mkdir -p ${log_dir}
+ns="baremetal-operator-system"
+pod_names=($(kubectl -n "${ns}" get pods -o json | jq -r ".items[].metadata.name"))
+for name in ${pod_names[@]}; do
+    containers=($(kubectl -n "${ns}" get pod ${name} -o json | jq -r ".spec.containers[].name"))
+    for c in ${containers[@]}; do
+        kubectl -n "${ns}" logs ${name} -c ${c} > "${log_dir}/${name}-${c}-log.txt"
+    done
+done