-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add CAPI+CAPM3 wf to multi-conductor experiment
- Loading branch information
Showing
30 changed files
with
1,378 additions
and
83 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,3 +9,5 @@ macaddrs | |
| uuids | ||
| sushy-tools-conf/* | ||
| bmc-*.yaml | ||
| ironic.env | ||
| ironic_logs/* | ||
31 changes: 31 additions & 0 deletions
31
Support/Multitenancy/Multiple-Ironic-conductors/Init-environment-v3.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| #!/bin/bash | ||
| set -e | ||
| trap 'trap - SIGTERM && kill -- -'$$'' SIGINT SIGTERM EXIT | ||
| __dir__=$(realpath "$(dirname "$0")") | ||
| # shellcheck disable=SC1091 | ||
| . ./config.sh | ||
| # This is temporarily required since https://review.opendev.org/c/openstack/sushy-tools/+/875366 has not been merged. | ||
| sudo ./vm-setup.sh | ||
| ./configure-minikube.sh | ||
| sudo ./handle-images.sh | ||
| ./build-sushy-tools-image.sh | ||
| ./generate_unique_nodes.sh | ||
| ./start_containers.sh | ||
| ./start-minikube.sh | ||
| ./build-api-server-container-image.sh | ||
|
|
||
| ./install-ironic.sh | ||
| ./install-bmo.sh | ||
|
|
||
| python create_nodes_v3.py | ||
|
|
||
| export CLUSTER_TOPOLOGY=true | ||
| clusterctl init --infrastructure=metal3 | ||
| # kubectl apply -f capim-modified.yaml | ||
| # yq -i "select(documentIndex == 0) | .spec.replicas = ${N_APISERVER_PODS}" apiserver-deployments.yaml | ||
| kubectl apply -f apiserver-deployments.yaml | ||
| ./generate-certificates.sh | ||
| # Wait for apiserver pod to exists | ||
| sleep 90 | ||
|
|
||
| ./create-cluster.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
40 changes: 40 additions & 0 deletions
40
Support/Multitenancy/Multiple-Ironic-conductors/apiserver-deployments.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| --- | ||
| apiVersion: apps/v1 | ||
| kind: Deployment | ||
| metadata: | ||
| name: capim-deployment | ||
| namespace: default | ||
| spec: | ||
| replicas: 2 | ||
| selector: | ||
| matchLabels: | ||
| app: capim | ||
| strategy: | ||
| type: Recreate | ||
| template: | ||
| metadata: | ||
| labels: | ||
| app: capim | ||
| spec: | ||
| containers: | ||
| - image: 172.22.0.1:5000/localimages/capim | ||
| imagePullPolicy: Always | ||
| name: capim | ||
| env: | ||
| - name: POD_IP | ||
| valueFrom: | ||
| fieldRef: | ||
| fieldPath: status.podIP | ||
| name: apiserver | ||
| --- | ||
| apiVersion: v1 | ||
| kind: Service | ||
| metadata: | ||
| name: apiserver-service | ||
| spec: | ||
| selector: | ||
| app: capim | ||
| ports: | ||
| - protocol: TCP | ||
| port: 3333 | ||
| targetPort: 3333 |
30 changes: 30 additions & 0 deletions
30
Support/Multitenancy/Multiple-Ironic-conductors/build-api-server-container-image.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| #!/bin/bash | ||
| # | ||
| __dir__=$(realpath $(dirname $0)) | ||
| IMAGE_NAME="172.22.0.1:5000/localimages/capim" | ||
|
|
||
| if [[ ${1:-""} == "-f" ]]; then | ||
| sudo podman rmi "${IMAGE_NAME}" | ||
| kubectl delete -f capim-modified.yaml | ||
| fi | ||
|
|
||
| if [[ $(sudo podman images | grep ${IMAGE_NAME}) != "" ]]; then | ||
| sudo podman push --tls-verify=false "${IMAGE_NAME}" | ||
| exit 0 | ||
| fi | ||
| CAPI_DIR="/tmp/cluster-api" | ||
| if [[ ! -d "${CAPI_DIR}" ]]; then | ||
| git clone https://github.com/kubernetes-sigs/cluster-api.git "${CAPI_DIR}" | ||
| fi | ||
|
|
||
| cd "${CAPI_DIR}" | ||
|
|
||
| INMEMORY_DIR="${CAPI_DIR}/test/infrastructure/inmemory" | ||
|
|
||
| cp "${__dir__}/main.go" "${INMEMORY_DIR}/main.go" | ||
|
|
||
| cd "${INMEMORY_DIR}" || exit | ||
|
|
||
| sudo podman build --build-arg=builder_image=docker.io/library/golang:1.20.8 --build-arg=goproxy=https://proxy.golang.org,direct --build-arg=ARCH=amd64 --build-arg=ldflags="-X 'sigs.k8s.io/cluster-api/version.buildDate=2023-10-10T11:47:30Z' -X 'sigs.k8s.io/cluster-api/version.gitCommit=8ba3f47b053da8bbf63cf407c930a2ee10bfd754' -X 'sigs.k8s.io/cluster-api/version.gitTreeState=dirty' -X 'sigs.k8s.io/cluster-api/version.gitMajor=1' -X 'sigs.k8s.io/cluster-api/version.gitMinor=0' -X 'sigs.k8s.io/cluster-api/version.gitVersion=v1.0.0-4041-8ba3f47b053da8-dirty' -X 'sigs.k8s.io/cluster-api/version.gitReleaseCommit=e09ed61cc9ba8bd37b0760291c833b4da744a985'" ../../.. -t "${IMAGE_NAME}" --file Dockerfile | ||
|
|
||
| sudo podman push --tls-verify=false "${IMAGE_NAME}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
17 changes: 17 additions & 0 deletions
17
Support/Multitenancy/Multiple-Ironic-conductors/capim-modified.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| apiVersion: v1 | ||
| kind: Pod | ||
| metadata: | ||
| name: apiserver | ||
| labels: | ||
| app: manager | ||
| spec: | ||
| containers: | ||
| - image: 172.22.0.1:5000/localimages/capim | ||
| imagePullPolicy: Always | ||
| name: capim | ||
| env: | ||
| - name: POD_IP | ||
| valueFrom: | ||
| fieldRef: | ||
| fieldPath: status.podIP | ||
| restartPolicy: Always |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
132 changes: 132 additions & 0 deletions
132
Support/Multitenancy/Multiple-Ironic-conductors/create-cluster.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,132 @@ | ||
| #!/bin/bash | ||
| # | ||
|
|
||
| source ./config.sh | ||
| CLUSTER_TEMPLATE=manifests/cluster-template.yaml | ||
| export CLUSTER_APIENDPOINT_PORT="6443" | ||
| export IMAGE_CHECKSUM="97830b21ed272a3d854615beb54cf004" | ||
| export IMAGE_CHECKSUM_TYPE="md5" | ||
| export IMAGE_FORMAT="raw" | ||
| export KUBERNETES_VERSION="v1.26.0" | ||
| export WORKERS_KUBEADM_EXTRA_CONFIG="" | ||
| export WORKER_MACHINE_COUNT="0" | ||
| export NODE_DRAIN_TIMEOUT="60s" | ||
| export CTLPLANE_KUBEADM_EXTRA_CONFIG="" | ||
|
|
||
|
|
||
| create_cluster() { | ||
| bmh_index="${1}" | ||
| cluster="test${bmh_index}" | ||
| namespace="${cluster}" | ||
| nodename="${cluster}" | ||
| fake_ipa_port=$(( 9901 + (( $bmh_index % ${N_FAKE_IPA} )) )) | ||
|
|
||
| export IMAGE_URL="http://192.168.111.1:${fake_ipa_port}/images/rhcos-ootpa-latest.qcow2" | ||
|
|
||
| kubectl port-forward service/apiserver-service 3333:3333 2>/dev/null& | ||
|
|
||
| echo "Creating cluster ${cluster} in namespace ${namespace}" | ||
| kubectl create namespace "${namespace}" | ||
| kubectl -n "${namespace}" apply -f bmc-${nodename}.yaml | ||
|
|
||
| caKeyEncoded=$(cat /tmp/ca.key | base64 -w 0) | ||
| caCertEncoded=$(cat /tmp/ca.crt | base64 -w 0) | ||
| etcdKeyEncoded=$(cat /tmp/etcd.key | base64 -w 0) | ||
| etcdCertEncoded=$(cat /tmp/etcd.crt | base64 -w 0) | ||
|
|
||
| cluster_endpoints=$(curl "localhost:3333/register?resource=${namespace}/${cluster}&caKey=${caKeyEncoded}&caCert=${caCertEncoded}&etcdKey=${etcdKeyEncoded}&etcdCert=${etcdCertEncoded}") | ||
| host=$(echo ${cluster_endpoints} | jq -r ".Host") | ||
| port=$(echo ${cluster_endpoints} | jq -r ".Port") | ||
|
|
||
| cat <<EOF > "/tmp/${cluster}-ca-secrets.yaml" | ||
| apiVersion: v1 | ||
| kind: Secret | ||
| metadata: | ||
| labels: | ||
| cluster.x-k8s.io/cluster-name: ${cluster} | ||
| name: ${cluster}-ca | ||
| namespace: ${namespace} | ||
| type: kubernetes.io/tls | ||
| data: | ||
| tls.crt: ${caCertEncoded} | ||
| tls.key: ${caKeyEncoded} | ||
| EOF | ||
|
|
||
| kubectl -n ${namespace} apply -f /tmp/${cluster}-ca-secrets.yaml | ||
|
|
||
| cat <<EOF > "/tmp/${cluster}-etcd-secrets.yaml" | ||
| apiVersion: v1 | ||
| kind: Secret | ||
| metadata: | ||
| labels: | ||
| cluster.x-k8s.io/cluster-name: ${cluster} | ||
| name: ${cluster}-etcd | ||
| namespace: ${namespace} | ||
| type: kubernetes.io/tls | ||
| data: | ||
| tls.crt: ${etcdCertEncoded} | ||
| tls.key: ${etcdKeyEncoded} | ||
| EOF | ||
|
|
||
| kubectl -n ${namespace} apply -f /tmp/${cluster}-etcd-secrets.yaml | ||
|
|
||
| # Generate metal3 cluster | ||
| export CLUSTER_APIENDPOINT_HOST="${host}" | ||
| export CLUSTER_APIENDPOINT_PORT="${port}" | ||
| echo "Generating cluster ${cluster} with clusterctl" | ||
| clusterctl generate cluster "${cluster}" \ | ||
| --from "${CLUSTER_TEMPLATE}" \ | ||
| --target-namespace "${namespace}" > /tmp/${cluster}-cluster.yaml | ||
| kubectl apply -f /tmp/${cluster}-cluster.yaml | ||
|
|
||
| sleep 10 | ||
|
|
||
| wait_for_resource() { | ||
| resource=$1 | ||
| jsonpath=${2:-"{.items[0].metadata.name}"} | ||
| while true; do | ||
| kubectl -n "${namespace}" get "${resource}" -o jsonpath="${jsonpath}" 2> /dev/null | ||
| if [ $? -eq 0 ]; then | ||
| return | ||
| fi | ||
| sleep 2 | ||
| done | ||
| } | ||
|
|
||
| bmh_name=$(wait_for_resource "bmh") | ||
| metal3machine=$(wait_for_resource "m3m") | ||
| machine=$(wait_for_resource "machine") | ||
|
|
||
| providerID="metal3://$namespace/$bmh_name/$metal3machine" | ||
| curl "localhost:3333/updateNode?resource=${namespace}/${cluster}&nodeName=${machine}&providerID=${providerID}" | ||
| echo "Done generating cluster ${cluster} with clusterctl" | ||
| } | ||
|
|
||
| for i in $(seq 1 $N_NODES); do | ||
| namespace="test${i}" | ||
| if [[ $(kubectl get ns | grep "${namespace}") != "" ]]; then | ||
| echo "ERROR: Namespace ${namespace} exists. Skip creating cluster" | ||
| continue | ||
| fi | ||
| create_cluster "${i}" | ||
| done | ||
|
|
||
| # Wait for all BMHs to be available. Clusters should be more or less ready by then. | ||
| desired_states=("available" "provisioning" "provisioned") | ||
| for i in $(seq 1 $N_NODES); do | ||
| namespace="test${i}" | ||
| bmh_name="$(kubectl -n ${namespace} get bmh -o jsonpath='{.items[0].metadata.name}')" | ||
| echo "Waiting for BMH ${bmh_name} to become available." | ||
| while true; do | ||
| bmh_state="$(kubectl -n ${namespace} get bmh -o jsonpath='{.items[0].status.provisioning.state}')" | ||
| if [[ "${desired_states[@]}" =~ "${bmh_state}" ]]; then | ||
| break | ||
| fi | ||
| sleep 3 | ||
| done | ||
| done | ||
|
|
||
| # Run describe for all clusters | ||
| for i in $(seq 1 $N_NODES); do | ||
| clusterctl -n "test${i}" describe cluster "test${i}" | ||
| done |
Oops, something went wrong.