From 42899b6b74d986ff87a3e375ad972af1a25853e5 Mon Sep 17 00:00:00 2001 From: Morven Cao Date: Tue, 5 Nov 2024 11:03:55 +0800 Subject: [PATCH] add setup scripts and templates for aro-hcp. (#214) * add setup scripts and templates for aro-hcp. Signed-off-by: morvencao * deploy aro-hcp env using upstream. Signed-off-by: morvencao * apply comments. Signed-off-by: morvencao --------- Signed-off-by: morvencao --- .gitignore | 4 +- test/e2e/setup/aro/Makefile | 61 ++++++++++++++++++++++++ test/e2e/setup/aro/README.md | 91 ++++++++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 test/e2e/setup/aro/Makefile create mode 100644 test/e2e/setup/aro/README.md diff --git a/.gitignore b/.gitignore index 4c06cbde..6df1bed4 100755 --- a/.gitignore +++ b/.gitignore @@ -58,4 +58,6 @@ test/e2e/.consumer_name test/e2e/.external_host_ip test/e2e/report/* unit-test-results.json -integration-test-results.json \ No newline at end of file +integration-test-results.json + +test/e2e/setup/aro/aro-hcp \ No newline at end of file diff --git a/test/e2e/setup/aro/Makefile b/test/e2e/setup/aro/Makefile new file mode 100644 index 00000000..0b42590e --- /dev/null +++ b/test/e2e/setup/aro/Makefile @@ -0,0 +1,61 @@ +# define the variables +REPO_URL = https://github.com/Azure/ARO-HCP.git +BRANCH = maestro-light-setup +CLONE_DIR = aro-hcp + +# clone the repo +clone: + @if [ -d $(CLONE_DIR) ]; then \ + echo "Removing existing directory $(CLONE_DIR)..."; \ + rm -rf $(CLONE_DIR); \ + fi; \ + echo "Cloning repository..."; \ + git clone $(REPO_URL) -b $(BRANCH) $(CLONE_DIR) +.PHONY: clone + +# create the cluster (svc-cluster or mgmt-cluster) +cluster: clone +ifndef AKSCONFIG + $(error "Must set AKSCONFIG") +endif + @$(MAKE) -C $(CLONE_DIR)/dev-infrastructure cluster +.PHONY: cluster + +# grant admin access to the cluster +aks.admin-access: +ifndef AKSCONFIG + $(error "Must set AKSCONFIG") +endif + @$(MAKE) -C $(CLONE_DIR)/dev-infrastructure aks.admin-access +.PHONY: aks.admin-access + +# retrieve the kubeconfig +aks.kubeconfig: +ifndef AKSCONFIG + $(error "Must set AKSCONFIG") +endif + @$(MAKE) -C $(CLONE_DIR)/dev-infrastructure aks.kubeconfig +.PHONY: aks.kubeconfig + +# deploy the maestro server +deploy-server: + @AKSCONFIG=svc-cluster $(MAKE) -C $(CLONE_DIR)/maestro deploy-server +.PHONY: deploy-server + +# deploy the maestro agent +deploy-agent: + @AKSCONFIG=mgmt-cluster $(MAKE) -C $(CLONE_DIR)/maestro deploy-agent +.PHONY: deploy-agent + +# register the maestro agent +register-agent: + @AKSCONFIG=svc-cluster $(MAKE) -C $(CLONE_DIR)/maestro register-agent +.PHONY: register-agent + +# clean up the resources +clean: +ifndef AKSCONFIG + $(error "Must set AKSCONFIG") +endif + @$(MAKE) -C $(CLONE_DIR)/dev-infrastructure clean +.PHONY: clean diff --git a/test/e2e/setup/aro/README.md b/test/e2e/setup/aro/README.md new file mode 100644 index 00000000..ec5e4524 --- /dev/null +++ b/test/e2e/setup/aro/README.md @@ -0,0 +1,91 @@ +# Maestro ARO-HCP Env Setup + +## Prerequisites + +* `az` version >= 2.60, `jq`, `make`, [kubelogin](https://azure.github.io/kubelogin/install.html), `kubectl` version >= 1.30, `helm` +* `az login` with service principal (azure AD user support is WIP) + +### Create Service Cluster + +Change those flags accordingly and then run the following command. Depending on the selected features, this may take a while: + + ```bash + AKSCONFIG=svc-cluster make cluster + ``` + +### Create Management Cluster + +A Management Cluster depends on certain resources found in the resource group of the Service Cluster. Therefore, a standalone Management Cluster can't be created right now and requires a Service Cluster + + ```bash + AKSCONFIG=mgmt-cluster make cluster + ``` + +### Access AKS Clusters + + ```bash + AKSCONFIG=svc-cluster make aks.admin-access # one time + AKSCONFIG=svc-cluster make aks.kubeconfig + AKSCONFIG=svc-cluster export KUBECONFIG=${HOME}/.kube/${AKSCONFIG}.kubeconfig + kubectl get ns + ``` + + (Replace `svc` with `mgmt` for management clusters) + +### Cleanup + +Setting the correct `AKSCONFIG`, this will cleanup all resources created in Azure + + ```bash + AKSCONFIG=svc-cluster make clean + ``` + + (Replace `svc` with `mgmt` for management clusters) + +## Deploy Maestro to AKS Clusters + +### Maestro Server + +> Make sure your `KUBECONFIG` points to the service cluster!!! + +> The service cluster has no ingress. To interact with the services you need to use `kubectl port-forward` + + ```bash + AKSCONFIG=svc-cluster make deploy-server + ``` + +To validate, have a look at the `maestro` namespace on the service cluster. Some pod restarts are expected in the first 1 minute until the containerized DB is ready. + +To access the HTTP and GRPC endpoints of maestro, run + + ```bash + kubectl port-forward svc/maestro 8001:8000 -n maestro + kubectl port-forward svc/maestro-grpc 8090 -n maestro + ``` + +If you need to restart the maestro server during testing and don't want the port-forward process to be broken, you can install the kubectl relay plugin from [https://github.com/knight42/krelay](https://github.com/knight42/krelay) and perform the port forward using the following steps: + + + ```bash + kubectl relay svc/maestro 8001:8000 -n maestro + kubectl relay svc/maestro-grpc 8090 -n maestro + ``` + +## Maestro Agent + +> Make sure your `KUBECONFIG` points to the management cluster!!! + +First install the agent + + ```bash + AKSCONFIG=mgmt-cluster make deploy-agent + ``` + +Then register it with the Maestro Server + +Make sure your `KUBECONFIG` points to the service cluster, then run + + ```bash + cd maestro + AKSCONFIG=svc-cluster make register-agent + ```