run maestro with rosa (#215)

Signed-off-by: Wei Liu <[email protected]>
openshift-online · Nov 6, 2024 · 8d6765a · 8d6765a
1 parent 42899b6
commit 8d6765a
Show file tree

Hide file tree

Showing 9 changed files with 564 additions and 2 deletions.
diff --git a/.gitignore b/.gitignore
@@ -52,12 +52,12 @@ hack/mosquitto-passwd.txt
 vendor/
 
 # Ignore test data
+_output
 test/e2e/.kubeconfig
 test/e2e/.consumer_id
 test/e2e/.consumer_name
 test/e2e/.external_host_ip
 test/e2e/report/*
 unit-test-results.json
 integration-test-results.json
-
-test/e2e/setup/aro/aro-hcp
+test/e2e/setup/aro/aro-hcp
diff --git a/test/e2e/setup/rosa/Makefile b/test/e2e/setup/rosa/Makefile
@@ -0,0 +1,30 @@
+SHELL:=/bin/bash
+
+e2e_dir=$(shell cd ${PWD}/../.. && pwd -P)
+
+rosa/setup-maestro:
+	./setup/maestro.sh
+.PHONY: rosa/setup-maestro
+
+rosa/setup-agent:
+	./setup/agent.sh
+.PHONY: rosa/setup-agent
+
+rosa/setup-e2e:
+	./setup/e2e.sh
+.PHONY: rosa/setup-e2e
+
+rosa/e2e-test: rosa/setup-e2e
+	ginkgo -v --fail-fast --label-filter="!(e2e-tests-spec-resync-reconnect||e2e-tests-status-resync-reconnect)" \
+	--output-dir="$(e2e_dir)/report" --json-report=report.json --junit-report=report.xml \
+	${e2e_dir}/pkg -- \
+	-api-server="http://127.0.0.1:8000" \
+	-grpc-server="127.0.0.1:8090" \
+	-server-kubeconfig=$(KUBECONFIG) \
+	-agent-kubeconfig=$(KUBECONFIG) \
+	-consumer-name=${PWD}/_output/consumer_id
+.PHONY: rosa/e2e-test
+
+rosa/teardown:
+	./setup/teardown.sh
+.PHONY: rosa/teardown
diff --git a/test/e2e/setup/rosa/README.md b/test/e2e/setup/rosa/README.md
@@ -0,0 +1,100 @@
+## Setup Maestro in ROSA env
+
+This demonstrates how to deploy the Maestro in ROSA env.
+
+### Prerequisites
+
+- Install the CLIs: `oc`, `rosa`, `aws` and `jq`
+- Ensue your `aws` CLI is logined with your AWS account and your AWS account should have the permissions to operate AWS IoT and AWS RDS  PostgreSQL in your provided region
+- Prepare two ROSA clusters, one is used as Service Cluster and the other is used as Management Cluster, e.g.
+
+```sh
+rosa create cluster --cluster-name=service --region=us-west-2 --sts --mode=auto
+rosa create cluster --cluster-name=management --region=us-west-2 --sts --mode=auto
+```
+
+### Setup Maestro server in your Service Cluster
+
+```sh
+export REGION="<your_rosa_cluster_region>" # e.g. us-west-2
+export CLUSTER_ID="<your_rosa_cluster_id_or_name>" # e.g. service
+export KUBECONFIG="<your_service_cluster_kubeconfig>"
+
+make rosa/setup-maestro
+```
+
+This will
+- Create AWS IoT client certs and policy for Maestro server in your region
+- Create AWS RDS PostgreSQL for Maestro server in your region
+- Deploy the Maestro server on the given cluster
+
+After the Maestro server is deployed, you can run following commands to start the Maestro RESTful service and GRPC service in your local host
+
+```sh
+oc port-forward svc/maestro 8000 -n maestro
+oc port-forward svc/maestro-grpc 8090 -n maestro
+```
+
+Then create a consumer in the Maestro, e.g.
+
+```sh
+curl -s -X POST -H "Content-Type: application/json" http://127.0.0.1:8000/api/maestro/v1/consumers -d '{"name": "management"}'
+```
+
+### Setup Maestro agent in your Management Cluster
+
+```sh
+export REGION="<your_rosa_cluster_region>" # e.g. us-west-2
+export CONSUMER_ID="<your_created_consumer_id_or_name>" # e.g. management
+export KUBECONFIG="<your_management_cluster_kubeconfig>"
+
+make rosa/setup-agent
+```
+
+This will
+- Create AWS IoT client certs and policy for Maestro agent in your region
+- Deploy the Maestro agent on the given cluster
+
+### Cleanup
+
+```sh
+export REGION="<your_cluster_region>"
+
+make rosa/teardown
+
+# delete your rosa clusters, e.g.
+rosa delete cluster --cluster=service
+rosa delete cluster --cluster=management
+```
+
+## Run Maestro e2e on a ROSA cluster
+
+### Prepare
+
+1. Install the following CLIs `oc`, `rosa`, `aws`, `jq` and [`krelay` plugin](https://github.com/knight42/krelay)
+2. Create a rosa cluster
+
+```sh
+rosa create cluster --cluster-name=maestro-e2e --region=us-west-2 --sts --mode=auto
+```
+
+### Run e2e
+
+```sh
+export KUBECONFIG="<your_rosa_cluster_kubeconfig>"
+export REGION="<your_rosa_cluster_region>"
+export CLUSTER_ID="<your_rosa_cluster_name_or_id>"
+
+make rosa/e2e-test
+```
+
+### Cleanup
+
+```sh
+export REGION="<your_rosa_cluster_region>"
+
+make rosa/teardown
+
+# delete your rosa clusters, e.g.
+rosa delete cluster --cluster=maestro-e2e 
+```
diff --git a/test/e2e/setup/rosa/setup/agent.sh b/test/e2e/setup/rosa/setup/agent.sh
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+
+#####################
+# Setup Maestro agent
+#####################
+
+PWD="$(cd "$(dirname ${BASH_SOURCE[0]})" ; pwd -P)"
+ROOT_DIR="$(cd ${PWD}/.. && pwd -P)"
+
+region=${REGION:-""}
+consumer_id=${CONSUMER_ID:-""}
+
+if [ -z "$region" ]; then
+    echo "region is required"
+    exit 1
+fi
+
+if [ -z "$consumer_id" ]; then
+    echo "consumer id is required"
+    exit 1
+fi
+
+echo "Setup Maestro agent in ${region} (consumer_id=${consumer_id})"
+
+IMAGE_REGISTRY=${IMAGE_REGISTRY:="quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro"}
+IMAGE_REPOSITORY="maestro"
+IMAGE_TAG=${IMAGE_TAG:-"1de63c6075f2c95c9661d790d164019f60d789f3"}
+
+output_dir=${ROOT_DIR}/_output
+certs_dir=${output_dir}/aws-certs
+consumer_cert_dir=${certs_dir}/iot/consumers
+policies_dir=${output_dir}/aws-policies
+
+mkdir -p ${consumer_cert_dir}
+mkdir -p ${policies_dir}
+
+# Download AWS IoT broker severing CA
+echo "Download AWS IoT broker severing CA ...."
+curl -s -o ${certs_dir}/iot-ca.pem https://www.amazontrust.com/repository/AmazonRootCA1.pem
+
+# Generated client certs for AWS IoT clients
+echo "Generate AWS IoT client certs for Maestro agent ...."
+consumer_cert_arn=$(aws iot create-keys-and-certificate \
+    --region ${region} \
+    --set-as-active \
+    --certificate-pem-outfile "${consumer_cert_dir}/${consumer_id}.crt" \
+    --public-key-outfile "${consumer_cert_dir}/${consumer_id}.public.key" \
+    --private-key-outfile "${consumer_cert_dir}/${consumer_id}.private.key" | jq -r '.certificateArn')
+echo "Maestro agent AWS IoT client certs are generated ($consumer_cert_arn)"
+
+# Attach policies for AWS IoT clients
+aws_account=$(aws sts get-caller-identity --region ${region} --output json | jq -r '.Account')
+
+echo "Generate AWS IoT policy for Maestro agent ...."
+cat $PWD/aws-iot-policies/consumer.template.json | sed "s/{region}/${region}/g" | sed "s/{aws_account}/${aws_account}/g" | sed "s/{consumer_id}/${consumer_id}/g" > $policies_dir/${consumer_id}.json
+policy_name=$(aws iot create-policy \
+    --region ${region} \
+    --policy-name maestro-${consumer_id} \
+    --policy-document "file://${policies_dir}/${consumer_id}.json" | jq -r '.policyName')
+aws iot attach-policy --region ${region} --policy-name maestro-${consumer_id} --target ${consumer_cert_arn}
+echo "Maestro agent AWS IoT policy $policy_name is generated"
+
+# Get AWS IoT broker endpoint
+mqtt_host=$(aws iot describe-endpoint --region ${region} --endpoint-type iot:Data-ATS | jq -r '.endpointAddress')
+echo "AWS IoT broke: ${mqtt_host}:8883"
+
+sleep 30
+
+# Deploy Maestro agent
+oc create namespace maestro-agent || true
+oc -n maestro-agent delete secrets maestro-agent-mqtt-creds --ignore-not-found
+oc -n maestro-agent create secret generic maestro-agent-mqtt-creds \
+    --from-file=ca.crt="${certs_dir}/iot-ca.pem" \
+    --from-file=client.crt="${consumer_cert_dir}/${consumer_id}.crt" \
+    --from-file=client.key="${consumer_cert_dir}/${consumer_id}.private.key"
+
+oc process --filename="https://raw.githubusercontent.com/openshift-online/maestro/refs/heads/main/templates/agent-template-rosa.yml" \
+    --local="true" \
+    --param="AGENT_NAMESPACE=maestro-agent" \
+    --param="CONSUMER_NAME=${consumer_id}" \
+    --param="IMAGE_REGISTRY=${IMAGE_REGISTRY}" \
+    --param="IMAGE_REPOSITORY=${IMAGE_REPOSITORY}" \
+    --param="IMAGE_TAG=${IMAGE_TAG}" \
+    --param="MQTT_HOST=${mqtt_host}" > ${output_dir}/maestro-${consumer_id}-rosa.json
+
+oc apply -f ${output_dir}/maestro-${consumer_id}-rosa.json
+oc -n maestro-agent wait deploy/maestro-agent --for condition=Available=True --timeout=300s
diff --git a/test/e2e/setup/rosa/setup/aws-iot-policies/consumer.template.json b/test/e2e/setup/rosa/setup/aws-iot-policies/consumer.template.json
@@ -0,0 +1,41 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iot:Connect"
+            ],
+            "Resource": [
+                "arn:aws:iot:{region}:{aws_account}:client/{consumer_id}-client"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iot:Publish"
+            ],
+            "Resource": [
+                "arn:aws:iot:{region}:{aws_account}:topic/sources/maestro/consumers/{consumer_id}/agentevents"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iot:Subscribe"
+            ],
+            "Resource": [
+                "arn:aws:iot:{region}:{aws_account}:topicfilter/sources/maestro/consumers/{consumer_id}/sourceevents"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iot:Receive"
+            ],
+            "Resource": [
+                "arn:aws:iot:{region}:{aws_account}:topic/sources/maestro/consumers/{consumer_id}/sourceevents"
+            ]
+        }
+    ]
+}
diff --git a/test/e2e/setup/rosa/setup/aws-iot-policies/source.template.json b/test/e2e/setup/rosa/setup/aws-iot-policies/source.template.json
@@ -0,0 +1,43 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iot:Connect"
+            ],
+            "Resource": [
+                "*"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iot:Publish"
+            ],
+            "Resource": [
+                "arn:aws:iot:{region}:{aws_account}:topic/sources/maestro/consumers/*/sourceevents"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iot:Subscribe"
+            ],
+            "Resource": [
+                "arn:aws:iot:{region}:{aws_account}:topicfilter/sources/maestro/consumers/+/agentevents",
+                "arn:aws:iot:{region}:{aws_account}:topicfilter/$share/statussubscribers/sources/maestro/consumers/+/agentevents"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iot:Receive"
+            ],
+            "Resource": [
+                "arn:aws:iot:{region}:{aws_account}:topic/sources/maestro/consumers/*/agentevents",
+                "arn:aws:iot:{region}:{aws_account}:topic/$share/statussubscribers/sources/maestro/consumers/*/agentevents"
+            ]
+        }
+    ]
+}
diff --git a/test/e2e/setup/rosa/setup/e2e.sh b/test/e2e/setup/rosa/setup/e2e.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+#####################
+# Setup Maestro e2e
+#####################
+
+PWD="$(cd "$(dirname ${BASH_SOURCE[0]})" ; pwd -P)"
+ROSA_DIR="$(cd ${PWD}/.. && pwd -P)"
+
+output_dir=${ROSA_DIR}/_output
+
+mkdir -p $output_dir
+
+echo "$output_dir"
+
+# Setup Maestro server
+CLUSTER_VPC=$vpc ${PWD}/maestro.sh
+sleep 90 # wait the maestro service ready
+
+# Start Maestro servers
+exec oc relay service/maestro 8000:8000 -n maestro > ${output_dir}/maestro.svc.log 2>&1 &
+maestro_server_pid=$!
+echo "Maestro server started: $maestro_server_pid"
+echo "$maestro_server_pid" > ${output_dir}/maestro_server.pid
+exec oc relay service/maestro-grpc 8090:8090 -n maestro > ${output_dir}/maestro-grpc.svc.log 2>&1 &
+maestro_grpc_server_pid=$!
+echo "Maestro GRPC server started: $maestro_grpc_server_pid"
+echo "$maestro_grpc_server_pid" > ${output_dir}/maestro_grpc_server.pid
+
+# need to wait the relay build the connection before we get the consumer id
+sleep 15
+
+# Prepare a consumer
+consumer_id=$(curl -s -X POST -H "Content-Type: application/json" http://127.0.0.1:8000/api/maestro/v1/consumers -d '{}' | jq -r '.id')
+echo $consumer_id > ${output_dir}/consumer_id
+echo "Consumer $consumer_id is created"
+
+# Setup Maestro agent
+oc apply -f https://raw.githubusercontent.com/open-cluster-management-io/api/release-0.14/work/v1/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml
+CONSUMER_ID=$consumer_id ${PWD}/agent.sh