From 8d6765a5acff491204d994504c49712b32ce24ad Mon Sep 17 00:00:00 2001 From: Wei Liu Date: Wed, 6 Nov 2024 16:46:55 +0800 Subject: [PATCH] run maestro with rosa (#215) Signed-off-by: Wei Liu --- .gitignore | 4 +- test/e2e/setup/rosa/Makefile | 30 +++ test/e2e/setup/rosa/README.md | 100 ++++++++++ test/e2e/setup/rosa/setup/agent.sh | 87 +++++++++ .../aws-iot-policies/consumer.template.json | 41 ++++ .../aws-iot-policies/source.template.json | 43 +++++ test/e2e/setup/rosa/setup/e2e.sh | 40 ++++ test/e2e/setup/rosa/setup/maestro.sh | 175 ++++++++++++++++++ test/e2e/setup/rosa/setup/teardown.sh | 46 +++++ 9 files changed, 564 insertions(+), 2 deletions(-) create mode 100644 test/e2e/setup/rosa/Makefile create mode 100644 test/e2e/setup/rosa/README.md create mode 100755 test/e2e/setup/rosa/setup/agent.sh create mode 100644 test/e2e/setup/rosa/setup/aws-iot-policies/consumer.template.json create mode 100644 test/e2e/setup/rosa/setup/aws-iot-policies/source.template.json create mode 100755 test/e2e/setup/rosa/setup/e2e.sh create mode 100755 test/e2e/setup/rosa/setup/maestro.sh create mode 100755 test/e2e/setup/rosa/setup/teardown.sh diff --git a/.gitignore b/.gitignore index 6df1bed4..af4e2dc4 100755 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,7 @@ hack/mosquitto-passwd.txt vendor/ # Ignore test data +_output test/e2e/.kubeconfig test/e2e/.consumer_id test/e2e/.consumer_name @@ -59,5 +60,4 @@ test/e2e/.external_host_ip test/e2e/report/* unit-test-results.json integration-test-results.json - -test/e2e/setup/aro/aro-hcp \ No newline at end of file +test/e2e/setup/aro/aro-hcp diff --git a/test/e2e/setup/rosa/Makefile b/test/e2e/setup/rosa/Makefile new file mode 100644 index 00000000..d9b3bc8e --- /dev/null +++ b/test/e2e/setup/rosa/Makefile @@ -0,0 +1,30 @@ +SHELL:=/bin/bash + +e2e_dir=$(shell cd ${PWD}/../.. && pwd -P) + +rosa/setup-maestro: + ./setup/maestro.sh +.PHONY: rosa/setup-maestro + +rosa/setup-agent: + ./setup/agent.sh +.PHONY: rosa/setup-agent + +rosa/setup-e2e: + ./setup/e2e.sh +.PHONY: rosa/setup-e2e + +rosa/e2e-test: rosa/setup-e2e + ginkgo -v --fail-fast --label-filter="!(e2e-tests-spec-resync-reconnect||e2e-tests-status-resync-reconnect)" \ + --output-dir="$(e2e_dir)/report" --json-report=report.json --junit-report=report.xml \ + ${e2e_dir}/pkg -- \ + -api-server="http://127.0.0.1:8000" \ + -grpc-server="127.0.0.1:8090" \ + -server-kubeconfig=$(KUBECONFIG) \ + -agent-kubeconfig=$(KUBECONFIG) \ + -consumer-name=${PWD}/_output/consumer_id +.PHONY: rosa/e2e-test + +rosa/teardown: + ./setup/teardown.sh +.PHONY: rosa/teardown diff --git a/test/e2e/setup/rosa/README.md b/test/e2e/setup/rosa/README.md new file mode 100644 index 00000000..075cef20 --- /dev/null +++ b/test/e2e/setup/rosa/README.md @@ -0,0 +1,100 @@ +## Setup Maestro in ROSA env + +This demonstrates how to deploy the Maestro in ROSA env. + +### Prerequisites + +- Install the CLIs: `oc`, `rosa`, `aws` and `jq` +- Ensue your `aws` CLI is logined with your AWS account and your AWS account should have the permissions to operate AWS IoT and AWS RDS PostgreSQL in your provided region +- Prepare two ROSA clusters, one is used as Service Cluster and the other is used as Management Cluster, e.g. + +```sh +rosa create cluster --cluster-name=service --region=us-west-2 --sts --mode=auto +rosa create cluster --cluster-name=management --region=us-west-2 --sts --mode=auto +``` + +### Setup Maestro server in your Service Cluster + +```sh +export REGION="" # e.g. us-west-2 +export CLUSTER_ID="" # e.g. service +export KUBECONFIG="" + +make rosa/setup-maestro +``` + +This will +- Create AWS IoT client certs and policy for Maestro server in your region +- Create AWS RDS PostgreSQL for Maestro server in your region +- Deploy the Maestro server on the given cluster + +After the Maestro server is deployed, you can run following commands to start the Maestro RESTful service and GRPC service in your local host + +```sh +oc port-forward svc/maestro 8000 -n maestro +oc port-forward svc/maestro-grpc 8090 -n maestro +``` + +Then create a consumer in the Maestro, e.g. + +```sh +curl -s -X POST -H "Content-Type: application/json" http://127.0.0.1:8000/api/maestro/v1/consumers -d '{"name": "management"}' +``` + +### Setup Maestro agent in your Management Cluster + +```sh +export REGION="" # e.g. us-west-2 +export CONSUMER_ID="" # e.g. management +export KUBECONFIG="" + +make rosa/setup-agent +``` + +This will +- Create AWS IoT client certs and policy for Maestro agent in your region +- Deploy the Maestro agent on the given cluster + +### Cleanup + +```sh +export REGION="" + +make rosa/teardown + +# delete your rosa clusters, e.g. +rosa delete cluster --cluster=service +rosa delete cluster --cluster=management +``` + +## Run Maestro e2e on a ROSA cluster + +### Prepare + +1. Install the following CLIs `oc`, `rosa`, `aws`, `jq` and [`krelay` plugin](https://github.com/knight42/krelay) +2. Create a rosa cluster + +```sh +rosa create cluster --cluster-name=maestro-e2e --region=us-west-2 --sts --mode=auto +``` + +### Run e2e + +```sh +export KUBECONFIG="" +export REGION="" +export CLUSTER_ID="" + +make rosa/e2e-test +``` + +### Cleanup + +```sh +export REGION="" + +make rosa/teardown + +# delete your rosa clusters, e.g. +rosa delete cluster --cluster=maestro-e2e +``` diff --git a/test/e2e/setup/rosa/setup/agent.sh b/test/e2e/setup/rosa/setup/agent.sh new file mode 100755 index 00000000..0ba6dd46 --- /dev/null +++ b/test/e2e/setup/rosa/setup/agent.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +##################### +# Setup Maestro agent +##################### + +PWD="$(cd "$(dirname ${BASH_SOURCE[0]})" ; pwd -P)" +ROOT_DIR="$(cd ${PWD}/.. && pwd -P)" + +region=${REGION:-""} +consumer_id=${CONSUMER_ID:-""} + +if [ -z "$region" ]; then + echo "region is required" + exit 1 +fi + +if [ -z "$consumer_id" ]; then + echo "consumer id is required" + exit 1 +fi + +echo "Setup Maestro agent in ${region} (consumer_id=${consumer_id})" + +IMAGE_REGISTRY=${IMAGE_REGISTRY:="quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro"} +IMAGE_REPOSITORY="maestro" +IMAGE_TAG=${IMAGE_TAG:-"1de63c6075f2c95c9661d790d164019f60d789f3"} + +output_dir=${ROOT_DIR}/_output +certs_dir=${output_dir}/aws-certs +consumer_cert_dir=${certs_dir}/iot/consumers +policies_dir=${output_dir}/aws-policies + +mkdir -p ${consumer_cert_dir} +mkdir -p ${policies_dir} + +# Download AWS IoT broker severing CA +echo "Download AWS IoT broker severing CA ...." +curl -s -o ${certs_dir}/iot-ca.pem https://www.amazontrust.com/repository/AmazonRootCA1.pem + +# Generated client certs for AWS IoT clients +echo "Generate AWS IoT client certs for Maestro agent ...." +consumer_cert_arn=$(aws iot create-keys-and-certificate \ + --region ${region} \ + --set-as-active \ + --certificate-pem-outfile "${consumer_cert_dir}/${consumer_id}.crt" \ + --public-key-outfile "${consumer_cert_dir}/${consumer_id}.public.key" \ + --private-key-outfile "${consumer_cert_dir}/${consumer_id}.private.key" | jq -r '.certificateArn') +echo "Maestro agent AWS IoT client certs are generated ($consumer_cert_arn)" + +# Attach policies for AWS IoT clients +aws_account=$(aws sts get-caller-identity --region ${region} --output json | jq -r '.Account') + +echo "Generate AWS IoT policy for Maestro agent ...." +cat $PWD/aws-iot-policies/consumer.template.json | sed "s/{region}/${region}/g" | sed "s/{aws_account}/${aws_account}/g" | sed "s/{consumer_id}/${consumer_id}/g" > $policies_dir/${consumer_id}.json +policy_name=$(aws iot create-policy \ + --region ${region} \ + --policy-name maestro-${consumer_id} \ + --policy-document "file://${policies_dir}/${consumer_id}.json" | jq -r '.policyName') +aws iot attach-policy --region ${region} --policy-name maestro-${consumer_id} --target ${consumer_cert_arn} +echo "Maestro agent AWS IoT policy $policy_name is generated" + +# Get AWS IoT broker endpoint +mqtt_host=$(aws iot describe-endpoint --region ${region} --endpoint-type iot:Data-ATS | jq -r '.endpointAddress') +echo "AWS IoT broke: ${mqtt_host}:8883" + +sleep 30 + +# Deploy Maestro agent +oc create namespace maestro-agent || true +oc -n maestro-agent delete secrets maestro-agent-mqtt-creds --ignore-not-found +oc -n maestro-agent create secret generic maestro-agent-mqtt-creds \ + --from-file=ca.crt="${certs_dir}/iot-ca.pem" \ + --from-file=client.crt="${consumer_cert_dir}/${consumer_id}.crt" \ + --from-file=client.key="${consumer_cert_dir}/${consumer_id}.private.key" + +oc process --filename="https://raw.githubusercontent.com/openshift-online/maestro/refs/heads/main/templates/agent-template-rosa.yml" \ + --local="true" \ + --param="AGENT_NAMESPACE=maestro-agent" \ + --param="CONSUMER_NAME=${consumer_id}" \ + --param="IMAGE_REGISTRY=${IMAGE_REGISTRY}" \ + --param="IMAGE_REPOSITORY=${IMAGE_REPOSITORY}" \ + --param="IMAGE_TAG=${IMAGE_TAG}" \ + --param="MQTT_HOST=${mqtt_host}" > ${output_dir}/maestro-${consumer_id}-rosa.json + +oc apply -f ${output_dir}/maestro-${consumer_id}-rosa.json +oc -n maestro-agent wait deploy/maestro-agent --for condition=Available=True --timeout=300s diff --git a/test/e2e/setup/rosa/setup/aws-iot-policies/consumer.template.json b/test/e2e/setup/rosa/setup/aws-iot-policies/consumer.template.json new file mode 100644 index 00000000..1e55c8a1 --- /dev/null +++ b/test/e2e/setup/rosa/setup/aws-iot-policies/consumer.template.json @@ -0,0 +1,41 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iot:Connect" + ], + "Resource": [ + "arn:aws:iot:{region}:{aws_account}:client/{consumer_id}-client" + ] + }, + { + "Effect": "Allow", + "Action": [ + "iot:Publish" + ], + "Resource": [ + "arn:aws:iot:{region}:{aws_account}:topic/sources/maestro/consumers/{consumer_id}/agentevents" + ] + }, + { + "Effect": "Allow", + "Action": [ + "iot:Subscribe" + ], + "Resource": [ + "arn:aws:iot:{region}:{aws_account}:topicfilter/sources/maestro/consumers/{consumer_id}/sourceevents" + ] + }, + { + "Effect": "Allow", + "Action": [ + "iot:Receive" + ], + "Resource": [ + "arn:aws:iot:{region}:{aws_account}:topic/sources/maestro/consumers/{consumer_id}/sourceevents" + ] + } + ] +} diff --git a/test/e2e/setup/rosa/setup/aws-iot-policies/source.template.json b/test/e2e/setup/rosa/setup/aws-iot-policies/source.template.json new file mode 100644 index 00000000..dff03a32 --- /dev/null +++ b/test/e2e/setup/rosa/setup/aws-iot-policies/source.template.json @@ -0,0 +1,43 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iot:Connect" + ], + "Resource": [ + "*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "iot:Publish" + ], + "Resource": [ + "arn:aws:iot:{region}:{aws_account}:topic/sources/maestro/consumers/*/sourceevents" + ] + }, + { + "Effect": "Allow", + "Action": [ + "iot:Subscribe" + ], + "Resource": [ + "arn:aws:iot:{region}:{aws_account}:topicfilter/sources/maestro/consumers/+/agentevents", + "arn:aws:iot:{region}:{aws_account}:topicfilter/$share/statussubscribers/sources/maestro/consumers/+/agentevents" + ] + }, + { + "Effect": "Allow", + "Action": [ + "iot:Receive" + ], + "Resource": [ + "arn:aws:iot:{region}:{aws_account}:topic/sources/maestro/consumers/*/agentevents", + "arn:aws:iot:{region}:{aws_account}:topic/$share/statussubscribers/sources/maestro/consumers/*/agentevents" + ] + } + ] +} diff --git a/test/e2e/setup/rosa/setup/e2e.sh b/test/e2e/setup/rosa/setup/e2e.sh new file mode 100755 index 00000000..d03d5377 --- /dev/null +++ b/test/e2e/setup/rosa/setup/e2e.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +##################### +# Setup Maestro e2e +##################### + +PWD="$(cd "$(dirname ${BASH_SOURCE[0]})" ; pwd -P)" +ROSA_DIR="$(cd ${PWD}/.. && pwd -P)" + +output_dir=${ROSA_DIR}/_output + +mkdir -p $output_dir + +echo "$output_dir" + +# Setup Maestro server +CLUSTER_VPC=$vpc ${PWD}/maestro.sh +sleep 90 # wait the maestro service ready + +# Start Maestro servers +exec oc relay service/maestro 8000:8000 -n maestro > ${output_dir}/maestro.svc.log 2>&1 & +maestro_server_pid=$! +echo "Maestro server started: $maestro_server_pid" +echo "$maestro_server_pid" > ${output_dir}/maestro_server.pid +exec oc relay service/maestro-grpc 8090:8090 -n maestro > ${output_dir}/maestro-grpc.svc.log 2>&1 & +maestro_grpc_server_pid=$! +echo "Maestro GRPC server started: $maestro_grpc_server_pid" +echo "$maestro_grpc_server_pid" > ${output_dir}/maestro_grpc_server.pid + +# need to wait the relay build the connection before we get the consumer id +sleep 15 + +# Prepare a consumer +consumer_id=$(curl -s -X POST -H "Content-Type: application/json" http://127.0.0.1:8000/api/maestro/v1/consumers -d '{}' | jq -r '.id') +echo $consumer_id > ${output_dir}/consumer_id +echo "Consumer $consumer_id is created" + +# Setup Maestro agent +oc apply -f https://raw.githubusercontent.com/open-cluster-management-io/api/release-0.14/work/v1/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml +CONSUMER_ID=$consumer_id ${PWD}/agent.sh diff --git a/test/e2e/setup/rosa/setup/maestro.sh b/test/e2e/setup/rosa/setup/maestro.sh new file mode 100755 index 00000000..21714aab --- /dev/null +++ b/test/e2e/setup/rosa/setup/maestro.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env bash + +###################### +# Setup Maestro server +###################### + +PWD="$(cd "$(dirname ${BASH_SOURCE[0]})" ; pwd -P)" +ROSA_DIR="$(cd ${PWD}/.. && pwd -P)" + +region=${REGION:-""} +cluster_id=${CLUSTER_ID:-""} + +if [ -z "$region" ]; then + echo "region is required" + exit 1 +fi + +if [ -z "$cluster_id" ]; then + echo "cluster id is required" + exit 1 +fi + +# Find Maestro server vpc +rosa_infra_id=$(rosa describe cluster --region=${region} --cluster=${cluster_id} -ojson | jq -r '.infra_id') +vpc=$(aws ec2 describe-vpcs --region=${region} \ + --filters Name=tag:Name,Values=${rosa_infra_id}-vpc | jq -r '.Vpcs[0].VpcId') + +echo "Setup Maestro in ${region} (cluster=$cluster_id,vpc=$vpc)" + +IMAGE_REGISTRY=${IMAGE_REGISTRY:-"quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro"} +IMAGE_REPOSITORY=${IMAGE_REPOSITORY:-"maestro"} +IMAGE_TAG=${IMAGE_TAG:-"1de63c6075f2c95c9661d790d164019f60d789f3"} + +output_dir=${ROSA_DIR}/_output +certs_dir=${output_dir}/aws-certs +source_cert_dir=${certs_dir}/iot/source +policies_dir=${output_dir}/aws-policies + +source_id="maestro" + +mkdir -p ${source_cert_dir} +mkdir -p ${policies_dir} + +db_pw=$(LC_CTYPE=C tr -dc 'a-zA-Z0-9' < /dev/urandom | head -c 16) +echo "$db_pw" > $output_dir/db.password + +# Download AWS IoT broker severing CA +echo "Download AWS IoT broker and database severing CA ...." +curl -s -o ${certs_dir}/iot-ca.pem https://www.amazontrust.com/repository/AmazonRootCA1.pem +curl -s -o ${certs_dir}/db-ca.pem "https://truststore.pki.rds.amazonaws.com/${region}/${region}-bundle.pem" + +# Generate client certs for AWS IoT clients +echo "Generate AWS IoT client certs for Maestro ...." +maestro_cert_arn=$(aws iot create-keys-and-certificate \ + --region ${region} \ + --set-as-active \ + --certificate-pem-outfile "${source_cert_dir}/${source_id}.crt" \ + --public-key-outfile "${source_cert_dir}/${source_id}.public.key" \ + --private-key-outfile "${source_cert_dir}/${source_id}.private.key" | jq -r '.certificateArn') +echo "Mastro AWS IoT client certs are generated ($maestro_cert_arn)" + +# Attach policies for AWS IoT clients +echo "Generate AWS IoT policy for Maestro ...." +aws_account=$(aws sts get-caller-identity --region ${region} --output json | jq -r '.Account') + +cat $PWD/aws-iot-policies/source.template.json | sed "s/{region}/${region}/g" | sed "s/{aws_account}/${aws_account}/g" > $policies_dir/source.json +policy_name=$(aws iot create-policy \ + --region ${region} \ + --policy-name ${source_id} \ + --policy-document "file://${policies_dir}/source.json" | jq -r '.policyName') +aws iot attach-policy --region ${region} --policy-name ${source_id} --target ${maestro_cert_arn} +echo "Maestro AWS IoT policy $policy_name is generated" + +# Allow AWS PostgrepSQL connection in the default security group +echo "Prepare AWS RDS PostgrepSQL for Maestro in ${region} (${vpc}) ...." +sg=$(aws ec2 get-security-groups-for-vpc \ + --region ${region} \ + --vpc-id ${vpc} \ + --query "SecurityGroupForVpcs[?GroupName=='default'].GroupId" | jq -r '.[0]') +result=$(aws ec2 authorize-security-group-ingress \ + --region ${region} \ + --group-id ${sg} \ + --protocol tcp --port 5432 --cidr 0.0.0.0/0 | jq -r '.Return') +echo "PostgrepSQL inbound rule is added to ${sg} (${result})" + +# Create a database subnet group for AWS PostgrepSQL +subnets="" +subnets_counts=0 +for subnet in $(aws ec2 describe-subnets --region ${region} --filters "Name=vpc-id,Values=${vpc}" | jq -r '.Subnets[].SubnetId'); do + subnets="$subnets,\"$subnet\"" + subnets_counts=$((subnets_counts+1)) +done + +if [ $subnets_counts -le 2 ]; then + # The DB subnet group doesn't meet Availability Zone (AZ) coverage requirement. Current AZ coverage: us-west-2a. Add subnets to cover at least 2 AZs. + current_az=$(aws ec2 describe-subnets --region ${region} --filters "Name=vpc-id,Values=${vpc}" | jq -r '.Subnets[0].AvailabilityZone') + for az in $(aws ec2 describe-availability-zones --region=${region} | jq -r '.AvailabilityZones[].ZoneName'); do + if [[ "$az" != "$current_az" ]]; then + subnet=$(aws ec2 create-subnet \ + --region=${region} \ + --vpc-id ${vpc} \ + --availability-zone ${az} \ + --cidr-block 10.0.64.0/18 \ + --tag-specifications "ResourceType=subnet,Tags=[{Key=Name,Value=maestro-db-subnet-${az}}]" | jq -r '.Subnet.SubnetId') + subnets="$subnets,\"$subnet\"" + break + fi + done +fi + +db_subnet_group=$(aws rds create-db-subnet-group \ + --region ${region} \ + --db-subnet-group-name maestrosubnetgroup \ + --db-subnet-group-description "Maestro DB subnet group" \ + --subnet-ids "[${subnets:1}]" | jq -r '.DBSubnetGroup.DBSubnetGroupName') +echo "PostgrepSQL subnet group ${db_subnet_group} is created" + +# Create AWS PostgrepSQL +db_id=$(aws rds create-db-instance \ + --region ${region} \ + --engine postgres \ + --engine-version 14.10 \ + --allocated-storage 20 \ + --db-instance-class db.t4g.large \ + --db-subnet-group-name ${db_subnet_group} \ + --db-instance-identifier maestro \ + --db-name maestro \ + --master-username maestro \ + --master-user-password "${db_pw}" | jq -r '.DBInstance.DBInstanceIdentifier') +db_id=maestro +i=1 +while [ $i -le 20 ] +do + db_status=$(aws rds describe-db-instances --region ${region} --db-instance-identifier ${db_id} | jq -r '.DBInstances[0].DBInstanceStatus') + echo "[$i] DB status: ${db_status}" + if [[ "$db_status" == "available" ]]; then + break + fi + i=$((i + 1)) + sleep 30 +done + +# Get AWS IoT broker and PostgrepSQL endpoints +mqtt_host=$(aws iot describe-endpoint --region ${region} --endpoint-type iot:Data-ATS | jq -r '.endpointAddress') +db_host=$(aws rds describe-db-instances --region ${region} --db-instance-identifier ${db_id} | jq -r '.DBInstances[0].Endpoint.Address') +echo "AWS IoT broke: ${mqtt_host}:8883" +echo "AWS RDS PostgreSQL: ${db_host}:5432 (${db_id})" + +# Deploy Maestro server +oc create namespace maestro || true + +oc -n maestro delete secrets mqtt-creds --ignore-not-found +oc -n maestro create secret generic mqtt-creds \ + --from-file=ca.crt="${certs_dir}/iot-ca.pem" \ + --from-file=client.crt="${source_cert_dir}/maestro.crt" \ + --from-file=client.key="${source_cert_dir}/maestro.private.key" + +oc -n maestro delete secret maestro-db --ignore-not-found +oc -n maestro create secret generic maestro-db \ + --from-literal=db.name=maestro \ + --from-literal=db.host=${db_host} \ + --from-literal=db.port=5432 \ + --from-literal=db.user=maestro \ + --from-literal=db.password="${db_pw}" \ + --from-file=db.ca_cert="${certs_dir}/db-ca.pem" + +oc process --filename="https://raw.githubusercontent.com/openshift-online/maestro/refs/heads/main/templates/service-template-rosa.yml" \ + --local="true" \ + --param="IMAGE_REGISTRY=${IMAGE_REGISTRY}" \ + --param="IMAGE_REPOSITORY=${IMAGE_REPOSITORY}" \ + --param="IMAGE_TAG=${IMAGE_TAG}" \ + --param="MQTT_HOST=${mqtt_host}" > ${output_dir}/maestro-rosa.json + +oc -n maestro apply -f ${output_dir}/maestro-rosa.json +oc -n maestro wait deploy/maestro --for condition=Available=True --timeout=300s diff --git a/test/e2e/setup/rosa/setup/teardown.sh b/test/e2e/setup/rosa/setup/teardown.sh new file mode 100755 index 00000000..33258709 --- /dev/null +++ b/test/e2e/setup/rosa/setup/teardown.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +region=${REGION:-""} + +if [ -z "$region" ]; then + echo "cluster region is required" + exit 1 +fi + +# Delete AWS PostgreSQL +db_status=$(aws rds delete-db-instance --region ${region} --db-instance-identifier maestro --skip-final-snapshot --delete-automated-backups | jq -r '.DBInstance.DBInstanceStatus') +echo "Deleting maestro db ($db_status)" + +i=1 +while [ $i -le 20 ] +do + db_status=$(aws rds describe-db-instances --region ${region} --db-instance-identifier maestro | jq -r '.DBInstances[0].DBInstanceStatus') + if [[ -z "$db_status" ]]; then + echo "DB is deleted" + break + fi + echo "[$i] DB status: ${db_status}" + i=$((i + 1)) + sleep 30 +done + +aws rds delete-db-subnet-group --region ${region} --db-subnet-group-name maestrosubnetgroup +echo "DB db subnet group is removed" + +# Remove AWS IoT polices and certificates +for cert_id in $(aws iot list-certificates --region ${region} | jq -r '.certificates[].certificateId'); do + cert_arn=$(aws iot describe-certificate --region ${region} --certificate-id $cert_id | jq -r '.certificateDescription.certificateArn') + # List all + for policy_name in $(aws iot list-attached-policies --region ${region} --target $cert_arn | jq -r '.policies[].policyName'); do + if [[ $policy_name == maestro* ]]; then + echo "delelet policy $policy_name" + aws iot detach-policy --region ${region} --target $cert_arn --policy-name $policy_name + aws iot delete-policy --region ${region} --policy-name $policy_name + + echo "delelet certificate $cert_id" + aws iot update-certificate --region ${region} --certificate-id $cert_id --new-status REVOKED + sleep 5 + aws iot delete-certificate --region ${region} --certificate-id $cert_id + fi + done +done