Skip to content

Commit

Permalink
QS AWS 3.2 (#59)
Browse files Browse the repository at this point in the history 
The new version will be v3.2.0 and  includes
* Fix the health-check port to be accessible to the AWS load balancer
* Quickstart support for Single Storage
* Support Private VPC Broker HA Deployment With Broker Services Exposed Internally
* Support for new Region Stockholm.
* Documentation updates
  • Loading branch information
@LewisKSaint
LewisKSaint authored Jul 28, 2022
1 parent 69d4f13 commit d4178f5
Show file tree
Hide file tree
Showing 22 changed files with 566 additions and 42 deletions.
11 changes: 10 additions & 1 deletion .github/workflows/build-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,14 @@ jobs:
timeout-minutes: 50

steps:
- name: Extract branch name for non PR
if: github.event_name != 'pull_request'
shell: bash
run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr _. - )" >> $GITHUB_ENV
id: extract_branch
- name: Extract branch name for PR
if: github.event_name == 'pull_request'
run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr _. -)" >> $GITHUB_ENV
- name: Set env
run: |
echo "AWS_DEFAULT_REGION=eu-west-2" >> $GITHUB_ENV
Expand Down Expand Up @@ -54,7 +62,8 @@ jobs:
export BUCKETREGION=`aws s3api get-bucket-location --bucket ${{ env.TEST_S3_BUCKET }} | grep LocationConstraint | awk -F' ' '{print $NF}' | tr -d '"'`
sed -i "s@SolaceStackRegionNAME@${AWS_DEFAULT_REGION}@g" ci/solace-aws-ha-3az-prod-test.json
sed -i "s@SolaceBucketRegionNAME@${BUCKETREGION}@g" ci/solace-aws-ha-3az-prod-test.json
aws s3 sync . s3://${{ env.TEST_S3_BUCKET }}/solace/eventbroker/latest --acl public-read
sed -i "s@SolaceBranchNAME@${{ env.BRANCH_NAME }}@g" ci/solace-aws-ha-3az-prod-test.json
aws s3 sync . s3://${{ env.TEST_S3_BUCKET }}/solace/eventbroker/${{ env.BRANCH_NAME }} --acl public-read
- name: Test with production deployment option (create public subnet)
run: |
Expand Down
160 changes: 160 additions & 0 deletions .github/workflows/build-vpc-test.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
name: build-private-vpc

# Controls when the action will run.
on:
#pull_request_target:

push:

jobs:
build:
runs-on: ubuntu-latest
timeout-minutes: 50

steps:
- name: Extract branch name for non PR
if: github.event_name != 'pull_request'
shell: bash
run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr _. - )" >> $GITHUB_ENV
id: extract_branch
- name: Extract branch name for PR
if: github.event_name == 'pull_request'
run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr _. -)" >> $GITHUB_ENV
- name: Set env
run: |
echo "AWS_DEFAULT_REGION=eu-west-1" >> $GITHUB_ENV
echo "TEST_S3_BUCKET=solace-cf-quickstart-ghtest" >> $GITHUB_ENV
TESTSTACKPREFIX=T$(echo $(date +%s) | rev)
echo "TESTSTACKPREFIX=$TESTSTACKPREFIX" >> $GITHUB_ENV
echo "TESTSTACKNAME=$TESTSTACKPREFIX-sol-aws-ghtest" >> $GITHUB_ENV
echo "TESTSTACK_VPCNAME=$TESTSTACKPREFIX-vpc-sol-aws-ghtest" >> $GITHUB_ENV
- name: Checkout
uses: actions/checkout@v2

- name: Recourse submodules
run: |
sed -i 's/[email protected]:/https:\/\/github.com\//' .gitmodules; cat .gitmodules
git submodule sync
git submodule init
git submodule update
- name: Configure AWS credentials from Test account
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_DEFAULT_REGION }}

- name: Lint yaml templates
run: |
sudo apt-get update -y; sudo apt-get install -y yamllint
aws cloudformation validate-template --template-body file://templates/solace-master.template
aws cloudformation validate-template --template-body file://templates/solace.template
aws cloudformation validate-template --template-body file://templates/nodecreate.template
aws s3 mb s3://${{ env.TEST_S3_BUCKET }} || echo "s3 bucket already existed"
- name: Copy test artifacts to test S3 bucket
run: |
MESSAGEBROKERNODEINSTANCETYPE=t2.medium
sed -i "s@SolaceDockerImageParameterValue@solace/solace-pubsub-standard:latest@g" ci/solace-aws-ha-3az-private-vpc-prod-test.json
sed -i "s@EventBrokerNodeInstanceTypeParameterValue@${MESSAGEBROKERNODEINSTANCETYPE}@g" ci/solace-aws-ha-3az-private-vpc-prod-test.json
aws s3 mb s3://${{ env.TEST_S3_BUCKET }} || echo "s3 bucket already existed"
export BUCKETREGION=`aws s3api get-bucket-location --bucket ${{ env.TEST_S3_BUCKET }} | grep LocationConstraint | awk -F' ' '{print $NF}' | tr -d '"'`
sed -i "s@SolaceStackRegionNAME@${AWS_DEFAULT_REGION}@g" ci/solace-aws-private-vpc-test.json
sed -i "s@SolaceBucketRegionNAME@${BUCKETREGION}@g" ci/solace-aws-ha-3az-private-vpc-prod-test.json
sed -i "s@SolaceBucketRegionNAME@${BUCKETREGION}@g" ci/solace-aws-private-vpc-test.json
sed -i "s@SolaceBranchNAME@${{ env.BRANCH_NAME }}@g" ci/solace-aws-ha-3az-private-vpc-prod-test.json
sed -i "s@SolaceBranchNAME@${{ env.BRANCH_NAME }}@g" ci/solace-aws-private-vpc-test.json
aws s3 sync . s3://${{ env.TEST_S3_BUCKET }}/solace/eventbroker/${{ env.BRANCH_NAME }} --acl public-read
- name: Test with production deployment option (create public subnet)
run: |
aws cloudformation create-stack --stack-name $TESTSTACK_VPCNAME --template-body file://templates/setup_sample_new_vpc.template --parameters file://ci/solace-aws-private-vpc-test.json --on-failure ROLLBACK --capabilities CAPABILITY_NAMED_IAM
echo "Waiting for stack create complete"
sleep 30; aws cloudformation describe-stacks --stack-name $TESTSTACK_VPCNAME
until aws cloudformation describe-stacks --stack-name $TESTSTACK_VPCNAME | grep -m 1 -E 'CREATE_COMPLETE'; do
sleep 10
if aws cloudformation describe-stacks --stack-name $TESTSTACK_VPCNAME | grep -m 1 -E 'ROLLBACK_IN_PROGRESS'; then
aws cloudformation describe-stack-events --stack-name $TESTSTACK_VPCNAME
aws cloudformation describe-stacks --stack-name $TESTSTACK_VPCNAME
exit 1
fi
sleep 10
done
aws cloudformation describe-stack-events --stack-name $TESTSTACK_VPCNAME
aws cloudformation describe-stacks --stack-name $TESTSTACK_VPCNAME --query "Stacks[0].Outputs"
echo "Pausing to get the VPC ready"; sleep 10
export private_vpc_conf="$(aws cloudformation describe-stacks --stack-name $TESTSTACK_VPCNAME --query Stacks[0].Outputs)"; echo $private_vpc_conf
echo "VPC set up complete"
sed -i "s@VPCIDParam@$(echo $private_vpc_conf | jq '.[] | select(.OutputKey == "VPCID").OutputValue' | tr -d '"')@g" ci/solace-aws-ha-3az-private-vpc-prod-test.json
sed -i "s@VPCSecurityGroupIDParam@$(echo $private_vpc_conf | jq '.[] | select(.OutputKey == "VPCSecurityGroupID").OutputValue' | tr -d '"')@g" ci/solace-aws-ha-3az-private-vpc-prod-test.json
sed -i "s@SolaceStackPublicSubnetID1A@$(echo $private_vpc_conf | jq '.[] | select(.OutputKey == "PublicSubnet1ID").OutputValue' | tr -d '"')@g" ci/solace-aws-ha-3az-private-vpc-prod-test.json
sed -i "s@SolaceStackPublicSubnetID2B@$(echo $private_vpc_conf | jq '.[] | select(.OutputKey == "PublicSubnet2ID").OutputValue' | tr -d '"')@g" ci/solace-aws-ha-3az-private-vpc-prod-test.json
sed -i "s@SolaceStackPrivateSubnetID1A@$(echo $private_vpc_conf | jq '.[] | select(.OutputKey == "PrivateSubnet1ID").OutputValue' | tr -d '"')@g" ci/solace-aws-ha-3az-private-vpc-prod-test.json
sed -i "s@SolaceStackPrivateSubnetID2B@$(echo $private_vpc_conf | jq '.[] | select(.OutputKey == "PrivateSubnet2ID").OutputValue' | tr -d '"')@g" ci/solace-aws-ha-3az-private-vpc-prod-test.json
sed -i "s@SolaceStackPrivateSubnetID3C@$(echo $private_vpc_conf | jq '.[] | select(.OutputKey == "PrivateSubnet3ID").OutputValue' | tr -d '"')@g" ci/solace-aws-ha-3az-private-vpc-prod-test.json
echo "BASTION_HOST=$(echo $private_vpc_conf | jq '.[] | select(.OutputKey == "EIP1").OutputValue' | tr -d '"')" >> $GITHUB_ENV
aws cloudformation create-stack --stack-name $TESTSTACKNAME --template-body file://templates/solace.template --parameters file://ci//solace-aws-ha-3az-private-vpc-prod-test.json --on-failure ROLLBACK --capabilities CAPABILITY_NAMED_IAM
echo "Waiting for stack create complete"
sleep 30; aws cloudformation describe-stacks --stack-name $TESTSTACKNAME
until aws cloudformation describe-stacks --stack-name $TESTSTACKNAME | grep -m 1 -E 'CREATE_COMPLETE'; do
sleep 10
if aws cloudformation describe-stacks --stack-name $TESTSTACKNAME | grep -m 1 -E 'ROLLBACK_IN_PROGRESS'; then
aws cloudformation describe-stack-events --stack-name $TESTSTACKNAME
aws cloudformation describe-stacks --stack-name $TESTSTACKNAME
exit 1
fi
sleep 10
done
aws cloudformation describe-stack-events --stack-name $TESTSTACKNAME
aws cloudformation describe-stacks --stack-name $TESTSTACKNAME
echo "Pausing to get the ELB ready"; sleep 10
export url="$(aws elbv2 describe-load-balancers | grep DNSName | grep `echo $TESTSTACKPREFIX | head -c4` | awk -F '"' '{print $4}')"; echo $url
echo "BROKER_ELB=$(echo $url)" >> $GITHUB_ENV
- name: Validate PubSub deployment
uses: appleboy/[email protected]
with:
timeout: 5m
command_timeout: 10m
host: ${{ env.BASTION_HOST }}
username: "ec2-user"
key: ${{ secrets.CI_KEY }}
script: |
echo "Validating Broker Setup with LB ${{ env.BROKER_ELB }}";
until curl http://${{ env.BROKER_ELB }}:8080; do sleep 10; done;
curl -O https://sftp.solace.com/download/SDKPERF_C_LINUX64;
tar -xvf SDKPERF_C_LINUX64;
sleep 120;
curl -sS -u admin:admin http://${{ env.BROKER_ELB }}:8080/SEMP -d "<rpc><show><redundancy></redundancy></show></rpc>";
curl -sS -u admin:admin http://${{ env.BROKER_ELB }}:8080/SEMP -d "<rpc><show><config-sync></config-sync></show></rpc>";
if [[ -z `curl -sS -u admin:admin http://${{ env.BROKER_ELB }}:8080/SEMP -d "<rpc><show><config-sync></config-sync></show></rpc>" | grep "<oper-status>Up</oper-status>"` ]] ; then echo "config-sync not up!"; exit 1; fi;
pubSubTools/sdkperf_c -cip=${{ env.BROKER_ELB }} -mn=100000 -mr=0 -ptl=t1 -stl=t1 | grep "Total Messages"
echo "Validation completed.";
- name: Delete test deployment (Cleanup)
if: ${{ always() }}
run: |
aws cloudformation delete-stack --stack-name $TESTSTACKNAME || echo "Couldn't delete stack $TESTSTACKNAME"
echo "Waiting for stack delete complete"
stackid="$(aws cloudformation describe-stacks --stack-name $TESTSTACKNAME | grep StackId | awk -F '"' '{print $4}')"
if [ -n "$stackid" ]; then until aws cloudformation describe-stacks --stack-name $stackid | grep -m 1 "DELETE_COMPLETE"; do sleep 10; done; fi
aws cloudformation delete-stack --stack-name $TESTSTACK_VPCNAME || echo "Couldn't delete stack $TESTSTACK_VPCNAME"
echo "Waiting for stack delete complete"
vpcStackid="$(aws cloudformation describe-stacks --stack-name $TESTSTACK_VPCNAME | grep StackId | awk -F '"' '{print $4}')"
if [ -n "$vpcStackid" ]; then until aws cloudformation describe-stacks --stack-name $vpcStackid | grep -m 1 "DELETE_COMPLETE"; do sleep 10; done; fi
- name: Update QuickStart S3 on success for SolaceProducts
if: ${{ success() }} && github.event_name == 'push'
run: |
if [ ${{ github.ref }} == 'refs/heads/master' ] && [ ${{ github.repository_owner }} == 'SolaceProducts' ] ; then
aws s3 sync . s3://solace-products/pubsubplus-aws-ha-quickstart/latest --acl public-read
fi
- name: Delete all unattached volumes (Cleanup)
if: ${{ always() }}
run: |
for volume in `aws ec2 describe-volumes --filter "Name=status,Values=available" --query "Volumes[*].{ID:VolumeId}" --region $AWS_DEFAULT_REGION | grep ID | awk -F '"' '{print $4}'`; do aws ec2 delete-volume --volume-id $volume --region $AWS_DEFAULT_REGION; done
Loading

0 comments on commit d4178f5

Please sign in to comment.