Merge pull request #185 from TykTechnologies/test-all-github-action

Test all GitHub action
TykTechnologies · Feb 21, 2023 · 91311fa · 91311fa
2 parents fff617a + 8f27805
commit 91311fa
Show file tree

Hide file tree

Showing 10 changed files with 116 additions and 68 deletions.
diff --git a/.github/workflows/tyk-demo-tests.yml b/.github/workflows/tyk-demo-tests.yml
@@ -7,27 +7,33 @@ jobs:
       - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event."
       - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!"
       - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
-      - name: Check out repository code
+      - name: Check Out Repository Code
         uses: actions/checkout@v3
       - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner."
-      - name: Create environment file
-        uses: "finnp/create-file-action@master"
-        env:
-          FILE_NAME: ".env"
-          FILE_DATA: "DASHBOARD_LICENCE=${{ secrets.DASH_LICENSE }}"
-      - name: Update hosts
+      - name: Create Environment File
+        run: |
+          echo "DASHBOARD_LICENCE=${{ secrets.DASH_LICENSE }}" >> .env
+          echo "MDCB_LICENCE=${{ secrets.MDCB_LICENSE }}" >> .env
+      - name: Update Hosts
         run: sudo ./scripts/update-hosts.sh
-      - name: Bootstrap Tyk
-        run: sudo ./up.sh
-      - name: Store bootstrap log
+      - name: Bootstrap and Test All Eligible Deployments
+        run: sudo ./scripts/test-all.sh
+      - name: Store Bootstrap Log
         if: success() || failure()
         uses: actions/upload-artifact@v3
         with:
           name: bootstrap-log
           path: bootstrap.log
-      - name: Show bootstrap log
+      - name: Store Test Log
+        if: success() || failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-log
+          path: test.log
+      - name: Show Bootstrap Log
         if: success() || failure()
         run: cat bootstrap.log
-      - name: Run Postman tests
-        run: sudo ./scripts/test.sh
+      - name: Show Test Log
+        if: success() || failure()
+        run: cat test.log
       - run: echo "🍏 This job's status is ${{ job.status }}."
diff --git a/.gitignore b/.gitignore
@@ -18,6 +18,7 @@ deployments/tyk/volumes/tyk-gateway/certs/tls-private-key.pem
 deployments/tyk/volumes/tyk-gateway/certs/private-key.pem
 deployments/tyk/volumes/tyk-gateway/certs/public-key.pem
 deployments/tyk/volumes/tyk-dashboard/certs/private-key.pem
+test.log
 
 ## Portal Deployment
 deployments/portal/volumes/database/portal.db
diff --git a/deployments/mdcb/README.md b/deployments/mdcb/README.md
@@ -12,7 +12,7 @@ MDCB architecture creates a Control Plane and a Data Plane:
 - MDCB is the link between the Planes
 
 The worker Gateway can be accessed here:
-- [Tyk Worker Gateway](http://tyk-worker-gateway.localhost:8084)
+- [Tyk Worker Gateway](http://tyk-worker-gateway.localhost:8090)
 
 ## Setup
 
@@ -79,7 +79,7 @@ These indicate that the Gateway has lost its connection to MDCB.
 You can then try accessing APIs via the Worker Gateway:
 
 ```
-curl http://tyk-worker-gateway.localhost:8084/basic-open-api/get
+curl http://tyk-worker-gateway.localhost:8090/basic-open-api/get
 ```
 
 You will receive a response, despite the Gateway being disconnected from MDCB. In a typical multi-data centre deployment, this would allow the Gateway to continue operating in the event of connectivity disuption between it and the primary data centre where MDCB is deployed.

diff --git a/deployments/mdcb/bootstrap.sh b/deployments/mdcb/bootstrap.sh
@@ -7,7 +7,7 @@ log_start_deployment
 bootstrap_progress
 
 log_message "Setting global variables"
-worker_gateway_base_url="http://tyk-worker-gateway.localhost:8084"
+worker_gateway_base_url="http://tyk-worker-gateway.localhost:8090"
 dashboard_base_url="http://tyk-dashboard.localhost:3000"
 log_ok
 bootstrap_progress

diff --git a/deployments/mdcb/docker-compose.yml b/deployments/mdcb/docker-compose.yml
@@ -15,13 +15,13 @@ services:
   tyk-worker-gateway:
     image: tykio/tyk-gateway:${GATEWAY_WORKER_VERSION:-v4.3.3-rc2}
     ports:
-      - 8084:8080
+      - 8090:8080
     networks:
       - tyk
     environment:
       - TYK_INSTRUMENTATION=${INSTRUMENTATION_ENABLED:-0}
       - TYK_GW_TRACER_ENABLED=${TRACING_ENABLED:-0}
-      - TYK_GW_SLAVEOPTIONS_APIKEY=${MDCB_USER_API_CREDENTIALS:?}
+      - TYK_GW_SLAVEOPTIONS_APIKEY=${MDCB_USER_API_CREDENTIALS:-placeholder}
     volumes:
       - ./deployments/mdcb/volumes/tyk-worker-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf
       - ./deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs

diff --git a/deployments/mdcb/tyk_demo_mdcb.postman_collection.json b/deployments/mdcb/tyk_demo_mdcb.postman_collection.json
@@ -44,7 +44,7 @@
 	"variable": [
 		{
 			"key": "tyk-worker-gateway.host",
-			"value": "tyk-worker-gateway.localhost:8084"
+			"value": "tyk-worker-gateway.localhost:8090"
 		}
 	]
 }
diff --git a/deployments/waf/docker-compose.yml b/deployments/waf/docker-compose.yml
@@ -1,13 +1,14 @@
 version: '3.3'
 services:
   waf:
-    image: owasp/modsecurity-crs:v3.0
+    image: owasp/modsecurity-crs:3.3.4-apache-202302060502
     networks:
       - tyk
     ports:
       - "8500:80"
     environment:
       - PARANOIA=1
+      - BACKEND=http://httpbin
     deploy:
       restart_policy:
         condition: any
diff --git a/deployments/waf/tyk_demo_waf.postman_collection.json b/deployments/waf/tyk_demo_waf.postman_collection.json
@@ -1,6 +1,6 @@
 {
 	"info": {
-		"_postman_id": "eeb6bf53-3960-4bb4-aaf0-0a1f1239dfb5",
+		"_postman_id": "9806a85d-289d-423f-be8f-ba8e2fdae08d",
 		"name": "Tyk Demo - WAF",
 		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
 	},
@@ -11,14 +11,14 @@
 				{
 					"listen": "test",
 					"script": {
-						"id": "b8e091fa-5bf4-429b-af47-84fbb05fddd6",
 						"exec": [
 							"pm.test(\"Response time is less than 200ms\", function () {",
 							"    pm.expect(pm.response.responseTime).to.be.below(200);",
 							"});",
 							"",
-							"pm.test(\"Body matches string\", function () {",
-							"    pm.expect(pm.response.text()).to.include(\"hello world\");",
+							"pm.test(\"Non-empty UUID response returned\", function () {",
+							"    var jsonData = pm.response.json();",
+							"    pm.expect(jsonData.uuid).is.not.empty;",
 							"});"
 						],
 						"type": "text/javascript"
@@ -29,13 +29,16 @@
 				"method": "GET",
 				"header": [],
 				"url": {
-					"raw": "http://{{waf.host}}",
+					"raw": "http://{{waf.host}}/uuid",
 					"protocol": "http",
 					"host": [
 						"{{waf.host}}"
+					],
+					"path": [
+						"uuid"
 					]
 				},
-				"description": "This request checks that the Web Application Firewall (WAF) is operational.\n\nIt should return a HTTP 200 response with the body text `hello world`."
+				"description": "This request checks that the Web Application Firewall (WAF) is operational.\n\nIt should return a HTTP 200 response with a JSON body containing a UUID."
 			},
 			"response": []
 		},
@@ -45,7 +48,6 @@
 				{
 					"listen": "test",
 					"script": {
-						"id": "e35f2848-c599-49c2-b2b6-271fe944c0bb",
 						"exec": [
 							"pm.test(\"Status code is 403\", function () {",
 							"    pm.response.to.have.status(403);",
@@ -83,7 +85,6 @@
 				{
 					"listen": "test",
 					"script": {
-						"id": "9497f7bc-b1be-4c0d-9832-7016ff2c61ac",
 						"exec": [
 							"pm.test(\"Status code is 200\", function () {",
 							"    pm.response.to.have.status(200);",
@@ -116,7 +117,6 @@
 				{
 					"listen": "test",
 					"script": {
-						"id": "a5902804-e15d-4cc8-b974-208783e9cb2f",
 						"exec": [
 							"pm.test(\"Status code is 400\", function () {",
 							"    pm.response.to.have.status(400);",
@@ -155,7 +155,6 @@
 			"response": []
 		}
 	],
-	"protocolProfileBehavior": {},
 	"variable": [
 		{
 			"key": "waf.host",

diff --git a/scripts/test-all.sh b/scripts/test-all.sh
@@ -11,8 +11,14 @@
 # A test is considered successful if a deployment can be created, tested and removed without error
 # The scope of testing is limited to the tests defined within the Postman collection
 # If no tests fail then this script will exit with a 0, otherwise it will be a non-zero value
+# Tests may fail due to environmental reasons, so if you experience a failure it's worth checking that it wasn't caused by an environmental error, such as lack of resources
 # The script must be run from the repository root i.e. ./scripts/test-all.sh
 
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+BLUE='\033[0;34m'
+NOCOLOUR='\033[0m'
+
 echo "Checking for active deployments"
 if [ ! -s .bootstrap/bootstrapped_deployments ]; then
    echo "  No active deployments found - proceeding with tests"
@@ -25,6 +31,10 @@ else
     ./down.sh
 fi
 
+# clear log files
+echo -n > test.log
+echo -n > bootstrap.log
+
 declare -a result_names
 declare -a result_codes
 
@@ -44,7 +54,7 @@ do
     echo "Validating deployment's Postman collection ($postman_collection_file_name)"
 
     if [ -z "$(ls -A $postman_collection_path 2>/dev/null)" ]; then
-        echo "  Collection not found. Skipping to next deployment."
+        echo -e "  Collection not found. ${BLUE}Skipping${NOCOLOUR} to next deployment."
         result_codes[${#result_codes[@]}]=2
         continue
     else
@@ -55,17 +65,17 @@ do
     # The jq command finds "listen" fields which have the value "test", if none are returned then the collection doesn't contain any tests
     postman_collection_tests=$(jq '..|.listen?|select(.=="test")' $postman_collection_path)
     if [[ "$postman_collection_tests" == "" ]]; then
-        echo "  Collection does not contain any tests. Skipping to next deployment."
+        echo -e "  Collection does not contain any tests. ${BLUE}Skipping${NOCOLOUR} to next deployment."
         result_codes[${#result_codes[@]}]=3
         continue
     else
         echo "  Collection contains tests."
     fi
 
     echo "Creating deployment: $deployment_name"
-    ./up.sh $deployment_name
+    ./up.sh $deployment_name persist-log
     if [ "$?" != "0" ]; then
-        echo "  Failed to create $deployment_name deployment"
+        echo -e "  ${RED}Failed${NOCOLOUR} to create $deployment_name deployment"
         result_codes[${#result_codes[@]}]=4
         echo "Removing deployment: $deployment_name"
         ./down.sh
@@ -84,21 +94,24 @@ do
         postman/newman:alpine \
         run "/etc/postman/tyk_demo.postman_collection.json" \
         --environment /etc/postman/test.postman_environment.json \
-        --insecure
+        --insecure \
+        | tee -a test.log
+    # output of above command is captured in test.log file
+    # file will contain control characters, so is advised to use command "less -r test.log", or similar, to view it
 
     if [ "$?" != "0" ]; then
-        echo "Tests failed for $deployment_name deployment"
+        echo -e "Tests ${RED}failed${NOCOLOUR} for $deployment_name deployment"
         result_codes[${#result_codes[@]}]=1
     else
-        echo "Tests passed for $deployment_name deployment"
+        echo -e "Tests ${GREEN}passed${NOCOLOUR} for $deployment_name deployment"
         result_codes[${#result_codes[@]}]=0
     fi
 
     echo "Removing deployment: $deployment_name"
     ./down.sh
 
     if [ "$?" != "0" ]; then
-        echo "  Failed to remove $deployment_name deployment"
+        echo -e "  ${RED}Failed${NOCOLOUR} to remove $deployment_name deployment"
         result_codes[${#result_codes[@]}]=5
         # failing to remove a deployment may negatively affect subsequent deployments and tests
         continue
@@ -115,42 +128,41 @@ test_fail_count=0
 test_skip_count=0
 for i in "${!result_codes[@]}"
 do 
-  result_print=""
   case ${result_codes[$i]} in
     0)
-        echo "$(tput setaf 2)Pass$(tput sgr 0) ${result_names[$i]} - Tests passed"
+        echo -e "${GREEN}Pass${NOCOLOUR} ${result_names[$i]} - Tests passed"
         test_pass_count=$((test_pass_count+1));;
     1) 
-        echo "$(tput setaf 1)Fail$(tput sgr 0) ${result_names[$i]} - Tests failed"
+        echo -e "${RED}Fail${NOCOLOUR} ${result_names[$i]} - Tests failed"
         test_fail_count=$((test_fail_count+1));;
     2) 
-        echo "$(tput setaf 4)Skip$(tput sgr 0) ${result_names[$i]} - No collection"
+        echo -e "${BLUE}Skip${NOCOLOUR} ${result_names[$i]} - No collection"
         test_skip_count=$((test_skip_count+1));;
     3) 
-        echo "$(tput setaf 4)Skip$(tput sgr 0) ${result_names[$i]} - No tests"
+        echo -e "${BLUE}Skip${NOCOLOUR} ${result_names[$i]} - No tests"
         test_skip_count=$((test_skip_count+1));;
     4) 
-        echo "$(tput setaf 1)Fail$(tput sgr 0) ${result_names[$i]} - Create failed"
-        test_skip_count=$((test_fail_count+1));;
+        echo -e "${RED}Fail${NOCOLOUR} ${result_names[$i]} - Create failed"
+        test_fail_count=$((test_fail_count+1));;
     5) 
-        echo "$(tput setaf 1)Fail$(tput sgr 0) ${result_names[$i]} - Remove failed"
-        test_skip_count=$((test_fail_count+1));;
+        echo -e "${RED}Fail${NOCOLOUR} ${result_names[$i]} - Remove failed"
+        test_fail_count=$((test_fail_count+1));;
     *) 
         echo "ERROR: Unexpected result code. Exiting."
         exit 2;;
     esac
 done
 
 echo -e "\nTest Result Totals:"
-echo "$(tput setaf 2)Pass$(tput sgr 0):$test_pass_count"
-echo "$(tput setaf 1)Fail$(tput sgr 0):$test_fail_count"
-echo "$(tput setaf 4)Skip$(tput sgr 0):$test_skip_count"
+echo -e "${GREEN}Pass${NOCOLOUR}:$test_pass_count"
+echo -e "${RED}Fail${NOCOLOUR}:$test_fail_count"
+echo -e "${BLUE}Skip${NOCOLOUR}:$test_skip_count"
 
 echo -e "\nExit Status:"
 if [ $test_fail_count = 0 ]; then
-    echo "Failure count is 0, exiting with code 0"
+    echo "No failures detected, exiting with code 0"
     exit 0
 else
-    echo "Failure count is not 0, exiting with code 1"
+    echo "Failures detected, exiting with code 1"
     exit 1
 fi