Release 5.3.0

.github/workflows/tyk-demo-tests.yml

@@ -11,7 +11,7 @@ jobs:
       - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!"
       - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
       - name: Check Out Repository Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner."
       - name: Create Environment File
         run: |
@@ -22,15 +22,15 @@ jobs:
       - name: Bootstrap and Test All Eligible Deployments
         run: sudo ./scripts/test-all.sh
       - name: Store Logs
-        if: success() || failure()
-        uses: actions/upload-artifact@v3
+        if: always()
+        uses: actions/upload-artifact@v4
         with:
           name: logs
           path: logs/
       - name: Show Bootstrap Log
-        if: success() || failure()
+        if: always()
         run: cat logs/bootstrap.log
       - name: Show Test Log
-        if: success() || failure()
+        if: always()
         run: cat logs/test.log
       - run: echo "🍏 This job's status is ${{ job.status }}."

CONTRIBUTING.md

@@ -137,7 +137,7 @@ These utility scripts are available in the `scripts` directory:
 * `add-gateway.sh`: Creates a new Tyk Gateway container, using the same configuration as the base Tyk deployment Gateway
 * `common.sh`: Contains functions useful for bootstrap scripts
 * `export.sh`: Uses the Dashboard API to export API and Policy definitions, overwriting data used to bootstrap the base Tyk deployment
-* `test.sh`: Uses a Newman container to run the Postman collection tests for only the **Tyk** deployment
+* `test.sh`: Uses a Newman container to run the Postman collection tests for all deployment that are currently bootstrapped
 * `test-all.sh`: As `test.sh`, but runs for all deployments
 * `update-hosts.sh`: Adds the necessary hosts to the `/etc/hosts` file
 
@@ -299,6 +299,14 @@ Once the tests are finished, the `delete` function can be called to remove the k
 tyk.dashboardApi.tools.apiKey.delete(pm);
 ```
 
+## Test Automation
+
+Deployments are included in test scripts (`scripts/test.sh` and `scripts/test-all.sh`) if all of the following criteria are met:
+
+- A Postman collection is found in the deployment
+- The collection does *not* contain a variable `test-runner-ignore` with the value `true`
+- The collection contains tests
+
 ## Testing Responses
 
 The Tyk Demo Postman collection contains many requests, each of which demonstrate a particular piece of functionality. Testing the responses generated by these requests provides validation that the desired result was achieved.

deployments/analytics-kibana/bootstrap.sh

@@ -6,6 +6,8 @@ log_start_deployment
 bootstrap_progress
 
 kibana_base_url="http://localhost:5601"
+gateway_base_url="http://$(jq -r '.host_config.override_hostname' deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf)"
+gateway_api_credentials=$(cat deployments/tyk/volumes/tyk-gateway/tyk.conf | jq -r .secret)
 
 log_message "Waiting for kibana to return desired response"
 wait_for_response "$kibana_base_url/app/kibana" "200"
@@ -45,6 +47,12 @@ fi
 log_ok
 bootstrap_progress
 
+log_message "Waiting for API availability"
+# this api id is for the 'basic open api' called by the next section
+wait_for_api_loaded "727dad853a8a45f64ab981154d1ffdad" "$gateway_base_url" "$gateway_api_credentials"
+log_ok
+bootstrap_progress        
+
 log_message "Sending a test request to provide Kibana with data, as Tyk bootstrap requests will not have been picked up by the Pump from this deployment"
 log_http_result "$(curl -s localhost:8080/basic-open-api/get -o /dev/null -w "%{http_code}" 2>> logs/bootstrap.log)"
 bootstrap_progress

deployments/analytics-splunk/bootstrap.sh

@@ -7,6 +7,8 @@ bootstrap_progress
 
 splunk_base_url="http://localhost:8000"
 splunk_base_mgmt_url="https://localhost:8089"
+gateway_base_url="http://$(jq -r '.host_config.override_hostname' deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf)"
+gateway_api_credentials=$(cat deployments/tyk/volumes/tyk-gateway/tyk.conf | jq -r .secret)
 
 log_message "Waiting for splunk to return desired response"
 wait_for_response "$splunk_base_url/en-GB/account/login" "200"
@@ -52,6 +54,14 @@ fi
 log_ok
 bootstrap_progress
 
+log_message "Waiting for API availability"
+# this api id is for the 'basic open api' called by the next section
+wait_for_api_loaded "727dad853a8a45f64ab981154d1ffdad" "$gateway_base_url" "$gateway_api_credentials"
+# this api id is for the 'httpbin acme' API called by the deployment tests
+wait_for_api_loaded "93fd5c15961041115974216e7a3e7175" "$gateway_base_url" "$gateway_api_credentials"
+log_ok
+bootstrap_progress        
+
 log_message "Sending a test request to provide Splunk with data"
 # since request sent in base bootstrap process will not have been picked up by elasticsearch-enabled pump
 log_http_result "$(curl -s localhost:8080/basic-open-api/get -o /dev/null -w "%{http_code}" 2>> logs/bootstrap.log)"

deployments/federation/bootstrap.sh

@@ -1,12 +1,13 @@
 #!/bin/bash
 
 source scripts/common.sh
-deployment="GraphQL Federations"
+deployment="GraphQL Federation"
 log_start_deployment
 
 # Setup variables
 dashboard_base_url="http://tyk-dashboard.localhost:$(jq -r '.listen_port' deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf)"
 gateway_base_url="http://$(jq -r '.host_config.override_hostname' deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf)"
+gateway_api_credentials=$(cat deployments/tyk/volumes/tyk-gateway/tyk.conf | jq -r .secret)
 dashboard_admin_api_credentials=$(cat deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf | jq -r .admin_secret)
 dashboard_user_api_key=$(get_context_data "1" "dashboard-user" "1" "api-key")
 
@@ -23,6 +24,17 @@ bootstrap_progress
 create_api "deployments/federation/data/apis-supergraph.json" "$dashboard_admin_api_credentials" "$dashboard_user_api_key"
 bootstrap_progress
 
+log_message "Waiting for API availability"
+for file in deployments/federation/data/*; do
+  if [[ -f $file ]]; then
+    target_api_id=$(cat $file | jq '.api_definition.api_id' --raw-output)
+    wait_for_api_loaded "$target_api_id" "$gateway_base_url" "$gateway_api_credentials"
+    bootstrap_progress        
+  fi
+done
+bootstrap_progress        
+log_ok
+
 log_end_deployment
 
 echo -e "\033[2K

deployments/keycloak-dcr/README.md

@@ -13,7 +13,7 @@ Run `keycloak-dcr` as one of your deployment configs, ie
 ./up.sh keycloak-dcr
 ```
 
-Visit `http://localhost:8180/auth/admin/` to access the keycloak admin console, and login with the credentials `admin/admin`
+Visit `http://keycloak:8180/` to access the keycloak admin console, and login with the credentials `admin/admin`
 
 Under the Client Registration tab you should find that an `initial access` token has been automagically created during the deployment bootstrap.
 

deployments/keycloak-dcr/bootstrap.sh

@@ -10,14 +10,13 @@ dashboard_admin_api_credentials=$(cat deployments/tyk/volumes/tyk-dashboard/tyk_
 dashboard_user_api_key=$(get_context_data "1" "dashboard-user" "1" "api-key")
 gateway_base_url="http://tyk-gateway.localhost:8080"
 gateway_api_credentials=$(cat deployments/tyk/volumes/tyk-gateway/tyk.conf | jq -r .secret)
-keycloak_base_url="http://keycloak:8180/auth"
-keycloak_admin_url="http://keycloak:8180/auth/admin"
+keycloak_base_url="http://keycloak:8180"
 
 log_message "Waiting for Dashboard API to be ready"
 wait_for_response "$dashboard_base_url/admin/organisations" "200" "admin-auth: $dashboard_admin_api_credentials"
 
 log_message "Waiting for Keycloak to respond ok"
-wait_for_response "$keycloak_base_url/" "200"
+wait_for_response "$keycloak_base_url/health/ready" "200"
 
 
 log_message "Obtaining keycloak user access token"
@@ -34,7 +33,7 @@ bootstrap_progress
 
 
 log_message "Creating a new initial access token"
-api_response="$(curl $keycloak_admin_url/realms/master/clients-initial-access -s \
+api_response="$(curl $keycloak_base_url/admin/realms/master/clients-initial-access -s \
   -H "Content-Type: application/json" \
   -H "Authorization: Bearer $access_token" \
   -d '{"count": 5}')"
@@ -107,7 +106,7 @@ bootstrap_progress
 log_end_deployment
 
 echo -e "\033[2K
-▼ IdP
+▼ Dynamic Client Registration
   ▽ Keycloak
-            Browser URL : $keycloak_admin_url
-                  Login : admin/admin"
+            Browser URL : $keycloak_base_url
+      Username/Password : admin/admin"

deployments/keycloak-dcr/data/tyk-dashboard/apis.json

@@ -177,7 +177,7 @@
             "use_go_plugin_auth": false,
             "enable_coprocess_auth": false,
             "jwt_signing_method": "rsa",
-            "jwt_source": "aHR0cDovL2tleWNsb2FrOjgxODAvYXV0aC9yZWFsbXMvbWFzdGVyL3Byb3RvY29sL29wZW5pZC1jb25uZWN0L2NlcnRz",
+            "jwt_source": "aHR0cDovL2tleWNsb2FrOjgxODAvcmVhbG1zL21hc3Rlci9wcm90b2NvbC9vcGVuaWQtY29ubmVjdC9jZXJ0cw==",
             "jwt_identity_base_field": "sub",
             "jwt_client_base_field": "",
             "jwt_policy_field_name": "pol",
@@ -258,7 +258,7 @@
             "proxy": {
                 "preserve_host_header": false,
                 "listen_path": "/keycloak-dcr-api/",
-                "target_url": "http://httpbin.org",
+                "target_url": "http://httpbin",
                 "disable_strip_slash": true,
                 "strip_listen_path": true,
                 "enable_load_balancing": false,

deployments/keycloak-dcr/data/tyk-dashboard/catalog.json

@@ -54,7 +54,7 @@
         "dcr_options": {
             "idp_host": "http://keycloak:8180/",
             "access_token": "ACCESS_TOKEN",
-            "registration_endpoint": "http://keycloak:8180/auth/realms/master/clients-registrations/openid-connect",
+            "registration_endpoint": "http://keycloak:8180/realms/master/clients-registrations/openid-connect",
             "provider": "keycloak",
             "grant_types": [
                 "client_credentials"

deployments/keycloak-dcr/docker-compose.yml

@@ -1,5 +1,5 @@
 ---
-version: '3.3'
+version: "3.3"
 services:
   keycloak-db:
     image: postgres:9.6
@@ -10,11 +10,15 @@ services:
     networks:
       - tyk
     ports:
-      - "25432:5432"
+      - 25432:5432
     volumes:
       - keycloak-data:/var/lib/postresql/data
     healthcheck:
-      test: [ "CMD", "pg_isready", "-U", "keycloak" ]
+      test:
+        - CMD
+        - pg_isready
+        - -U
+        - keycloak
       interval: 30s
       timeout: 30s
       retries: 3
@@ -23,23 +27,25 @@ services:
       restart_policy:
         condition: on-failure
   keycloak:
-    image: jboss/keycloak:11.0.2
+    image: keycloak/keycloak:24.0.1
     depends_on:
       - keycloak-db
-    command: [ "-Djboss.http.port=8180" ]
     networks:
       - tyk
     ports:
-      - "8180:8180"
+      - 8180:8180
+    command:
+      - start-dev
     environment:
-      DB_VENDOR: POSTGRES
-      DB_ADDR: keycloak-db
-      DB_PORT: 5432
-      DB_DATABASE: keycloak
-      DB_USER: keycloak
-      DB_PASSWORD: keycloak
-      KEYCLOAK_USER: admin
-      KEYCLOAK_PASSWORD: admin
-
+      KC_DB: postgres
+      KC_DB_URL_HOST: keycloak-db
+      KC_DB_URL_PORT: 5432
+      KC_DB_URL_DATABASE: keycloak
+      KC_DB_USERNAME: keycloak
+      KC_DB_PASSWORD: keycloak
+      KEYCLOAK_ADMIN: admin
+      KEYCLOAK_ADMIN_PASSWORD: admin
+      KC_HEALTH_ENABLED: true
+      KC_HTTP_PORT: 8180
 volumes:
-  keycloak-data:
+  keycloak-data: null

deployments/keycloak-dcr/tyk_demo_keycloak_dcr.postman_collection.json

@@ -1,12 +1,13 @@
 {
 	"info": {
-		"_postman_id": "536c8d57-c0d1-4ed3-b934-812b0e84809e",
+		"_postman_id": "a9424867-b45b-4bdc-b2c9-06c88e548e25",
 		"name": "Tyk Demo - Keycloak DCR",
-		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
+		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
+		"_exporter_id": "6131676"
 	},
 	"item": [
 		{
-			"name": "Obtain access token from keycloak",
+			"name": "Part 1 - Obtain access token from keycloak",
 			"event": [
 				{
 					"listen": "test",
@@ -55,33 +56,32 @@
 					]
 				},
 				"url": {
-					"raw": "http://{{keycloak.host}}/auth/realms/{{realm}}/protocol/openid-connect/token",
+					"raw": "http://{{keycloak.host}}/realms/{{realm}}/protocol/openid-connect/token",
 					"protocol": "http",
 					"host": [
 						"{{keycloak.host}}"
 					],
 					"path": [
-						"auth",
 						"realms",
 						"{{realm}}",
 						"protocol",
 						"openid-connect",
 						"token"
 					]
 				},
-				"description": "Obtain SAT (service account token)"
+				"description": "Obtain SAT (service account token)\n\nTo use this example, you must update the `client_id` and `client_secret` values, using those of the OAuth client created by following the guidance as described in the _usage_ section of the deployment readme."
 			},
 			"response": []
 		},
 		{
-			"name": "Call API",
+			"name": "Part 2 - Call API",
 			"request": {
 				"auth": {
 					"type": "bearer",
 					"bearer": [
 						{
 							"key": "token",
-							"value": "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJwVGNPWFZRUHNMRzNSSVpzUFpHc2lET2JUTjZBNkluV0UtMi1COGUxR1pZIn0.eyJleHAiOjE2NDMzNjQxMDMsImlhdCI6MTY0MzM2NDA0MywianRpIjoiZWMyMTg5NWYtZjE3Zi00MzMzLWIwNjgtNjhhZWFmYjdkNTExIiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MTgwL2F1dGgvcmVhbG1zL21hc3RlciIsImF1ZCI6ImFjY291bnQiLCJzdWIiOiJjZTczNjM5Yy04YjRiLTQzYzEtYWMyYi04ZTgyOWMwMjYzZWYiLCJ0eXAiOiJCZWFyZXIiLCJhenAiOiI5NDQwNzkyMS04N2Y2LTQzZWYtYmQyNi04Zjg3ZTFlYTJkNDYiLCJzZXNzaW9uX3N0YXRlIjoiY2MyMjI1OTctNTA2NC00MzJiLWIxNzctZjcwNTIwOTkwYjZlIiwiYWNyIjoiMSIsInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJvZmZsaW5lX2FjY2VzcyIsInVtYV9hdXRob3JpemF0aW9uIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiYWNjb3VudCI6eyJyb2xlcyI6WyJtYW5hZ2UtYWNjb3VudCIsIm1hbmFnZS1hY2NvdW50LWxpbmtzIiwidmlldy1wcm9maWxlIl19fSwic2NvcGUiOiJlbWFpbCBwcm9maWxlIiwiY2xpZW50SWQiOiI5NDQwNzkyMS04N2Y2LTQzZWYtYmQyNi04Zjg3ZTFlYTJkNDYiLCJjbGllbnRIb3N0IjoiMTkyLjE2OC4wLjEiLCJlbWFpbF92ZXJpZmllZCI6ZmFsc2UsInByZWZlcnJlZF91c2VybmFtZSI6InNlcnZpY2UtYWNjb3VudC05NDQwNzkyMS04N2Y2LTQzZWYtYmQyNi04Zjg3ZTFlYTJkNDYiLCJjbGllbnRBZGRyZXNzIjoiMTkyLjE2OC4wLjEifQ.c3QWZHTTeRiENTvZ9Yf2ncsK414MxyJgbpI3zAOQRoycAv_8UYXTSx_24PyHem-SIPA8gQo7FHA9uOmWQIpgBgS2Qudqv7tQuvuiYl-QCMYHSBGLb_kFZVCC8d7tt7jPx136YmoEhxAUodVZxw0rHPg5k4ljt67LCZjudwbpBXk87oxuL5cdem623DNamSqr7ecLuRgcksdpO-e1qEKVwN9TCCToyOP8t4Lu4rIW4ju30OnJlACvM8XN0I5jRf0YAVfODFkO5PKGivtuZcPbDnIBzn0fn3PAW17XnL12brtaIw726hsouNZdHWiDWcl4h057BJuqBfdnraT8F2p3Vg",
+							"value": "<token>",
 							"type": "string"
 						}
 					]
@@ -98,7 +98,8 @@
 						"keycloak-dcr-api",
 						"get"
 					]
-				}
+				},
+				"description": "Set the authorization token using the token generated in Part 1. This will grant access to the API."
 			},
 			"response": []
 		}
@@ -131,13 +132,18 @@
 		},
 		{
 			"key": "keycloak.host",
-			"value": "localhost:8180",
+			"value": "keycloak:8180",
 			"type": "string"
 		},
 		{
 			"key": "realm",
 			"value": "master",
 			"type": "string"
+		},
+		{
+			"key": "test-runner-ignore",
+			"value": "true",
+			"type": "string"
 		}
 	]
 }

deployments/mdcb/bootstrap.sh

@@ -77,11 +77,17 @@ bootstrap_progress
 log_message "Checking that worker Gateway is accessible (tyk-worker-gateway)"
 worker_gateway_api_credentials=$(cat deployments/tyk/volumes/tyk-gateway/tyk.conf | jq -r .secret)
 result=""
+reload_attempt=0
 while [ "$result" != "0" ]; do
   wait_for_response "$worker_gateway_base_url/basic-open-api/get" "200" "" 3
   result="$?"
   if [ "$result" != "0" ]; then
-    log_message "  Gateway not returning desired response, attempting hot reload"
+    if [ "$reload_attempt" -gt "3" ]; then
+      log_message "  ERROR: Unable to access API via tyk-worker-gateway (max retry count reached)"
+      exit 1
+    fi
+    reload_attempt=$((reload_attempt+1))
+    log_message "  Gateway not returning desired response, attempting hot reload (attempt #$reload_attempt)"
     hot_reload "$worker_gateway_base_url" "$worker_gateway_api_credentials" 
     sleep 2
   fi

deployments/mdcb/docker-compose.yml

@@ -2,7 +2,7 @@
 version: '3.3'
 services:
   tyk-mdcb:
-    image: tykio/tyk-mdcb-docker:${MDCB_VERSION:-v2.4.1}
+    image: tykio/tyk-mdcb-docker:${MDCB_VERSION:-v2.5.0}
     ports:
       - 9091:9091
     networks:
@@ -16,7 +16,7 @@ services:
       - tyk-redis
       - tyk-mongo
   tyk-worker-gateway:
-    image: tykio/tyk-gateway:${GATEWAY_WORKER_VERSION:-v5.2.5}
+    image: tykio/tyk-gateway:${GATEWAY_WORKER_VERSION:-v5.3.0}
     ports:
       - 8090:8080
     networks: