TT-1961: add support for multiple plugin bundles

.github/workflows/test.yaml

@@ -12,38 +12,20 @@ jobs:
     - name: Setup Go
       uses: actions/setup-go@v5
       with:
-        go-version: '1.20.x'
-
-    - name: Setup Golang caches
-      uses: actions/cache@v4
-      with:
-        path: |
-          ~/.cache/go-build
-          ~/go/pkg/mod
-        key: ${{ runner.os }}-golang-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-golang-
+        go-version: "1.20"
 
     - name: Test
       run: make test
 
-    # - name: Lint
-    #  run: make lint
-
     - name: Build server and client
       run: make build
 
-    - name: Cache Docker images.
-      uses: ScribeMD/[email protected]
-      with:
-        key: docker-${{ runner.os }}-${{ hashFiles('docker-compose.yaml') }}
-
     - name: Start Docker Compose with MongoDB and Tyk Mserv
       run: make start
 
     - name: Build and bundle plugins
       run: |
-        make plugin
+        make plugins
         make bundles
 
     - name: Install Venom command line tool

Makefile

@@ -10,7 +10,7 @@ SHELL := bash
 MAKEFLAGS += --no-builtin-rules
 MAKEFLAGS += --warn-undefined-variables
 
-export TYK_VERSION := v5.2.2
+export TYK_VERSION := v5.6.1
 
 ifeq ($(origin .RECIPEPREFIX), undefined)
   $(error This Make does not support .RECIPEPREFIX. Please use GNU Make 4.0 or later.)
@@ -92,19 +92,21 @@ mservctl:
 > CGO_ENABLED=0 go build -o ../bin/mservctl
 .PHONY: mservctl
 
-start: ## Start runs development environment with mserv and mongo in docker-compose.
-> docker-compose up -d
+start: ## Start runs development environment with mserv and mongo in docker compose.
+> docker compose up -d
 
-stop: ## Stop runs development environment with mserv and mongo in docker-compose.
-> docker-compose stop
+stop: ## Stop runs development environment with mserv and mongo in docker compose.
+> docker compose stop
 
-# Builds Go plugin and moves it into local Tyk instance.
-plugin:
-> docker-compose run --rm tyk-plugin-compiler plugin.go _$$(date +%s)
-.PHONY: plugin
+# Builds multiple Go plugins and moves them into local Tyk instance.
+plugins:
+> @for plugin in plugin_1.go plugin_2.go; do \
+>	docker compose run --rm tyk-plugin-compiler $$plugin _$$(date +%s); \
+> done
+.PHONY: plugins
 
 bundles:
-> docker-compose run --rm --workdir /plugin-source --entrypoint "/opt/tyk-gateway/tyk bundle build -y -o bundle.zip" tyk-gateway
+> docker compose run --rm --workdir /plugin-source --entrypoint "/opt/tyk-gateway/tyk bundle build -y -o bundle.zip" tyk-gateway
 .PHONY: bundles
 
 integration: ## Runs integration test for mserv and mservctl it needs services running.

api/api.go

@@ -168,40 +168,87 @@ func (a *API) HandleNewBundle(ctx context.Context, filePath, apiID, bundleName s
 		Added:    time.Now(),
 	}
 
-	if len(bdl.Manifest.FileList) != 1 {
-		return nil, errors.New("only one plugin file file allowed per bundle")
-	}
-
 	pluginContainerID := fmt.Sprintf(FmtPluginContainer, bundleName)
 
 	fCont, err := getContainer(pluginContainerID)
 	if err != nil {
 		return nil, fmt.Errorf("get container error: %w", err)
 	}
 
-	// Parse name and path.
-	fName := bdl.Manifest.FileList[0]
-	pluginPath := path.Join(bdl.Path, fName)
+	// Iterate over plugin files.
+	for _, fName := range bdl.Manifest.FileList {
+		pluginPath := path.Join(bdl.Path, fName)
 
-	f, err := os.Open(pluginPath)
-	if err != nil {
-		return nil, fmt.Errorf("error opening file: %w", err)
-	}
+		f, err := os.Open(pluginPath)
+		if err != nil {
+			return nil, fmt.Errorf("error opening file: %w", err)
+		}
 
-	fInfo, err := f.Stat()
-	if err != nil {
-		return nil, fmt.Errorf("error stat file: %w", err)
-	}
+		fInfo, err := f.Stat()
+		if err != nil {
+			return nil, fmt.Errorf("error stat file: %w", err)
+		}
 
-	r := bufio.NewReader(f)
+		r := bufio.NewReader(f)
 
-	item, err := fCont.Put(fInfo.Name(), r, fInfo.Size(), nil)
-	if err != nil {
-		return nil, fmt.Errorf("error uploading file: %w", err)
-	}
+		item, err := fCont.Put(fInfo.Name(), r, fInfo.Size(), nil)
+		if err != nil {
+			return nil, fmt.Errorf("error uploading file: %w", err)
+		}
 
-	// This is an internal URL, must be interpreted by Stow
-	ref := item.URL().String()
+		// This is an internal URL, must be interpreted by Stow.
+		ref := item.URL().String()
+
+		log.Info("completed storage")
+
+		for _, f := range bdl.Manifest.CustomMiddleware.Pre {
+			p := &storage.Plugin{
+				UID:      uuid.NewString(),
+				FileName: fName,
+				FileRef:  ref,
+				Name:     f.Name,
+				Type:     coprocess.HookType_Pre,
+			}
+
+			mw.Plugins = append(mw.Plugins, p)
+		}
+
+		for _, f := range bdl.Manifest.CustomMiddleware.Post {
+			p := &storage.Plugin{
+				UID:      uuid.NewString(),
+				FileName: fName,
+				FileRef:  ref,
+				Name:     f.Name,
+				Type:     coprocess.HookType_Post,
+			}
+
+			mw.Plugins = append(mw.Plugins, p)
+		}
+
+		for _, f := range bdl.Manifest.CustomMiddleware.PostKeyAuth {
+			p := &storage.Plugin{
+				UID:      uuid.NewString(),
+				FileName: fName,
+				FileRef:  ref,
+				Name:     f.Name,
+				Type:     coprocess.HookType_PostKeyAuth,
+			}
+
+			mw.Plugins = append(mw.Plugins, p)
+		}
+
+		if bdl.Manifest.CustomMiddleware.AuthCheck.Name != "" {
+			p := &storage.Plugin{
+				UID:      uuid.NewString(),
+				FileName: fName,
+				FileRef:  ref,
+				Name:     bdl.Manifest.CustomMiddleware.AuthCheck.Name,
+				Type:     coprocess.HookType_CustomKeyCheck,
+			}
+
+			mw.Plugins = append(mw.Plugins, p)
+		}
+	}
 
 	// Store the bundle zip file too, because we can use it again
 	bF, err := os.Open(filepath.Clean(filePath))
@@ -222,58 +269,7 @@ func (a *API) HandleNewBundle(ctx context.Context, filePath, apiID, bundleName s
 	// This is an internal URL, must be interpreted by Stow
 	mw.BundleRef = bundleData.URL().String()
 
-	log.Info("completed storage")
-
-	for _, f := range bdl.Manifest.CustomMiddleware.Pre {
-		p := &storage.Plugin{
-			UID:      uuid.NewString(),
-			FileName: fName,
-			FileRef:  ref,
-			Name:     f.Name,
-			Type:     coprocess.HookType_Pre,
-		}
-
-		mw.Plugins = append(mw.Plugins, p)
-	}
-
-	for _, f := range bdl.Manifest.CustomMiddleware.Post {
-		p := &storage.Plugin{
-			UID:      uuid.NewString(),
-			FileName: fName,
-			FileRef:  ref,
-			Name:     f.Name,
-			Type:     coprocess.HookType_Post,
-		}
-
-		mw.Plugins = append(mw.Plugins, p)
-	}
-
-	for _, f := range bdl.Manifest.CustomMiddleware.PostKeyAuth {
-		p := &storage.Plugin{
-			UID:      uuid.NewString(),
-			FileName: fName,
-			FileRef:  ref,
-			Name:     f.Name,
-			Type:     coprocess.HookType_PostKeyAuth,
-		}
-
-		mw.Plugins = append(mw.Plugins, p)
-	}
-
-	if bdl.Manifest.CustomMiddleware.AuthCheck.Name != "" {
-		p := &storage.Plugin{
-			UID:      uuid.NewString(),
-			FileName: fName,
-			FileRef:  ref,
-			Name:     bdl.Manifest.CustomMiddleware.AuthCheck.Name,
-			Type:     coprocess.HookType_CustomKeyCheck,
-		}
-
-		mw.Plugins = append(mw.Plugins, p)
-	}
-
 	log.Warning("not loading into dispatcher")
-	// a.LoadMWIntoDispatcher(mw, bdl.Path)
 
 	// store in mongo
 	_, err = a.store.CreateMW(ctx, &mw)

bundles/simple/go.mod

@@ -1,3 +1,3 @@
 module tyk-plugin
 
-go 1.20
+go 1.20.14

bundles/simple/manifest.json

@@ -1,12 +1,19 @@
 {
   "file_list": [
-    "plugin_v5.2.2_linux_amd64.so"
+    "plugin_1_v5.6.1_linux_amd64.so",
+    "plugin_2_v5.6.1_linux_amd64.so"
   ],
   "custom_middleware": {
     "pre": [
       {
         "name": "AddFooBarHeader",
-        "path": "plugin_v5.2.2_linux_amd64.so",
+        "path": "plugin_1_v5.6.1_linux_amd64.so",
+        "require_session": false,
+        "raw_body_only": false
+      },
+      {
+        "name": "AddHelloWorldHeader",
+        "path": "plugin_2_v5.6.1_linux_amd64.so",
         "require_session": false,
         "raw_body_only": false
       }

bundles/simple/plugin_2.go

@@ -0,0 +1,10 @@
+package main
+
+import (
+	"net/http"
+)
+
+// AddHelloWorldHeader adds custom "Hello: World" header to the request
+func AddHelloWorldHeader(_ http.ResponseWriter, r *http.Request) {
+	r.Header.Add("Hello", "World")
+}

docker-compose.yaml

@@ -19,7 +19,7 @@ services:
       - tyk
 
   mongodb:
-    image: mongo:4.0
+    image: mongo:6.0
     restart: always
     environment:
       - AUTH=no