Improvement: add additional information to environment services

DEPRECATIONS.md

@@ -13,6 +13,7 @@ All deprecations are listed below, with the most recent announcements at the top
 ### Lagoon v2.18.0
 release link: https://github.com/uselagoon/lagoon/releases/tag/v2.18.0
 * The standard drupal based tasks that Lagoon ships with (drush ....) have been flagged as deprecated and should not be used anymore. These will need to be replaced with [custom tasks](https://docs.lagoon.sh/using-lagoon-advanced/custom-tasks/). Example replacement tasks will be provided prior to their removal.
+* This release introduces a deprecation of the `setEnvironmentServices` mutation to updated services for an environment, it is being replaced with `addOrUpdateEnvironmentService` and `deleteEnvironmentService`. This is becaues the type is being refactored to support additional information, and eventually additional functionality. For now, the actions-handler service will still support the older `setEnvironmentServices` for backwards compatability for a short period to allow older versions of `lagoon-remote` to still work, but a new version of `lagoon-remote` will be available that will no longer provides the payload that the actions-handler uses.
 
 ### Lagoon v2.17.0
 

local-dev/api-data-watcher-pusher/api-data/01-populate-api-data-lagoon-demo.gql

@@ -320,13 +320,41 @@ mutation PopulateApi {
     id
   }
 
-  UIProject1Environment1addServices: setEnvironmentServices(
-    input:{
-      environment:3,
-      services:["cli", "nginx", "mariadb"]
+  UIProject1Environment1addServices1:   addOrUpdateEnvironmentService(
+    input: {
+      environment: 3
+      name: "cli"
+      type: "cli-persistent"
+      containers: [{name: "cli"}]
     }
-  ) {
+  ){
+    id
+    name
+    type
+  }
+  UIProject1Environment1addServices2:   addOrUpdateEnvironmentService(
+    input: {
+      environment: 3
+      name: "nginx"
+      type: "nginx-php-persistent"
+      containers: [{name: "nginx"},{name:"php"}]
+    }
+  ){
+    id
+    name
+    type
+  }
+  UIProject1Environment1addServices3:   addOrUpdateEnvironmentService(
+    input: {
+      environment: 3
+      name: "mariadb"
+      type: "mariadb-single"
+      containers: [{name: "mariadb"}]
+    }
+  ){
     id
+    name
+    type
   }
 
   UIProject1Environment1addFacts: addFacts(

services/actions-handler/go.mod

@@ -4,14 +4,14 @@ go 1.21
 
 require (
 	github.com/cheshir/go-mq/v2 v2.0.1
-	github.com/uselagoon/machinery v0.0.9
+	github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3
 	gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e
 )
 
 require (
 	github.com/NeowayLabs/wabbit v0.0.0-20210927194032-73ad61d1620e // indirect
 	github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect
-	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
+	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/guregu/null v4.0.0+incompatible // indirect

services/actions-handler/go.sum

@@ -303,7 +303,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
 github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
@@ -407,6 +406,8 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
+github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -838,8 +839,20 @@ github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/uselagoon/machinery v0.0.9 h1:wj9CVSUtneh/ynt2by5UsaUoveFxkI7MY1c/EbR90p8=
-github.com/uselagoon/machinery v0.0.9/go.mod h1:IXLxlkahEAEgpCmu9Xa/Wmjo6ja4Aoq7tf8G7VrileE=
+github.com/uselagoon/machinery v0.0.17-0.20240108020517-e6622621374c h1:xEUsDgNpM3ZaD0S6dsxQ+j/8hrnMM1HLEhbDUxrqFHE=
+github.com/uselagoon/machinery v0.0.17-0.20240108020517-e6622621374c/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240108024302-9784c2db7f1b h1:eF3O0RMETo+Bk3/I1SLjwkTvqbn/SkwCyTPWOFAl6ek=
+github.com/uselagoon/machinery v0.0.17-0.20240108024302-9784c2db7f1b/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240108025245-bf15fc15ab08 h1:SRCSGYkw6G77d20f5aA0Q/TJYhczybgEU4g7SodVsNg=
+github.com/uselagoon/machinery v0.0.17-0.20240108025245-bf15fc15ab08/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240108043534-c066518389be h1:cOBkrQa7DYDnIVZebJViIc9XEggFKYjfSKzqHC9VckE=
+github.com/uselagoon/machinery v0.0.17-0.20240108043534-c066518389be/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240108043655-6cf0c0b42884 h1:33GmXqKEkpPzsspIse3xx/aN/Leh+Jdl9QcRrbnEktg=
+github.com/uselagoon/machinery v0.0.17-0.20240108043655-6cf0c0b42884/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240108050446-30ff0a7df794 h1:2LP/ytk7sY6BrVY67PizVVYF6EnZKPzwKEOJg8JFY1I=
+github.com/uselagoon/machinery v0.0.17-0.20240108050446-30ff0a7df794/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3 h1:DYklzy44C1s1a1O6LqAi8RUpuqDzTzJTnW9IRQ8J91k=
+github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
 github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
@@ -1152,8 +1165,9 @@ golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220405210540-1e041c57c461/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

services/actions-handler/handler/controller_builds.go

@@ -39,7 +39,7 @@ func (m *Messenger) handleBuild(ctx context.Context, messageQueue *mq.MessageQue
 
 	// set up a lagoon client for use in the following process
 	l := lclient.New(m.LagoonAPI.Endpoint, "actions-handler", &token, false)
-	deployment, err := lagoon.GetDeploymentByName(ctx, message.Namespace, message.Meta.BuildName, l)
+	deployment, err := lagoon.GetDeploymentByName(ctx, message.Meta.Project, message.Meta.Environment, message.Meta.BuildName, false, l)
 	if err != nil {
 		m.toLagoonLogs(messageQueue, map[string]interface{}{
 			"severity": "error",
@@ -61,6 +61,7 @@ func (m *Messenger) handleBuild(ctx context.Context, messageQueue *mq.MessageQue
 		return nil
 	}
 	var environmentID uint
+	environment := &schema.Environment{}
 	// determine the environment id from the message
 	if message.Meta.ProjectID == nil && message.Meta.EnvironmentID == nil {
 		project, err := lagoon.GetMinimalProjectByName(ctx, message.Meta.Project, l)
@@ -77,24 +78,38 @@ func (m *Messenger) handleBuild(ctx context.Context, messageQueue *mq.MessageQue
 			}
 			return err
 		}
-		environment, err := lagoon.GetEnvironmentByName(ctx, message.Meta.Environment, project.ID, l)
+		environment, err = lagoon.GetEnvironmentByName(ctx, message.Meta.Environment, project.ID, l)
 		if err != nil {
 			// send the log to the lagoon-logs exchange to be processed
 			m.toLagoonLogs(messageQueue, map[string]interface{}{
 				"severity": "error",
 				"event":    fmt.Sprintf("actions-handler:%s:failed", "updateDeployment"),
-				"meta":     project,
+				"meta":     environment,
 				"message":  err.Error(),
 			})
 			if m.EnableDebug {
-				log.Println(fmt.Sprintf("%sERROR: unable to get project - %v", prefix, err))
+				log.Println(fmt.Sprintf("%sERROR: unable to get environment - %v", prefix, err))
 			}
 			return err
 		}
 		environmentID = environment.ID
 	} else {
 		// pull the id from the message
 		environmentID = *message.Meta.EnvironmentID
+		environment, err = lagoon.GetEnvironmentByID(ctx, environmentID, l)
+		if err != nil {
+			// send the log to the lagoon-logs exchange to be processed
+			m.toLagoonLogs(messageQueue, map[string]interface{}{
+				"severity": "error",
+				"event":    fmt.Sprintf("actions-handler:%s:failed", "updateDeployment"),
+				"meta":     environment,
+				"message":  err.Error(),
+			})
+			if m.EnableDebug {
+				log.Println(fmt.Sprintf("%sERROR: unable to get environment - %v", prefix, err))
+			}
+			return err
+		}
 	}
 
 	// prepare the deployment patch for later step
@@ -169,7 +184,8 @@ func (m *Messenger) handleBuild(ctx context.Context, messageQueue *mq.MessageQue
 			return err
 		}
 		log.Println(fmt.Sprintf("%supdated environment", prefix))
-		if message.Meta.Services != nil {
+		// @TODO START @DEPRECATED this should be removed when the `setEnvironmentServices` mutation gets removed from the API
+		if message.Meta.Services != nil { // @DEPRECATED
 			existingServices := []string{}
 			for _, s := range updateEnvironment.Services {
 				existingServices = append(existingServices, s.Name)
@@ -191,6 +207,72 @@ func (m *Messenger) handleBuild(ctx context.Context, messageQueue *mq.MessageQue
 				}
 				log.Println(fmt.Sprintf("%supdated environment services - %v", prefix, strings.Join(message.Meta.Services, ",")))
 			}
+		} // END @DEPRECATED
+		// services now provide additional information
+		if message.Meta.EnvironmentServices != nil {
+			// collect all the errors as this process runs through
+			errs := []error{}
+			// run through the environments services that currently exist
+			for _, eService := range environment.Services {
+				exists := false
+				for _, mService := range message.Meta.EnvironmentServices {
+					if eService.Name == mService.Name {
+						exists = true
+					}
+				}
+				// remove any that don't exist in the environment anymore
+				if !exists {
+					// delete it
+					s2del := schema.DeleteEnvironmentServiceInput{EnvironmentID: environmentID, Name: eService.Name}
+					setServices, err := lagoon.DeleteEnvironmentService(ctx, s2del, l)
+					if err != nil {
+						// send the log to the lagoon-logs exchange to be processed
+						m.toLagoonLogs(messageQueue, map[string]interface{}{
+							"severity": "error",
+							"event":    fmt.Sprintf("actions-handler:%s:failed", "updateDeployment"),
+							"meta":     setServices,
+							"message":  err.Error(),
+						})
+						if m.EnableDebug {
+							log.Println(fmt.Sprintf("%sERROR: unable to delete environment services - %v", prefix, err))
+						}
+						errs = append(errs, err)
+					}
+				}
+			}
+			// then update all the existing services
+			for _, mService := range message.Meta.EnvironmentServices {
+				var containers []schema.ServiceContainerInput
+				s2add := schema.AddEnvironmentServiceInput{EnvironmentID: environmentID, Name: mService.Name, Type: mService.Type, Containers: containers}
+				// add or update it
+				setServices, err := lagoon.AddOrUpdateEnvironmentService(ctx, s2add, l)
+				if err != nil {
+					// send the log to the lagoon-logs exchange to be processed
+					m.toLagoonLogs(messageQueue, map[string]interface{}{
+						"severity": "error",
+						"event":    fmt.Sprintf("actions-handler:%s:failed", "updateDeployment"),
+						"meta":     setServices,
+						"message":  err.Error(),
+					})
+					if m.EnableDebug {
+						log.Println(fmt.Sprintf("%sERROR: unable to update environment services - %v", prefix, err))
+					}
+					errs = append(errs, err)
+				}
+			}
+			// consolidate error messages down to a single error to return
+			var errMsg []string
+			errMsgs := false
+			for _, err := range errs {
+				if err != nil {
+					errMsgs = true
+					errMsg = append(errMsg, err.Error())
+				}
+			}
+			if errMsgs {
+				return fmt.Errorf(strings.Join(errMsg, ","))
+			}
+			log.Println(fmt.Sprintf("%supdated environment services", prefix))
 		}
 	}
 	return nil

services/actions-handler/handler/example-payloads/builds/buildcomplete.json

@@ -0,0 +1,27 @@
+{
+	"type": "build",
+	"namespace": "lagoon-demo-staging",
+	"meta": {
+		"buildName": "lagoon-build-wxcu6",
+		"buildPhase": "complete",
+		"buildStatus": "complete",
+		"buildStep": "complete",
+		"environment": "staging",
+		"environmentId": 5,
+		"logLink": "https://lagoon-ui.example.com/projects/example-com/example-com-main/deployments/lagoon-build-unmdse",
+		"project": "lagoon-demo",
+		"projectId": 18,
+		"remoteId": "cf1af8da-4988-4a00-b579-5ce299cb6ae7",
+		"route": "https://example.com",
+		"routes": ["https://active.example.com", "https://example.com", "https://main.example.com", "https://nginx-example-com-main.example.com", "https://varnish-example-com-main.example.com"],
+		"startTime": "2022-10-28 10:26:47",
+        "environmentServices": [
+            {"name":"nginx", "type":"nginx-php-persistent", "containers":[{"name":"nginx"},{"name": "php"}]},
+            {"name":"redis", "type":"redis", "containers":[{"name":"redis"}]},
+            {"name":"varnish", "type":"varnish", "containers":[{"name":"varnish"}]},
+            {"name":"solr", "type":"solr-php-persistent", "containers":[{"name":"solr"}]},
+            {"name":"cli", "type":"cli-persistent", "containers":[{"name":"cli"}]}
+        ],
+		"clusterName": "cluster-01"
+	}
+}

services/actions-handler/handler/example-payloads/builds/buildpending.json

@@ -15,7 +15,13 @@
 		"route": "https://example.com",
 		"routes": ["https://active.example.com", "https://example.com", "https://main.example.com", "https://nginx-example-com-main.example.com", "https://varnish-example-com-main.example.com"],
 		"startTime": "2022-10-28 10:26:47",
-		"services": ["nginx", "php", "varnish", "cli", "solr-php-persistent", "redis"],
+        "environmentServices": [
+            {"name":"nginx", "type":"nginx-php-persistent", "containers":[{"name":"nginx"},{"name": "php"}]},
+            {"name":"redis", "type":"redis", "containers":[{"name":"redis"}]},
+            {"name":"varnish", "type":"varnish", "containers":[{"name":"varnish"}]},
+            {"name":"solr", "type":"solr-php-persistent", "containers":[{"name":"solr"}]},
+            {"name":"cli", "type":"cli-persistent", "containers":[{"name":"cli"}]}
+        ],
 		"clusterName": "cluster-01"
 	}
 }

services/actions-handler/handler/example-payloads/builds/buildrunning.json

@@ -15,7 +15,6 @@
 		"route": "https://example.com",
 		"routes": ["https://active.example.com", "https://example.com", "https://main.example.com", "https://nginx-example-com-main.example.com", "https://varnish-example-com-main.example.com"],
 		"startTime": "2022-10-28 10:26:47",
-		"services": ["varnish", "cli", "solr-php-persistent", "redis", "nginx", "php"],
 		"clusterName": "cluster-01"
 	}
 }

services/api/database/migrations/20240114000000_environment_services.js

@@ -0,0 +1,32 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.up = async function(knex) {
+    return knex.schema
+    .alterTable('environment_service', function (table) {
+        table.string('type', 300);
+        table.timestamp('updated').notNullable().defaultTo(knex.fn.now());
+        table.timestamp('created').notNullable().defaultTo(knex.fn.now());
+        table.unique(['name', 'environment'], {indexName: 'service_environment'});
+    })
+    .createTable('environment_service_container', function (table) {
+        table.integer('service_id');
+        table.string('name', 300);
+        table.unique(['service_id', 'name'], {indexName: 'service_container'});
+    })
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = async function(knex) {
+    return knex.schema
+    .alterTable('environment_service', (table) => {
+        table.dropColumn('type');
+        table.dropColumn('updated');
+        table.dropColumn('created');
+    })
+    .dropTable('environment_service_container')
+};

services/api/src/resolvers.js

@@ -124,6 +124,10 @@ const {
   userCanSshToEnvironment,
   getEnvironmentUrl,
   getEnvironmentsByKubernetes,
+  addOrUpdateEnvironmentService,
+  getEnvironmentByServiceId,
+  getServiceContainersByServiceId,
+  deleteEnvironmentService,
 } = require('./resources/environment/resolvers');
 
 const {
@@ -466,6 +470,9 @@ const resolvers = {
   Fact: {
     references: getFactReferencesByFactId,
   },
+  EnvironmentService: {
+    containers: getServiceContainersByServiceId,
+  },
   Deployment: {
     environment: getEnvironmentByDeploymentId,
     uiLink: getDeploymentUrl,
@@ -714,7 +721,9 @@ const resolvers = {
     removeDeployTargetFromOrganization,
     updateEnvironmentDeployTarget,
     removeUserFromOrganizationGroups,
-    bulkImportProjectsAndGroupsToOrganization
+    bulkImportProjectsAndGroupsToOrganization,
+    addOrUpdateEnvironmentService,
+    deleteEnvironmentService
   },
   Subscription: {
     backupChanged: backupSubscriber,