chore: merge main and fix conflicts

uselagoon · Feb 14, 2024 · 143186c · 143186c
2 parents 8a86dc4 + 06a6e44
commit 143186c
Show file tree

Hide file tree

Showing 52 changed files with 1,130 additions and 119 deletions.
diff --git a/.github/workflows/test-db-migrations.yml b/.github/workflows/test-db-migrations.yml
@@ -0,0 +1,62 @@
+name: Test database migration and rollback
+
+on:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - 'services/api/database/**'
+      - '.github/workflows/test-db-migrations.yml'
+    tags:
+      - 'v*.*.*'
+  pull_request:
+    branches:
+      - 'main'
+    paths:
+      - 'services/api/database/**'
+      - '.github/workflows/test-db-migrations.yml'
+
+jobs:
+  makeup:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout PR
+        uses: actions/checkout@v4
+        if: ${{ github.event_name == 'pull_request' }}
+        with:
+          fetch-depth: "0"
+          ref: ${{ github.event.pull_request.head.sha }}
+      -
+        name: Checkout Branch or Tag
+        uses: actions/checkout@v4
+        if: ${{ github.event_name != 'pull_request' }}
+        with:
+          fetch-depth: "0"
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      -
+        name: Build necessary images
+        run: |
+          make build
+      -
+        name: Bring up a docker compose lagoon
+        run: |
+          make up
+      -
+        name: Show initial migration logs
+        run: |
+          docker compose -p lagoon logs api-init
+      -
+        name: Initiate rollback
+        run: |
+          docker compose -p lagoon exec api sh -c './node_modules/.bin/knex migrate:rollback --all --cwd /app/services/api/database'
+      -
+        name: Reperform initial migration
+        run: |
+          docker compose -p lagoon exec api sh -c './node_modules/.bin/knex migrate:latest --cwd /app/services/api/database'
+      -
+        name: Remove testing setup
+        run: |
+          make down
diff --git a/DEPRECATIONS.md b/DEPRECATIONS.md
@@ -13,6 +13,7 @@ All deprecations are listed below, with the most recent announcements at the top
 ### Lagoon v2.18.0
 release link: https://github.com/uselagoon/lagoon/releases/tag/v2.18.0
 * The standard drupal based tasks that Lagoon ships with (drush ....) have been flagged as deprecated and should not be used anymore. These will need to be replaced with [custom tasks](https://docs.lagoon.sh/using-lagoon-advanced/custom-tasks/). Example replacement tasks will be provided prior to their removal.
+* This release introduces a deprecation of the `setEnvironmentServices` mutation to updated services for an environment, it is being replaced with `addOrUpdateEnvironmentService` and `deleteEnvironmentService`. This is becaues the type is being refactored to support additional information, and eventually additional functionality. For now, the actions-handler service will still support the older `setEnvironmentServices` for backwards compatability for a short period to allow older versions of `lagoon-remote` to still work, but a new version of `lagoon-remote` will be available that will no longer provides the payload that the actions-handler uses.
 
 ### Lagoon v2.17.0
 

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -45,7 +45,9 @@ services:
       - CONSOLE_LOGGING_LEVEL=trace
   api-db-init:
     image: ${IMAGE_REPO:-lagoon}/api
-    command: ./node_modules/.bin/knex migrate:latest --cwd /app/services/api/database
+    command: >
+      sh -c "./node_modules/.bin/knex migrate:list --cwd /app/services/api/database
+      && ./node_modules/.bin/knex migrate:latest --cwd /app/services/api/database"
     volumes:
       - ./services/api/database:/app/services/api/database
       - ./node-packages:/app/node-packages:delegated
@@ -213,10 +215,6 @@ services:
     environment:
       - MINIO_ROOT_USER=minio
       - MINIO_ROOT_PASSWORD=minio123
-  local-registry:
-    image: ${IMAGE_REPO:-lagoon}/local-registry
-    ports:
-      - '5000:5000'
   drush-alias:
     image: uselagoon/drush-alias:latest
     ports:

diff --git a/docs/using-lagoon-the-basics/index.md b/docs/using-lagoon-the-basics/index.md
@@ -1,6 +1,6 @@
 # Using Lagoon - Overview
 
-This section covers some of the basic features and functionality in Lagoon. If you're familiar with these, move on to [Using Lagoon - Advanced](../concepts-advanced/index.md).
+This section covers some of the basic features and functionality in Lagoon. If you're familiar with these, move on to [Using Lagoon - Advanced](../using-lagoon-advanced/index.md).
 
 If you need help, contact your Lagoon administrator or reach out to the community and maintainers in our [Discord](../community/discord.md).
 

diff --git a/local-dev/api-data-watcher-pusher/api-data/01-populate-api-data-lagoon-demo.gql b/local-dev/api-data-watcher-pusher/api-data/01-populate-api-data-lagoon-demo.gql
@@ -320,13 +320,41 @@ mutation PopulateApi {
     id
   }
 
-  UIProject1Environment1addServices: setEnvironmentServices(
-    input:{
-      environment:3,
-      services:["cli", "nginx", "mariadb"]
+  UIProject1Environment1addServices1:   addOrUpdateEnvironmentService(
+    input: {
+      environment: 3
+      name: "cli"
+      type: "cli-persistent"
+      containers: [{name: "cli"}]
     }
-  ) {
+  ){
+    id
+    name
+    type
+  }
+  UIProject1Environment1addServices2:   addOrUpdateEnvironmentService(
+    input: {
+      environment: 3
+      name: "nginx"
+      type: "nginx-php-persistent"
+      containers: [{name: "nginx"},{name:"php"}]
+    }
+  ){
+    id
+    name
+    type
+  }
+  UIProject1Environment1addServices3:   addOrUpdateEnvironmentService(
+    input: {
+      environment: 3
+      name: "mariadb"
+      type: "mariadb-single"
+      containers: [{name: "mariadb"}]
+    }
+  ){
     id
+    name
+    type
   }
 
   UIProject1Environment1addFacts: addFacts(

diff --git a/node-packages/commons/src/api.ts b/node-packages/commons/src/api.ts
@@ -113,24 +113,7 @@ let transportOptions: {
   timeout: 60000
 };
 
-if (!envHasConfig('JWTSECRET') || !envHasConfig('JWTAUDIENCE')) {
-  logger.error(
-    'Unable to create api token due to missing `JWTSECRET`/`JWTAUDIENCE` environment variables'
-  );
-} else {
-  const apiAdminToken = createJWTWithoutUserId({
-    payload: {
-      role: 'admin',
-      iss: 'lagoon-commons',
-      aud: getConfigFromEnv('JWTAUDIENCE')
-    },
-    jwtSecret: getConfigFromEnv('JWTSECRET')
-  });
-
-  transportOptions.headers.Authorization = `Bearer ${apiAdminToken}`;
-}
-
-const transport = new Transport(`${getConfigFromEnv('API_HOST', 'http://api:3000')}/graphql`, transportOptions);
+const transport = new Transport(`${getConfigFromEnv('API_HOST', 'http://api:3000')}/graphql`, {transportOptions});
 
 export const graphqlapi = new Lokka({ transport });
 
@@ -1384,11 +1367,15 @@ export const addDeployment = (
   completed: string = null,
   priority: number = null,
   bulkId: string = null,
-  bulkName: string = null
+  bulkName: string = null,
+  sourceUser = null,
+  sourceType = null,
 ): Promise<any> =>
   graphqlapi.mutate(
     `
-  ($name: String!, $status: DeploymentStatusType!, $created: String!, $environment: Int!, $id: Int, $remoteId: String, $started: String, $completed: String, $priority: Int, $bulkId: String, $bulkName: String) {
+  ($name: String!, $status: DeploymentStatusType!, $created: String!, $environment: Int!, $id: Int, $remoteId: String,
+    $started: String, $completed: String, $priority: Int, $bulkId: String, $bulkName: String,
+    $sourceUser: String, $sourceType: DeploymentSourceType) {
     addDeployment(input: {
         name: $name
         status: $status
@@ -1401,6 +1388,8 @@ export const addDeployment = (
         priority: $priority
         bulkId: $bulkId
         bulkName: $bulkName
+        sourceUser: $sourceUser
+        sourceType: $sourceType
     }) {
       ...${deploymentFragment}
     }
@@ -1417,7 +1406,9 @@ export const addDeployment = (
       completed,
       priority,
       bulkId,
-      bulkName
+      bulkName,
+      sourceUser,
+      sourceType,
     }
   );
 
@@ -1433,10 +1424,14 @@ export const addDeployment = (
     service = null,
     command = null,
     execute = false,
+    sourceUser = null,
+    sourceType = null,
   ) =>
     graphqlapi.mutate(
       `
-    ($name: String!, $status: TaskStatusType!, $created: String!, $environment: Int!, $id: Int, $remoteId: String, $started: String, $completed: String, $service: String, $command: String, $execute: Boolean) {
+    ($name: String!, $status: TaskStatusType!, $created: String!, $environment: Int!, $id: Int, $remoteId: String,
+      $started: String, $completed: String, $service: String, $command: String, $execute: Boolean,
+      $sourceUser: String, $sourceType: TaskSourceType) {
       addTask(input: {
           name: $name
           status: $status
@@ -1449,6 +1444,8 @@ export const addDeployment = (
           service: $service
           command: $command
           execute: $execute
+          sourceUser: $sourceUser
+          sourceType: $sourceType
       }) {
         ...${taskFragment}
       }
@@ -1466,6 +1463,8 @@ export const addDeployment = (
         service,
         command,
         execute,
+        sourceUser,
+        sourceType,
       },
     );
 

diff --git a/node-packages/commons/src/lokka-transport-http-retry.ts b/node-packages/commons/src/lokka-transport-http-retry.ts
@@ -1,13 +1,41 @@
 import { Transport as LokkaTransportHttp } from '@lagoon/lokka-transport-http';
 import fetchUrl from 'node-fetch';
+import { createJWTWithoutUserId } from './jwt';
+import { logger } from './logs/local-logger';
+import { envHasConfig, getConfigFromEnv } from './util/config';
+
+// generate a fresh token for each operation
+const generateToken = () => {
+  if (!envHasConfig('JWTSECRET') || !envHasConfig('JWTAUDIENCE')) {
+    logger.error(
+      'Unable to create api token due to missing `JWTSECRET`/`JWTAUDIENCE` environment variables'
+    );
+  } else {
+    const apiAdminToken = createJWTWithoutUserId({
+      payload: {
+        role: 'admin',
+        iss: 'lagoon-internal',
+        aud: getConfigFromEnv('JWTAUDIENCE'),
+        // set a 60s expiry on the token
+        exp: Math.floor(Date.now() / 1000) + 60
+      },
+      jwtSecret: getConfigFromEnv('JWTSECRET')
+    });
+
+    return `Bearer ${apiAdminToken}`;
+  }
+  return ""
+}
 
 class NetworkError extends Error {}
 class ApiError extends Error {}
 
 // Retries the fetch if operational/network errors occur
 const retryFetch = (endpoint, options, retriesLeft = 5, interval = 1000) =>
-  new Promise((resolve, reject) =>
-    fetchUrl(endpoint, options)
+  new Promise((resolve, reject) => {
+    // get a fresh token for every request
+    options.headers.Authorization = generateToken()
+    return fetchUrl(endpoint, options)
       .then(response => {
         if (response.status !== 200 && response.status !== 400) {
           throw new NetworkError(`Invalid status code: ${response.status}`);
@@ -38,6 +66,7 @@ const retryFetch = (endpoint, options, retriesLeft = 5, interval = 1000) =>
           retryFetch(endpoint, options, retriesLeft - 1).then(resolve, reject);
         }, interval);
       })
+    }
   );
 
 export class Transport extends LokkaTransportHttp {

diff --git a/node-packages/commons/src/tasks.ts b/node-packages/commons/src/tasks.ts
@@ -372,7 +372,9 @@ export const getControllerBuildData = async function(deployData: any) {
     buildPriority,
     bulkId,
     bulkName,
-    buildVariables
+    buildVariables,
+    sourceUser,
+    sourceType,
   } = deployData;
 
   var environmentName = makeSafe(branchName)
@@ -593,7 +595,9 @@ export const getControllerBuildData = async function(deployData: any) {
       null, null, null, null,
       buildPriority,
       bulkId,
-      bulkName
+      bulkName,
+      sourceUser,
+      sourceType,
     );
   } catch (error) {
     logger.error(`Could not save deployment for project ${lagoonProjectData.id}. Message: ${error}`);

diff --git a/services/actions-handler/go.mod b/services/actions-handler/go.mod
@@ -4,14 +4,14 @@ go 1.21
 
 require (
 	github.com/cheshir/go-mq/v2 v2.0.1
-	github.com/uselagoon/machinery v0.0.9
+	github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3
 	gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e
 )
 
 require (
 	github.com/NeowayLabs/wabbit v0.0.0-20210927194032-73ad61d1620e // indirect
 	github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect
-	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
+	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/guregu/null v4.0.0+incompatible // indirect

diff --git a/services/actions-handler/go.sum b/services/actions-handler/go.sum
@@ -303,7 +303,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
 github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
@@ -407,6 +406,8 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
+github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -838,8 +839,20 @@ github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/uselagoon/machinery v0.0.9 h1:wj9CVSUtneh/ynt2by5UsaUoveFxkI7MY1c/EbR90p8=
-github.com/uselagoon/machinery v0.0.9/go.mod h1:IXLxlkahEAEgpCmu9Xa/Wmjo6ja4Aoq7tf8G7VrileE=
+github.com/uselagoon/machinery v0.0.17-0.20240108020517-e6622621374c h1:xEUsDgNpM3ZaD0S6dsxQ+j/8hrnMM1HLEhbDUxrqFHE=
+github.com/uselagoon/machinery v0.0.17-0.20240108020517-e6622621374c/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240108024302-9784c2db7f1b h1:eF3O0RMETo+Bk3/I1SLjwkTvqbn/SkwCyTPWOFAl6ek=
+github.com/uselagoon/machinery v0.0.17-0.20240108024302-9784c2db7f1b/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240108025245-bf15fc15ab08 h1:SRCSGYkw6G77d20f5aA0Q/TJYhczybgEU4g7SodVsNg=
+github.com/uselagoon/machinery v0.0.17-0.20240108025245-bf15fc15ab08/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240108043534-c066518389be h1:cOBkrQa7DYDnIVZebJViIc9XEggFKYjfSKzqHC9VckE=
+github.com/uselagoon/machinery v0.0.17-0.20240108043534-c066518389be/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240108043655-6cf0c0b42884 h1:33GmXqKEkpPzsspIse3xx/aN/Leh+Jdl9QcRrbnEktg=
+github.com/uselagoon/machinery v0.0.17-0.20240108043655-6cf0c0b42884/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240108050446-30ff0a7df794 h1:2LP/ytk7sY6BrVY67PizVVYF6EnZKPzwKEOJg8JFY1I=
+github.com/uselagoon/machinery v0.0.17-0.20240108050446-30ff0a7df794/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
+github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3 h1:DYklzy44C1s1a1O6LqAi8RUpuqDzTzJTnW9IRQ8J91k=
+github.com/uselagoon/machinery v0.0.17-0.20240108054822-78639cc0a1f3/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU=
 github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
@@ -1152,8 +1165,9 @@ golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220405210540-1e041c57c461/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=