Merge branch 'main' into testing/ansible-core

Jenkinsfile

@@ -171,7 +171,7 @@ pipeline {
         TOKEN = credentials('git-amazeeio-helmfile-ci-trigger')
       }
       steps {
-        sh script: "curl -X POST -F token=$TOKEN -F ref=keycloak_settings https://git.amazeeio.cloud/api/v4/projects/86/trigger/pipeline", label: "Trigger lagoon-core helmfile sync on amazeeio-test6"
+        sh script: "curl -X POST -F token=$TOKEN -F ref=master https://git.amazeeio.cloud/api/v4/projects/86/trigger/pipeline", label: "Trigger lagoon-core helmfile sync on amazeeio-test6"
       }
     }
     stage ('push images to uselagoon/*') {

Makefile

@@ -50,7 +50,7 @@ UPSTREAM_TAG ?= latest
 # BUILD_DEPLOY_IMAGE_TAG is the docker tag from uselagoon/build-deploy-image to use -
 # latest is the most current release
 # edge is the most current merged change
-BUILD_DEPLOY_IMAGE_TAG ?= latest
+BUILD_DEPLOY_IMAGE_TAG ?= edge
 
 # To build k3d with Calico instead of Flannel, set this to true. Note that the Calico install in lagoon-charts is always
 # disabled for use with k3d, as the cluster needs it on creation.
@@ -366,7 +366,7 @@ STERN_VERSION = v2.6.1
 CHART_TESTING_VERSION = v3.9.0
 K3D_IMAGE = docker.io/rancher/k3s:v1.26.6-k3s1
 TESTS = [nginx,api,features-kubernetes,bulk-deployment,features-kubernetes-2,features-variables,active-standby-kubernetes,tasks,drush,python,gitlab,github,bitbucket,services,workflows]
-CHARTS_TREEISH = organizations
+CHARTS_TREEISH = main
 TASK_IMAGES = task-activestandby
 
 # Symlink the installed kubectl client if the correct version is already

docs/drupal/automatic-updates.md

@@ -0,0 +1,33 @@
+# Automatic Updates
+
+Lagoon deploys applications in a way that is not compatible with some methods of
+updating Drupal core and contrib. Lagoon expects to build immutable images and
+run immutable containers. When the code of the application is changed at
+runtime, it can cause any of the following problems:
+
+1. Containers are managed automatically by Kubernetes and may be moved,
+   restarted, or scaled at any time. When this happens, the original built
+   container image will be ran and any changes that happened at runtime are
+   lost.
+2. Tasks and cronjobs may run with the orignal built container image and won't
+   have access to any updated code.
+3. Updating requires write permissions to the filesystem, but it is possible to
+   configure an environment that forces a read-only filesystem.
+4. Best practices is to deploy small containers that each do one thing. For a
+   typical Drupal project this means there is a `cli`, `php`, and `nginx`
+   container which each contain a copy of the code. Updating only one of these
+   containers will cause issues with code mismatches.
+
+The following update methods been disabled by Lagoon.
+
+### Drupal Automatic Updates
+
+The [Automatic Updates](https://www.drupal.org/project/automatic_updates)
+contrib module is disabled by and it will also be disabled when it moves
+into Drupal core.
+
+### Drush
+
+Using `drush pm-install` or `drush pm-update` is disabled by default as part of
+the [amazeeio/drupal-integrations](https://github.com/amazeeio/drupal-integrations)
+package.

docs/drupal/integrate-drupal-and-fastly.md

@@ -2,11 +2,23 @@
 
 ## Prerequisites
 
-* A Drupal `7`, `8` or `9` site
+* Drupal 7+
 * A Fastly service ID
 * A Fastly API token with the permission to purge
 
-## Drupal 8 or 9 with cache tag purging
+## Drupal 7 with URL based purging
+
+1. Download and install the [Fastly Drupal module](https://www.drupal.org/project/fastly).
+2. Configure the Fastly service ID and API token.
+3. Optionally configure the webhooks (so you can ping Slack for instance when a cache purge is sent)
+4. Only URL based purging can be done in Drupal 7 (simple purging).
+5. Alter Drupal's client IP in `settings.php`:
+
+```php title="settings.php changes for Drupal 7"
+$conf['reverse_proxy_header'] = 'HTTP_TRUE_CLIENT_IP';
+```
+
+## Drupal 8+ with cache tag purging
 
 Use Composer to get the latest version of the module:
 
@@ -20,11 +32,11 @@ You will need to enable the following modules:
 * `fastlypurger`
 * `http_cache_control` \(2.x\)
 * `purge`
-* `purge_ui` \(technically optional, but this is really handy to have enabled on production\)
+* `purge_ui` (technically optional, but this is really handy to have enabled on production)
 * `purge_processor_lateruntime`
 * `purge_processor_cron`
 * `purge_queuer_coretags`
-* `purge_drush` \(useful for purge via Drush, here is a [list of commands](https://git.drupalcode.org/project/purge/-/blob/8.x-3.x/README.md#drush-commands)\)
+* `purge_drush` (useful for purge via Drush, here is a [list of commands](https://git.drupalcode.org/project/purge/-/blob/8.x-3.x/README.md#drush-commands))
 
 ### Configure the Fastly module in Drupal
 
@@ -40,15 +52,16 @@ A `site ID` is required, the module will generate one for you when you first ins
 * Cache tag hash length: 4
 * Purge method: Use soft purge
 
-A `4` character cache tag is plenty for most sites, a `5` character cache tag is likely better for sites with _millions_ of entities \(to reduce cache tag collisions\).
+A `4` character cache tag is plenty for most sites, a `5` character cache tag is likely better for sites with _millions_ of entities (to reduce cache tag collisions).
 
-##### Soft purging should be used, this means the item in Fastly is marked as stale, rather than being purged so that it can be used in the event the origin is down \(with the feature 'serve while stale'\).
+!!! Note
+    Soft purging should be used, this means the object in Fastly is marked as stale, rather than being evicted entirely so that it can be used in the event the origin is down (with the feature [serve while stale](https://developer.fastly.com/solutions/tutorials/stale/)).
 
-![Fastly admin UI for purging](https://camo.githubusercontent.com/8d0fb54560570966c2387c9b88c76d366f5d85e2f8901b644a06ca3f41210ed2/68747470733a2f2f692e6962622e636f2f34677777644c762f666173746c792d70757267652e706e67)
+![Fastly admin UI for purging. Shows the configuration options for cache tag length and using soft purge](https://camo.githubusercontent.com/8d0fb54560570966c2387c9b88c76d366f5d85e2f8901b644a06ca3f41210ed2/68747470733a2f2f692e6962622e636f2f34677777644c762f666173746c792d70757267652e706e67)
 
 #### Set the Stale Content Options
 
-Set the options to what makes sense for your site. Minimum 1 hour \(`3600`\), maximum 1 week `604800`\). Generally something like the following will be fine:
+Set the options to what makes sense for your site. Minimum 1 hour (`3600`), maximum 1 week (`604800`). Generally something like the following will be fine:
 
 1. Stale while revalidate - on, 14440 seconds
 2. Stale if error - on, 604800 seconds
@@ -72,13 +85,13 @@ Set up the following options:
 
 #### Queue
 
-* Queuers: Core tags queuer, Purge block\(s\)
+* Queuers: Core tags queuer, Purge block(s)
 * Queue: Database
-* Processors: Core processor, Late runtime processor, Purge block\(s\)
+* Processors: Core processor, Late runtime processor, Purge block(s)
 
 ![Fastly Admin UI configuration](https://camo.githubusercontent.com/c4020852ed509b77dc52ae16572b048f49a38b0f88a901ae4593ed970d481f95/68747470733a2f2f692e6962622e636f2f4659304c7363432f666173746c792e706e67)
 
-What this means is that we will be using Drupal's built in core tag queuer \(add tags to the queue\), the queue will be stored in the database \(default\), and the queue will be processed by
+What this means is that we will be using Drupal's built in core tag queuer \(add tags to the queue\), the queue will be stored in the database (default), and the queue will be processed by
 
 * Cron processor
 * Late runtime processor
@@ -87,8 +100,8 @@ In order for the cron processor to run, you need to ensure that cron is running
 
 ```bash title="start cron"
 [drupal8]production@cli-drupal:/app$ drush cron -v
- ...
- [notice] Starting execution of purge_processor_cron_cron(), execution of node_cron() took 21.16ms.
+...
+[notice] Starting execution of purge_processor_cron_cron(), execution of node_cron() took 21.16ms.
 ```
 
 The `Late runtime processor` will run in `hook_exit()` for every page load, this can be useful to process the purges nearly as quickly as they come into the queue.
@@ -111,9 +124,38 @@ For most sites, a sensible default could be
 !!! Note
     This relies on your site having accurate cache tags represented for all the content that exists on the page.
 
-### Viewing caching headers using cURL
+### True client IPs
+
+We configure Fastly to send the actual client IP back on the HTTP header `True-Client-IP`, you can make Drupal respect this header with the following changes in `settings.php`:
+
+```php title="settings.php changes for Drupal < 8.7.0"
+$settings['reverse_proxy'] = TRUE;
+$settings['reverse_proxy_header'] = 'HTTP_TRUE_CLIENT_IP';
+```
+
+However, with Drupal 8.7.0, [there was a change to remove this functionality](https://www.drupal.org/node/3030558). You can achieve the same goal with the following snippet
+
+```php title="settings.php changes for Drupal >= 8.7.0"
+/**
+ * Tell Drupal to use the True-Client-IP HTTP header.
+ */
+if (isset($_SERVER['HTTP_TRUE_CLIENT_IP'])) {
+  $_SERVER['REMOTE_ADDR'] = $_SERVER['HTTP_TRUE_CLIENT_IP'];
+}
+```
+
+### Drush integration
+
+```php title="settings.php"
+ fastly:
+   fastly:purge:all (fpall)                                                    Purge whole service.
+   fastly:purge:key (fpkey)                                                    Purge cache by key.
+   fastly:purge:url (fpurl)                                                    Purge cache by Url.
+```
+
+## Viewing Fastly caching headers using cURL
 
-Use this function: \(works in Linux and Mac OSX\)
+Use this function: (works in Linux and Mac OSX)
 
 ```bash title="cURL function"
 function curlf() { curl -sLIXGET -H 'Fastly-Debug:1' "$@" | grep -iE 'X-Cache|Cache-Control|Set-Cookie|X-Varnish|X-Hits|Vary|Fastly-Debug|X-Served|surrogate-control|surrogate-key' }
@@ -136,61 +178,20 @@ From the above headers we can see that:
 
 * The HTML page is cacheable
 * Browsers will cache the page for 601 seconds
-* Fastly will cache the page for 32 days \(`2764800` seconds\)
-* Tiered caching is in effect \(edge PoP in Wellington, and shield PoP in France\)
+* Fastly will cache the page for 32 days (`2764800` seconds)
+* Tiered caching is in effect (edge PoP in Wellington, and shield PoP in France)
 * The HTML page was a cache hit at the edge PoP
 
 ### Sending manual purge requests to Fastly
 
 If you ever want to remove a specific page from cache manually, there are ways to do this.
 
-For a single page, you do not need any authentication:
-
-```bash title="Single page cURL"
-curl -Ssi -XPURGE -H 'Fastly-Soft-Purge:1' https://www.example.com/subpage
+```bash title="Single page U"
+curl -Ssi -XPURGE -H 'Fastly-Soft-Purge:1' -H "Fastly-Key:$FASTLY_API_TOKEN" https://www.example.com/subpage
 ```
 
-For cache tags, you need to supply your API token for authentication:
+You can also purge by cache tag:
 
 ```bash title="Cache tags"
-curl -XPOST -H "Fastly-Key:<Fastly API Key>" https://api.fastly.com/service/<serviceID>/purge/<surrogatekey>
-```
-
-You can always find what your site ID cache tag is by using PHP
-
-```php title="Find site ID cache tag"
-php > var_dump(substr(base64_encode(md5('bananasite', true)), 0, 4));
-string(4) "DTRk"
-```
-
-So you can purge your entire site from Fastly fairly easily.
-
-### True client IPs
-
-We configure Fastly to send the actual client IP back on the HTTP header `True-Client-IP`, you can make Drupal respect this header with the following changes in `settings.php`:
-
-```php title="settings.php"
-$settings['reverse_proxy'] = TRUE;
-$settings['reverse_proxy_header'] = 'HTTP_TRUE_CLIENT_IP';
-```
-
-### Drush integration
-
-```php title="settings.php"
- fastly:
-   fastly:purge:all (fpall)                                                    Purge whole service.
-   fastly:purge:key (fpkey)                                                    Purge cache by key.
-   fastly:purge:url (fpurl)                                                    Purge cache by Url.
-```
-
-## Drupal 7 with URL based purging
-
-1. Download and install the [Fastly Drupal module](https://www.drupal.org/project/fastly).
-2. Configure the Fastly service ID and API token.
-3. Optionally configure the webhooks \(so you can ping Slack for instance when a cache purge is sent\)
-4. Only URL based purging can be done in Drupal 7 \(simple purging\).
-5. Alter Drupal's client IP in `settings.php`:
-
-```php title="settings.php"
-$conf['reverse_proxy_header'] = 'HTTP_TRUE_CLIENT_IP';
+curl -XPOST -H 'Fastly-Soft-Purge:1' -H "Fastly-Key:$FASTLY_API_TOKEN" https://api.fastly.com/service/$FASTLY_API_SERVICE/purge/<surrogatekey>
 ```

docs/using-lagoon-the-basics/docker-compose-yml.md

@@ -212,14 +212,36 @@ By default, Lagoon expects that services from custom templates are rolled out vi
 
 You can also overwrite the rollout for just one specific environment. This is done in [`.lagoon.yml`](lagoon-yml.md#environments-name-rollouts).
 
-## BuildKit and Docker Compose v2
+## Docker Compose v2 compatibility
 
-BuildKit is a toolkit for converting source code to build artifacts in an efficient, expressive and repeatable manner.
+!!! bug
+    Note that while using older versions of Docker Compose V2 locally, you may experience some known issues - these have been resolved in later releases (v2.17.3 onwards).
 
-With the release of Lagoon v2.11.0, Lagoon now provides support for BuildKit-based docker-compose builds. To enable BuildKit for your Project or Environment, add `DOCKER_BUILDKIT=1` as a build-time variable.
+The resolution for these errors is usually to update (or [install a later version](https://docs.docker.com/compose/install/) of) the version of Docker Compose you are using, either standalone or by upgrading the version of Docker Desktop you're using.  See the Docker Desktop [release notes](https://docs.docker.com/desktop/release-notes/) for more information
 
-!!! bug
-    Note that while using BuildKit locally, you may experience some known issues.
+``` shell title="Docker Compose output indicating depends_on error"
+Failed to solve with frontend dockerfile.v0: failed to create LLB definition: pull access denied, repository does not exist or may require authorization
+
+or
+
+Failed to solve: drupal9-base-cli: pull access denied, repository does not exist or may require authorization: server message: insufficient_scope: authorization failed`
+```
+
+* These are resolved in Docker Compose v2.13.0
+* This message means that your build has tried to access a Docker image that hasn't been built yet. As BuildKit builds in parallel, if you have a Docker image that inherits another one (as we do in Drupal with the CLI).
+* You can also use the [target](https://docs.docker.com/compose/compose-file/build/#target) field inside the build to reconfigure as a multi-stage build
+* If you are already running a newer Docker Compose version, this error may be because you're defaulting to using a docker-container build context with buildx. You should make sure that `docker buildx ls` shows the docker builder as default, not a docker-container based one. Check the docs on docker buildx [here](https://docs.docker.com/engine/reference/commandline/buildx_use/)
+
+``` shell title="Docker Compose output indicating volumes_from error"
+no such service: container:amazeeio-ssh-agent
+```
+
+* This is resolved in Docker Compose v2.17.3
+* This message means that the service that provides SSH access into locally running containers runs outside of your Docker Compose stack and is inaccessible.
+* The section can also be removed from your `docker-compose.yml` file if you don't require SSH access from inside your local environment.
+
+## BuildKit and Lagoon
+
+BuildKit is a toolkit for converting source code to build artifacts in an efficient, expressive and repeatable manner.
 
-* `Failed to solve with frontend dockerfile.v0: failed to create LLB definition: pull access denied, repository does not exist or may require authorization`: This message means that your build has tried to access a Docker image that hasn't been built yet. As BuildKit builds in parallel, if you have a Docker image that inherits another one (as we do in Drupal with the CLI). You can use the [target](https://docs.docker.com/compose/compose-file/build/#target) field inside the build to reconfigure as a multi-stage build
-* issues with `volumes_from` in Docker Compose v2 - this service (that provides SSH access into locally running containers) has been deprecated by Docker Compose. The section can be removed from your `docker-compose.yml` file if you don't require SSH access from inside your local environment, or can be worked around on a project-by-project basis - see https://github.com/pygmystack/pygmy/issues/333#issuecomment-1274091375 for more information.
+With the release of Lagoon v2.11.0, Lagoon now provides support for more advanced BuildKit-based docker-compose builds. To enable BuildKit for your Project or Environment, add `DOCKER_BUILDKIT=1` as a build-time variable to your Lagoon project or environment.

local-dev/api-data-watcher-pusher/api-data/04-populate-api-data-organizations.gql

@@ -7,7 +7,7 @@ mutation PopulateApi {
     friendlyName: "Test Organization"
     description: "An organization for testing"
     quotaProject: 5
-    quotaEnvironment: 15
+    quotaEnvironment: 4
     quotaGroup: 10
     quotaNotification: 10
   }) {
@@ -44,7 +44,7 @@ mutation PopulateApi {
     id
   }
 
-  UIOrganizationGroup: addGroup(
+  UIOrganizationGroup: addGroupToOrganization(
     input: { name: "organization-group1", organization: 1}
   ) {
     id

mkdocs.yml

@@ -52,6 +52,7 @@ nav:
     - Subfolders: drupal/subfolders.md
     - Integrate Drupal & Fastly: drupal/integrate-drupal-and-fastly.md
     - PHPUnit and PhpStorm: drupal/phpunit-and-phpstorm.md
+    - Automatic Updates: drupal/automatic-updates.md
   - WordPress:
     - Overview: applications/wordpress.md
   - Node.js-based:

node-packages/commons/src/api.ts

@@ -1256,10 +1256,10 @@ export const getEnvironmentsForProject = (
   }
 `);
 
-export async function getOrganizationById(organization: number): Promise<any> {
+export async function getOrganizationById(id: number): Promise<any> {
   const result = await graphqlapi.query(`
     {
-      organization:organizationById(organization: ${organization}) {
+      organization:organizationById(id: ${id}) {
         id
         name
         friendlyName
@@ -1282,7 +1282,7 @@ export async function getOrganizationById(organization: number): Promise<any> {
   `);
 
   if (!result || !result.organization) {
-    throw new OrganizationNotFound(`Cannot find organization ${organization}`);
+    throw new OrganizationNotFound(`Cannot find organization ${id}`);
   }
 
   return result.organization;

node-packages/commons/src/tasks.ts

@@ -768,12 +768,15 @@ export const createDeployTask = async function(deployData: any) {
   const environments = await getEnvironmentsForProject(projectName);
 
   if (project.organization) {
-    // check the environment quota, this prevents environments being deployed by the api or webhooks
-    const curOrg = await getOrganizationById(project.organization);
-    if (curOrg.environments.length >= curOrg.quotaEnvironment) {
-      throw new OrganizationEnvironmentLimit(
-        `'${branchName}' would exceed the organization environment quota of ${curOrg.quotaEnvironment}`
-      );
+    // if this would be a new environment, check it against the environment quota
+    if (!environments.project.environments.map(e => e.name).find(i => i === branchName)) {
+      // check the environment quota, this prevents environments being deployed by the api or webhooks
+      const curOrg = await getOrganizationById(project.organization);
+      if (curOrg.environments.length >= curOrg.quotaEnvironment && curOrg.quotaEnvironment != -1) {
+        throw new OrganizationEnvironmentLimit(
+          `'${branchName}' would exceed organization environment quota: ${curOrg.environments.length}/${curOrg.quotaEnvironment}`
+        );
+      }
     }
   }
 

services/actions-handler/Dockerfile

@@ -1,7 +1,7 @@
 # build the binary
 ARG UPSTREAM_REPO
 ARG UPSTREAM_TAG
-FROM golang:1.18-alpine3.17 AS builder
+FROM golang:1.21-alpine AS builder
 # bring in all the packages
 COPY . /go/src/github.com/uselagoon/lagoon/services/actions-handler/
 WORKDIR /go/src/github.com/uselagoon/lagoon/services/actions-handler/

services/actions-handler/go.mod

@@ -1,6 +1,6 @@
 module github.com/uselagoon/lagoon/services/actions-handler
 
-go 1.18
+go 1.21
 
 require (
 	github.com/cheshir/go-mq/v2 v2.0.1