Merge branch 'main' into docs/nfs-provisioner

.github/workflows/mkdocs.yml

@@ -11,12 +11,24 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout main
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
-      - name: Deploy docs
-        uses: mhausenblas/mkdocs-deploy-gh-pages@master
-        # Or use mhausenblas/mkdocs-deploy-gh-pages@nomaterial to build without the mkdocs-material theme
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_ACTION_PAT }}
-          CONFIG_FILE: ./mkdocs.yml
-          REQUIREMENTS: docs/requirements.txt
+      # - name: Deploy docs
+      #   uses: mhausenblas/mkdocs-deploy-gh-pages@master
+      #   # Or use mhausenblas/mkdocs-deploy-gh-pages@nomaterial to build without the mkdocs-material theme
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GH_ACTION_PAT }}
+      #     CONFIG_FILE: ./mkdocs.yml
+      #     REQUIREMENTS: docs/requirements.txt
+      - name: Build
+        uses: Tiryoh/actions-mkdocs@v0
+        with:
+          mkdocs_version: 'latest' # option
+          #mkdocs_version: '1.1' # option
+          requirements: 'docs/requirements.txt' # option
+          configfile: './mkdocs.yml' # option
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./site

.github/workflows/test-db-migrations.yml

@@ -0,0 +1,62 @@
+name: Test database migration and rollback
+
+on:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - 'services/api/database/**'
+      - '.github/workflows/test-db-migrations.yml'
+    tags:
+      - 'v*.*.*'
+  pull_request:
+    branches:
+      - 'main'
+    paths:
+      - 'services/api/database/**'
+      - '.github/workflows/test-db-migrations.yml'
+
+jobs:
+  makeup:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout PR
+        uses: actions/checkout@v4
+        if: ${{ github.event_name == 'pull_request' }}
+        with:
+          fetch-depth: "0"
+          ref: ${{ github.event.pull_request.head.sha }}
+      -
+        name: Checkout Branch or Tag
+        uses: actions/checkout@v4
+        if: ${{ github.event_name != 'pull_request' }}
+        with:
+          fetch-depth: "0"
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      -
+        name: Build necessary images
+        run: |
+          make build
+      -
+        name: Bring up a docker compose lagoon
+        run: |
+          make up
+      -
+        name: Show initial migration logs
+        run: |
+          docker compose -p lagoon logs api-init
+      -
+        name: Initiate rollback
+        run: |
+          docker compose -p lagoon exec api sh -c './node_modules/.bin/knex migrate:rollback --all --cwd /app/services/api/database'
+      -
+        name: Reperform initial migration
+        run: |
+          docker compose -p lagoon exec api sh -c './node_modules/.bin/knex migrate:latest --cwd /app/services/api/database'
+      -
+        name: Remove testing setup
+        run: |
+          make down

.github/workflows/test-mkdocs.yml

@@ -0,0 +1,38 @@
+name: Test mkdocs
+on:
+  push:
+    branches:
+      - 'docs/**'
+  pull_request:
+    branches:
+      - 'main'
+    paths:
+      - 'docs/**'
+      - 'mkdocs.yml'
+      - '.github/workflows/test-mkdocs.yml'
+
+jobs:
+  mkdocs:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout PR
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      -
+        name: Install mkdocs and dependencies
+        run: |
+          pip install --upgrade pip
+          pip install mkdocs
+          pip install -r docs/requirements.txt
+      -
+        name: Check docs build ok in native mkdocs
+        run: |
+          mkdocs -V
+          mkdocs build -s
+      -
+        name: Check docs build ok with amazee.io mkdocs image
+        run: |
+          docker run --rm ghcr.io/amazeeio/mkdocs-material -V
+          docker run --rm -p 8000:8000 -v ${PWD}:/docs ghcr.io/amazeeio/mkdocs-material build -s

.gitignore

@@ -32,4 +32,5 @@ kindconfig.*
 k3dconfig.*
 lagoon-charts.*
 calico.yaml
-.devcontainer
+.devcontainer
+docker-compose.override.yml

DEPRECATIONS.md

@@ -0,0 +1,7 @@
+# Lagoon Project Deprecation Announcements
+
+Deprecations will be tracked by the release they are announced in, and then updated when the actual deprecation occurs. All deprecations should provide a rough timeline (in months or releases).
+
+## Deprecation and Upgrades policy
+
+See https://docs.lagoon.sh/releases/policy

Jenkinsfile

@@ -1,3 +1,5 @@
+def skipRemainingStages = false
+
 pipeline {
   agent { label 'lagoon' }
   environment {
@@ -20,6 +22,20 @@ pipeline {
         sh 'env'
       }
     }
+    stage ('skip on docs commit') {
+      when {
+        anyOf {
+          changeRequest branch: 'docs\\/.*', comparator: 'REGEXP'
+          branch pattern: "docs\\/.*", comparator: "REGEXP"
+        }
+      }
+      steps {
+        script {
+          skipRemainingStages = true
+          echo "Docs only update, no build needed."
+        }
+      }
+    }
     // in order to have the newest images from upstream (with all the security
     // updates) we clean our local docker cache on tag deployments
     // we don't do this all the time to still profit from image layer caching
@@ -28,12 +44,20 @@ pipeline {
     stage ('clean docker image cache') {
       when {
         buildingTag()
+        expression {
+            !skipRemainingStages
+        }
       }
       steps {
         sh script: "docker image prune -af", label: "Pruning images"
       }
     }
     stage ('build and push images') {
+      when {
+        expression {
+            !skipRemainingStages
+        }
+      }
       environment {
         PASSWORD = credentials('amazeeiojenkins-dockerhub-password')
       }
@@ -46,11 +70,21 @@ pipeline {
       }
     }
     stage ('show trivy scan results') {
+      when {
+        expression {
+            !skipRemainingStages
+        }
+      }
       steps {
         sh script: "cat scan.txt", label: "Display scan results"
       }
     }
     stage ('setup test cluster') {
+      when {
+        expression {
+            !skipRemainingStages
+        }
+      }
       parallel {
         stage ('0: setup test cluster') {
           steps {
@@ -72,10 +106,15 @@ pipeline {
       }
     }
     stage ('run first test suite') {
+      when {
+        expression {
+            !skipRemainingStages
+        }
+      }
       parallel {
         stage ('1: run first test suite') {
           steps {
-            sh script: "make -j$NPROC k3d/retest TESTS=[api,deploytarget,active-standby-kubernetes,features-kubernetes,features-kubernetes-2,features-variables,tasks] BRANCH_NAME=${SAFEBRANCH_NAME}", label: "Running first test suite on k3d cluster"
+            sh script: "make -j$NPROC k3d/retest TESTS=[api,deploytarget,active-standby-kubernetes,features-kubernetes,features-kubernetes-2,features-variables] BRANCH_NAME=${SAFEBRANCH_NAME}", label: "Running first test suite on k3d cluster"
             sh script: "pkill -f './local-dev/stern'", label: "Closing off test-suite-1 log after test completion"
           }
         }
@@ -106,11 +145,16 @@ pipeline {
       }
     }
     stage ('run second test suite') {
+      when {
+        expression {
+            !skipRemainingStages
+        }
+      }
       parallel {
         stage ('2: run second test suite') {
           steps {
             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
-                sh script: "make -j$NPROC k3d/retest TESTS=[bulk-deployment,gitlab,github,bitbucket,python,image-cache,workflows,ssh-legacy] BRANCH_NAME=${SAFEBRANCH_NAME}", label: "Running second test suite on k3d cluster"
+                sh script: "make -j$NPROC k3d/retest TESTS=[bulk-deployment,image-cache,services,ssh-legacy,tasks,workflows] BRANCH_NAME=${SAFEBRANCH_NAME}", label: "Running second test suite on k3d cluster"
             }
             sh script: "pkill -f './local-dev/stern'", label: "Closing off test-suite-2 log after test completion"
           }
@@ -126,11 +170,16 @@ pipeline {
       }
     }
     stage ('run third test suite') {
+      when {
+        expression {
+            !skipRemainingStages
+        }
+      }
       parallel {
         stage ('3: run third test suite') {
           steps {
             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
-                sh script: "make -j$NPROC k3d/retest TESTS=[services,drush] BRANCH_NAME=${SAFEBRANCH_NAME}", label: "Running third test suite on k3d cluster"
+                sh script: "make -j$NPROC k3d/retest TESTS=[gitlab,github,bitbucket,drush] BRANCH_NAME=${SAFEBRANCH_NAME}", label: "Running third test suite on k3d cluster"
             }
             sh script: "pkill -f './local-dev/stern'", label: "Closing off test-suite-3 log after test completion"
           }
@@ -146,11 +195,14 @@ pipeline {
       }
     }
     stage ('push images to testlagoon/* with :latest tag') {
-      when {
+       when {
         branch 'main'
         not {
           environment name: 'SKIP_IMAGE_PUBLISH', value: 'true'
         }
+        expression {
+            !skipRemainingStages
+        }
       }
       environment {
         PASSWORD = credentials('amazeeiojenkins-dockerhub-password')
@@ -166,12 +218,15 @@ pipeline {
         not {
           environment name: 'SKIP_IMAGE_PUBLISH', value: 'true'
         }
+        expression {
+            !skipRemainingStages
+        }
       }
       environment {
         TOKEN = credentials('git-amazeeio-helmfile-ci-trigger')
       }
       steps {
-        sh script: "curl -X POST -F token=$TOKEN -F ref=master https://git.amazeeio.cloud/api/v4/projects/86/trigger/pipeline", label: "Trigger lagoon-core helmfile sync on amazeeio-test6"
+        sh script: "curl -X POST -F token=$TOKEN -F ref=main https://git.amazeeio.cloud/api/v4/projects/86/trigger/pipeline", label: "Trigger lagoon-core helmfile sync on amazeeio-test6"
       }
     }
     stage ('push images to uselagoon/*') {
@@ -180,6 +235,9 @@ pipeline {
         not {
           environment name: 'SKIP_IMAGE_PUBLISH', value: 'true'
         }
+        expression {
+            !skipRemainingStages
+        }
       }
       environment {
         PASSWORD = credentials('amazeeiojenkins-dockerhub-password')
@@ -195,6 +253,9 @@ pipeline {
           branch 'testing/scans'
           buildingTag()
         }
+        expression {
+            !skipRemainingStages
+        }
       }
       steps {
         sh script: 'make scan-images', label: "perform scan routines"