From 09cb268f0a4e747bfd26108926834a9d8ad6c8c0 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Mon, 21 Oct 2024 07:36:00 +1100 Subject: [PATCH] test: ability to install stable chart versions --- Jenkinsfile | 6 +- Makefile | 211 ++++++++++++------ .../developing-lagoon.md | 132 ++++++++++- k3d.config.yaml.tpl | 9 +- 4 files changed, 280 insertions(+), 78 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dce95f47df..17a636f509 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -62,7 +62,7 @@ pipeline { PASSWORD = credentials('amazeeiojenkins-dockerhub-password') } steps { - sh script: "make -O build", label: "Building images" + sh script: "make -j$NPROC -O build", label: "Building images" retry(3) { sh script: 'docker login -u amazeeiojenkins -p $PASSWORD', label: "Docker login" sh script: "make -O publish-testlagoon-images PUBLISH_PLATFORM_ARCH=linux/amd64 BRANCH_NAME=${SAFEBRANCH_NAME}", label: "Publishing built amd64 images to testlagoon/*" @@ -88,9 +88,9 @@ pipeline { parallel { stage ('0: setup test cluster') { steps { - sh script: "make -j$NPROC local-dev/k3d", label: "Configure k3d" + sh script: "make local-dev-tools", label: "Configure k3d" sh script: "./local-dev/k3d cluster delete --all", label: "Delete any remnant clusters" - sh script: "make -j$NPROC k3d/test TESTS=[nginx] BRANCH_NAME=${SAFEBRANCH_NAME}", label: "Setup cluster and run nginx smoketest" + sh script: "make k3d/test TESTS=[nginx] BRANCH_NAME=${SAFEBRANCH_NAME}", label: "Setup cluster and run nginx smoketest" sh script: "pkill -f './local-dev/stern'", label: "Closing off test-suite-0 log after test completion" // script { // skipRemainingStages = true diff --git a/Makefile b/Makefile index a19f43d704..47f8165da8 100644 --- a/Makefile +++ b/Makefile @@ -407,15 +407,62 @@ compose-api-logs-development: KUBECTL_VERSION := v1.31.0 HELM_VERSION := v3.16.1 -K3D_VERSION = v5.7.3 +K3D_VERSION = v5.7.4 GOJQ_VERSION = v0.12.16 STERN_VERSION = v2.6.1 CHART_TESTING_VERSION = v3.11.0 -K3D_IMAGE = docker.io/rancher/k3s:v1.31.0-k3s1 +K3D_IMAGE = docker.io/rancher/k3s:v1.31.1-k3s1 TESTS = [nginx,api,features-kubernetes,bulk-deployment,features-kubernetes-2,features-variables,active-standby-kubernetes,tasks,drush,python,gitlab,github,bitbucket,services,workflows] -CHARTS_TREEISH = main +CHARTS_TREEISH = stable-chart-install +CHARTS_REPOSITORY = https://github.com/uselagoon/lagoon-charts.git +#CHARTS_REPOSITORY = ../lagoon-charts TASK_IMAGES = task-activestandby +# the following can be used to install stable versions of lagoon directly from chart versions +# rather than the bleeding edge from CHARTS_TREEISH +# these will not build or use built images from this repository, just what the charts provide +# this can be used to install a known version and then revert to test upgrading from a stable version +INSTALL_STABLE_CORE = false +INSTALL_STABLE_REMOTE = false +INSTALL_STABLE_BUILDDEPLOY = false +INSTALL_STABLE_LAGOON = false +ifeq ($(INSTALL_STABLE_LAGOON), true) + INSTALL_STABLE_CORE = true + INSTALL_STABLE_REMOTE = true + INSTALL_STABLE_BUILDDEPLOY = true +endif +STABLE_CORE_CHART_VERSION = +STABLE_REMOTE_CHART_VERSION = +STABLE_STABLE_BUILDDEPLOY_CHART_VERSION = + +# the following can be used to selectively leave out the installation of certain +# dbaas provider types +INSTALL_MARIADB_PROVIDER = +INSTALL_POSTGRES_PROVIDER = +INSTALL_MONGODB_PROVIDER = +INSTALL_DBAAS_PROVIDERS = true +ifeq ($(INSTALL_DBAAS_PROVIDERS), false) + INSTALL_MARIADB_PROVIDER = false + INSTALL_POSTGRES_PROVIDER = false + INSTALL_MONGODB_PROVIDER = false +endif +# mongo currently doesn't work on arm based systems, so just disable the provider entirely for now +ifeq ($(ARCH), darwin) + INSTALL_MONGODB_PROVIDER = false +endif +ifeq ($(MACHINE), arm64) + INSTALL_MONGODB_PROVIDER = false +endif + +INSTALL_UNAUTHENTICATED_REGISTRY = false +# harbor currently doesn't work on arm based systems, so install an unauthenticated registry instead +ifeq ($(ARCH), darwin) + INSTALL_UNAUTHENTICATED_REGISTRY = true +endif +ifeq ($(MACHINE), arm64) + INSTALL_UNAUTHENTICATED_REGISTRY = true +endif + # the name of the docker network to create DOCKER_NETWORK = k3d @@ -427,7 +474,7 @@ ifeq ($(KUBECTL_VERSION), $(shell kubectl version --client 2>/dev/null | grep Cl $(info linking local kubectl version $(KUBECTL_VERSION)) ln -sf $(shell command -v kubectl) ./local-dev/kubectl else -ifneq ($(KUBECTL_VERSION), v$(shell ./local-dev/kubectl version --client 2>/dev/null | grep Client | sed -E 's/Client Version: (v[0-9.]+).*/\1/')) +ifneq ($(KUBECTL_VERSION), $(shell ./local-dev/kubectl version --client 2>/dev/null | grep Client | sed -E 's/Client Version: (v[0-9.]+).*/\1/')) $(info downloading kubectl version $(KUBECTL_VERSION) for $(ARCH)) rm local-dev/kubectl || true curl -sSLo local-dev/kubectl https://storage.googleapis.com/kubernetes-release/release/$(KUBECTL_VERSION)/bin/$(ARCH)/amd64/kubectl @@ -443,7 +490,7 @@ ifeq ($(HELM_VERSION), $(shell helm version --short --client 2>/dev/null | sed - $(info linking local helm version $(HELM_VERSION)) ln -sf $(shell command -v helm) ./local-dev/helm else -ifneq ($(HELM_VERSION), v$(shell ./local-dev/helm version --short --client 2>/dev/null | sed -nE 's/(v[0-9.]+).*/\1/p')) +ifneq ($(HELM_VERSION), $(shell ./local-dev/helm version --short --client 2>/dev/null | sed -nE 's/(v[0-9.]+).*/\1/p')) $(info downloading helm version $(HELM_VERSION) for $(ARCH)) rm local-dev/helm || true curl -sSL https://get.helm.sh/helm-$(HELM_VERSION)-$(ARCH)-amd64.tar.gz | tar -xzC local-dev --strip-components=1 $(ARCH)-amd64/helm @@ -506,6 +553,9 @@ ifneq ($(STERN_VERSION), v$(shell ./local-dev/stern --version 2>/dev/null | sed endif endif +.PHONY: local-dev-tools +local-dev-tools: local-dev/k3d local-dev/jq local-dev/helm local-dev/kubectl local-dev/stern + .PHONY: helm/repos helm/repos: local-dev/helm # install repo dependencies required by the charts @@ -520,11 +570,12 @@ helm/repos: local-dev/helm $(HELM) repo add metallb https://metallb.github.io/metallb $(HELM) repo add jetstack https://charts.jetstack.io $(HELM) repo add jouve https://jouve.github.io/charts/ + $(HELM) repo add twuni https://helm.twun.io $(HELM) repo update # stand up a k3d cluster configured appropriately for lagoon testing .PHONY: k3d/cluster -k3d/cluster: local-dev/k3d local-dev/jq local-dev/helm local-dev/kubectl +k3d/cluster: local-dev/k3d local-dev/jq local-dev/helm local-dev/kubectl local-dev/stern $(K3D) cluster list | grep -q "$(CI_BUILD_TAG)" && exit; \ docker network create $(DOCKER_NETWORK) || true \ && export KUBECONFIG=$$(mktemp) \ @@ -559,59 +610,97 @@ K3D_TOOLS = k3d helm kubectl jq stern # install lagoon charts and run lagoon test suites in a k3d cluster .PHONY: k3d/test -k3d/test: k3d/setup - export KUBECONFIG="$$(pwd)/kubeconfig.k3d.$(CI_BUILD_TAG)" \ - && cd lagoon-charts.k3d.lagoon \ - && export IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \ - && $(MAKE) fill-test-ci-values DOCKER_NETWORK=$(DOCKER_NETWORK) TESTS=$(TESTS) IMAGE_TAG=$(SAFE_BRANCH_NAME) DISABLE_CORE_HARBOR=true \ - HELM=$(HELM) KUBECTL=$(KUBECTL) JQ=$(JQ) \ - OVERRIDE_BUILD_DEPLOY_DIND_IMAGE=uselagoon/build-deploy-image:${BUILD_DEPLOY_IMAGE_TAG} \ - OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \ - IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \ - SKIP_ALL_DEPS=true \ - LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \ - USE_CALICO_CNI=false \ - LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \ - LAGOON_FEATURE_FLAG_DEFAULT_ROOTLESS_WORKLOAD=enabled \ - && docker run --rm --network host --name ct-$(CI_BUILD_TAG) \ - --volume "$$(pwd)/test-suite-run.ct.yaml:/etc/ct/ct.yaml" \ - --volume "$$(pwd):/workdir" \ - --volume "$$(realpath ../kubeconfig.k3d.$(CI_BUILD_TAG)):/root/.kube/config" \ - --workdir /workdir \ - "quay.io/helmpack/chart-testing:$(CHART_TESTING_VERSION)" \ - ct install --helm-extra-args "--timeout 60m" +k3d/test: k3d/setup k3d/install-lagoon k3d/retest LOCAL_DEV_SERVICES = api auth-server actions-handler api-sidecar-handler logs2notifications webhook-handler webhooks2tasks -# install lagoon charts in a Kind cluster +# install lagoon dependencies in a k3d cluster .PHONY: k3d/setup -k3d/setup: k3d/cluster helm/repos $(addprefix local-dev/,$(K3D_TOOLS)) build +k3d/setup: k3d/cluster helm/repos $(addprefix local-dev/,$(K3D_TOOLS)) k3d/checkout-charts + export KUBECONFIG="$$(realpath kubeconfig.k3d.$(CI_BUILD_TAG))" \ + && cd lagoon-charts.k3d.lagoon \ + && $(MAKE) install-lagoon-dependencies \ + INSTALL_UNAUTHENTICATED_REGISTRY=$(INSTALL_UNAUTHENTICATED_REGISTRY) \ + DOCKER_NETWORK=$(DOCKER_NETWORK) \ + JQ=$(JQ) HELM=$(HELM) KUBECTL=$(KUBECTL) \ + USE_CALICO_CNI=false \ + $$([ $(INSTALL_MARIADB_PROVIDER) ] && echo 'INSTALL_MARIADB_PROVIDER=$(INSTALL_MARIADB_PROVIDER)') \ + $$([ $(INSTALL_POSTGRES_PROVIDER) ] && echo 'INSTALL_POSTGRES_PROVIDER=$(INSTALL_POSTGRES_PROVIDER)') \ + $$([ $(INSTALL_MONGODB_PROVIDER) ] && echo 'INSTALL_MONGODB_PROVIDER=$(INSTALL_MONGODB_PROVIDER)') + +# k3d/dev can only be run once a cluster is up and running (run k3d/test or k3d/local-stack first) - it doesn't rebuild the cluster at all +# just checks out the repository and rebuilds and pushes the built images +# into the image registry and reinstalls lagoon charts +.PHONY: k3d/dev +k3d/dev: k3d/checkout-charts k3d/install-lagoon + @$(MAKE) k3d/get-lagoon-details + +# this is used to checkout the chart repo/branch again if required. otherwise will use the symbolic link +# that is created for subsequent commands +.PHONY: k3d/checkout-charts +k3d/checkout-charts: export CHARTSDIR=$$(mktemp -d ./lagoon-charts.XXX) \ && ln -sfn "$$CHARTSDIR" lagoon-charts.k3d.lagoon \ - && git clone https://github.com/uselagoon/lagoon-charts.git "$$CHARTSDIR" \ + && git clone $(CHARTS_REPOSITORY) "$$CHARTSDIR" \ && cd "$$CHARTSDIR" \ - && git checkout $(CHARTS_TREEISH) \ - && export KUBECONFIG="$$(realpath ../kubeconfig.k3d.$(CI_BUILD_TAG))" \ + && git checkout $(CHARTS_TREEISH) + +# this just installs lagoon-core, lagoon-remote, and lagoon-build-deploy +# doing this allows for lagoon to be installed with a known stable chart version with the INSTALL_STABLE_X overrides +# and then running just the install-lagoon target without the INSTALL_STABLE_X overrides to verify if an upgrade is likely to succeed +# this sets INSTALL_LAGOON_DEPENDENCIES=false so that the dependencies aren't installed again, allowing for this target to be run multiple time +# to install or upgrade lagoon +.PHONY: k3d/install-lagoon +k3d/install-lagoon: +ifneq ($(INSTALL_STABLE_CORE),true) + $(MAKE) build + export KUBECONFIG="$$(realpath kubeconfig.k3d.$(CI_BUILD_TAG))" \ && export IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \ - && $(MAKE) install-registry DOCKER_NETWORK=$(DOCKER_NETWORK) JQ=$(JQ) HELM=$(HELM) KUBECTL=$(KUBECTL) USE_CALICO_CNI=false \ - && cd .. && $(MAKE) k3d/push-images JQ=$(JQ) HELM=$(HELM) KUBECTL=$(KUBECTL) && cd "$$CHARTSDIR" \ - && $(MAKE) fill-test-ci-values DOCKER_NETWORK=$(DOCKER_NETWORK) TESTS=$(TESTS) IMAGE_TAG=$(SAFE_BRANCH_NAME) DISABLE_CORE_HARBOR=true \ - HELM=$(HELM) KUBECTL=$(KUBECTL) JQ=$(JQ) \ - OVERRIDE_BUILD_DEPLOY_DIND_IMAGE=uselagoon/build-deploy-image:${BUILD_DEPLOY_IMAGE_TAG} \ - $$([ $(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG) ] && echo 'OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG=$(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG)') \ - $$([ $(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY) ] && echo 'OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY=$(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY)') \ - OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \ - IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \ - SKIP_INSTALL_REGISTRY=true \ - LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \ - USE_CALICO_CNI=false \ - LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \ - LAGOON_FEATURE_FLAG_DEFAULT_ROOTLESS_WORKLOAD=enabled + && $(MAKE) k3d/push-images JQ=$(JQ) HELM=$(HELM) KUBECTL=$(KUBECTL) +endif + export KUBECONFIG="$$(realpath kubeconfig.k3d.$(CI_BUILD_TAG))" \ + && export IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \ + && cd lagoon-charts.k3d.lagoon \ + && $(MAKE) install-lagoon \ + INSTALL_UNAUTHENTICATED_REGISTRY=$(INSTALL_UNAUTHENTICATED_REGISTRY) \ + INSTALL_LAGOON_DEPENDENCIES=false DOCKER_NETWORK=$(DOCKER_NETWORK) \ + TESTS=$(TESTS) IMAGE_TAG=$(SAFE_BRANCH_NAME) DISABLE_CORE_HARBOR=true \ + HELM=$(HELM) KUBECTL=$(KUBECTL) JQ=$(JQ) \ + OVERRIDE_BUILD_DEPLOY_DIND_IMAGE=uselagoon/build-deploy-image:${BUILD_DEPLOY_IMAGE_TAG} \ + $$([ $(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG) ] && echo 'OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG=$(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG)') \ + $$([ $(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY) ] && echo 'OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY=$(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY)') \ + OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \ + IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \ + SKIP_INSTALL_REGISTRY=true \ + LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \ + USE_CALICO_CNI=false \ + LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \ + LAGOON_FEATURE_FLAG_DEFAULT_ROOTLESS_WORKLOAD=enabled \ + $$([ $(INSTALL_STABLE_CORE) ] && echo 'INSTALL_STABLE_CORE=$(INSTALL_STABLE_CORE)') \ + $$([ $(INSTALL_STABLE_REMOTE) ] && echo 'INSTALL_STABLE_REMOTE=$(INSTALL_STABLE_REMOTE)') \ + $$([ $(INSTALL_STABLE_BUILDDEPLOY) ] && echo 'INSTALL_STABLE_BUILDDEPLOY=$(INSTALL_STABLE_BUILDDEPLOY)') \ + $$([ $(STABLE_CORE_CHART_VERSION) ] && echo 'STABLE_CORE_CHART_VERSION=$(STABLE_CORE_CHART_VERSION)') \ + $$([ $(STABLE_REMOTE_CHART_VERSION) ] && echo 'STABLE_REMOTE_CHART_VERSION=$(STABLE_REMOTE_CHART_VERSION)') \ + $$([ $(STABLE_BUILDDEPLOY_CHART_VERSION) ] && echo 'STABLE_BUILDDEPLOY_CHART_VERSION=$(STABLE_BUILDDEPLOY_CHART_VERSION)') \ + $$([ $(INSTALL_MARIADB_PROVIDER) ] && echo 'INSTALL_MARIADB_PROVIDER=$(INSTALL_MARIADB_PROVIDER)') \ + $$([ $(INSTALL_POSTGRES_PROVIDER) ] && echo 'INSTALL_POSTGRES_PROVIDER=$(INSTALL_POSTGRES_PROVIDER)') \ + $$([ $(INSTALL_MONGODB_PROVIDER) ] && echo 'INSTALL_MONGODB_PROVIDER=$(INSTALL_MONGODB_PROVIDER)') + +# k3d/stable-install-lagoon is the same as k3d/install-lagoon except that it starts it with the latest stable chart versions +.PHONY: k3d/stable-install-lagoon +k3d/stable-install-lagoon: + $(MAKE) k3d/install-lagoon INSTALL_STABLE_LAGOON=true # k3d/local-stack will deploy and seed a lagoon-core with a lagoon-remote and all basic services to get you going # and will provide some initial seed data for a user to jump right in and start using lagoon .PHONY: k3d/local-stack -k3d/local-stack: k3d/setup k3d/seed-data k3d/get-lagoon-details +k3d/local-stack: k3d/setup k3d/install-lagoon k3d/seed-data k3d/get-lagoon-details + +# k3d/stable-local-stack is the same as k3d/local-stack except that it starts it with the latest stable chart versions +# a helper without having to remember to specify the stable option to the target +.PHONY: k3d/stable-local-stack +k3d/stable-local-stack: + $(MAKE) k3d/local-stack INSTALL_STABLE_LAGOON=true # k3d/local-dev-patch will build the services in LOCAL_DEV_SERVICES on your machine, and then use kubectl patch to mount the folders into Kubernetes # the deployments should be restarted to trigger any updated code changes @@ -652,23 +741,7 @@ k3d/local-dev-logging: && echo -e 'You will need to create a default index at http://0.0.0.0:5601/app/management/kibana/indexPatterns/create \n' \ && echo -e 'with a default `container-logs-*` pattern' -# k3d/dev can only be run once a cluster is up and running (run k3d/test first) - it doesn't rebuild the cluster at all, just pushes the built images -# into the image registry and reinstalls the lagoon-core helm chart. -.PHONY: k3d/dev -k3d/dev: build - @export KUBECONFIG="$$(realpath ./kubeconfig.k3d.$(CI_BUILD_TAG))" \ - && export IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \ - && $(MAKE) k3d/push-images && cd lagoon-charts.k3d.lagoon \ - && $(MAKE) install-lagoon-core DOCKER_NETWORK=$(DOCKER_NETWORK) IMAGE_TAG=$(SAFE_BRANCH_NAME) DISABLE_CORE_HARBOR=true \ - HELM=$(HELM) KUBECTL=$(KUBECTL) \ - JQ=$(JQ) \ - OVERRIDE_BUILD_DEPLOY_DIND_IMAGE=uselagoon/build-deploy-image:${BUILD_DEPLOY_IMAGE_TAG} \ - $$([ $(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG) ] && echo 'OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG=$(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG)') \ - $$([ $(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY) ] && echo 'OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY=$(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY)') \ - OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE=$$IMAGE_REGISTRY/task-activestandby:$(SAFE_BRANCH_NAME) \ - LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \ - IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" - @$(MAKE) k3d/get-lagoon-details + # k3d/push-images pushes locally build images into the k3d cluster registry. IMAGES = $(K3D_SERVICES) $(LOCAL_DEV_SERVICES) $(TASK_IMAGES) @@ -696,9 +769,9 @@ k3d/get-lagoon-details: uselagoon/tests \ python3 /ansible/tasks/api/admin_token.py)" \ && echo "Lagoon webhook URL: http://lagoon-webhook.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io" \ - && echo "SSH Core Service: $$($(KUBECTL) -n lagoon-core get services lagoon-core-ssh -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):$$($(KUBECTL) -n lagoon-core get services lagoon-core-ssh -o jsonpath='{.spec.ports[0].port}')" \ - && echo "SSH Portal Service: $$($(KUBECTL) -n lagoon get services lagoon-remote-ssh-portal -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):$$($(KUBECTL) -n lagoon get services lagoon-remote-ssh-portal -o jsonpath='{.spec.ports[0].port}')" \ - && echo "SSH Token Service: $$($(KUBECTL) -n lagoon-core get services lagoon-core-ssh-token -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):$$($(KUBECTL) -n lagoon-core get services lagoon-core-ssh-token -o jsonpath='{.spec.ports[0].port}')" \ + && echo "SSH Core Service: lagoon-ssh.$$($(KUBECTL) -n lagoon-core get services lagoon-core-ssh -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io:$$($(KUBECTL) -n lagoon-core get services lagoon-core-ssh -o jsonpath='{.spec.ports[0].port}')" \ + && echo "SSH Portal Service: lagoon-ssh-portal.$$($(KUBECTL) -n lagoon get services lagoon-remote-ssh-portal -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io:$$($(KUBECTL) -n lagoon get services lagoon-remote-ssh-portal -o jsonpath='{.spec.ports[0].port}')" \ + && echo "SSH Token Service: lagoon-token.$$($(KUBECTL) -n lagoon-core get services lagoon-core-ssh-token -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io:$$($(KUBECTL) -n lagoon-core get services lagoon-core-ssh-token -o jsonpath='{.spec.ports[0].port}')" \ && echo "Keycloak admin URL: http://lagoon-keycloak.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/auth" \ && echo "Keycloak admin password: $$($(KUBECTL) get secret -n lagoon-core lagoon-core-keycloak -o jsonpath="{.data.KEYCLOAK_ADMIN_PASSWORD}" | base64 --decode)" \ && echo "MailPit (email catching service): http://mailpit.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io" \ @@ -718,7 +791,7 @@ k3d/get-lagoon-cli-details: -e JWTUSER=localadmin \ uselagoon/tests \ python3 /ansible/tasks/api/admin_token.py) \\" \ - && echo "--hostname $$($(KUBECTL) -n lagoon-core get services lagoon-core-ssh-token -o jsonpath='{.status.loadBalancer.ingress[0].ip}') \\" \ + && echo "--hostname lagoon-token.$$($(KUBECTL) -n lagoon-core get services lagoon-core-ssh-token -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io \\" \ && echo "--port $$($(KUBECTL) -n lagoon-core get services lagoon-core-ssh-token -o jsonpath='{.spec.ports[0].port}')" \ && echo "" \ && echo "When interacting with this Lagoon via the lagoon-cli, you will need to add the flag '--lagoon local-k3d' to commands" \ @@ -777,6 +850,8 @@ k3d/port-forwards: .PHONY: k3d/retest k3d/retest: export KUBECONFIG="$$(pwd)/kubeconfig.k3d.$(CI_BUILD_TAG)" \ + && $(MAKE) build/tests \ + && $(MAKE) k3d/push-images JQ=$(JQ) HELM=$(HELM) KUBECTL=$(KUBECTL) IMAGES="tests" \ && cd lagoon-charts.k3d.lagoon \ && export IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \ && $(MAKE) fill-test-ci-values DOCKER_NETWORK=$(DOCKER_NETWORK) TESTS=$(TESTS) IMAGE_TAG=$(SAFE_BRANCH_NAME) DISABLE_CORE_HARBOR=true \ diff --git a/docs/contributing-to-lagoon/developing-lagoon.md b/docs/contributing-to-lagoon/developing-lagoon.md index 94b80167d4..d844a941ff 100644 --- a/docs/contributing-to-lagoon/developing-lagoon.md +++ b/docs/contributing-to-lagoon/developing-lagoon.md @@ -73,20 +73,113 @@ make -j8 build The make file offers a command that allows you to spin up Lagoon inside of a k3d cluster locally and explore its functionality. -Using the following make command will create a k3d cluster, install Lagoon and all of the necessary components to get you up and running and ready to explore. +Using the following make command will create a k3d cluster and install any dependencies (ingress, harbor, s3 compatible storage, etc) that are required by Lagoon. It will then build any images required and push them to the registry, and install `lagoon-core`, `lagoon-remote`, and `lagoon-build-deploy` charts. ```bash title="Deploy local stack" make k3d/local-stack ``` +!!! warning + Once you have a running `make k3d/local-stack` you should not run it again. There are other targets that can be run safely after the initial stack is running. + +At the end of the process, the command will provide some useful information that will help to get you up and running and able to log in to the UI or using the API with tools like the Lagoon CLI. + !!! warning This can take some time to complete as it will install a lot of components necessary to make Lagoon work. This includes things like ingress-nginx, harbor, and all the additional services to make exploring Lagoon easy. -At the end of the process, the command will provide some useful information that will get you up and running and able to log in to the UI or using the API with tools like the Lagoon CLI. + +### Local stack setup options + +There are a number of variables that can be used with `make k3d/local-stack` that can enable or disable certain options when setting up the initial cluster. + +#### DBaaS providers + +There are some Database as a Service (DBaaS) providers that are installed by default, to speed up time with a local stack, or for compatability reasons, you can disable them individually. + +```bash title="Deploy local stack without specific provider" +# disable all of them +make k3d/local-stack INSTALL_DBAAS_PROVIDERS=false + +# just disable mongodb +make k3d/local-stack INSTALL_MONGODB_PROVIDER=false +# just disable mariadb +make k3d/local-stack INSTALL_MARIADB_PROVIDER=false +# just disable postgres +make k3d/local-stack INSTALL_POSTGRES_PROVIDER=false +# other combinations as needed +make k3d/local-stack INSTALL_POSTGRES_PROVIDER=false INSTALL_MONGODB_PROVIDER=false +``` + +!!! info + If you're using an arm64 based operating system (MacOS M* for example), then `INSTALL_MONGODB_PROVIDER` will always be set to `false`, this is because there are some issues due to MongoDB not working properly. + +#### Stable chart installation and upgrading chart versions + +It is possible to run the local-stack as it would be directly from a stable chart version. + +This can be useful if you want to get a Lagoon up and running without having to build all the service images. + +Using this feature also allows for upgrading from a stable chart version to a feature chart. + +##### All stable components + +This will install lagoon-core, lagoon-remote, and lagoon-build-deploy charts with their current stable versions + +```bash title="Deploy local stack with stable core, remote, build-deploy" +make k3d/stable-local-stack + +# which is a shorter target for +make k3d/local-stack INSTALL_STABLE_LAGOON=true +``` + +##### Specific stable components + +It's also possible to selectively install certain components with a stable version. Anything not set to `true` will install the version from the charts repository. + +```bash title="Deploy local stack with stable versions" +make k3d/local-stack INSTALL_STABLE_CORE=true +make k3d/local-stack INSTALL_STABLE_REMOTE=true +make k3d/local-stack INSTALL_STABLE_BUILDDEPLOY=true +``` + +### Upgrading Lagoon components only + +Once you have a `k3d/local-stack` running, if you need to re-run the Lagoon installation to test an upgrade you can use the following target. + +```bash title="Install or upgrade all Lagoon components" +make k3d/install-lagoon + +# or shortcut to a stable lagoon +make k3d/stable-install-lagoon +# which is a shorter target for +make k3d/install-lagoon INSTALL_STABLE_LAGOON=true +``` + +This can be used if you started an initial `local-stack` with stable chart versions. Running this without any of the `INSTALL_STABLE_X` flags, will build and install Lagoon with the versions of images in the current `uselagoon/lagoon` branch, and install using the charts from the repository listed under `CHARTS_REPOSITORY` and `CHARTS_TREEISH`. + +You can also retain any stable versions of other components if you wish, for example only installing the bleeding edge lagoon-core. This can be used to verify an upgrade path from a stable chart + +```bash title="Install all stable then upgrade Lagoon core only" +# install a local stack with all stable components +make k3d/local-stack INSTALL_STABLE_LAGOON=true + +# optionally checkout a new version of the charts branch if there are changes made +make k3d/checkout-charts + +# make changes to a lagoon service or the lagoon-core charts and re-install lagoon +make k3d/install-lagoon INSTALL_STABLE_REMOTE=true INSTALL_STABLE_BUILDDEPLOY=true +``` + +!!! warning + The upstream charts `CHARTS_REPOSITORY` and `CHARTS_TREEISH` are only checked out when the cluster is first created and installed. + If you make changes to the charts branch, you can use `make k3d/dev` to clone and checkout the latest changes. Otherwise the existing checked out chart branch would be used. `make k3d/dev` is a short cut to `make k3d/checkout-charts` and `make k3d/install-lagoon`, which can also be used independently. + +!!! warning + Starting a `make k3d/local-stack` without any stable versions, then using `make k3d/install-lagoon` with `INSTALL_STABLE_X` flags may result in a broken Lagoon installation, however, this can be useful for verifying any downgrade or rollback paths. ### Run the Lagoon test-suite -If you're developing new functionality in Lagoon and want to make sure the tests complete, you can run the entire test suite using the following options +If you're developing new functionality in Lagoon and want to make sure the tests complete, you can run the entire test suite using the following options. 1. Start Lagoon test routine using the defaults in the Makefile \(all tests\). @@ -164,7 +257,7 @@ make k3d/get-lagoon-cli-details #### Rebuild Lagoon core and push images -This will re-push the images listed in `KIND_SERVICES` with the correct tag, and redeploy the lagoon-core chart. This is useful for testing small changes to Lagoon services, but does not support "live" development. You will need to rebuild these images locally first, e.g `rm build/api && make build/api`. +This will checkout `CHARTS_REPOSITORY` and `CHARTS_TREEISH` again, then build and re-push the images listed in `KIND_SERVICES` with the correct tag, and redeploy the lagoon charts. This is useful for testing small changes to Lagoon services, but does not support "live" development. You will need to rebuild these images locally first, e.g `rm build/api && make build/api`. ```bash title="Re-push images" make k3d/dev @@ -222,6 +315,21 @@ This will remove the K3D Lagoon cluster from your local Docker. make k3d/clean ``` +##### Additional cleanup commands + +There could be a number of old resources hanging around after you've run the stack a few times, you can use the following to clean up further. + +```bash title="Cleanup commands" +# remove any unused generated kubeconfigs +make k3d/clean-k3dconfigs + +# remove any unused cloned chart directories +make k3d/clean-charts + +# combination of tear down, remove any unused cloned chart directories, and unused generated kubeconfigs +make k3d/clean-all +``` + ### Ansible The Lagoon test uses Ansible to run the test suite. Each range of tests for a specific function has been split into its own routine. If you are performing development work locally, select which tests to run, and update the `$TESTS` variable in the Makefile to reduce the concurrent tests running. @@ -329,3 +437,19 @@ make k3d/push-images IMAGES=tests ```bash title="Re-run tests" make k3d/retest TESTS=[features-variables] ``` + +### k3d internal DNS cluster service resolution issues + +If you encounter an issue whe running a local-stack that seems to indicate that the cluster can't resolve an internal service within the cluster, you may need to run +```bash title="br_netfilter" +sudo modprobe br_netfilter +``` + +### k3d containers reporting too many open files + +You may need to bump `fs.inotify.max_user_instances` and `fs.inotify.max_user_watches` + +```bash title="sysctl" +sudo sysctl fs.inotify.max_user_instances=8192 +sudo sysctl fs.inotify.max_user_watches=524288 +``` diff --git a/k3d.config.yaml.tpl b/k3d.config.yaml.tpl index ef5cba805b..dfa4a9a511 100644 --- a/k3d.config.yaml.tpl +++ b/k3d.config.yaml.tpl @@ -8,11 +8,14 @@ volumes: - volume: services:/lagoon/services - volume: node_packages:/lagoon/node-packages registries: + create: # pass through for k3d node to share images with the host + image: ligfx/k3d-registry-dockerd:v0.5 + proxy: + remoteURL: "*" + volumes: + - /var/run/docker.sock:/var/run/docker.sock config: | mirrors: - docker.io: - endpoint: - - "https://imagecache.amazeeio.cloud" "registry.${LAGOON_K3D_NETWORK}.nip.io": endpoint: - https://registry.${LAGOON_K3D_NETWORK}.nip.io