From 1031fbff2685efa090341ba4999259b0d7068e4d Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Mon, 24 Jun 2024 09:44:28 +1000 Subject: [PATCH 1/4] refactor: use compose from uselagoon/lagoon --- .dockerignore | 12 +++- .github/workflows/ui-workflow.yaml | 37 ++++++++---- .gitignore | 3 + Makefile | 62 +++++++++++++++++++ cypress/cypress.config.ts | 2 +- cypress/e2e/general/navigation.cy.ts | 2 +- cypress/run.sh | 11 ++++ docker-compose.yml | 4 +- server.js | 2 +- test/Makefile | 34 ----------- test/README.md | 15 ----- test/docker-compose.yaml | 90 ---------------------------- test/keycloak/configure-keycloak.sh | 52 ---------------- 13 files changed, 117 insertions(+), 209 deletions(-) create mode 100644 Makefile create mode 100755 cypress/run.sh delete mode 100644 test/Makefile delete mode 100644 test/README.md delete mode 100644 test/docker-compose.yaml delete mode 100755 test/keycloak/configure-keycloak.sh diff --git a/.dockerignore b/.dockerignore index 6e9b80f9..0a3b6ecd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,7 +7,17 @@ README.md **/yarn-error.log **/.gitignore node_modules +cypress_cache +cypress +.vscode +.github +.storybook + build # Envs -**/.env.local \ No newline at end of file +**/.env.local + +lagoon-core.* +Makefile +cypress/screenshots \ No newline at end of file diff --git a/.github/workflows/ui-workflow.yaml b/.github/workflows/ui-workflow.yaml index 8313cf87..294af344 100644 --- a/.github/workflows/ui-workflow.yaml +++ b/.github/workflows/ui-workflow.yaml @@ -53,13 +53,13 @@ jobs: - name: Build and start Lagoon-minimal run: | - cd test - make up + make development-api - name: Start ui env: - GRAPHQL_API: http://0.0.0.0:33000/graphql - KEYCLOAK_API: http://0.0.0.0:38088/auth + GRAPHQL_API: http://localhost:3000/graphql + KEYCLOAK_API: http://localhost:8088/auth + NODE_PORT: 3003 run: | yarn install yarn build @@ -70,26 +70,37 @@ jobs: with: config-file: ./cypress/cypress.config.ts auto-cancel-after-failures: 1 - wait-on: 'http://localhost:3000' + wait-on: 'http://localhost:3003' command: yarn cypress:runRbac + env: + cypress_api: http://localhost:3000/graphql + cypress_keycloak: http://localhost:8088/auth + cypress_url: http://localhost:3003 - - name: Run General Cypress Tests + - name: Run Organization Cypress Tests uses: cypress-io/github-action@v6 with: config-file: ./cypress/cypress.config.ts auto-cancel-after-failures: 1 - wait-on: 'http://localhost:3000' - command: yarn cypress:runGeneral + wait-on: 'http://localhost:3003' + command: yarn cypress:runOrganizations + env: + cypress_api: http://localhost:3000/graphql + cypress_keycloak: http://localhost:8088/auth + cypress_url: http://localhost:3003 - - name: Run Organization Cypress Tests + - name: Run General Cypress Tests uses: cypress-io/github-action@v6 with: config-file: ./cypress/cypress.config.ts auto-cancel-after-failures: 1 - wait-on: 'http://localhost:3000' - command: yarn cypress:runOrganizations + wait-on: 'http://localhost:3003' + command: yarn cypress:runGeneral + env: + cypress_api: http://localhost:3000/graphql + cypress_keycloak: http://localhost:8088/auth + cypress_url: http://localhost:3003 - name: Stop Docker containers run: | - cd test - make down + make development-api-down diff --git a/.gitignore b/.gitignore index 3b669ab8..57d854c1 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,9 @@ build-storybook.log .env .python-version docker-compose.override.yml +cypress_cache +lagoon-core.* +cypress/screenshots # ts linter *.tsbuildinfo \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..fa2c67b1 --- /dev/null +++ b/Makefile @@ -0,0 +1,62 @@ +# upstream +CI_BUILD_TAG ?= lagoon-ui +CORE_REPO=https://github.com/uselagoon/lagoon.git +CORE_TREEISH=make-export-refactoring +CYPRESS_BASE=cypress/base:20.13.1 + +LAGOON_CORE_IMAGE_REPO=testlagoon +LAGOON_CORE_IMAGE_TAG=main + +.PHONY: yarn-start-ui +yarn-start-ui: + export GRAPHQL_API=http://localhost:3000/graphql \ + && export KEYCLOAK_API=http://localhost:8088/auth \ + && yarn install \ + && yarn build \ + && yarn start \ + +# run-cypress: +.PHONY: start-ui +start-ui: development-api + export GRAPHQL_API=http://localhost:3000/graphql \ + && export KEYCLOAK_API=http://localhost:8088/auth \ + && export NODE_ENV=production \ + && export NODE_PORT=3003 \ + && export LAGOON_UI_TOURS_ENABLED=disabled \ + && docker compose -p $(CI_BUILD_TAG) --compatibility up --build -d ui + +.PHONY: development-api +development-api: + export LAGOON_CORE=$$(mktemp -d ./lagoon-core.XXX) \ + && export GRAPHQL_API=http://localhost:3000/graphql \ + && export KEYCLOAK_API=http://localhost:8088/auth \ + && git clone $(CORE_REPO) "$$LAGOON_CORE" \ + && cd "$$LAGOON_CORE" \ + && git checkout $(CORE_TREEISH) \ + && IMAGE_REPO=$(LAGOON_CORE_IMAGE_REPO) IMAGE_REPO_TAG=$(LAGOON_CORE_IMAGE_TAG) COMPOSE_STACK_NAME=core-$(CI_BUILD_TAG) docker compose -p core-$(CI_BUILD_TAG) pull \ + && IMAGE_REPO=$(LAGOON_CORE_IMAGE_REPO) IMAGE_REPO_TAG=$(LAGOON_CORE_IMAGE_TAG) COMPOSE_STACK_NAME=core-$(CI_BUILD_TAG) $(MAKE) compose-api-logs-development + +.PHONY: run-cypress-with-development-api +run-cypress-with-development-api: start-ui + $(MAKE) run-cypress + +.PHONY: run-cypress +run-cypress: + export GRAPHQL_API=http://localhost:3000/graphql \ + && export KEYCLOAK_API=http://localhost:8088/auth \ + && export UI_URL=http://localhost:3003 \ + && docker run --rm -it --network host --name ct-$(CI_BUILD_TAG) \ + --volume "$$(pwd):/workdir" -w /workdir \ + -e cypress_api=$$GRAPHQL_API \ + -e cypress_keycloak=$$KEYCLOAK_API \ + -e cypress_url=$$UI_URL \ + --entrypoint=/bin/bash $(CYPRESS_BASE) ./cypress/run.sh + +.PHONY: development-api-down +development-api-down: + docker-compose -p core-$(CI_BUILD_TAG) --compatibility down -v --remove-orphans + +.PHONY: down +down: + $(MAKE) development-api-down + docker-compose -p $(CI_BUILD_TAG) --compatibility down -v --remove-orphans \ No newline at end of file diff --git a/cypress/cypress.config.ts b/cypress/cypress.config.ts index c5ca49da..b9b5ccae 100644 --- a/cypress/cypress.config.ts +++ b/cypress/cypress.config.ts @@ -6,7 +6,7 @@ export default defineConfig({ e2e: { env: { api: 'http://0.0.0.0:33000/graphql', - keycloak: 'http://0.0.0.0:38088', + keycloak: 'http://0.0.0.0:38088/auth', url: 'http://0.0.0.0:3000', user_guest: 'guest@example.com', user_reporter: 'reporter@example.com', diff --git a/cypress/e2e/general/navigation.cy.ts b/cypress/e2e/general/navigation.cy.ts index 68fed588..d2e931b1 100644 --- a/cypress/e2e/general/navigation.cy.ts +++ b/cypress/e2e/general/navigation.cy.ts @@ -46,7 +46,7 @@ describe('Navigation tests', () => { navigation.getLinkElement('account').click(); - const redirect = `${Cypress.env('keycloak')}/auth/realms/lagoon/account/`; + const redirect = `${Cypress.env('keycloak')}/realms/lagoon/account/`; cy.origin(redirect, { args: { redirect } }, ({ redirect }) => { cy.location().should(loc => { expect(loc.toString()).to.eq(redirect); diff --git a/cypress/run.sh b/cypress/run.sh new file mode 100755 index 00000000..a18432fc --- /dev/null +++ b/cypress/run.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# create cache for repeat runs +mkdir -p /workdir/cypress_cache +ln -s /workdir/cypress_cache /root/.cache + +yarn --frozen-lockfile +# run the tests +yarn cypress:runRbac +yarn cypress:runGeneral +yarn cypress:runOrganizations \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 6100b46d..e184e1ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,9 @@ x-environment: &default-environment LAGOON_ENVIRONMENT_TYPE: production GRAPHQL_API: "${GRAPHQL_API:-http://0.0.0.0:3000/graphql}" KEYCLOAK_API: "${KEYCLOAK_API:-http://0.0.0.0:8088/auth}" - LAGOON_UI_TOURS_ENABLED: enabled + LAGOON_UI_TOURS_ENABLED: ${LAGOON_UI_TOURS_ENABLED:-enabled} + NODE_ENV: "${NODE_ENV}" + NODE_PORT: "${NODE_PORT:-3003}" services: ui: diff --git a/server.js b/server.js index e88f9cef..7e18a30b 100644 --- a/server.js +++ b/server.js @@ -2,7 +2,7 @@ const express = require('express'); const next = require('next'); const dev = process.env.NODE_ENV !== 'production'; -const port = dev ? 3003 : 3000; +const port = process.env.NODE_PORT || 3000; const app = next({ dev, dir: 'src', diff --git a/test/Makefile b/test/Makefile deleted file mode 100644 index 415e23e5..00000000 --- a/test/Makefile +++ /dev/null @@ -1,34 +0,0 @@ -SHELL := /bin/bash - -KCADM = /opt/jboss/keycloak/bin/kcadm.sh -KCADM_CONFIG = /tmp/.keycloak/kcadm.config -KCADM_STRING = --server http://$$(hostname -i):8080/auth --user $$KEYCLOAK_ADMIN_USER --password $$KEYCLOAK_ADMIN_PASSWORD --realm master -KCADM_LOGIN = $(KCADM) config credentials --config $(KCADM_CONFIG) $(KCADM_STRING) - -.PHONY: build-lagoon -build-lagoon: - docker compose up -d --quiet-pull - $(MAKE) wait-for-keycloak - @echo "\nYour Lagoon stack is now running locally - use 'GRAPHQL_API=http://0.0.0.0:33000/graphql KEYCLOAK_API=http://0.0.0.0:38088/auth' as the variables to access it\n" - -.PHONY: wait-for-keycloak -wait-for-keycloak: - $(info Waiting for Keycloak to be ready....) - grep -m 1 "Config of Keycloak done." <(docker compose --compatibility logs -f keycloak 2>&1) - -.PHONY: keycloak-setup -keycloak-setup: - docker compose exec keycloak bash -c "/upload/configure-keycloak.sh" - -.PHONY: reload-data -reload-data: - docker compose up -d local-api-data-watcher-pusher - -.PHONY: down -down: - docker compose down --remove-orphans --volumes - -.PHONY: up -up: - $(MAKE) build-lagoon - $(MAKE) keycloak-setup diff --git a/test/README.md b/test/README.md deleted file mode 100644 index b0e80d21..00000000 --- a/test/README.md +++ /dev/null @@ -1,15 +0,0 @@ -## Lagoon minimal setup - -Use this docker-compose.yml to start a minimal Lagoon Core for testing purposes. - -### Running it - -Simple! Just run `make up` - this will pull the images, start Lagoon, and auto-configure the passwords. - -Lagoon comes built-in with organizations, groups, projects and users. - -All usernames have matching passwords (eg user:owner@example.com pass:owner@example.com) - -The file is configured to start the API and keycloak on non-usual ports to avoid any collisions - -Use `GRAPHQL_API=http://0.0.0.0:33000/graphql KEYCLOAK_API=http://0.0.0.0:38088/auth` with any tools. diff --git a/test/docker-compose.yaml b/test/docker-compose.yaml deleted file mode 100644 index 2914f24c..00000000 --- a/test/docker-compose.yaml +++ /dev/null @@ -1,90 +0,0 @@ -name: lagoon-minimal - -services: - api-db: - image: testlagoon/api-db:main - networks: - - default - broker: - image: testlagoon/broker:main - restart: on-failure - networks: - - default - api-init: - image: testlagoon/api:main - command: ./node_modules/.bin/knex migrate:latest --cwd /app/services/api/database - depends_on: - - api-db - - keycloak - api-lagoon-migrations: - image: testlagoon/api:main - command: sh -c "./node_modules/.bin/tsc && node -r dotenv-extended/config dist/migrations/lagoon/migration.js" - environment: - - KEYCLOAK_URL=http://172.17.0.1:38088 - depends_on: - api-init: - condition: service_completed_successfully # don't start the lagoon migrations until the db migrations is completed - keycloak: - condition: service_started - api-sidecar-handler: - # this is neded for the internal dns references - container_name: apisidecarhandler - image: testlagoon/api-sidecar-handler:main - ports: - - '3333:3333' - api: - image: testlagoon/api:main - ports: - - '33000:3000' - networks: - - default - environment: - - KEYCLOAK_FRONTEND_URL=http://172.17.0.1:38088 - - NODE_ENV=development - - OPENSEARCH_INTEGRATION_ENABLED=false - - DISABLE_CORE_HARBOR=true - - CI=${CI:-true} - - S3_FILES_HOST=http://172.17.0.1:39000 - - S3_BAAS_ACCESS_KEY_ID=minio - - S3_BAAS_SECRET_ACCESS_KEY=minio123 - - CONSOLE_LOGGING_LEVEL=trace - - SIDECAR_HANDLER_HOST=apisidecarhandler - depends_on: - api-lagoon-migrations: - condition: service_started - keycloak: - condition: service_started - api-sidecar-handler: - condition: service_started - api-redis: - image: testlagoon/api-redis:main - keycloak: - image: testlagoon/keycloak:main - depends_on: - - keycloak-db - ports: - - '38088:8080' - volumes: - - ./keycloak:/upload - environment: - - KEYCLOAK_FRONTEND_URL=http://0.0.0.0:38088/auth - keycloak-db: - image: testlagoon/keycloak-db:main - local-minio: - image: minio/minio - entrypoint: sh - command: -c 'mkdir -p /export/restores && mkdir -p /export/lagoon-files && mkdir -p /export/harbor-images && minio server /export --console-address ":9001" ' - ports: - - '39000:9000' - - '39001:9001' - environment: - - MINIO_ROOT_USER=minio - - MINIO_ROOT_PASSWORD=minio123 - local-api-data-watcher-pusher: - image: testlagoon/local-api-data-watcher-pusher:main - depends_on: - - api - command: ["bash", "-c", " - wait-for api:3000 -t 600; - /home/data-init-push.sh; - "] diff --git a/test/keycloak/configure-keycloak.sh b/test/keycloak/configure-keycloak.sh deleted file mode 100755 index aee9bf06..00000000 --- a/test/keycloak/configure-keycloak.sh +++ /dev/null @@ -1,52 +0,0 @@ -function is_keycloak_running { - local http_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/auth/admin/realms) - if [[ $http_code -eq 401 ]]; then - return 0 - else - return 1 - fi -} - -function configure_user_passwords { - - LAGOON_DEMO_USERS=("guest@example.com" "reporter@example.com" "developer@example.com" "maintainer@example.com" "owner@example.com") - LAGOON_DEMO_ORG_USERS=("orguser@example.com" "orgviewer@example.com" "orgadmin@example.com" "orgowner@example.com" "platformowner@example.com") - - for i in ${LAGOON_DEMO_USERS[@]} - do - echo Configuring password for $i - /opt/keycloak/bin/kcadm.sh set-password --config $CONFIG_PATH --username $i -p $i --target-realm Lagoon - done - - for i in ${LAGOON_DEMO_ORG_USERS[@]} - do - echo Configuring password for $i - /opt/keycloak/bin/kcadm.sh set-password --config $CONFIG_PATH --username $i -p $i --target-realm Lagoon - done -} - -function configure_platformowner { - echo Configuring platform owner role - /opt/keycloak/bin/kcadm.sh add-roles --uusername platformowner@example.com --rolename platform-owner --config $CONFIG_PATH --target-realm Lagoon -} - -function configure_keycloak { - until is_keycloak_running; do - echo Keycloak still not running, waiting 5 seconds - sleep 5 - done - - # Set the config file path because $HOME/.keycloak/kcadm.config resolves to /opt/jboss/?/.keycloak/kcadm.config for some reason, causing it to fail - CONFIG_PATH=/tmp/kcadm.config - - echo Keycloak is running, proceeding with configuration - - /opt/keycloak/bin/kcadm.sh config credentials --config $CONFIG_PATH --server http://localhost:8080/auth --user $KEYCLOAK_USER --password $KEYCLOAK_PASSWORD --realm master - - configure_user_passwords - configure_platformowner - - echo "Config of Keycloak users done" -} - -configure_keycloak \ No newline at end of file From dddcf87318917a5ea39dbf23f5ef351d4346694a Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Tue, 25 Jun 2024 17:14:28 +1000 Subject: [PATCH 2/4] test: remove needs --- .github/workflows/ui-workflow.yaml | 2 +- Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ui-workflow.yaml b/.github/workflows/ui-workflow.yaml index 294af344..ecdafbbf 100644 --- a/.github/workflows/ui-workflow.yaml +++ b/.github/workflows/ui-workflow.yaml @@ -40,7 +40,7 @@ jobs: cypress-tests: runs-on: ubuntu-latest - needs: [format-check, lint] + # needs: [format-check, lint] steps: - name: Checkout Repository diff --git a/Makefile b/Makefile index fa2c67b1..1b64af3a 100644 --- a/Makefile +++ b/Makefile @@ -54,9 +54,9 @@ run-cypress: .PHONY: development-api-down development-api-down: - docker-compose -p core-$(CI_BUILD_TAG) --compatibility down -v --remove-orphans + docker compose -p core-$(CI_BUILD_TAG) --compatibility down -v --remove-orphans .PHONY: down down: $(MAKE) development-api-down - docker-compose -p $(CI_BUILD_TAG) --compatibility down -v --remove-orphans \ No newline at end of file + docker compose -p $(CI_BUILD_TAG) --compatibility down -v --remove-orphans \ No newline at end of file From d538b86e8a2985186b595605992aaa00e3c551fc Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Mon, 16 Sep 2024 10:22:16 +1000 Subject: [PATCH 3/4] test: matrix versions --- .github/workflows/ui-workflow.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ui-workflow.yaml b/.github/workflows/ui-workflow.yaml index ecdafbbf..499de01c 100644 --- a/.github/workflows/ui-workflow.yaml +++ b/.github/workflows/ui-workflow.yaml @@ -39,7 +39,17 @@ jobs: cypress-tests: runs-on: ubuntu-latest - + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + lagoon_image_tag: ["main"] + lagoon_image_repo: ["testlagoon"] + experimental: [false] + # include: + # - lagoon_image_tag: "v2.20.1" + # lagoon_image_repo: "uselagoon" + # experimental: true # needs: [format-check, lint] steps: @@ -53,7 +63,7 @@ jobs: - name: Build and start Lagoon-minimal run: | - make development-api + make development-api LAGOON_CORE_IMAGE_REPO=${{matrix.lagoon_image_repo}} LAGOON_CORE_IMAGE_TAG=${{matrix.lagoon_image_tag}} - name: Start ui env: From ba8478cf4dddc175b6a2963ed7a778720448cc33 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Mon, 16 Sep 2024 12:03:04 +1000 Subject: [PATCH 4/4] test: use main --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1b64af3a..3c729690 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # upstream CI_BUILD_TAG ?= lagoon-ui CORE_REPO=https://github.com/uselagoon/lagoon.git -CORE_TREEISH=make-export-refactoring +CORE_TREEISH=main CYPRESS_BASE=cypress/base:20.13.1 LAGOON_CORE_IMAGE_REPO=testlagoon