diff --git a/Makefile b/Makefile index 83bfc5abce..466246e9d8 100644 --- a/Makefile +++ b/Makefile @@ -273,16 +273,16 @@ webhooks-test-services = webhook-handler webhooks2tasks backup-handler # These targets are used as dependencies to bring up containers in the right order. .PHONY: main-test-services-up main-test-services-up: $(foreach image,$(main-test-services),build/$(image)) - IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d $(main-test-services) + IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) -f docker-compose.local-dev.yaml --compatibility up -d $(main-test-services) $(MAKE) wait-for-keycloak .PHONY: drupaltest-services-up drupaltest-services-up: main-test-services-up $(foreach image,$(drupal-test-services),build/$(image)) - IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d $(drupal-test-services) + IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) -f docker-compose.local-dev.yaml --compatibility up -d $(drupal-test-services) .PHONY: webhooks-test-services-up webhooks-test-services-up: main-test-services-up $(foreach image,$(webhooks-test-services),build/$(image)) - IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d $(webhooks-test-services) + IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) -f docker-compose.local-dev.yaml --compatibility up -d $(webhooks-test-services) ####### ####### Publishing Images @@ -329,13 +329,13 @@ logs: # Start all Lagoon Services up: ifeq ($(ARCH), darwin) - IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d + IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) -f docker-compose.development.yaml --compatibility up -d else # once this docker issue is fixed we may be able to do away with this # linux-specific workaround: https://github.com/docker/cli/issues/2290 KEYCLOAK_URL=$$(docker network inspect -f '{{(index .IPAM.Config 0).Gateway}}' bridge):8088 \ IMAGE_REPO=$(CI_BUILD_TAG) \ - docker compose -p $(CI_BUILD_TAG) --compatibility up -d + docker compose -p $(CI_BUILD_TAG) -f docker-compose.development.yaml --compatibility up -d endif $(MAKE) wait-for-keycloak @@ -363,23 +363,32 @@ local-dev-yarn-stop: .PHONY: ui-development ui-development: build-ui-logs-development - IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db api-sidecar-handler local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis + IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) -f docker-compose.local-dev.yaml --compatibility up -d api api-db api-sidecar-handler local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis $(MAKE) wait-for-keycloak .PHONY: api-development api-development: build-ui-logs-development - IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db api-sidecar-handler local-api-data-watcher-pusher keycloak keycloak-db broker api-redis + IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) -f docker-compose.local-dev.yaml --compatibility up -d api api-db api-sidecar-handler local-api-data-watcher-pusher keycloak keycloak-db broker api-redis $(MAKE) wait-for-keycloak .PHONY: ui-logs-development ui-logs-development: build-ui-logs-development - IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db api-sidecar-handler actions-handler local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis logs2notifications local-minio mailhog - $(MAKE) wait-for-keycloak + IMAGE_REPO=$(CI_BUILD_TAG) COMPOSE_STACK_NAME=$(CI_BUILD_TAG) ADDITIONAL_FLAGS="-f docker-compose.local-dev.yaml" ADDITIONAL_SERVICES="ui" $(MAKE) compose-api-logs-development .PHONY: api-logs-development api-logs-development: build-ui-logs-development - IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db api-sidecar-handler actions-handler local-api-data-watcher-pusher keycloak keycloak-db broker api-redis logs2notifications local-minio mailhog - $(MAKE) wait-for-keycloak + IMAGE_REPO=$(CI_BUILD_TAG) COMPOSE_STACK_NAME=$(CI_BUILD_TAG) ADDITIONAL_FLAGS="-f docker-compose.local-dev.yaml" ADDITIONAL_SERVICES="" $(MAKE) compose-api-logs-development + +# compose-api-logs-development can be consumed by other repositories to start a local api +# supported make variable passthrough are +# IMAGE_REPO - the docker repostory to use (uselagoon, testlagoon, other) +# IMAGE_REPO_TAG - the tag to use for (main, pr-1234, vX.X.X, other) (each service also has a specific tag override, see the docker-compose.yaml file) +# COMPOSE_STACK_NAME - the name of the stack that will be started +# ADDITIONAL_SERVICES - a way to pass through additional services ("ui", "ui ssh", etc..) +.PHONY: compose-api-logs-development +compose-api-logs-development: + docker compose -p $(COMPOSE_STACK_NAME) $(ADDITIONAL_FLAGS) --compatibility up -d $(ADDITIONAL_SERVICES) api api-db api-sidecar-handler actions-handler local-api-data-watcher-pusher keycloak keycloak-db broker api-redis logs2notifications local-minio mailhog + $(MAKE) CI_BUILD_TAG=$(COMPOSE_STACK_NAME) wait-for-keycloak ## CI targets diff --git a/docker-compose.local-dev.yaml b/docker-compose.local-dev.yaml new file mode 100644 index 0000000000..f5e00b0ce5 --- /dev/null +++ b/docker-compose.local-dev.yaml @@ -0,0 +1,46 @@ +version: '3.2' + +volumes: + broker: {} + +services: + webhook-handler: + command: yarn run dev + volumes: + - ./services/webhook-handler/src:/app/services/webhook-handler/src + - ./node-packages:/app/node-packages:delegated + environment: + - CONSOLE_LOGGING_LEVEL=trace + webhooks2tasks: + command: yarn run dev + volumes: + - ./services/webhooks2tasks/src:/app/services/webhooks2tasks/src + - ./node-packages:/app/node-packages:delegated + environment: + - CONSOLE_LOGGING_LEVEL=trace + api-db-init: + volumes: + - ./services/api/database:/app/services/api/database + - ./node-packages:/app/node-packages:delegated + - /app/node-packages/commons/dist + api-lagoon-migrations: + volumes: + - ./services/api/src:/app/services/api/src + api: + command: ./node_modules/.bin/tsc-watch --build --incremental --onSuccess "node -r dotenv-extended/config dist/index" + volumes: + - ./services/api/src:/app/services/api/src + - ./node-packages:/app/node-packages:delegated + - /app/node-packages/commons/dist + auth-server: + command: yarn run dev + volumes: + - ./services/auth-server/src:/app/services/auth-server/src + keycloak: + volumes: + - "./services/keycloak/startup-scripts:/opt/keycloak/startup-scripts" + - "./services/keycloak/themes/lagoon:/opt/keycloak/themes/lagoon" + - "./local-dev/keycloak:/lagoon/keycloak" + local-api-data-watcher-pusher: + volumes: + - ./local-dev/api-data-watcher-pusher:/home diff --git a/docker-compose.yaml b/docker-compose.yaml index f4c7108e9d..8b2aad5bbd 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,12 +5,11 @@ volumes: services: api-db: - image: ${IMAGE_REPO:-lagoon}/api-db + image: ${IMAGE_REPO:-lagoon}/api-db:${IMAGE_REPO_API_DB_TAG:-${IMAGE_REPO_TAG:-latest}} ports: - '3366:3306' webhook-handler: - image: ${IMAGE_REPO:-lagoon}/webhook-handler - command: yarn run dev + image: ${IMAGE_REPO:-lagoon}/webhook-handler:${IMAGE_REPO_WEBHOOK_HANDLER_TAG:-${IMAGE_REPO_TAG:-latest}} ports: - '7777:3000' volumes: @@ -19,14 +18,14 @@ services: environment: - CONSOLE_LOGGING_LEVEL=trace backup-handler: - image: ${IMAGE_REPO:-lagoon}/backup-handler + image: ${IMAGE_REPO:-lagoon}/backup-handler:${IMAGE_REPO_BACKUP_HANDLER_TAG:-${IMAGE_REPO_TAG:-latest}} restart: on-failure ports: - '7778:3000' depends_on: - broker broker: - image: ${IMAGE_REPO:-lagoon}/broker + image: ${IMAGE_REPO:-lagoon}/broker:${IMAGE_REPO_BROKER_TAG:-${IMAGE_REPO_TAG:-latest}} ports: - '15672:15672' - '5672:5672' @@ -34,7 +33,7 @@ services: volumes: - broker:/var/lib/rabbitmq broker-ff-enable: - image: ${IMAGE_REPO:-lagoon}/broker + image: ${IMAGE_REPO:-lagoon}/broker:${IMAGE_REPO_BROKER_TAG:-${IMAGE_REPO_TAG:-latest}} entrypoint: /enable-feature-flags.sh environment: - SERVICE_NAME=broker @@ -44,11 +43,11 @@ services: api-sidecar-handler: # this is neded for the internal dns references container_name: apisidecarhandler - image: ${IMAGE_REPO:-lagoon}/api-sidecar-handler + image: ${IMAGE_REPO:-lagoon}/api-sidecar-handler:${IMAGE_REPO_BROKER_TAG:-${IMAGE_REPO_TAG:-latest}} ports: - '3333:3333' logs2notifications: - image: ${IMAGE_REPO:-lagoon}/logs2notifications + image: ${IMAGE_REPO:-lagoon}/logs2notifications:${IMAGE_REPO_LOGS2NOTIFICATIONS_TAG:-${IMAGE_REPO_TAG:-latest}} environment: - S3_FILES_HOST=http://172.17.0.1:9000 mailhog: @@ -57,33 +56,23 @@ services: ports: - '32025:8025' webhooks2tasks: - image: ${IMAGE_REPO:-lagoon}/webhooks2tasks - command: yarn run dev - volumes: - - ./services/webhooks2tasks/src:/app/services/webhooks2tasks/src - - ./node-packages:/app/node-packages:delegated + image: ${IMAGE_REPO:-lagoon}/webhooks2tasks:${IMAGE_REPO_WEBHOOKS2TASKS_TAG:-${IMAGE_REPO_TAG:-latest}} environment: - CONSOLE_LOGGING_LEVEL=trace - SIDECAR_HANDLER_HOST=apisidecarhandler api-db-init: - image: ${IMAGE_REPO:-lagoon}/api + image: ${IMAGE_REPO:-lagoon}/api:${IMAGE_REPO_API_TAG:-${IMAGE_REPO_TAG:-latest}} command: > sh -c "./node_modules/.bin/knex migrate:list --cwd /app/services/api/database && ./node_modules/.bin/knex migrate:latest --cwd /app/services/api/database" - volumes: - - ./services/api/database:/app/services/api/database - - ./node-packages:/app/node-packages:delegated - - /app/node-packages/commons/dist depends_on: api-db: condition: service_started keycloak: condition: service_started api-lagoon-migrations: - image: ${IMAGE_REPO:-lagoon}/api + image: ${IMAGE_REPO:-lagoon}/api:${IMAGE_REPO_API_TAG:-${IMAGE_REPO_TAG:-latest}} command: sh -c "./node_modules/.bin/tsc && node -r dotenv-extended/config dist/migrations/lagoon/migration.js" - volumes: - - ./services/api/src:/app/services/api/src environment: - NODE_ENV=development - CONSOLE_LOGGING_LEVEL=trace @@ -95,12 +84,7 @@ services: api-sidecar-handler: condition: service_started api: - image: ${IMAGE_REPO:-lagoon}/api - command: ./node_modules/.bin/tsc-watch --build --incremental --onSuccess "node -r dotenv-extended/config dist/index" - volumes: - - ./services/api/src:/app/services/api/src - - ./node-packages:/app/node-packages:delegated - - /app/node-packages/commons/dist + image: ${IMAGE_REPO:-lagoon}/api:${IMAGE_REPO_API_TAG:-${IMAGE_REPO_TAG:-latest}} environment: - KEYCLOAK_FRONTEND_URL=http://localhost:8088/ - NODE_ENV=development @@ -138,12 +122,12 @@ services: ports: - '8888:3000' actions-handler: - image: ${IMAGE_REPO:-lagoon}/actions-handler + image: ${IMAGE_REPO:-lagoon}/actions-handler:${IMAGE_REPO_ACTIONS_HANDLER_TAG:-${IMAGE_REPO_TAG:-latest}} restart: on-failure depends_on: - broker ssh: - image: ${IMAGE_REPO:-lagoon}/ssh + image: ${IMAGE_REPO:-lagoon}/ssh:${IMAGE_REPO_SSH_TAG:-${IMAGE_REPO_TAG:-latest}} depends_on: - api - auth-server @@ -154,18 +138,15 @@ services: - ./services/ssh/home/command.sh:/home/command.sh - ./services/ssh/home/rsh.sh:/home/rsh.sh auth-server: - image: ${IMAGE_REPO:-lagoon}/auth-server - command: yarn run dev + image: ${IMAGE_REPO:-lagoon}/auth-server:${IMAGE_REPO_AUTH_SERVER_TAG:-${IMAGE_REPO_TAG:-latest}} environment: - JWTISSUER=auth-server.dev - JWTAUDIENCE=api.dev - CONSOLE_LOGGING_LEVEL=trace - volumes: - - ./services/auth-server/src:/app/services/auth-server/src ports: - '3001:3000' keycloak: - image: ${IMAGE_REPO:-lagoon}/keycloak + image: ${IMAGE_REPO:-lagoon}/keycloak:${IMAGE_REPO_KEYCLOAK_TAG:-${IMAGE_REPO_TAG:-latest}} user: '111111111' depends_on: - keycloak-db @@ -178,16 +159,12 @@ services: # Uncomment for local new relic tracking # - NEW_RELIC_LICENSE_KEY= # - NEW_RELIC_APP_NAME=keycloak-local - volumes: - - "./services/keycloak/startup-scripts:/opt/keycloak/startup-scripts" - - "./services/keycloak/themes/lagoon:/opt/keycloak/themes/lagoon" - - "./local-dev/keycloak:/lagoon/keycloak" keycloak-db: - image: ${IMAGE_REPO:-lagoon}/keycloak-db + image: ${IMAGE_REPO:-lagoon}/keycloak-db:${IMAGE_REPO_KEYCLOAK_DB_TAG:-${IMAGE_REPO_TAG:-latest}} ports: - '3336:3306' tests-kubernetes: - image: ${IMAGE_REPO:-lagoon}/tests + image: ${IMAGE_REPO:-lagoon}/tests:${IMAGE_REPO_TESTS_TAG:-${IMAGE_REPO_TAG:-latest}} environment: - CLUSTER_TYPE=control-k8s - WEBHOOK_HOST=webhook-handler @@ -212,11 +189,9 @@ services: local-api-data-watcher-pusher: depends_on: - api - image: ${IMAGE_REPO:-lagoon}/local-api-data-watcher-pusher + image: ${IMAGE_REPO:-lagoon}/local-api-data-watcher-pusher:${IMAGE_REPO_API_DATA_WATCHER_TAG:-${IMAGE_REPO_TAG:-latest}} environment: - CLEAR_API_DATA=false - volumes: - - ./local-dev/api-data-watcher-pusher:/home local-minio: image: minio/minio entrypoint: sh @@ -234,4 +209,4 @@ services: labels: version: '2' api-redis: - image: ${IMAGE_REPO:-lagoon}/api-redis + image: ${IMAGE_REPO:-lagoon}/api-redis:${IMAGE_REPO_API_REDIS_TAG:-${IMAGE_REPO_TAG:-latest}}