From 19d97b10c94c55a4d34648932278cb5061b9b0b2 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Fri, 14 Feb 2025 12:18:05 +0100 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=90=9B(nginx)=20increase=20nginx=20bu?= =?UTF-8?q?ffer=20size=20when=20proxifying=20keycloak?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nginx is used to proxify keycloak in our development configuration. When a new user is created keycloak is send a large amount of headers in its response and the default nginx config is not enough to handle this amount of headers. We have to increase the proxy buffer size to handle them. --- docker/files/etc/nginx/conf.d/default.conf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/files/etc/nginx/conf.d/default.conf b/docker/files/etc/nginx/conf.d/default.conf index 2644822ce..e5967e333 100644 --- a/docker/files/etc/nginx/conf.d/default.conf +++ b/docker/files/etc/nginx/conf.d/default.conf @@ -88,5 +88,11 @@ server { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # Increase proxy buffer size to allow keycloak to send large + # header responses when a user is created. + proxy_buffer_size 128k; + proxy_buffers 4 256k; + proxy_busy_buffers_size 256k; } } From e048fa0b61ccec4c49f94f3d0889941525100709 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Fri, 14 Feb 2025 12:42:02 +0100 Subject: [PATCH 2/4] =?UTF-8?q?=E2=99=BB=EF=B8=8F(docker)=20remove=20usage?= =?UTF-8?q?=20of=20dockerize?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We remove dockerize and use healthcheck on docker compose services instead. --- .github/workflows/impress-frontend.yml | 22 --------- Makefile | 5 -- docker-compose.yml | 68 ++++++++++++++++++++------ 3 files changed, 52 insertions(+), 43 deletions(-) diff --git a/.github/workflows/impress-frontend.yml b/.github/workflows/impress-frontend.yml index f869517de..ab7fcce9e 100644 --- a/.github/workflows/impress-frontend.yml +++ b/.github/workflows/impress-frontend.yml @@ -88,28 +88,6 @@ jobs: - name: Start Docker services run: make bootstrap FLUSH_ARGS='--no-input' cache= - # Tool to wait for a service to be ready - - name: Install Dockerize - run: | - curl -sSL https://github.com/jwilder/dockerize/releases/download/v0.8.0/dockerize-linux-amd64-v0.8.0.tar.gz | sudo tar -C /usr/local/bin -xzv - - - name: Wait for services to be ready - run: | - printf "Minio check...\n" - dockerize -wait tcp://localhost:9000 -timeout 20s - printf "Keyclock check...\n" - dockerize -wait tcp://localhost:8080 -timeout 20s - printf "Server collaboration check...\n" - dockerize -wait tcp://localhost:4444 -timeout 20s - printf "Ngnix check...\n" - dockerize -wait tcp://localhost:8083 -timeout 20s - printf "DRF check...\n" - dockerize -wait tcp://localhost:8071 -timeout 20s - printf "Postgres Keyclock check...\n" - dockerize -wait tcp://localhost:5433 -timeout 20s - printf "Postgres back check...\n" - dockerize -wait tcp://localhost:15432 -timeout 20s - - name: Run e2e tests run: cd src/frontend/ && yarn e2e:test --project='chromium' diff --git a/Makefile b/Makefile index 0e81087b3..996662465 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,6 @@ COMPOSE_EXEC_APP = $(COMPOSE_EXEC) app-dev COMPOSE_RUN = $(COMPOSE) run --rm COMPOSE_RUN_APP = $(COMPOSE_RUN) app-dev COMPOSE_RUN_CROWDIN = $(COMPOSE_RUN) crowdin crowdin -WAIT_DB = @$(COMPOSE_RUN) dockerize -wait tcp://$(DB_HOST):$(DB_PORT) -timeout 60s # -- Backend MANAGE = $(COMPOSE_RUN_APP) python manage.py @@ -124,8 +123,6 @@ run: ## start the wsgi (production) and development server @$(COMPOSE) up --force-recreate -d celery-dev @$(COMPOSE) up --force-recreate -d y-provider @$(COMPOSE) up --force-recreate -d nginx - @echo "Wait for postgresql to be up..." - @$(WAIT_DB) .PHONY: run run-with-frontend: ## Start all the containers needed (backend to frontend) @@ -188,14 +185,12 @@ test-back-parallel: ## run all back-end tests in parallel makemigrations: ## run django makemigrations for the impress project. @echo "$(BOLD)Running makemigrations$(RESET)" @$(COMPOSE) up -d postgresql - @$(WAIT_DB) @$(MANAGE) makemigrations .PHONY: makemigrations migrate: ## run django migrations for the impress project. @echo "$(BOLD)Running migrations$(RESET)" @$(COMPOSE) up -d postgresql - @$(WAIT_DB) @$(MANAGE) migrate .PHONY: migrate diff --git a/docker-compose.yml b/docker-compose.yml index e10221b92..7de863354 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,11 @@ services: postgresql: image: postgres:16 + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 1s + timeout: 2s + retries: 300 env_file: - env.d/development/postgresql ports: @@ -15,7 +20,7 @@ services: - "1081:1080" minio: - # user: ${DOCKER_USER:-1000} + user: ${DOCKER_USER:-1000} image: minio/minio environment: - MINIO_ROOT_USER=impress @@ -23,6 +28,11 @@ services: ports: - '9000:9000' - '9001:9001' + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 1s + timeout: 20s + retries: 300 entrypoint: "" command: minio server --console-address :9001 /data volumes: @@ -31,7 +41,9 @@ services: createbuckets: image: minio/mc depends_on: - - minio + minio: + condition: service_healthy + restart: true entrypoint: > sh -c " /usr/bin/mc alias set impress http://minio:9000 impress password && \ @@ -59,10 +71,15 @@ services: - ./src/backend:/app - ./data/static:/data/static depends_on: - - postgresql - - mailcatcher - - redis - - createbuckets + postgresql: + condition: service_healthy + restart: true + mailcatcher: + condition: service_started + redis: + condition: service_started + createbuckets: + condition: service_started celery-dev: user: ${DOCKER_USER:-1000} @@ -93,9 +110,13 @@ services: - env.d/development/common - env.d/development/postgresql depends_on: - - postgresql - - redis - - minio + postgresql: + condition: service_healthy + restart: true + redis: + condition: service_started + minio: + condition: service_started celery: user: ${DOCKER_USER:-1000} @@ -116,9 +137,13 @@ services: volumes: - ./docker/files/etc/nginx/conf.d:/etc/nginx/conf.d:ro depends_on: - - keycloak - - app-dev - - y-provider + app-dev: + condition: service_started + y-provider: + condition: service_started + keycloak: + condition: service_healthy + restart: true frontend-dev: user: "${DOCKER_USER:-1000}" @@ -135,9 +160,6 @@ services: ports: - "3000:3000" - dockerize: - image: jwilder/dockerize - crowdin: image: crowdin/cli:3.16.0 volumes: @@ -169,6 +191,11 @@ services: kc_postgresql: image: postgres:14.3 + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 1s + timeout: 2s + retries: 300 ports: - "5433:5432" env_file: @@ -187,6 +214,13 @@ services: - --hostname-admin-url=http://localhost:8083/ - --hostname-strict=false - --hostname-strict-https=false + - --health-enabled=true + - --metrics-enabled=true + healthcheck: + test: ["CMD", "curl", "--head", "-fsS", "http://localhost:8080/health/ready"] + interval: 1s + timeout: 2s + retries: 300 environment: KEYCLOAK_ADMIN: admin KEYCLOAK_ADMIN_PASSWORD: admin @@ -200,4 +234,6 @@ services: ports: - "8080:8080" depends_on: - - kc_postgresql + kc_postgresql: + condition: service_healthy + restart: true From 704c4d7f9ac1cca646bbe62059e4a4ca8ed16618 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Fri, 14 Feb 2025 12:43:49 +0100 Subject: [PATCH 3/4] =?UTF-8?q?=E2=99=BB=EF=B8=8F(make)=20make=20run=20com?= =?UTF-8?q?mand=20starting=20everything?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The run command is not starting the frontend application. We change the run commands. The run command is strating everything. The run-backend command is starting all services needed to use the backend application. --- Makefile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 996662465..176908379 100644 --- a/Makefile +++ b/Makefile @@ -80,12 +80,12 @@ bootstrap: \ data/static \ create-env-files \ build \ - run-with-frontend \ migrate \ demo \ back-i18n-compile \ mails-install \ - mails-build + mails-build \ + run .PHONY: bootstrap # -- Docker/compose @@ -119,16 +119,17 @@ logs: ## display app-dev logs (follow mode) @$(COMPOSE) logs -f app-dev .PHONY: logs -run: ## start the wsgi (production) and development server +run-backend: ## Start only the backend application and all needed services @$(COMPOSE) up --force-recreate -d celery-dev @$(COMPOSE) up --force-recreate -d y-provider @$(COMPOSE) up --force-recreate -d nginx -.PHONY: run +.PHONY: run-backend -run-with-frontend: ## Start all the containers needed (backend to frontend) - @$(MAKE) run +run: ## start the wsgi (production) and development server +run: + @$(MAKE) run-backend @$(COMPOSE) up --force-recreate -d frontend-dev -.PHONY: run-with-frontend +.PHONY: run status: ## an alias for "docker compose ps" @$(COMPOSE) ps From ee4402efbbd8db370db652f67eceb816f634bdd2 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Fri, 14 Feb 2025 13:10:35 +0100 Subject: [PATCH 4/4] =?UTF-8?q?=E2=99=BB=EF=B8=8F(docker)=20rename=20front?= =?UTF-8?q?end-dev=20service=20in=20frontend?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The frontend-dev service is in fact using the production image. We rename it in frontend accordingly with what it really does. We also have to change name rules in Makefile to be consistent. --- Makefile | 10 +++++----- README.md | 10 +++++----- docker-compose.yml | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 176908379..ea5fdbc5b 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,7 @@ build-yjs-provider: ## build the y-provider container build-frontend: cache ?= build-frontend: ## build the frontend container - @$(COMPOSE) build frontend-dev $(cache) + @$(COMPOSE) build frontend $(cache) .PHONY: build-frontend down: ## stop and remove containers, networks, images, and volumes @@ -128,7 +128,7 @@ run-backend: ## Start only the backend application and all needed services run: ## start the wsgi (production) and development server run: @$(MAKE) run-backend - @$(COMPOSE) up --force-recreate -d frontend-dev + @$(COMPOSE) up --force-recreate -d frontend .PHONY: run status: ## an alias for "docker compose ps" @@ -306,16 +306,16 @@ help: .PHONY: help # Front -frontend-install: ## install the frontend locally +frontend-development-install: ## install the frontend locally cd $(PATH_FRONT_IMPRESS) && yarn -.PHONY: frontend-install +.PHONY: frontend-development-install frontend-lint: ## run the frontend linter cd $(PATH_FRONT) && yarn lint .PHONY: frontend-lint run-frontend-development: ## Run the frontend in development mode - @$(COMPOSE) stop frontend-dev + @$(COMPOSE) stop frontend cd $(PATH_FRONT_IMPRESS) && yarn dev .PHONY: run-frontend-development diff --git a/README.md b/README.md index a1bf9e6cf..c024440b1 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,9 @@ $ docker -v Docker version 20.10.2, build 2291f61 -$ docker compose -v +$ docker compose version -docker compose version 1.27.4, build 40524192 +Docker Compose version v2.32.4 ``` > ⚠️ You may need to run the following commands with sudo but this can be avoided by adding your user to the `docker` group. @@ -97,7 +97,7 @@ password: impress 📝 Note that if you need to run them afterwards, you can use the eponym Make rule: ```shellscript -$ make run-with-frontend +$ make run ``` ⚠️ For the frontend developer, it is often better to run the frontend in development mode locally. @@ -105,7 +105,7 @@ $ make run-with-frontend To do so, install the frontend dependencies with the following command: ```shellscript -$ make frontend-install +$ make frontend-development-install ``` And run the frontend locally in development mode with the following command: @@ -117,7 +117,7 @@ $ make run-frontend-development To start all the services, except the frontend container, you can use the following command: ```shellscript -$ make run +$ make run-backend ``` **Adding content** diff --git a/docker-compose.yml b/docker-compose.yml index 7de863354..84ad1b037 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -145,7 +145,7 @@ services: condition: service_healthy restart: true - frontend-dev: + frontend: user: "${DOCKER_USER:-1000}" build: context: .