From 390eacb01a710239e69f71d0c52134b7e87d17d4 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 24 Jan 2024 15:44:22 +0100 Subject: [PATCH] Remove duplicated definitions of random volumes (#36993) Docker Compose 2.24 started to error out on duplicated (but otherwise identical) volume definitions coming from multiple docker compose files. We had urandom volume defined not only in base yaml but also in all the backend and integration yaml files which new docker compose does not like (and I agree with it). That resulted with error: ``services.airflow.volumes array items[0,2] must be unique`` This PR removes the duplicated volume definitions, as side effect it makes our docker compose files cleaner. --- scripts/ci/docker-compose/backend-mysql.yml | 1 - scripts/ci/docker-compose/backend-none.yml | 2 -- scripts/ci/docker-compose/backend-postgres.yml | 1 - scripts/ci/docker-compose/backend-sqlite-no-volume.yml | 2 -- scripts/ci/docker-compose/backend-sqlite.yml | 1 - scripts/ci/docker-compose/integration-cassandra.yml | 1 - scripts/ci/docker-compose/integration-celery.yml | 2 -- scripts/ci/docker-compose/integration-kerberos.yml | 1 - scripts/ci/docker-compose/integration-mongo.yml | 1 - scripts/ci/docker-compose/integration-pinot.yml | 2 -- scripts/ci/docker-compose/integration-trino.yml | 1 - 11 files changed, 15 deletions(-) diff --git a/scripts/ci/docker-compose/backend-mysql.yml b/scripts/ci/docker-compose/backend-mysql.yml index d74a607003fd0..f857cda9bfc7a 100644 --- a/scripts/ci/docker-compose/backend-mysql.yml +++ b/scripts/ci/docker-compose/backend-mysql.yml @@ -32,7 +32,6 @@ services: - MYSQL_DATABASE=airflow volumes: - ../mysql/conf.d:/etc/mysql/conf.d:ro - - /dev/urandom:/dev/random # Required to get non-blocking entropy source - mysql-db-volume:/var/lib/mysql healthcheck: test: ["CMD", "mysql", "-h", "localhost", "-P", "3306", "-u", "root", "-e", "SELECT 1"] diff --git a/scripts/ci/docker-compose/backend-none.yml b/scripts/ci/docker-compose/backend-none.yml index 7ce2a8e0a82cd..59e691469abd1 100644 --- a/scripts/ci/docker-compose/backend-none.yml +++ b/scripts/ci/docker-compose/backend-none.yml @@ -20,5 +20,3 @@ services: environment: - BACKEND=none - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=none-backend:// - volumes: - - /dev/urandom:/dev/random # Required to get non-blocking entropy source diff --git a/scripts/ci/docker-compose/backend-postgres.yml b/scripts/ci/docker-compose/backend-postgres.yml index 26731dcec3020..5a3b2e12d3661 100644 --- a/scripts/ci/docker-compose/backend-postgres.yml +++ b/scripts/ci/docker-compose/backend-postgres.yml @@ -33,7 +33,6 @@ services: - POSTGRES_DB=airflow - POSTGRES_HOST_AUTH_METHOD=password volumes: - - /dev/urandom:/dev/random # Required to get non-blocking entropy source - postgres-db-volume:/var/lib/postgresql/data healthcheck: test: ["CMD", "psql", "-h", "localhost", "-U", "postgres", "-c", "select 1", "airflow"] diff --git a/scripts/ci/docker-compose/backend-sqlite-no-volume.yml b/scripts/ci/docker-compose/backend-sqlite-no-volume.yml index 10fa2b82dd5b0..76f0026a1d7ef 100644 --- a/scripts/ci/docker-compose/backend-sqlite-no-volume.yml +++ b/scripts/ci/docker-compose/backend-sqlite-no-volume.yml @@ -20,5 +20,3 @@ services: environment: - BACKEND=sqlite - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=${SQLITE_URL} - volumes: - - /dev/urandom:/dev/random # Required to get non-blocking entropy source diff --git a/scripts/ci/docker-compose/backend-sqlite.yml b/scripts/ci/docker-compose/backend-sqlite.yml index d160f819e663a..427ca3723e1ea 100644 --- a/scripts/ci/docker-compose/backend-sqlite.yml +++ b/scripts/ci/docker-compose/backend-sqlite.yml @@ -21,7 +21,6 @@ services: - BACKEND=sqlite - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=${SQLITE_URL} volumes: - - /dev/urandom:/dev/random # Required to get non-blocking entropy source - sqlite-db-volume:/root/airflow/sqlite volumes: sqlite-db-volume: diff --git a/scripts/ci/docker-compose/integration-cassandra.yml b/scripts/ci/docker-compose/integration-cassandra.yml index 665637217c31a..9e73f24adb6f2 100644 --- a/scripts/ci/docker-compose/integration-cassandra.yml +++ b/scripts/ci/docker-compose/integration-cassandra.yml @@ -25,7 +25,6 @@ services: HEAP_NEWSIZE: 128M MAX_HEAP_SIZE: 256M volumes: - - /dev/urandom:/dev/random # Required to get non-blocking entropy source - cassandra-db-volume:/var/lib/cassandra healthcheck: # We use IPv6 variant of the check to workaround the problem with 3.0.26 version diff --git a/scripts/ci/docker-compose/integration-celery.yml b/scripts/ci/docker-compose/integration-celery.yml index 11383198b86c3..ce23391f1ae6c 100644 --- a/scripts/ci/docker-compose/integration-celery.yml +++ b/scripts/ci/docker-compose/integration-celery.yml @@ -22,7 +22,6 @@ services: labels: breeze.description: "Integration required for Celery executor tests." volumes: - - /dev/urandom:/dev/random # Required to get non-blocking entropy source - rabbitmq-db-volume:/var/lib/rabbitmq healthcheck: test: rabbitmq-diagnostics -q ping @@ -33,7 +32,6 @@ services: redis: image: redis:5.0.1 volumes: - - /dev/urandom:/dev/random # Required to get non-blocking entropy source - redis-db-volume:/data/redis ports: - "${REDIS_HOST_PORT}:6379" diff --git a/scripts/ci/docker-compose/integration-kerberos.yml b/scripts/ci/docker-compose/integration-kerberos.yml index 8f32d65074078..28e8262636568 100644 --- a/scripts/ci/docker-compose/integration-kerberos.yml +++ b/scripts/ci/docker-compose/integration-kerberos.yml @@ -29,7 +29,6 @@ services: volumes: - kerberos-keytabs:/root/kerberos-keytabs - - /dev/urandom:/dev/random # Required to get non-blocking entropy source environment: - KRB5_TRACE=/dev/stderr diff --git a/scripts/ci/docker-compose/integration-mongo.yml b/scripts/ci/docker-compose/integration-mongo.yml index 207e727f1aec9..f7ce42e837aa0 100644 --- a/scripts/ci/docker-compose/integration-mongo.yml +++ b/scripts/ci/docker-compose/integration-mongo.yml @@ -22,7 +22,6 @@ services: labels: breeze.description: "Integration required for MongoDB hooks." volumes: - - /dev/urandom:/dev/random # Required to get non-blocking entropy source - mongo-db-volume:/data/db healthcheck: test: echo 'db.runCommand("ping").ok' | mongo localhost:27017/test --quiet diff --git a/scripts/ci/docker-compose/integration-pinot.yml b/scripts/ci/docker-compose/integration-pinot.yml index b0d1964eabe4d..dd985b19da7ba 100644 --- a/scripts/ci/docker-compose/integration-pinot.yml +++ b/scripts/ci/docker-compose/integration-pinot.yml @@ -23,8 +23,6 @@ services: breeze.description: "Integration required for Apache Pinot hooks." ports: - "9080:9080" - volumes: - - /dev/urandom:/dev/random # Required to get non-blocking entropy source command: QuickStart -type batch healthcheck: test: curl -f http://localhost:8000/health diff --git a/scripts/ci/docker-compose/integration-trino.yml b/scripts/ci/docker-compose/integration-trino.yml index 3c57f6222404f..6c49c6e9e17ee 100644 --- a/scripts/ci/docker-compose/integration-trino.yml +++ b/scripts/ci/docker-compose/integration-trino.yml @@ -42,7 +42,6 @@ services: - "37778-37786:7778" volumes: - - /dev/urandom:/dev/random # Required to get non-blocking entropy source - ../dockerfiles/krb5-kdc-server/krb5.conf:/etc/krb5.conf:ro - trino-db-volume:/data/trino - kerberos-keytabs:/home/trino/kerberos-keytabs