feat: improve docker-compose services boot sequence (#31747)

apache · Jan 10, 2025 · 7bd53a8 · 7bd53a8
1 parent 5f18e84
commit 7bd53a8
Show file tree

Hide file tree

Showing 28 changed files with 205 additions and 88 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -134,4 +134,4 @@ jobs:
         if: steps.check.outputs.docker
         shell: bash
         run: |
-          docker compose -f docker-compose-image-tag.yml up --exit-code-from superset-init
+          docker compose -f docker-compose-image-tag.yml up superset-init --exit-code-from superset-init
diff --git a/docker-compose-image-tag.yml b/docker-compose-image-tag.yml
@@ -22,9 +22,6 @@
 # unique random secure passwords and SECRET_KEY.
 # -----------------------------------------------------------------------
 x-superset-image: &superset-image apachesuperset.docker.scarf.sh/apache/superset:${TAG:-latest-dev}
-x-superset-depends-on: &superset-depends-on
-  - db
-  - redis
 x-superset-volumes:
   &superset-volumes # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
   - ./docker:/app/docker
@@ -64,8 +61,12 @@ services:
     restart: unless-stopped
     ports:
       - 8088:8088
-    depends_on: *superset-depends-on
+    depends_on:
+      superset-init:
+        condition: service_completed_successfully
     volumes: *superset-volumes
+    environment:
+      SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
 
   superset-init:
     image: *superset-image
@@ -76,11 +77,18 @@ services:
         required: true
       - path: docker/.env-local # optional override
         required: false
-    depends_on: *superset-depends-on
+    depends_on:
+      db:
+        condition: service_started
+      redis:
+        condition: service_started
     user: "root"
     volumes: *superset-volumes
     healthcheck:
       disable: true
+    environment:
+      SUPERSET_LOAD_EXAMPLES: "${SUPERSET_LOAD_EXAMPLES:-yes}"
+      SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
 
   superset-worker:
     image: *superset-image
@@ -92,7 +100,9 @@ services:
       - path: docker/.env-local # optional override
         required: false
     restart: unless-stopped
-    depends_on: *superset-depends-on
+    depends_on:
+      superset-init:
+        condition: service_completed_successfully
     user: "root"
     volumes: *superset-volumes
     healthcheck:
@@ -101,6 +111,8 @@ services:
           "CMD-SHELL",
           "celery -A superset.tasks.celery_app:app inspect ping -d celery@$$HOSTNAME",
         ]
+    environment:
+      SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
 
   superset-worker-beat:
     image: *superset-image
@@ -112,11 +124,15 @@ services:
       - path: docker/.env-local # optional override
         required: false
     restart: unless-stopped
-    depends_on: *superset-depends-on
+    depends_on:
+      superset-init:
+        condition: service_completed_successfully
     user: "root"
     volumes: *superset-volumes
     healthcheck:
       disable: true
+    environment:
+      SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
 
 volumes:
   superset_home:

diff --git a/docker-compose-non-dev.yml b/docker-compose-non-dev.yml
@@ -21,9 +21,6 @@
 # create you own docker environment file (docker/.env) with your own
 # unique random secure passwords and SECRET_KEY.
 # -----------------------------------------------------------------------
-x-superset-depends-on: &superset-depends-on
-  - db
-  - redis
 x-superset-volumes:
   &superset-volumes # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
   - ./docker:/app/docker
@@ -70,8 +67,12 @@ services:
     restart: unless-stopped
     ports:
       - 8088:8088
-    depends_on: *superset-depends-on
+    depends_on:
+      superset-init:
+        condition: service_completed_successfully
     volumes: *superset-volumes
+    environment:
+      SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
 
   superset-init:
     container_name: superset_init
@@ -83,11 +84,18 @@ services:
         required: true
       - path: docker/.env-local # optional override
         required: false
-    depends_on: *superset-depends-on
+    depends_on:
+      db:
+        condition: service_started
+      redis:
+        condition: service_started
     user: "root"
     volumes: *superset-volumes
     healthcheck:
       disable: true
+    environment:
+      SUPERSET_LOAD_EXAMPLES: "${SUPERSET_LOAD_EXAMPLES:-yes}"
+      SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
 
   superset-worker:
     build:
@@ -100,7 +108,9 @@ services:
       - path: docker/.env-local # optional override
         required: false
     restart: unless-stopped
-    depends_on: *superset-depends-on
+    depends_on:
+      superset-init:
+        condition: service_completed_successfully
     user: "root"
     volumes: *superset-volumes
     healthcheck:
@@ -109,6 +119,8 @@ services:
           "CMD-SHELL",
           "celery -A superset.tasks.celery_app:app inspect ping -d celery@$$HOSTNAME",
         ]
+    environment:
+      SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
 
   superset-worker-beat:
     build:
@@ -121,11 +133,15 @@ services:
       - path: docker/.env-local # optional override
         required: false
     restart: unless-stopped
-    depends_on: *superset-depends-on
+    depends_on:
+      superset-init:
+        condition: service_completed_successfully
     user: "root"
     volumes: *superset-volumes
     healthcheck:
       disable: true
+    environment:
+      SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
 
 volumes:
   superset_home:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -22,9 +22,6 @@
 # unique random secure passwords and SECRET_KEY.
 # -----------------------------------------------------------------------
 x-superset-user: &superset-user root
-x-superset-depends-on: &superset-depends-on
-  - db
-  - redis
 x-superset-volumes: &superset-volumes
   # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
   - ./docker:/app/docker
@@ -95,10 +92,13 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
     user: *superset-user
-    depends_on: *superset-depends-on
+    depends_on:
+      superset-init:
+        condition: service_completed_successfully
     volumes: *superset-volumes
     environment:
       CYPRESS_CONFIG: "${CYPRESS_CONFIG:-}"
+      SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
 
   superset-websocket:
     container_name: superset_websocket
@@ -143,11 +143,17 @@ services:
         required: true
       - path: docker/.env-local # optional override
         required: false
-    depends_on: *superset-depends-on
+    depends_on:
+      db:
+        condition: service_started
+      redis:
+        condition: service_started
     user: *superset-user
     volumes: *superset-volumes
     environment:
       CYPRESS_CONFIG: "${CYPRESS_CONFIG:-}"
+      SUPERSET_LOAD_EXAMPLES: "${SUPERSET_LOAD_EXAMPLES:-yes}"
+      SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
     healthcheck:
       disable: true
 
@@ -174,7 +180,6 @@ services:
         required: true
       - path: docker/.env-local # optional override
         required: false
-    depends_on: *superset-depends-on
     volumes: *superset-volumes
 
   superset-worker:
@@ -189,8 +194,12 @@ services:
         required: false
     environment:
       CELERYD_CONCURRENCY: 2
+      CYPRESS_CONFIG: "${CYPRESS_CONFIG:-}"
+      SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
     restart: unless-stopped
-    depends_on: *superset-depends-on
+    depends_on:
+      superset-init:
+        condition: service_completed_successfully
     user: *superset-user
     volumes: *superset-volumes
     extra_hosts:
@@ -212,11 +221,15 @@ services:
       - path: docker/.env-local # optional override
         required: false
     restart: unless-stopped
-    depends_on: *superset-depends-on
+    depends_on:
+      - superset-worker
     user: *superset-user
     volumes: *superset-volumes
     healthcheck:
       disable: true
+    environment:
+      CYPRESS_CONFIG: "${CYPRESS_CONFIG:-}"
+      SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
 
   superset-tests-worker:
     build:
@@ -237,8 +250,11 @@ services:
       REDIS_RESULTS_DB: 3
       REDIS_HOST: localhost
       CELERYD_CONCURRENCY: 8
+      SUPERSET_LOG_LEVEL: "${SUPERSET_LOG_LEVEL:-info}"
     network_mode: host
-    depends_on: *superset-depends-on
+    depends_on:
+      superset-init:
+        condition: service_completed_successfully
     user: *superset-user
     volumes: *superset-volumes
     healthcheck:

diff --git a/docker/.env b/docker/.env
@@ -66,3 +66,4 @@ SUPERSET_SECRET_KEY=TEST_NON_DEV_SECRET
 ENABLE_PLAYWRIGHT=false
 PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
 BUILD_SUPERSET_FRONTEND_IN_DOCKER=true
+SUPERSET_LOG_LEVEL=info
diff --git a/docker/docker-init.sh b/docker/docker-init.sh
@@ -76,7 +76,7 @@ if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
         superset load_test_users
         superset load_examples --load-test-data
     else
-        superset load_examples --force
+        superset load_examples
     fi
     echo_step "4" "Complete" "Loading examples"
 fi
diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py
@@ -104,6 +104,9 @@ class CeleryConfig:
 WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL
 SQLLAB_CTAS_NO_LIMIT = True
 
+log_level_text = os.getenv("SUPERSET_LOG_LEVEL", "INFO")
+LOG_LEVEL = getattr(logging, log_level_text.upper(), logging.INFO)
+
 #
 # Optionally import superset_config_docker.py (which will have been included on
 # the PYTHONPATH) in order to allow for local settings to be overridden

diff --git a/docs/docs/contributing/development.mdx b/docs/docs/contributing/development.mdx
@@ -72,13 +72,18 @@ documentation.
 configured to be secure.
 :::
 
+
 ### Supported environment variables
 
 Affecting the Docker build process:
 - **SUPERSET_BUILD_TARGET (default=dev):** which --target to build, either `lean` or `dev` are commonly used
 - **INCLUDE_FIREFOX (default=false):** whether to include the Firefox headless browser in the build
 - **INCLUDE_CHROMIUM (default=false):** whether to include the Firefox headless browser in the build
 - **BUILD_TRANSLATIONS(default=false):** whether to compile the translations from the .po files available
+- **SUPERSET_LOAD_EXAMPLES (default=yes):** whether to load the examples into the database upon startup,
+  save some precious time on startup by `SUPERSET_LOAD_EXAMPLES=no docker compose up`
+- **SUPERSET_LOG_LEVEL (default=info)**: Can be set to debug, info, warning, error, critical
+  for more verbose logging
 
 For more env vars that affect your configuration, see this
 [superset_config.py](https://github.com/apache/superset/blob/master/docker/pythonpath_dev/superset_config.py)

diff --git a/superset/cli/examples.py b/superset/cli/examples.py
@@ -32,56 +32,56 @@ def load_examples_run(
     force: bool = False,
 ) -> None:
     if only_metadata:
-        print("Loading examples metadata")
+        logger.info("Loading examples metadata")
     else:
         examples_db = database_utils.get_example_database()
-        print(f"Loading examples metadata and related data into {examples_db}")
+        logger.info(f"Loading examples metadata and related data into {examples_db}")
 
     # pylint: disable=import-outside-toplevel
     import superset.examples.data_loading as examples
 
     examples.load_css_templates()
 
     if load_test_data:
-        print("Loading energy related dataset")
+        logger.info("Loading energy related dataset")
         examples.load_energy(only_metadata, force)
 
-    print("Loading [World Bank's Health Nutrition and Population Stats]")
+    logger.info("Loading [World Bank's Health Nutrition and Population Stats]")
     examples.load_world_bank_health_n_pop(only_metadata, force)
 
-    print("Loading [Birth names]")
+    logger.info("Loading [Birth names]")
     examples.load_birth_names(only_metadata, force)
 
     if load_test_data:
-        print("Loading [Tabbed dashboard]")
+        logger.info("Loading [Tabbed dashboard]")
         examples.load_tabbed_dashboard(only_metadata)
 
-        print("Loading [Supported Charts Dashboard]")
+        logger.info("Loading [Supported Charts Dashboard]")
         examples.load_supported_charts_dashboard()
     else:
-        print("Loading [Random long/lat data]")
+        logger.info("Loading [Random long/lat data]")
         examples.load_long_lat_data(only_metadata, force)
 
-        print("Loading [Country Map data]")
+        logger.info("Loading [Country Map data]")
         examples.load_country_map_data(only_metadata, force)
 
-        print("Loading [San Francisco population polygons]")
+        logger.info("Loading [San Francisco population polygons]")
         examples.load_sf_population_polygons(only_metadata, force)
 
-        print("Loading [Flights data]")
+        logger.info("Loading [Flights data]")
         examples.load_flights(only_metadata, force)
 
-        print("Loading [BART lines]")
+        logger.info("Loading [BART lines]")
         examples.load_bart_lines(only_metadata, force)
 
-        print("Loading [Misc Charts] dashboard")
+        logger.info("Loading [Misc Charts] dashboard")
         examples.load_misc_dashboard()
 
-        print("Loading DECK.gl demo")
+        logger.info("Loading DECK.gl demo")
         examples.load_deck_dash()
 
     if load_big_data:
-        print("Loading big synthetic data for tests")
+        logger.info("Loading big synthetic data for tests")
         examples.load_big_data()
 
     # load examples that are stored as YAML config files