docker-compose.pipeline.yaml

services:
  etl-pipeline:
    platform: linux/amd64
    image: ${DOCKER_NAMESPACE:-ghcr.io/zenysis}/${DOCKER_IMAGE_PREFIX:-harmony}-etl-pipeline:${DOCKER_TAG:-latest}
    pull_policy: ${PULL_POLICY:-always}
    volumes:
      - ${DRUID_SHARED_FOLDER:-/home/share}:/home/share
      - ${OUTPUT_PATH:-/data/output}:/zenysis/pipeline/out
      # Map minio config folder
      # When pipeline runs, mc should pick up $USER and point to this:
      - ${MC_CONFIG_PATH:-/home/zenysis/.mc}:/home/zenysis/.mc
      # When you bash into the pipeline, mc should point to this:
      - ${MC_CONFIG_PATH:-/home/zenysis/.mc}:/.mc
    environment:
      - DATABASE_URL=${DATABASE_URL}
      # Alembic uses POSTGRES_DB_URI
      - POSTGRES_DB_URI:=${POSTGRES_DB_URI}
      - ZEN_ENV=${ZEN_ENV}
      - DRUID_HOST=${DRUID_HOST}
      - COMMAND=${COMMAND-echo Specify a pipeline command, e.g. "./pipeline/harmony_demo/generate/zeus_generate run"}
      - DEFAULT_SECRET_KEY=${DEFAULT_SECRET_KEY:-changeme}
      - SQLALCHEMY_DATABASE_URI=${DATABASE_URL}
    deploy:
      resources:
        limits:
          memory: 50g
    user: ${PIPELINE_USER:-1000}:${PIPELINE_GROUP:-1000}
    command: [ "/bin/bash", "-c", "./docker/entrypoint_pipeline.sh" ]
volumes:
  mc_config:
    driver: local
    driver_opts:
      type: none
      o: bind
      device: ${MC_CONFIG_PATH:-/home/zenysis/.mc}