docker-compose.yml

# compose "namespace", prepended to all network names, container names, etc.
name: ipl

# Define placeholder for running a container with the same UID/GID as your local user
x-local-user: &local-user ${DOCKER_LOCAL_USER:?Variable needs to be set in .env (e.g. "DOCKER_LOCAL_USER=1000:1000")}

x-ocpdb-defaults: &ocpdb-defaults
  image: ${OCPDB_IMAGE}
  networks: [ipl]
  user: *local-user
  environment:
    OCPDB_POSTGRES_DB: ${OCPDB_POSTGRES_DB:?'missing/empty'}
    OCPDB_POSTGRES_USER: ${OCPDB_POSTGRES_USER:?'missing/empty'}
    OCPDB_POSTGRES_PASSWORD: ${OCPDB_POSTGRES_PASSWORD:?'missing/empty'}
    OCPDB_POSTGRES_HOST: ${OCPDB_POSTGRES_HOST}
    OCPDB_PROJECT_URL: ${OCPDB_PROJECT_URL}
    OCPDB_CELERY_BROKER_URL: ${OCPDB_CELERY_BROKER_URL}
  depends_on:
    ocpdb-db:
      condition: service_healthy
    ocpdb-rabbitmq:
      condition: service_healthy
  volumes:
    - ./etc/ocpdb/config.yaml:/app/config.yaml
    - ./etc/ocpdb/config.secrets.yaml:/app/config.secrets.yaml
    - ./var/ocpdb/logs:/app/logs
    - ./var/ocpdb/temp:/app/temp

x-park-api-defaults: &park-api-defaults
  image: ${PARK_API_IMAGE}
  networks: [ipl]
  user: ${DOCKER_LOCAL_USER}
  environment:
    PARK_API_POSTGRES_DB: ${PARK_API_POSTGRES_DB:?'missing/empty'}
    PARK_API_POSTGRES_USER: ${PARK_API_POSTGRES_USER:?'missing/empty'}
    PARK_API_POSTGRES_PASSWORD: ${PARK_API_POSTGRES_PASSWORD:?'missing/empty'}
    PARK_API_POSTGRES_HOST: ${PARK_API_POSTGRES_HOST}
    PARK_API_PROJECT_URL: ${PARK_API_PROJECT_URL:?'missing/empty'}
    PARK_API_CELERY_BROKER_URL: ${PARK_API_CELERY_BROKER_URL}
  depends_on:
    park-api-db:
      condition: service_healthy
    park-api-rabbitmq:
      condition: service_healthy
  volumes:
    - ./etc/park-api/config.yaml:/app/config.yaml
    - ./etc/park-api/config.secrets.yaml:/app/config.secrets.yaml
    - ./etc/park-api/files:/app/files
    - ./var/park-api/logs:/app/logs
    - ./var/park-api/temp:/app/temp

x-generic-rabbitmq: &generic-rabbitmq
  networks: [ipl]
  image: rabbitmq:3.13
  user: rabbitmq  # required due eacces-issue: https://github.com/docker-library/rabbitmq/issues/318
  environment:
    # Disable spammy logging
    RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: '-rabbit log [{console,[{level,warning}]}]'
  healthcheck:
    test: rabbitmq-diagnostics -q ping
    interval: 3s
    timeout: 1s
    retries: 20
  restart: unless-stopped

services:
  # Traefik is our ingress: requests enter our infrastructure through it.
  # It is an HTTP reverse proxy, discovering available services (and their containers)
  # automatically by using the Docker API.
  # Trafik can't run as $DOCKER_LOCAL_USER (see &local-user), because service discovery just works with root.
  ingress:
    networks: [ipl]
    image: traefik:v3.2
    ports:
      # todo: change to 80:80
      - "8080:80"
      # # todo: change to 443:443
      # - "8443:443"
      # todo: change to 8080:8080
      # web UI, Prometheus, etc.
      - "8081:8080"
    environment:
      TZ: ${TZ:?'missing/empty $TZ'}
    volumes:
      - ./etc/traefik:/etc/traefik
      - ./var/log/traefik:/var/log/traefik
      # So that Traefik can listen to the Docker events.
      - /var/run/docker.sock:/var/run/docker.sock
    healthcheck:
      test: "traefik healthcheck"
      interval: 3s
      timeout: 1s
      retries: 10
    restart: unless-stopped

  goaccess:
    # > profiles defines a list of named profiles for the service to be enabled under. If unassigned, the service is always started but if assigned, it is only started if the profile is activated.
    # https://github.com/compose-spec/compose-spec/blob/77cc0f9575b560c94ca2a3b94db126c54c9e4759/spec.md#profiles
    profiles:
      - import-logs
    networks: [ipl]
    user: *local-user
    image: ${GOACCESS_IMAGE}
    environment:
      - TZ=${TZ:?missing/empty}
      - GOACCESS_KEEP_LAST_DAYS=${GOACCESS_KEEP_LAST_DAYS:?missing/empty}
    volumes:
      # entrypoint script
      - ./bin/goaccess/import.sh:/opt/import.sh:ro
      # GoAccess configuration file
      - ./etc/goaccess.conf:/etc/goaccess.conf:ro
      # data written by GoAccess itself: statistics database, generated HTML report, processing logs
      - ./var/goaccess/data:/srv/goaccess/data
      - ./var/goaccess/report:/srv/goaccess/report
      - ./var/log/goaccess:/srv/goaccess/logs
      # Traefik access logs
      - ./var/log/traefik:/var/log/traefik:ro
    entrypoint: ["/opt/import.sh"]

  # This service applies a couple of conversions for certain GBFS feeds to fix
  # syntax issues which prevent importing in Lamassu.
  # It needs to be configured as proxy for Lamassu and feeds to be filtered should
  # explicitly be requests using http. Proxy will take care of https requesting and
  # necessary response rewrites.
  transformer-proxy:
    networks: [ipl]
    image: ${TRANSFORMER_PROXY_IMAGE}
    environment:
      # mapped to the `termlog_verbosity` config underneath
      # https://docs.mitmproxy.org/stable/concepts-options/#available-options
      LOG_LEVEL: warn
    volumes:
      # Mount config file which declares hosts to proxy
      - ./etc/transformer-proxy/config.yaml:/app/config.yaml
    ports:
      # We export the port for debugging purposes. Lamassu acceses via docker network
      - ${TRANSFORMER_PROXY_PORT}:8080
    restart: unless-stopped

  caddy:
    networks: [ipl]
    image: ${CADDY_IMAGE}
    restart: unless-stopped
    ports:
      # public:
      # - index page
      # - IPL assets, e.g. traffic data
      # - well-known URIs (security.txt, favicon.ico, etc.)
      - 6999:80
      # GoAccess access statistics reports
      - "${GOACCESS_PORT:?'missing/empty env var $GOACCESS_PORT'}:81"
    volumes:
      # Caddy configuration file
      - ./etc/caddy/Caddyfile:/etc/caddy/Caddyfile:ro
      # data stored by Caddy itself: certificates, parsed & normalized config, etc.
      - ./var/caddy/data:/data
      - ./var/caddy/config:/config

      # index page (served on :80)
      - ./etc/index-page:/var/www/index-page:ro
      # public IPL assets, e.g. traffic data (served on :80)
      - ./var/www/datasets:/var/www/datasets:ro
      # webcam data (served on :80, password-protected by the `ingress` service)
      - ./var/webcam:/var/www/webcam:ro
      # non-public gbfs feeds
      - ./var/gbfs/feeds/free2move_stuttgart:/var/www/gbfs/free2move_stuttgart:ro
      # well-known URIs: security.txt, favicon.ico, etc. (served on :80)
      - ./etc/well-known:/var/www/well-known:ro
      # GoAccess access statistics reports (served on :81)
      - ./var/goaccess/report:/var/goaccess/report:ro
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.caddy.rule=Path(`/`) || PathPrefix(`/datasets`) || PathPrefix(`/.well-known/`) || Path(`/robots.txt`) || Path(`/favicon.ico`)"
      - "traefik.http.routers.caddy-webcam.rule=PathPrefix(`/webcam`)"
      - "traefik.http.services.caddy.loadbalancer.server.port=80"
      - "traefik.http.routers.caddy-webcam.middlewares=caddy-webcam"
      - "traefik.http.middlewares.caddy-webcam.basicauth.users=${WEBCAM_BASIC_AUTH:?missing/empty}"
      - "traefik.http.routers.caddy-gbfs.rule=PathPrefix(`/gbfs`)"
      - "traefik.http.routers.caddy-gbfs.middlewares=caddy-gbfs"
      - "traefik.http.middlewares.caddy-gbfs.basicauth.users=${GBFS_BASIC_AUTH:?missing/empty}"

  # This service requests non-standard sharing provider APIs and generates
  # a gbfs feed intended for local import into Lamassu. The volume needs to be
  # mounted by the lamassu container also.
  x2gbfs:
    networks: [ipl]
    image: ${X2GBFS_IMAGE}
    volumes:
      - ./var/gbfs/feeds:/app/out/
      - ./var/gbfs/temp:/app/temp/
    command: -p ${X2GBFS_PROVIDERS:?missing/empty} -b file:///var/gbfs/feeds -i ${X2GBFS_UPDATE_INTERVAL_SECONDS:?missing/empty}
    environment:
      - DEER_API_URL
      - DEER_USER=${DEER_USER:?missing/empty}
      - DEER_PASSWORD=${DEER_PASSWORD:?missing/empty}
      - CANTAMEN_IXSI_API_URL=${CANTAMEN_IXSI_API_URL:?missing/empty}
      - CANTAMEN_IXSI_API_TIMEOUT=${CANTAMEN_IXSI_API_TIMEOUT:?missing/empty}
      - MOQO_API_TOKEN=${MOQO_API_TOKEN:?missing/empty}
      - FLINKSTER_CLIENT_ID=${FLINKSTER_CLIENT_ID:?missing/empty}
      - FLINKSTER_SECRET=${FLINKSTER_SECRET:?missing/empty}
      - FREE2MOVE_USER=${X2GBFS_FREE2MOVE_USER:?missing/empty}
      - FREE2MOVE_PASSWORD=${X2GBFS_FREE2MOVE_PASSWORD:?missing/empty}
      - CACHE_DIR=/app/temp
    restart: unless-stopped
    healthcheck:
      test: ps aux | grep -q 'x2gbfs\.x2gbfs' || exit 1
      # If you want to wait until all feeds have been generated 
      # before depending lamassu starts subscribing them, 
      # configure an appropriate startup_period, e.g. 60s
      # Otherwise, lamassu might update them only after the 
      # update interval period has passed, which for test might
      # last several hours.
      start_interval: ${X2GBFS_HEALTHCHECK_START_INTERVAL:-1s}
      start_period: 1s
      interval: 60s

  lamassu:
    networks: [ipl]
    image: ${LAMASSU_IMAGE}
    ports:
      - ${LAMASSU_PORT}:8080
      - ${LAMASSU_ADMIN_PORT}:9001
    environment:
      LAMASSU_DB_CONNECT_CLIENT_ID: ${LAMASSU_DB_CONNECT_CLIENT_ID:?missing/empty}
      LAMASSU_DB_CONNECT_API_KEY: ${LAMASSU_DB_CONNECT_API_KEY:?missing/empty}
      LAMASSU_BOLT_AUTH_URL: ${LAMASSU_BOLT_AUTH_URL:?missing/empty}
      LAMASSU_BOLT_CLIENT_ID: ${LAMASSU_BOLT_CLIENT_ID:?missing/empty}
      LAMASSU_BOLT_CLIENT_PASSWORD: ${LAMASSU_BOLT_CLIENT_PASSWORD:?missing/empty}
      LAMASSU_TIER_API_URL: ${LAMASSU_TIER_API_URL:?missing/empty}
      LAMASSU_TIER_USER: ${LAMASSU_TIER_USER:?missing/empty}
      LAMASSU_TIER_PASSWORD: ${LAMASSU_TIER_PASSWORD:?missing/empty}
      LAMASSU_VOI_AUTH_URL: ${LAMASSU_VOI_AUTH_URL:?missing/empty}
      LAMASSU_VOI_CLIENT_ID: ${LAMASSU_VOI_CLIENT_ID:?missing/empty}
      LAMASSU_VOI_CLIENT_PASSWORD: ${LAMASSU_VOI_CLIENT_PASSWORD:?missing/empty}
      LAMASSU_SHAREDMOBILITY_CH_AUTHORIZATION: ${LAMASSU_SHAREDMOBILITY_CH_AUTHORIZATION:?missing/empty}
      LAMASSU_BASE_URL: ${LAMASSU_BASE_URL}
      LAMASSU_FEED_UPATE_INTERVAL_IN_MS: ${LAMASSU_FEED_UPATE_INTERVAL_IN_MS:?missing/empty}
      LAMASSU_FEED_CACHE_MINIMUM_INTERVAL_IN_S: ${LAMASSU_FEED_CACHE_MINIMUM_INTERVAL_IN_S:?missing/empty}
      # depending on base image, JAVA_TOOL_OPTIONS must be used
      JDK_JAVA_OPTIONS:
        -Dspring.config.location=/etc/application-config/application.properties
        -Dspring.profiles.active=leader
        -Dorg.entur.lamassu.adminPassword=${LAMASSU_ADMIN_PASSWORD}
        -Dhttp.proxyHost=transformer-proxy
        -Dhttp.proxyPort=8080
    volumes:
      - ./etc/lamassu/:/etc/application-config/
      - ./var/gbfs/feeds:/var/gbfs/feeds
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.lamassu.rule=PathPrefix(`/sharing`)"
      - "traefik.http.routers.lamassu.middlewares=sharing-stripprefix"
      - "traefik.http.middlewares.sharing-stripprefix.stripprefix.prefixes=/sharing"
      - "traefik.http.services.lamassu.loadbalancer.server.port=8080"
      # Note: /admin is shifted to the management port in application.properties'
      # management.endpoints.web.exposure.include (currently including admin,info,health,prometheus)
    depends_on:
        redis:
          condition: service_healthy
        transformer-proxy:
          condition: service_started
        x2gbfs:
          condition: service_started
    restart: unless-stopped

  redis:
    networks: [ipl]
    image: redis:7-alpine
    # Redis tries to use all memory it gets, and doesn't query the container's memory limit.
    # see also https://stackoverflow.com/a/70779280
    # --save '' should explicitly disable saving to volume data, avoiding no space left on devide errors
    command: --maxmemory ${REDIS_MEMORY_MB:-256}mb --save ''
    deploy:
      resources:
        limits:
          memory: ${REDIS_MEMORY_MB:-256}M
    # The redis image doesn't provide a health check by itself.
    # see also https://github.com/docker-library/redis/issues/91
    healthcheck:
      test: "redis-cli --raw incr ping"
      interval: 1s
      timeout: 3s
      retries: 20
    restart: unless-stopped

  ipl-db:
    networks: [ipl]
    image: ${IPL_POSTGIS_IMAGE}
    volumes:
      - ./var/ipl-db/data:/var/lib/postgresql/data
    environment:
      - POSTGRES_DB=${IPL_POSTGRES_DB}
      - POSTGRES_USER=${IPL_POSTGRES_USER}
      - POSTGRES_PASSWORD=${IPL_POSTGRES_PASSWORD:?missing/empty}
    healthcheck:
      test: "PGPASSWORD=${IPL_POSTGRES_PASSWORD:?missing/empty} pg_isready -h 127.0.0.1 -U ${IPL_POSTGRES_USER} -d ${IPL_POSTGRES_DB}"
      interval: 5s
      timeout: 3s
      retries: 12
    restart: unless-stopped

  geoserver:
    networks: [ipl]
    image: ${GEOSERVER_IMAGE}
    user: *local-user
    volumes:
      # var/geoserver defaults to geoserver/data_dir, custom config shall explicitly be defined in etc/geoserver
      - ./var/geoserver/datadir/:/var/geoserver/datadir/
      - ./var/geoserver/logs/:/var/geoserver/logs/
      - ./var/geoserver/gwc_cache_dir/:/var/geoserver/gwc_cache_dir/
      - ./etc/geoserver/:/var/geoserver/datadir_template/
      # global.xml is written by geoserver before config reload, so we need to have it mounted already at start
      - ./etc/geoserver/global.xml:/var/geoserver/datadir/global.xml
      - ./etc/geoserver/logging.xml:/var/geoserver/datadir/logging.xml
      - ./bin/geoserver/geoserver-rest-config.sh:/usr/local/bin/geoserver-rest-config.sh
      - ./bin/geoserver/geoserver-rest-reload.sh:/usr/local/bin/geoserver-rest-reload.sh
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.geoserver.rule=PathPrefix(`/geoserver`)"
      - "traefik.http.services.geoserver.loadbalancer.server.port=8080"
    ports:
      - ${GEOSERVER_PORT}:8080
    environment:
      # Special env vars picked up by the kartoza/geoserver Docker image, getting passed in as Java system properties (`-D`).
      - ADMIN_PASSWORD=${GEOSERVER_ADMIN_PASSWORD}
      - INITIAL_MEMORY=${GEOSERVER_INITIAL_MEMORY}
      - MAXIMUM_MEMORY=${GEOSERVER_MAXIMUM_MEMORY}
      - GEOSERVER_DATA_DIR_CUSTOM=/var/geoserver/datadir_template/
      - GEOSERVER_LOGGING_PROFILE=${GEOSERVER_LOGGING_PROFILE}
      - GEOSERVER_CSRF_WHITELIST=${GEOSERVER_CSRF_WHITELIST:?missing/empty}
      - PLUGIN_DYNAMIC_URLS=${GEOSERVER_PLUGIN_DYNAMIC_URLS:?missing/empty}
      - PROXY_BASE_URL=${GEOSERVER_PROXY_BASE_URL:?missing/empty}
      # All we want to do is set user.timezone and DALLOW_ENV_PARAMETRIZATION. Due to https://github.com/geosolutions-it/docker-geoserver/issues/133
      # we currently need to pass in all default properties already defined in the dockerfile
      - JAVA_OPTS= >-
          -Duser.timezone=${TZ:?'missing/empty $TZ'}
          -DALLOW_ENV_PARAMETRIZATION=true
          -Xms$${INITIAL_MEMORY}
          -Xmx$${MAXIMUM_MEMORY}
          -Djava.awt.headless=true
          -server
          -Dfile.encoding=UTF8
          -Djavax.servlet.request.encoding=UTF-8
          -Djavax.servlet.response.encoding=UTF-8
          -XX:SoftRefLRUPolicyMSPerMB=36000
          -XX:+UseG1GC
          -XX:MaxGCPauseMillis=200
          -XX:ParallelGCThreads=20
          -XX:ConcGCThreads=5
          -Dorg.geotools.coverage.jaiext.enabled=$${JAIEXT_ENABLED}
          -Dorg.geotools.shapefile.datetime=true
          -DGEOSERVER_LOG_LOCATION=$${GEOSERVER_LOG_LOCATION}
          -DGEOWEBCACHE_CONFIG_DIR=$${GEOWEBCACHE_CONFIG_DIR}
          -DGEOWEBCACHE_CACHE_DIR=$${GEOWEBCACHE_CACHE_DIR}
          -DNETCDF_DATA_DIR=$${NETCDF_DATA_DIR}
          -DGRIB_CACHE_DIR=$${GRIB_CACHE_DIR}
      # The following parameters are *not* picked up by geosoultionsit/geoserver, but instead passed through as "regular" env vars, and then read by Geoserver whenever one of the config files references them.
      - PGBOUNCER_POSTGRES_USER=${PGBOUNCER_POSTGRES_USER:?missing/empty}
      - PGBOUNCER_POSTGRES_PASSWORD=${PGBOUNCER_POSTGRES_PASSWORD:?missing/empty}
      - PARK_API_POSTGRES_PASSWORD=${PARK_API_POSTGRES_PASSWORD?missing/empty}
      - OCPDB_POSTGRES_PASSWORD=${OCPDB_POSTGRES_PASSWORD:?missing/empty}
    depends_on:
      pgbouncer:
        # For sharing & transit layers
        condition: service_healthy
      park-api-db:
        # For parking_sites layer
        condition: service_healthy
      ocpdb-db:
        # For ocpdb layer
        condition: service_healthy
    healthcheck:
      test: "curl -fsS -o /dev/null -u 'admin:${GEOSERVER_ADMIN_PASSWORD}' http://localhost:8080/geoserver/rest/about/version.xml"
      interval: 0m10s
      timeout: 10s
      retries: 22
    restart: unless-stopped

  # This service runs the postgres DB used by dagster for run storage, schedule storage,
  # and event log storage.
  dagster-postgresql:
    networks: [ipl]
    image: ${DAGSTER_POSTGRES_IMAGE}
    environment:
      POSTGRES_USER: "${DAGSTER_POSTGRES_USER}"
      POSTGRES_PASSWORD: "${DAGSTER_POSTGRES_PASSWORD}"
      POSTGRES_DB: "${DAGSTER_POSTGRES_DB}"
    volumes:
        - ./var/dagster-postgresql/data:/var/lib/postgresql/data
    healthcheck:
        test: "PGPASSWORD=${DAGSTER_POSTGRES_PASSWORD} pg_isready -h 127.0.0.1 -U ${DAGSTER_POSTGRES_USER} -d ${DAGSTER_POSTGRES_DB}"
        interval: 5s
        timeout: 3s
        retries: 12
    restart: unless-stopped

  # This service runs the gRPC server that loads your user code, in both dagster-webserver
  # and dagster-daemon. Multiple containers like this can be deployed separately - each just needs to run on
  # its own port, and have its own entry in the workspace.yaml file that's loaded by
  # dagster-webserver.
  dagster-pipeline:
    networks: [ipl]
    image: ${DAGSTER_PIPELINE_IMAGE}
    environment:
      # connect via pgbouncer to improve performance
      - PGHOST=pgbouncer
      - PGPORT=6432
      - PGUSER=${PGBOUNCER_POSTGRES_USER:?missing/empty}
      - PGPASSWORD=${PGBOUNCER_POSTGRES_PASSWORD:?missing/empty}
      - PGDATABASE=dagster # determined by pgbouncer!
      # Use docker hostname of lamassu service to avoid roundtrip via proxy service
      # todo [breaking]: remove `IPL_LAMASSU_BASE_URL` to prevent confusion with `LAMASSU_BASE_URL`
      # temporarilly remove trainling slash as pipeline adds another / which causes the spring firewall to block
      - IPL_LAMASSU_BASE_URL=http://lamassu:8080
      - IPL_LAMASSU_INTERNAL_BASE_URL=http://lamassu:8080/
      - IPL_POSTGRES_HOST=ipl-db
      - IPL_POSTGRES_PORT=5432
      - IPL_POSTGRES_DB
      - IPL_POSTGRES_USER
      - IPL_POSTGRES_PASSWORD
      # Variables required for GTFS-Import
      - IPL_GTFS_IMPORTER_IMAGE
      - IPL_GTFS_IMPORTER_NETWORK
      - IPL_GTFS_IMPORTER_GTFS_DOWNLOAD_URL
      - IPL_GTFS_IMPORTER_GTFS_DOWNLOAD_USER_AGENT
      - IPL_GTFS_DB_POSTGRES_HOST
      - IPL_GTFS_DB_POSTGRES_USER
      - IPL_GTFS_DB_POSTGRES_PASSWORD
      - IPL_GTFS_DB_POSTGRES_DB
      - IPL_GTFS_DB_POSTGRES_DB_PREFIX
      - IPL_GTFS_IMPORTER_SCHEMA=${IPL_GTFS_IMPORTER_SCHEMA:?missing/empty}
      - IPL_GTFS_DB_POSTGREST_USER
      - IPL_GTFS_DB_POSTGREST_PASSWORD
      # Required to mount volumes for docker-in-docker executions
      - IPL_GTFS_IMPORTER_HOST_CUSTOM_SCRIPTS_DIR=${PWD}/etc/gtfs/
      - IPL_GTFS_IMPORTER_HOST_GTFS_OUTPUT_DIR=${PWD}/var/gtfs/
      # directory where datasets should stored for publishing (Note: should be mounted as volume)
      - WWW_ROOT_DIR=/var/www/
      # RadVIS Download credentials
      - RADVIS_WFS_USER=${RADVIS_WFS_USER:?missing/empty}
      - RADVIS_WFS_PASSWORD=${RADVIS_WFS_PASSWORD:?missing/empty}
      # Roadworks download url
      - ROADWORKS_SVZBW_DATEX2_DOWNLOAD_URL=${ROADWORKS_SVZBW_DATEX2_DOWNLOAD_URL:?missing/empty}
      # Webcam settings
      - IPL_WEBCAM_SERVER=${WEBCAM_FTP_SERVER:?missing/empty}
      - IPL_WEBCAM_USER=${WEBCAM_FTP_USER:?missing/empty}
      - IPL_WEBCAM_PASSWORD=${WEBCAM_FTP_PASSWORD:?missing/empty}
      - IPL_WEBCAM_KEEP_DAYS=${WEBCAM_KEEP_DAYS:?missing/empty}
      - IPL_WEBCAM_WORKER=${WEBCAM_WORKER:?missing/empty}
    volumes:
      # Make docker API accessible so we can start containers in jobs
      - /var/run/docker.sock:/var/run/docker.sock
      # mount storage dir so logs/assets are written to volume shared with dagaster-daemon/dagit
      - ./var/dagster/storage/:/opt/dagster/dagster_home/storage/
      # mount www dir to republish downloaded datasets
      - ./var/www/datasets/:/var/www/
      # Mount webcam dir
      - ./var/webcam:/var/webcam
    depends_on:
        pgbouncer:
          condition: service_healthy
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "/bin/bash", "printf", "'GET / HTTP/1.1\n\n'", ">/dev/tcp/127.0.0.1/4000"]
      timeout: 1s
      interval: 1s
      start_period: 2s
      retries: 15
  # This service runs dagster-webserver, which loads your user code from the user code container.
  # Since our instance uses the QueuedRunCoordinator, any runs submitted from dagster-webserver
  # will be put on a queue and later dequeued and launched by dagster-daemon.
  dagster-dagit:
    networks: [ipl]
    image: ${DAGSTER_DAGIT_IMAGE}
    # expose:
    #   - "3000"
    ports:
      - "3000:3000"
    environment:
      # connect via pgbouncer to improve performance
      PGHOST: pgbouncer
      PGPORT: 6432
      PGUSER: ${PGBOUNCER_POSTGRES_USER:?missing/empty}
      PGPASSWORD: ${PGBOUNCER_POSTGRES_PASSWORD:?missing/empty}
      PGDATABASE: dagster # determined by pgbouncer!
    volumes: # Make docker API accessible so we can terminate containers from dagster-webserver
      - /var/run/docker.sock:/var/run/docker.sock
      - /tmp/io_manager_storage:/tmp/io_manager_storage
      - ./etc/dagster:/opt/dagster/dagster_home/
      - ./var/dagster/storage/:/opt/dagster/dagster_home/storage/
    depends_on:
      pgbouncer:
        condition: service_healthy
      dagster-pipeline:
        condition: service_started
    restart: unless-stopped

  # This service runs the dagster-daemon process, which is responsible for taking runs
  # off of the queue and launching them, as well as creating runs from schedules or sensors.
  dagster-daemon:
    networks: [ipl]
    image: ${DAGSTER_DAEMON_IMAGE}
    environment:
      # connect via pgbouncer to improve performance
      PGHOST: pgbouncer
      PGPORT: 6432
      PGUSER: ${PGBOUNCER_POSTGRES_USER:?missing/empty}
      PGPASSWORD: ${PGBOUNCER_POSTGRES_PASSWORD:?missing/empty}
      PGDATABASE: dagster # determined by pgbouncer!
    volumes: # Make docker API accessible so we can launch containers using host docker
      - /var/run/docker.sock:/var/run/docker.sock
      - /tmp/io_manager_storage:/tmp/io_manager_storage
      - ./etc/dagster:/opt/dagster/dagster_home/
      - ./var/dagster/storage/:/opt/dagster/dagster_home/storage/
    depends_on:
      pgbouncer:
        condition: service_healthy
      dagster-pipeline:
        condition: service_started
    restart: unless-stopped

  # This service stores GTFS data imported by gtfs-importer.
  gtfs-db:
    networks: [ipl]
    image: ${IPL_GTFS_DB_IMAGE}
    volumes:
      - ./var/gtfs/gtfs-db:/var/lib/postgresql/data
    environment:
      POSTGRES_USER: ${IPL_GTFS_DB_POSTGRES_USER}
      POSTGRES_PASSWORD: ${IPL_GTFS_DB_POSTGRES_PASSWORD}
      POSTGRES_DB: ${IPL_GTFS_DB_POSTGRES_DB}
    healthcheck:
      test: "PGPASSWORD=${IPL_GTFS_DB_POSTGRES_PASSWORD} pg_isready -h 127.0.0.1 -U ${IPL_GTFS_DB_POSTGRES_USER} -d ${IPL_GTFS_DB_POSTGRES_DB}"
      timeout: 3s
      interval: 5s
      start_period: 10s
      retries: 10
    restart: unless-stopped

  # pgbouncer acts like a facade in front of PostgreSQL instances (ipl-db, gtfs-db, dagster-postgresql), providing
  # - programmatically configurable query routing (used by `make import-new-gtfs`)
  # - improved connect performance for clients, especially Dagster workers
  pgbouncer:
    networks: [ipl]
    depends_on:
      ipl-db:
        condition: service_started
      gtfs-db:
        condition: service_started
      dagster-postgresql:
        condition: service_started
    links:
      - ipl-db
      - gtfs-db
      - dagster-postgresql
    image: ${PGBOUNCER_IMAGE}
    volumes:
      # contains the latest import's DB name
      - ./var/gtfs/pgbouncer-dsn.txt:/var/gtfs-pgbouncer-dsn.txt
      - ./etc/reload-pgbouncer-databases.sh:/reload-pgbouncer-databases.sh
    environment:
      # Even if we define all upstream database connections "manually" using $PGBOUNCER_DSN_* (see below), the bitnami/pgbouncer Docker image expects 1 connection to be specified via $POSTGRESQL_*, which it implicitly adds to the `[database]` section of pgbouncer.ini. Note that the specified database credentials must be valid.
      # However, the $POSTGRESQL_* env vars are *also* used to configure client access to pgbouncer: They get implicitly added to the generated userlist.txt, meaning that clients *must* use them (if there are no other user/password pairs defined, which we don't do) to connect to *any* exposed connection.
      # Therefore, we duplicate the connection to ipl-db and expose it as `meta`, clearly denoting it as a connection *not* intended for pgbouncer clients.
      PGBOUNCER_DATABASE: meta
      POSTGRESQL_HOST: ipl-db
      POSTGRESQL_DATABASE: ${IPL_POSTGRES_DB:?missing/empty}
      POSTGRESQL_USERNAME: ${PGBOUNCER_POSTGRES_USER:?missing/empty}
      POSTGRESQL_PASSWORD: ${PGBOUNCER_POSTGRES_PASSWORD:?missing/empty}
      # > Most polite method. When a client connects, a server connection will be assigned to it for the whole duration the client stays connected. When the client disconnects, the server connection will be put back into the pool. This is the default method.
      # We hardcode this, because we definitely don't want the other pooling modes.
      PGBOUNCER_POOL_MODE: session
      # > Disconnect a server in session pooling mode immediately or after the end of the current transaction if it is in “close_needed” mode (set by RECONNECT, RELOAD that changes connection settings, or DNS change), rather than waiting for the session end. […]
      # > If because of this setting a server connection is closed before the end of the client session, the client connection is also closed. This ensures that the client notices that the session has been interrupted.
      # > This setting makes connection configuration changes take effect sooner if session pooling and long-running sessions are used. […]
      PGBOUNCER_SERVER_FAST_CLOSE: '1'
      # todo: remove? should work without 🤔
      # see also https://github.com/bitnami/containers/issues/48636#issuecomment-1722518107
      PGBOUNCER_AUTH_TYPE: md5
      # Geoserver connects with `extra_float_digits` set.
      # see https://github.com/bitnami/containers/issues/25394#issuecomment-1457893192
      # dagster-* connect with `statement_timeout` set.
      PGBOUNCER_IGNORE_STARTUP_PARAMETERS: extra_float_digits, statement_timeout
      # reduce logging verbosity
      PGBOUNCER_LOG_CONNECTIONS: '0'
      PGBOUNCER_LOG_DISCONNECTIONS: '0'

      # upsteam connections
      # see also https://www.pgbouncer.org/config.html#section-databases
      # see also https://www.postgresql.org/docs/15/libpq-connect.html#id-1.7.3.8.3.5
      # expose ipl-db as `ipl`
      PGBOUNCER_DSN_0: "ipl=host=ipl-db dbname=${IPL_POSTGRES_DB:?missing/empty} user=${IPL_POSTGRES_USER:?missing/empty} password=${IPL_POSTGRES_PASSWORD:?missing/empty}"
      # expose dagster-postgresql as `dagster`
      PGBOUNCER_DSN_1: "dagster=host=dagster-postgresql dbname=${DAGSTER_POSTGRES_DB:?missing/empty} user=${DAGSTER_POSTGRES_USER:?missing/empty} password=${DAGSTER_POSTGRES_PASSWORD:?missing/empty}"
      # expose the latest GTFS import within gtfs-db as `gtfs`
      # $PGBOUNCER_DSN_2 gets generated from /var/gtfs-pgbouncer-dsn.txt (which is written by gtfs-importer) by the Docker image's entrypoint script.
      # see https://github.com/bitnami/containers/issues/46152#issuecomment-1695320501
      # todo: not yet, push & PR our changes, see `image` field above!
      PGBOUNCER_DSN_2_FILE: /var/gtfs-pgbouncer-dsn.txt
    restart: unless-stopped
    healthcheck:
      # pgbouncer exposes a `pgbouncer` "meta database", providing an interface for statistics and to administer the instance. We use it here to check if pgbouncer is working properly.
      test: 'env PGPASSWORD="$$POSTGRESQL_PASSWORD" psql -p 6432 -U "$$POSTGRESQL_USERNAME" pgbouncer -b -c "SHOW USERS" >/dev/null'
      interval: 0m15s
      timeout: 5s
      retries: 10

  # todo /arrivals_departures: enforce stop_id filtering?
  gtfs-api:
    networks: [ipl]
    depends_on:
      pgbouncer:
        condition: service_healthy
    links:
      - pgbouncer
    image: ${IPL_GTFS_API_IMAGE:?missing/empty}
    ports:
      - ${IPL_GTFS_API_PORT}:3000
      - ${IPL_GTFS_API_ADMIN_PORT:?missing/empty}:3001
    read_only: true
    environment:
      # connect via pgbouncer to regardless of the GTFS DBs' suffixes & improve performance
      PGHOST: pgbouncer
      PGPORT: 6432
      PGUSER: ${PGBOUNCER_POSTGRES_USER:?missing/empty}
      PGPASSWORD: ${PGBOUNCER_POSTGRES_PASSWORD:?missing/empty}
      PGDATABASE: gtfs # determined by pgbouncer!
      # PostgREST-specific env vars
      PGRST_OPENAPI_SERVER_PROXY_URI: ${IPL_GTFS_API_PUBLIC_BASE_URL}
      PGRST_DB_SCHEMAS: api
      PGRST_DB_ANON_ROLE: web_anon
      PGRST_DB_MAX_ROWS: ${IPL_GTFS_API_MAX_ROWS:?missing/empty}
      PGRST_ADMIN_SERVER_PORT: 3001 # exposes health check & metrics

      # > Also set db-channel-enabled to false since LISTEN is not compatible with transaction pooling. Although it should not give any errors if left enabled.
      # https://postgrest.org/en/stable/references/connection_pool.html#external-connection-poolers
      PGRST_DB_CHANNEL_ENABLED: 'False'
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.gtfs-api.rule=PathPrefix(`/gtfs`)"
      - "traefik.http.routers.gtfs-api.middlewares=gtfs-stripprefix"
      - "traefik.http.middlewares.gtfs-stripprefix.stripprefix.prefixes=/gtfs"
      - "traefik.http.services.gtfs-api.loadbalancer.server.port=3000"
    restart: unless-stopped

  gtfs-importer:
    networks: [ipl]
    # gtfstidy currently only provides a linux/amd64 binary
    # see also https://github.com/patrickbr/gtfstidy/pull/21
    platform: linux/amd64
    # gtfs-imports will be performed via dagster nevertheless, we keep it here for now
    # i.e. to have it downloaded
    image: ${IPL_GTFS_IMPORTER_IMAGE:?missing/empty}
    profiles:
      - import-new-gtfs
    depends_on:
      gtfs-db:
        condition: service_healthy
    links:
      - gtfs-db
    volumes:
      - ./var/gtfs:/var/gtfs
      - ./etc/gtfs:/etc/gtfs
      # mount modified download script
      - ./etc/gtfs/download.sh:/importer/download.sh
    environment:
      PGHOST: gtfs-db
      PGPORT: 5432
      PGUSER: ${IPL_GTFS_DB_POSTGRES_USER:?missing/empty}
      PGPASSWORD: ${IPL_GTFS_DB_POSTGRES_PASSWORD:?missing/empty}
      # the default DB, used by gtfs-importer to keep track of imports
      PGDATABASE: ${IPL_GTFS_DB_POSTGRES_DB:?missing/empty}
      GTFS_DOWNLOAD_URL: ${IPL_GTFS_IMPORTER_GTFS_DOWNLOAD_URL:?missing/empty}
      GTFS_DOWNLOAD_USER_AGENT: ${IPL_GTFS_IMPORTER_GTFS_DOWNLOAD_USER_AGENT:?missing/empty}
      # the prefix of all DBs created for imports by gtfs-importer
      GTFS_IMPORTER_DB_PREFIX: ${IPL_GTFS_DB_POSTGRES_DB_PREFIX:?missing/empty}
      GTFS_IMPORTER_SCHEMA: ${IPL_GTFS_IMPORTER_SCHEMA:?missing/empty}
      # gtfstidy: Don't remove stops duplicates (--remove-red-stops).
      GTFSTIDY_REMOVE_REDUNDANT_STOPS: 'false'
      # path to the file containing the latest import's DB name
      GTFS_IMPORTER_DSN_FILE: /var/gtfs/pgbouncer-dsn.txt
      GTFS_TMP_DIR: /var/gtfs
      POSTGREST_USER: ${IPL_GTFS_DB_POSTGREST_USER:?missing/empty}
      POSTGREST_PASSWORD: ${IPL_GTFS_DB_POSTGREST_PASSWORD:?missing/empty}
      POSTGREST_STATEMENT_COST_LIMIT: ${IPL_GTFS_API_COST_LIMIT:?missing/empty}

  gtfs-api-docs:
    networks: [ipl]
    depends_on:
      gtfs-api:
        condition: service_started
    # This is a 3rd-part Elements CLI built by the community.
    # https://github.com/skriptfabrik/elements-cli
    # see also https://github.com/stoplightio/elements/issues/765#issuecomment-1152620255
    image: skriptfabrik/elements-cli:0.5.3
    command:
      - "preview"
      # listen on container port 8080
      - "--port=8080"
      # communicate host port to clients
      - "--virtual-host=${IPL_GTFS_API_DOCS_PUBLIC_HOST:?missing/empty}"
      - "--virtual-port=${IPL_GTFS_API_DOCS_PUBLIC_PORT:?missing/empty}"
      - "--base-path=/docs/gtfs"
      # *public* URL of the gtfs-api OpenAPI spec
      - "${IPL_GTFS_API_PUBLIC_BASE_URL:?missing/empty}"
    ports:
      - ${IPL_GTFS_API_DOCS_PORT}:8080
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.gtfs-api-docs.rule=PathPrefix(`/docs/gtfs`)"
      - "traefik.http.services.gtfs-api-docs.loadbalancer.server.port=8080"
    restart: unless-stopped

  ocpdb-flask:
    <<: *ocpdb-defaults
    command: ["gunicorn", "--bind", "0.0.0.0:5000", "webapp.entry_point_gunicorn:app"]
    restart: unless-stopped
    ports:
      - ${OCPDB_API_PORT}:5000
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.ocpdb-flask.rule=PathPrefix(`/ocpdb`)"
      - "traefik.http.services.ocpdb-flask.loadbalancer.server.port=5000"
      - "traefik.http.routers.ocpdb-flask.middlewares=ocpdb-stripprefix"
      - "traefik.http.middlewares.ocpdb-stripprefix.stripprefix.prefixes=/ocpdb"

  ocpdb-worker:
    <<: *ocpdb-defaults
    restart: unless-stopped
    command: ["celery", "-A", "webapp.entry_point_celery", "worker", "-c", "2"]

  ocpdb-init:
    <<: *ocpdb-defaults
    command: ["sh", "-c", "flask db upgrade && flask bnetza import-web"]

  ocpdb-db:
    networks: [ipl]
    image: ${OCPDB_DB_IMAGE}
    volumes:
      - ./var/ocpdb/ocpdb-db:/var/lib/postgresql/data
    environment:
      - POSTGRES_USER=${OCPDB_POSTGRES_USER}
      - POSTGRES_PASSWORD=${OCPDB_POSTGRES_PASSWORD}
      - POSTGRES_DB=${OCPDB_POSTGRES_DB}
    healthcheck:
      test: "PGPASSWORD=${OCPDB_POSTGRES_PASSWORD} pg_isready -h 127.0.0.1 -U ${OCPDB_POSTGRES_USER} -d ${OCPDB_POSTGRES_DB}"
      interval: 10s
      timeout: 3s
      retries: 3
    restart: unless-stopped

  ocpdb-rabbitmq:
    <<: *generic-rabbitmq

  park-api-flask:
    <<: *park-api-defaults
    restart: unless-stopped
    command: ["gunicorn", "--bind", "0.0.0.0:5000", "--timeout", "300", "webapp.entry_point_gunicorn:app"]
    ports:
      - ${PARK_API_API_PORT}:5000
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.park-api-flask.rule=PathPrefix(`/park-api`)"
      - "traefik.http.services.park-api-flask.loadbalancer.server.port=5000"
      - "traefik.http.routers.park-api-flask.middlewares=park-api-stripprefix"
      - "traefik.http.middlewares.park-api-stripprefix.stripprefix.prefixes=/park-api"

  park-api-init:
    <<: *park-api-defaults
    command: ["bash", "-c", "flask db upgrade && flask source init-converters"]

  park-api-worker:
    <<: *park-api-defaults
    restart: unless-stopped
    command: ["celery", "-A", "webapp.entry_point_celery", "worker", "-c", "2"]

  park-api-heartbeat:
    <<: *park-api-defaults
    restart: unless-stopped
    command: ["celery", "-A", "webapp.entry_point_celery", "beat", "-s", "/tmp/celerybeat-schedule"]

  park-api-db:
    networks: [ipl]
    image: ${PARK_API_DB_IMAGE}
    volumes:
      - ./var/park-api/park-api-db:/var/lib/postgresql/data
    environment:
      - POSTGRES_USER=${PARK_API_POSTGRES_USER}
      - POSTGRES_PASSWORD=${PARK_API_POSTGRES_PASSWORD}
      - POSTGRES_DB=${PARK_API_POSTGRES_DB}
    healthcheck:
      test: "PGPASSWORD=${PARK_API_POSTGRES_PASSWORD} pg_isready -h 127.0.0.1 -U ${PARK_API_POSTGRES_USER} -d ${PARK_API_POSTGRES_DB}"
      interval: 10s
      timeout: 3s
      retries: 3
    restart: unless-stopped

  park-api-rabbitmq:
    <<: *generic-rabbitmq

  sftp:
    image: ${SFTP_IMAGE}
    networks: [ipl]
    ports:
      - "${SFTP_PORT}:22"
    volumes:
      - ./etc/sftp/users.conf:/etc/sftp/users.conf:ro
      - ./etc/sftp/ssh_host_ed25519_key:/etc/ssh/ssh_host_ed25519_key:ro
      - ./etc/sftp/ssh_host_rsa_key:/etc/ssh/ssh_host_rsa_key:ro
      - ./etc/sftp/polizeibw/keys:/home/polizeibw/.ssh/keys:ro
      - ./var/www/datasets/traffic/incidents-bw:/home/polizeibw/upload
    restart: unless-stopped

networks:
  ipl:
    driver: bridge
    ipam:
      driver: default
      config:
      - subnet: 172.28.0.0/16
        ip_range: 172.28.5.0/24
        # Traefikforwards X-Forwarded-* headers only from trusted IPs, so we need a specific IP here.
        gateway: 172.28.5.254