diff --git a/.env b/.env index f95ba078..fae61aa3 100644 --- a/.env +++ b/.env @@ -99,15 +99,18 @@ GEOSERVER_MAXIMUM_MEMORY=4G # In prod, should switch to PRODUCTION_LOGGING. For other logging profiles see: https://docs.geoserver.org/stable/en/user/configuration/globalsettings.html#config-globalsettings-log-profile GEOSERVER_LOGGING_PROFILE=DEFAULT_LOGGING +# RabbitMQ for OCPDB and ParkAPI +RABBITMQ_VERSION=rabbitmq:3.13 + # ocpdb OCPDB_API_PORT=7000 OCPDB_POSTGRES_USER=ocpdb OCPDB_POSTGRES_DB=ocpdb OCPDB_POSTGRES_HOST=ocpdb-db -OCPDB_CELERY_BROKER_URL=amqp://ocpdb-rabbitmq +OCPDB_CELERY_BROKER_URL=amqp://rabbitmq/ocpdb OCPDB_PRIVACY_URL=https://www.nvbw.de/datenschutz OCPDB_LEGAL_NOTICE_URL=https://www.nvbw.de/impressum -OCPDB_IMAGE=ghcr.io/binary-butterfly/ocpdb:1.0.0 +OCPDB_IMAGE=ghcr.io/binary-butterfly/ocpdb:1.2.1 OCPDB_DB_IMAGE=postgis/postgis:15-3.4-alpine # ParkAPI @@ -115,7 +118,7 @@ PARK_API_API_PORT=7500 PARK_API_POSTGRES_USER=park-api PARK_API_POSTGRES_DB=park-api PARK_API_POSTGRES_HOST=park-api-db -PARK_API_CELERY_BROKER_URL=amqp://park-api-rabbitmq +PARK_API_CELERY_BROKER_URL=amqp://rabbitmq/park-api PARK_API_IMAGE=ghcr.io/parkendd/park-api-v3:0.16.2 PARK_API_DB_IMAGE=postgis/postgis:15-3.4-alpine diff --git a/CHANGELOG.md b/CHANGELOG.md index bfec7b30..15193807 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ## Unreleased ### Added -- GeoServer: added ferry and furnicular to `mdbw_transit_stations_default` and `mdbw_transit_routes_default` gtfs styles +- GeoServer: added ferry and funicular to `mdbw_transit_stations_default` and `mdbw_transit_routes_default` gtfs styles +- OCPDB 1.2 with Stadtwerke Stuttgart +- OCPDB 1.2 with cron mechanisms for permanent data pulling + +### Changed + +- Centralize rabbitmq for better performance +- Improve rabbitmq healthcheck for better performance +- upgrade OCPDB to 1.2.1 with BNetzA Import Fix + ## 2024-12-20 @@ -18,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Changed - remove obsolete GBFS providers tier_basel, tier_bern, tier_stgallen, tier_winterthur, tier_zurich + ## 2024-12-11 ### Added @@ -26,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Changed - VRS parking providers obtain new Mobilithek subscription ids + ## 2024-12-10 ### Added @@ -36,6 +47,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - new namespace for GBFS feed voi_de to avoid conflict with voi_ch + ## 2024-12-04 ### Changed diff --git a/docker-compose.yml b/docker-compose.yml index c02b8e94..0e6984c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,7 @@ x-ocpdb-defaults: &ocpdb-defaults depends_on: ocpdb-db: condition: service_healthy - ocpdb-rabbitmq: + rabbitmq: condition: service_healthy volumes: - ./etc/ocpdb/config.yaml:/app/config.yaml @@ -40,7 +40,7 @@ x-park-api-defaults: &park-api-defaults depends_on: park-api-db: condition: service_healthy - park-api-rabbitmq: + rabbitmq: condition: service_healthy volumes: - ./etc/park-api/config.yaml:/app/config.yaml @@ -49,20 +49,6 @@ x-park-api-defaults: &park-api-defaults - ./var/park-api/logs:/app/logs - ./var/park-api/temp:/app/temp -x-generic-rabbitmq: &generic-rabbitmq - networks: [ipl] - image: rabbitmq:3.13 - user: rabbitmq # required due eacces-issue: https://github.com/docker-library/rabbitmq/issues/318 - environment: - # Disable spammy logging - RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: '-rabbit log [{console,[{level,warning}]}]' - healthcheck: - test: rabbitmq-diagnostics -q ping - interval: 3s - timeout: 1s - retries: 20 - restart: unless-stopped - services: # Traefik is our ingress: requests enter our infrastructure through it. # It is an HTTP reverse proxy, discovering available services (and their containers) @@ -204,10 +190,10 @@ services: restart: unless-stopped healthcheck: test: ps aux | grep -q 'x2gbfs\.x2gbfs' || exit 1 - # If you want to wait until all feeds have been generated - # before depending lamassu starts subscribing them, + # If you want to wait until all feeds have been generated + # before depending lamassu starts subscribing them, # configure an appropriate startup_period, e.g. 60s - # Otherwise, lamassu might update them only after the + # Otherwise, lamassu might update them only after the # update interval period has passed, which for test might # last several hours. start_interval: ${X2GBFS_HEALTHCHECK_START_INTERVAL:-1s} @@ -711,6 +697,24 @@ services: - "traefik.http.services.gtfs-api-docs.loadbalancer.server.port=8080" restart: unless-stopped + rabbitmq: + networks: [ipl] + image: ${RABBITMQ_VERSION} + user: rabbitmq # required due eacces-issue: https://github.com/docker-library/rabbitmq/issues/318 + environment: + # Disable spammy logging + RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: '-rabbit log [{console,[{level,warning}]}]' + healthcheck: + test: rabbitmq-diagnostics -q ping + interval: 30s + start_period: 20s + start_interval: 5s + timeout: 10s + volumes: + - ./etc/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf + - ./etc/rabbitmq/definitions.json:/etc/rabbitmq/definitions.json + restart: unless-stopped + ocpdb-flask: <<: *ocpdb-defaults command: ["gunicorn", "--bind", "0.0.0.0:5000", "webapp.entry_point_gunicorn:app"] @@ -729,9 +733,14 @@ services: restart: unless-stopped command: ["celery", "-A", "webapp.entry_point_celery", "worker", "-c", "2"] + ocpdb-heartbeat: + <<: *ocpdb-defaults + restart: unless-stopped + command: ["celery", "-A", "webapp.entry_point_celery", "beat", "-s", "/tmp/celerybeat-schedule"] + ocpdb-init: <<: *ocpdb-defaults - command: ["sh", "-c", "flask db upgrade && flask bnetza import-web"] + command: ["sh", "-c", "flask db upgrade && flask source fetch-all"] ocpdb-db: networks: [ipl] @@ -749,9 +758,6 @@ services: retries: 3 restart: unless-stopped - ocpdb-rabbitmq: - <<: *generic-rabbitmq - park-api-flask: <<: *park-api-defaults restart: unless-stopped @@ -795,9 +801,6 @@ services: retries: 3 restart: unless-stopped - park-api-rabbitmq: - <<: *generic-rabbitmq - sftp: image: ${SFTP_IMAGE} networks: [ipl] diff --git a/etc/ocpdb/config.yaml b/etc/ocpdb/config.yaml index c8baae21..9c94b065 100644 --- a/etc/ocpdb/config.yaml +++ b/etc/ocpdb/config.yaml @@ -10,4 +10,9 @@ OPENAPI_SERVERS: # So far, we don't have credentials, so we can put REMOTE_SERVERS here REMOTE_SERVERS: BNETZA: - url: https://data.bundesnetzagentur.de/Bundesnetzagentur/SharedDocs/Downloads/DE/Sachgebiete/Energie/Unternehmen_Institutionen/E_Mobilitaet/Ladesaeulenregister.xlsx + url: https://data.bundesnetzagentur.de/Bundesnetzagentur/SharedDocs/Downloads/DE/Sachgebiete/Energie/Unternehmen_Institutionen/E_Mobilitaet/Ladesaeulenregister_01122024.xlsx + SW_STUTTGART: + url: 'https://new-poi.chargecloud.de' + +AUTO_FETCH_SOURCES: + - sw_stuttgart diff --git a/etc/rabbitmq/definitions.json b/etc/rabbitmq/definitions.json new file mode 100644 index 00000000..0b39b3ab --- /dev/null +++ b/etc/rabbitmq/definitions.json @@ -0,0 +1,57 @@ +{ + "permissions": [ + { + "configure": ".*", + "read": ".*", + "user": "guest", + "vhost": "/", + "write": ".*" + }, + { + "configure": ".*", + "read": ".*", + "user": "guest", + "vhost": "ocpdb", + "write": ".*" + }, + { + "configure": ".*", + "read": ".*", + "user": "guest", + "vhost": "park-api", + "write": ".*" + } + ], + "rabbit_version": "3.13.7", + "rabbitmq_version": "3.13.7", + "users": [ + { + "hashing_algorithm": "rabbit_password_hashing_sha256", + "name": "guest", + "password_hash": "jJ6i6IgumvqlsjiFXbmB57Kr1K4fXCsNVk+Wrw4r9Rx+/6kq", + "tags": [ + "administrator" + ] + } + ], + "vhosts": [ + { + "metadata": { + "description": "Default virtual host" + }, + "name": "/" + }, + { + "metadata": { + "description": "Virtual host for ParkAPI" + }, + "name": "park-api" + }, + { + "metadata": { + "description": "Virtual host for OCPDB" + }, + "name": "ocpdb" + } + ] +} diff --git a/etc/rabbitmq/rabbitmq.conf b/etc/rabbitmq/rabbitmq.conf new file mode 100644 index 00000000..b408e668 --- /dev/null +++ b/etc/rabbitmq/rabbitmq.conf @@ -0,0 +1,5 @@ +# Just reload definitions if they are changed +definitions.skip_if_unchanged = true +# Point to definitions file +definitions.import_backend = local_filesystem +definitions.local.path = /etc/rabbitmq/definitions.json