From 856f0e789c9862f9081cc032f554e1124549946d Mon Sep 17 00:00:00 2001 From: Jocke Date: Thu, 21 Mar 2019 13:50:16 +0100 Subject: [PATCH 01/42] Update Docker file Add labels for metadata and move to an alpine base image --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e9fe3db..49bfc9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,9 @@ -FROM rabbitmq:3.6.14-management +FROM rabbitmq:3.6-management-alpine +LABEL maintainer "EGA System Developers", \ + org.label-schema.schema-version="1.0", \ + org.label-schema.build-date=$BUILD_DATE, \ + org.label-schema.vcs-url="https://github.com/EGA-archive/LocalEGA-mq", \ + org.label-schema.vcs-ref=$SOURCE_COMMIT EXPOSE 5672 15672 From 38d60c0c027c54488d343b148d8953f86d9e622c Mon Sep 17 00:00:00 2001 From: Jocke Date: Thu, 21 Mar 2019 13:50:45 +0100 Subject: [PATCH 02/42] Add build-hooks for docker-hub This is used to inject the metadata variables --- Dockerfile | 14 +++++++++----- hooks/build | 6 ++++++ 2 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 hooks/build diff --git a/Dockerfile b/Dockerfile index 49bfc9b..fa22579 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,13 @@ FROM rabbitmq:3.6-management-alpine -LABEL maintainer "EGA System Developers", \ - org.label-schema.schema-version="1.0", \ - org.label-schema.build-date=$BUILD_DATE, \ - org.label-schema.vcs-url="https://github.com/EGA-archive/LocalEGA-mq", \ - org.label-schema.vcs-ref=$SOURCE_COMMIT + +ARG BUILD_DATE +ARG SOURCE_COMMIT + +LABEL maintainer "EGA System Developers" +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.build-date=$BUILD_DATE +LABEL org.label-schema.vcs-url="https://github.com/EGA-archive/LocalEGA-mq" +LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT EXPOSE 5672 15672 diff --git a/hooks/build b/hooks/build new file mode 100644 index 0000000..9867ce3 --- /dev/null +++ b/hooks/build @@ -0,0 +1,6 @@ +#!/bin/bash +docker build \ + --cache-from egarchive/lega-mq:stable \ + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg SOURCE_COMMIT=$(git rev-parse --short HEAD) \ + --tag $IMAGE_NAME . From c97e1770a528f4b473c4edaaba5c87e74348561f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Haziza?= Date: Mon, 25 Mar 2019 19:02:39 +0100 Subject: [PATCH 03/42] Loading configurations as new admin user --- README.md | 1 + entrypoint.sh | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ae786c8..7f48f86 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ The following environment variables can be used to configure the broker: | Variable | Description | |---------:|:------------| | `MQ_USER` | Default user (with admin rights) | +| `MQ_PASSWORD` | Password for the above user | | `MQ_PASSWORD_HASH` | Password hash for the above user | | `CEGA_CONNECTION` | DSN URL for the shovels and federated queues with CentralEGA | diff --git a/entrypoint.sh b/entrypoint.sh index 709f08e..32f6b69 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,13 +5,14 @@ set -x [[ -z "${CEGA_CONNECTION}" ]] && echo 'Environment CEGA_CONNECTION is empty' 1>&2 && exit 1 [[ -z "${MQ_USER}" ]] && echo 'Environment MQ_USER is empty' 1>&2 && exit 1 +[[ -z "${MQ_PASSWORD}" ]] && echo 'Environment MQ_PASSWORD is empty' 1>&2 && exit 1 [[ -z "${MQ_PASSWORD_HASH}" ]] && echo 'Environment MQ_PASSWORD_HASH is empty' 1>&2 && exit 1 cat > /etc/rabbitmq/defs.json < /etc/rabbitmq/defs-cega.json <&1 && exit 1 ROUND=30 - until rabbitmqadmin import /etc/rabbitmq/defs-cega.json || ((ROUND<0)) - do + until rabbitmqadmin -u "${MQ_USER}" -p "${MQ_PASSWORD}" import /etc/rabbitmq/defs-cega.json || ((ROUND<0)) + do sleep 1 $((ROUND--)) done From fc1f66e8d7f9bfb3e4ac9f580cfcfd360eaf8508 Mon Sep 17 00:00:00 2001 From: Dmytro Titov Date: Wed, 27 Mar 2019 12:13:55 +0100 Subject: [PATCH 04/42] Use static shovels --- Dockerfile | 23 ++++---- README.md | 27 ++++++++- advanced.config | 107 ++++++++++++++++++++++++++++++++++ definitions.json | 129 +++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 14 +++++ entrypoint.sh | 139 ++------------------------------------------- rabbitmq.conf | 4 ++ 7 files changed, 294 insertions(+), 149 deletions(-) create mode 100644 advanced.config create mode 100644 definitions.json create mode 100644 docker-compose.yml create mode 100644 rabbitmq.conf diff --git a/Dockerfile b/Dockerfile index e9fe3db..c0f9e69 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,17 @@ -FROM rabbitmq:3.6.14-management +FROM rabbitmq:3.7.8-management-alpine -EXPOSE 5672 15672 +RUN rabbitmq-plugins enable --offline rabbitmq_federation rabbitmq_federation_management rabbitmq_shovel rabbitmq_shovel_management -VOLUME /var/lib/rabbitmq +COPY entrypoint.sh /usr/local/bin/ega-entrypoint.sh -RUN mkdir -p /etc/rabbitmq/ && \ - chown -R rabbitmq:rabbitmq /etc/rabbitmq +RUN chmod +x /usr/local/bin/ega-entrypoint.sh -# Initialization -RUN rabbitmq-plugins enable --offline rabbitmq_federation && \ - rabbitmq-plugins enable --offline rabbitmq_federation_management && \ - rabbitmq-plugins enable --offline rabbitmq_shovel && \ - rabbitmq-plugins enable --offline rabbitmq_shovel_management +COPY definitions.json /etc/rabbitmq/definitions.json -COPY entrypoint.sh /usr/local/bin/entrypoint.sh -RUN chmod 755 /usr/local/bin/entrypoint.sh +COPY advanced.config /etc/rabbitmq/advanced.config + +COPY rabbitmq.conf /etc/rabbitmq/rabbitmq.conf + +ENTRYPOINT ["/usr/local/bin/ega-entrypoint.sh"] -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] CMD ["rabbitmq-server"] diff --git a/README.md b/README.md index 7f48f86..e736d86 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,37 @@ # LocalEGA internal message broker in a docker image -We use [RabbitMQ 3.6.14](https://hub.docker.com/_/rabbitmq) including the management plugins. +We use [RabbitMQ 3.7.8](https://hub.docker.com/_/rabbitmq) including the management plugins. + +## Configuration The following environment variables can be used to configure the broker: | Variable | Description | |---------:|:------------| | `MQ_USER` | Default user (with admin rights) | -| `MQ_PASSWORD` | Password for the above user | | `MQ_PASSWORD_HASH` | Password hash for the above user | | `CEGA_CONNECTION` | DSN URL for the shovels and federated queues with CentralEGA | If you want persistent data, you can use a named volume or a bind-mount and make it point to `/var/lib/rabbitmq`. + +## Sample Docker Compose definition + +``` +version: '3.3' + +services: + + mq: + image: egarchive/lega-mq:latest + hostname: mq + ports: + - "5672:5672" + - "15672:15672" + environment: + - USER_NAME=admin + - PASSWORD_HASH=4tHURqDiZzypw0NTvoHhpn8/MMgONWonWxgRZ4NXgR8nZRBz + - CEGA_CONNECTION + +``` + +Run `docker-compose up -d` to test it. diff --git a/advanced.config b/advanced.config new file mode 100644 index 0000000..26c953c --- /dev/null +++ b/advanced.config @@ -0,0 +1,107 @@ +[ + {rabbitmq_shovel, + [{shovels, [ + {to_cega, + [{source, + [{protocol, amqp091}, + {uris, ["amqp://"]}, + {declarations, [{'queue.declare', [{exclusive, true}]}, + {'queue.bind', + [{exchange, <<"cega">>}, + {queue, <<>>}, + {routing_key, <<"#">>} + ]} + ]}, + {queue, <<>>}, + {prefetch_count, 10} + ]}, + {destination, + [{protocol, amqp091}, + {uris, ["CEGA_CONNECTION"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"localega.v1">>}]}]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} + ]}, + {cega_processing, + [{source, + [{protocol, amqp091}, + {uris, ["amqp://"]}, + {declarations, [{'queue.declare', [{exclusive, true}]}, + {'queue.bind', + [{exchange, <<"cega">>}, + {queue, <<>>}, + {routing_key, <<"files.processing">>} + ]} + ]}, + {queue, <<>>}, + {prefetch_count, 10} + ]}, + {destination, + [{protocol, amqp091}, + {uris, ["amqp://"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"cega">>}, + {routing_key, <<"files.processing">>} + ]} + ]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} + ]}, + {cega_error, + [{source, + [{protocol, amqp091}, + {uris, ["amqp://"]}, + {declarations, [{'queue.declare', [{exclusive, true}]}, + {'queue.bind', + [{exchange, <<"cega">>}, + {queue, <<>>}, + {routing_key, <<"files.error">>} + ]} + ]}, + {queue, <<>>}, + {prefetch_count, 10} + ]}, + {destination, + [{protocol, amqp091}, + {uris, ["amqp://"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"cega">>}, + {routing_key, <<"files.error">>} + ]} + ]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} + ]}, + {cega_completion, + [{source, + [{protocol, amqp091}, + {uris, ["amqp://"]}, + {declarations, [{'queue.declare', [{exclusive, true}]}, + {'queue.bind', + [{exchange, <<"lega">>}, + {queue, <<>>}, + {routing_key, <<"completed">>} + ]} + ]}, + {queue, <<>>}, + {prefetch_count, 10} + ]}, + {destination, + [{protocol, amqp091}, + {uris, ["amqp://"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"cega">>}, + {routing_key, <<"files.completed">>} + ]} + ]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} + ]} + ]} + ]} +]. diff --git a/definitions.json b/definitions.json new file mode 100644 index 0000000..a4c9168 --- /dev/null +++ b/definitions.json @@ -0,0 +1,129 @@ +{ + "users": [ + { + "name": "MQ_USER", + "password_hash": "MQ_PASSWORD_HASH", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "administrator" + } + ], + "vhosts": [ + { + "name": "/" + } + ], + "permissions": [ + { + "user": "MQ_USER", + "vhost": "/", + "configure": ".*", + "write": ".*", + "read": ".*" + } + ], + "parameters": [ + { + "value": { + "ack-mode": "on-confirm", + "queue": "v1.stableIDs", + "trust-user-id": false, + "uri": "CEGA_CONNECTION" + }, + "vhost": "/", + "component": "federation-upstream", + "name": "CEGA-ids" + }, + { + "value": { + "ack-mode": "on-confirm", + "queue": "v1.files", + "trust-user-id": false, + "uri": "CEGA_CONNECTION" + }, + "vhost": "/", + "component": "federation-upstream", + "name": "CEGA-files" + } + ], + "policies": [ + { + "vhost": "/", + "name": "CEGA-files", + "pattern": "files", + "apply-to": "queues", + "definition": { + "federation-upstream": "CEGA-files" + }, + "priority": 0 + }, + { + "vhost": "/", + "name": "CEGA-ids", + "pattern": "stableIDs", + "apply-to": "queues", + "definition": { + "federation-upstream": "CEGA-ids" + }, + "priority": 0 + } + ], + "queues": [ + { + "name": "stableIDs", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": { + } + }, + { + "name": "files", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": { + } + }, + { + "name": "archived", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": { + } + } + ], + "exchanges": [ + { + "name": "cega", + "vhost": "/", + "type": "topic", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": { + } + }, + { + "name": "lega", + "vhost": "/", + "type": "topic", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": { + } + } + ], + "bindings": [ + { + "source": "lega", + "vhost": "/", + "destination": "archived", + "destination_type": "queue", + "routing_key": "archived", + "arguments": { + } + } + ] +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..981ea38 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,14 @@ +version: '3.3' + +services: + + mq: + image: egarchive/lega-mq:latest + hostname: mq + ports: + - "5672:5672" + - "15672:15672" + environment: + - USER_NAME=admin + - PASSWORD_HASH=4tHURqDiZzypw0NTvoHhpn8/MMgONWonWxgRZ4NXgR8nZRBz + - CEGA_CONNECTION diff --git a/entrypoint.sh b/entrypoint.sh index 32f6b69..7d9838e 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,137 +1,8 @@ #!/bin/bash -set -e -set -x +sed -i 's%MQ_USER%'${USER_NAME}'%g' /etc/rabbitmq/definitions.json +sed -i 's%MQ_PASSWORD_HASH%'${PASSWORD_HASH}'%g' /etc/rabbitmq/definitions.json +sed -i 's%CEGA_CONNECTION%'${CEGA_CONNECTION}'%g' /etc/rabbitmq/definitions.json +sed -i 's%CEGA_CONNECTION%'${CEGA_CONNECTION}'%g' /etc/rabbitmq/advanced.config -[[ -z "${CEGA_CONNECTION}" ]] && echo 'Environment CEGA_CONNECTION is empty' 1>&2 && exit 1 -[[ -z "${MQ_USER}" ]] && echo 'Environment MQ_USER is empty' 1>&2 && exit 1 -[[ -z "${MQ_PASSWORD}" ]] && echo 'Environment MQ_PASSWORD is empty' 1>&2 && exit 1 -[[ -z "${MQ_PASSWORD_HASH}" ]] && echo 'Environment MQ_PASSWORD_HASH is empty' 1>&2 && exit 1 - -cat > /etc/rabbitmq/defs.json < /etc/rabbitmq/rabbitmq.config < /etc/rabbitmq/defs-cega.json <&1 && exit 1 - - ROUND=30 - until rabbitmqadmin -u "${MQ_USER}" -p "${MQ_PASSWORD}" import /etc/rabbitmq/defs-cega.json || ((ROUND<0)) - do - sleep 1 - $((ROUND--)) - done - ((ROUND<0)) && echo "Central EGA connections *_not_* loaded" 2>&1 && exit 1 - echo "Central EGA connections loaded" -} & - -exec "$@" # ie CMD rabbitmq-server +exec /usr/local/bin/docker-entrypoint.sh "$@" diff --git a/rabbitmq.conf b/rabbitmq.conf new file mode 100644 index 0000000..0bcab88 --- /dev/null +++ b/rabbitmq.conf @@ -0,0 +1,4 @@ +listeners.tcp.default = 5672 +disk_free_limit.absolute = 1GB +management.listener.port = 15672 +management.load_definitions = /etc/rabbitmq/definitions.json From c31cd9889036b5e2523d399c80d8f81503b5746b Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 28 Mar 2019 11:26:24 +0100 Subject: [PATCH 05/42] Fix MQ version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fa22579..e636195 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rabbitmq:3.6-management-alpine +FROM rabbitmq:3.6.16-management-alpine ARG BUILD_DATE ARG SOURCE_COMMIT From d8ce07325e8982aebbf8335285654468b09181fe Mon Sep 17 00:00:00 2001 From: Dmytro Titov Date: Thu, 28 Mar 2019 13:26:08 +0100 Subject: [PATCH 06/42] Fix env-vars naming --- README.md | 4 ++-- docker-compose.yml | 4 ++-- entrypoint.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e736d86..5203506 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,8 @@ services: - "5672:5672" - "15672:15672" environment: - - USER_NAME=admin - - PASSWORD_HASH=4tHURqDiZzypw0NTvoHhpn8/MMgONWonWxgRZ4NXgR8nZRBz + - MQ_USER=admin + - MQ_PASSWORD_HASH=4tHURqDiZzypw0NTvoHhpn8/MMgONWonWxgRZ4NXgR8nZRBz - CEGA_CONNECTION ``` diff --git a/docker-compose.yml b/docker-compose.yml index 981ea38..80364a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,6 @@ services: - "5672:5672" - "15672:15672" environment: - - USER_NAME=admin - - PASSWORD_HASH=4tHURqDiZzypw0NTvoHhpn8/MMgONWonWxgRZ4NXgR8nZRBz + - MQ_USER=admin + - MQ_PASSWORD_HASH=4tHURqDiZzypw0NTvoHhpn8/MMgONWonWxgRZ4NXgR8nZRBz - CEGA_CONNECTION diff --git a/entrypoint.sh b/entrypoint.sh index 7d9838e..cc49844 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/bash -sed -i 's%MQ_USER%'${USER_NAME}'%g' /etc/rabbitmq/definitions.json -sed -i 's%MQ_PASSWORD_HASH%'${PASSWORD_HASH}'%g' /etc/rabbitmq/definitions.json +sed -i 's%MQ_USER%'${MQ_USER}'%g' /etc/rabbitmq/definitions.json +sed -i 's%MQ_PASSWORD_HASH%'${MQ_PASSWORD_HASH}'%g' /etc/rabbitmq/definitions.json sed -i 's%CEGA_CONNECTION%'${CEGA_CONNECTION}'%g' /etc/rabbitmq/definitions.json sed -i 's%CEGA_CONNECTION%'${CEGA_CONNECTION}'%g' /etc/rabbitmq/advanced.config From e3d88bd78c770008c0b2c0051936f3588d9c9835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Haziza?= Date: Thu, 4 Apr 2019 18:33:01 +0200 Subject: [PATCH 07/42] Adding templates --- .github/ISSUE_TEMPLATE/bug-report.md | 28 ++++++++++++++++ .github/ISSUE_TEMPLATE/feature-request.md | 14 ++++++++ .github/ISSUE_TEMPLATE/new-sprint-task.md | 14 ++++++++ .github/PULL_REQUEST_TEMPLATE.md | 39 +++++++++++++++++++++++ 4 files changed, 95 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug-report.md create mode 100644 .github/ISSUE_TEMPLATE/feature-request.md create mode 100644 .github/ISSUE_TEMPLATE/new-sprint-task.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 0000000..c9c17f7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,28 @@ +--- +name: Bug Report +about: Use this to report a new bug + +--- + +### Expected Behavior + + +### Current Behavior + + +### Possible Solution + + +### Steps to Reproduce + + +1. +2. +3. +4. + +### Context (Environment) + + + + diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 0000000..d82023b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,14 @@ +--- +name: Feature Request +about: Suggest an idea for this project + +--- + +### Description + + +### Proposed solution + + +### Definition of Done + diff --git a/.github/ISSUE_TEMPLATE/new-sprint-task.md b/.github/ISSUE_TEMPLATE/new-sprint-task.md new file mode 100644 index 0000000..5d9134e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/new-sprint-task.md @@ -0,0 +1,14 @@ +--- +name: Sprint Task +about: Use when creating a new task in a sprint + +--- + +### Description + + +### Definition of Done + + +### How to test + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..9b51b4e --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,39 @@ + + + + +### Describe the pull request: + +- [ ] Bug fix +- [ ] Functional change +- [ ] New feature +- [ ] Code cleanup +- [ ] Build system change +- [ ] Documentation change +- [ ] Language translation + +### Pull request long description: + + +### Changes made: + + + +### Related issues: + + + +### Additional information: + + +### Release note: + + + +### Documentation change: + + + + +### Mentions: + From 115d84e32d8861d265c0b94dd741fdfb351769ff Mon Sep 17 00:00:00 2001 From: Dmytro Titov Date: Fri, 5 Apr 2019 13:43:58 +0200 Subject: [PATCH 08/42] Remove cega_processing and cega_error shovels, we don't need them --- advanced.config | 52 ------------------------------------------------- 1 file changed, 52 deletions(-) diff --git a/advanced.config b/advanced.config index 26c953c..be5f035 100644 --- a/advanced.config +++ b/advanced.config @@ -24,58 +24,6 @@ {ack_mode, on_confirm}, {reconnect_delay, 5} ]}, - {cega_processing, - [{source, - [{protocol, amqp091}, - {uris, ["amqp://"]}, - {declarations, [{'queue.declare', [{exclusive, true}]}, - {'queue.bind', - [{exchange, <<"cega">>}, - {queue, <<>>}, - {routing_key, <<"files.processing">>} - ]} - ]}, - {queue, <<>>}, - {prefetch_count, 10} - ]}, - {destination, - [{protocol, amqp091}, - {uris, ["amqp://"]}, - {declarations, []}, - {publish_properties, [{delivery_mode, 2}]}, - {publish_fields, [{exchange, <<"cega">>}, - {routing_key, <<"files.processing">>} - ]} - ]}, - {ack_mode, on_confirm}, - {reconnect_delay, 5} - ]}, - {cega_error, - [{source, - [{protocol, amqp091}, - {uris, ["amqp://"]}, - {declarations, [{'queue.declare', [{exclusive, true}]}, - {'queue.bind', - [{exchange, <<"cega">>}, - {queue, <<>>}, - {routing_key, <<"files.error">>} - ]} - ]}, - {queue, <<>>}, - {prefetch_count, 10} - ]}, - {destination, - [{protocol, amqp091}, - {uris, ["amqp://"]}, - {declarations, []}, - {publish_properties, [{delivery_mode, 2}]}, - {publish_fields, [{exchange, <<"cega">>}, - {routing_key, <<"files.error">>} - ]} - ]}, - {ack_mode, on_confirm}, - {reconnect_delay, 5} - ]}, {cega_completion, [{source, [{protocol, amqp091}, From c3094381e604e400f8cb0cbbd13dc1dd012f806b Mon Sep 17 00:00:00 2001 From: Dmytro Titov Date: Fri, 5 Apr 2019 14:10:15 +0200 Subject: [PATCH 09/42] Add missing lines from Jocke's commit --- Dockerfile | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Dockerfile b/Dockerfile index c0f9e69..356e23e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,18 @@ FROM rabbitmq:3.7.8-management-alpine +ARG BUILD_DATE +ARG SOURCE_COMMIT + +LABEL maintainer "EGA System Developers" +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.build-date=$BUILD_DATE +LABEL org.label-schema.vcs-url="https://github.com/EGA-archive/LocalEGA-mq" +LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT + +EXPOSE 5672 15672 + +VOLUME /var/lib/rabbitmq + RUN rabbitmq-plugins enable --offline rabbitmq_federation rabbitmq_federation_management rabbitmq_shovel rabbitmq_shovel_management COPY entrypoint.sh /usr/local/bin/ega-entrypoint.sh From 9fdce39a1df78433e76cb05aa57d5c628b2ec5f5 Mon Sep 17 00:00:00 2001 From: Dmytro Titov Date: Thu, 11 Apr 2019 10:37:38 +0200 Subject: [PATCH 10/42] Don't call /usr/local/bin/docker-entrypoint.sh --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index cc49844..9b260f9 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,4 +5,4 @@ sed -i 's%MQ_PASSWORD_HASH%'${MQ_PASSWORD_HASH}'%g' /etc/rabbitmq/definitions.js sed -i 's%CEGA_CONNECTION%'${CEGA_CONNECTION}'%g' /etc/rabbitmq/definitions.json sed -i 's%CEGA_CONNECTION%'${CEGA_CONNECTION}'%g' /etc/rabbitmq/advanced.config -exec /usr/local/bin/docker-entrypoint.sh "$@" +exec "$@" From 7185170d97c10e344f8159668025d932102385c5 Mon Sep 17 00:00:00 2001 From: Dmytro Titov Date: Thu, 11 Apr 2019 14:23:29 +0200 Subject: [PATCH 11/42] Add checks of env-vars --- entrypoint.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/entrypoint.sh b/entrypoint.sh index 9b260f9..6e6df95 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,5 +1,9 @@ #!/bin/bash +[[ -z "${MQ_USER}" ]] && echo 'Environment variable MQ_USER is empty' 1>&2 && exit 1 +[[ -z "${MQ_PASSWORD_HASH}" ]] && echo 'Environment variable MQ_PASSWORD_HASH is empty' 1>&2 && exit 1 +[[ -z "${CEGA_CONNECTION}" ]] && echo 'Environment variable CEGA_CONNECTION is empty' 1>&2 && exit 1 + sed -i 's%MQ_USER%'${MQ_USER}'%g' /etc/rabbitmq/definitions.json sed -i 's%MQ_PASSWORD_HASH%'${MQ_PASSWORD_HASH}'%g' /etc/rabbitmq/definitions.json sed -i 's%CEGA_CONNECTION%'${CEGA_CONNECTION}'%g' /etc/rabbitmq/definitions.json From 088f5faa33493209ead648577146cb14d58496a0 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 2 May 2019 15:43:58 +0200 Subject: [PATCH 12/42] Add TLS config Update rabbitmq config to enable TLS for local connections. The following restrictions apply; verification of both the server and client is enforced. Connection is not allowed unless mutual verification is ok. TLS version 1.2 or above is required. --- rabbitmq.conf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rabbitmq.conf b/rabbitmq.conf index 0bcab88..1b53668 100644 --- a/rabbitmq.conf +++ b/rabbitmq.conf @@ -1,4 +1,10 @@ -listeners.tcp.default = 5672 +listeners.ssl.default = 5671 +ssl_options.cacertfile = /etc/rabbitmq/ssl/ca-root.pem +ssl_options.certfile = /etc/rabbitmq/ssl/mq-server.pem +ssl_options.keyfile = /etc/rabbitmq/ssl/mq-server-key.pem +ssl_options.verify = verify_peer +ssl_options.fail_if_no_peer_cert = true +ssl_options.versions.1 = tlsv1.2 disk_free_limit.absolute = 1GB management.listener.port = 15672 management.load_definitions = /etc/rabbitmq/definitions.json From c349c0aee4216b89dd829f57d9abcac34244279a Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 10 May 2019 09:24:49 +0200 Subject: [PATCH 13/42] Replace `sed` in entrypoint script This is due to how the new connection string looks like. --- Dockerfile | 4 -- advanced.config | 55 ------------------- definitions.json | 129 --------------------------------------------- docker-compose.yml | 14 ----- entrypoint.sh | 118 +++++++++++++++++++++++++++++++++++++++-- 5 files changed, 114 insertions(+), 206 deletions(-) delete mode 100644 advanced.config delete mode 100644 definitions.json delete mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 356e23e..1fa7774 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,10 +19,6 @@ COPY entrypoint.sh /usr/local/bin/ega-entrypoint.sh RUN chmod +x /usr/local/bin/ega-entrypoint.sh -COPY definitions.json /etc/rabbitmq/definitions.json - -COPY advanced.config /etc/rabbitmq/advanced.config - COPY rabbitmq.conf /etc/rabbitmq/rabbitmq.conf ENTRYPOINT ["/usr/local/bin/ega-entrypoint.sh"] diff --git a/advanced.config b/advanced.config deleted file mode 100644 index be5f035..0000000 --- a/advanced.config +++ /dev/null @@ -1,55 +0,0 @@ -[ - {rabbitmq_shovel, - [{shovels, [ - {to_cega, - [{source, - [{protocol, amqp091}, - {uris, ["amqp://"]}, - {declarations, [{'queue.declare', [{exclusive, true}]}, - {'queue.bind', - [{exchange, <<"cega">>}, - {queue, <<>>}, - {routing_key, <<"#">>} - ]} - ]}, - {queue, <<>>}, - {prefetch_count, 10} - ]}, - {destination, - [{protocol, amqp091}, - {uris, ["CEGA_CONNECTION"]}, - {declarations, []}, - {publish_properties, [{delivery_mode, 2}]}, - {publish_fields, [{exchange, <<"localega.v1">>}]}]}, - {ack_mode, on_confirm}, - {reconnect_delay, 5} - ]}, - {cega_completion, - [{source, - [{protocol, amqp091}, - {uris, ["amqp://"]}, - {declarations, [{'queue.declare', [{exclusive, true}]}, - {'queue.bind', - [{exchange, <<"lega">>}, - {queue, <<>>}, - {routing_key, <<"completed">>} - ]} - ]}, - {queue, <<>>}, - {prefetch_count, 10} - ]}, - {destination, - [{protocol, amqp091}, - {uris, ["amqp://"]}, - {declarations, []}, - {publish_properties, [{delivery_mode, 2}]}, - {publish_fields, [{exchange, <<"cega">>}, - {routing_key, <<"files.completed">>} - ]} - ]}, - {ack_mode, on_confirm}, - {reconnect_delay, 5} - ]} - ]} - ]} -]. diff --git a/definitions.json b/definitions.json deleted file mode 100644 index a4c9168..0000000 --- a/definitions.json +++ /dev/null @@ -1,129 +0,0 @@ -{ - "users": [ - { - "name": "MQ_USER", - "password_hash": "MQ_PASSWORD_HASH", - "hashing_algorithm": "rabbit_password_hashing_sha256", - "tags": "administrator" - } - ], - "vhosts": [ - { - "name": "/" - } - ], - "permissions": [ - { - "user": "MQ_USER", - "vhost": "/", - "configure": ".*", - "write": ".*", - "read": ".*" - } - ], - "parameters": [ - { - "value": { - "ack-mode": "on-confirm", - "queue": "v1.stableIDs", - "trust-user-id": false, - "uri": "CEGA_CONNECTION" - }, - "vhost": "/", - "component": "federation-upstream", - "name": "CEGA-ids" - }, - { - "value": { - "ack-mode": "on-confirm", - "queue": "v1.files", - "trust-user-id": false, - "uri": "CEGA_CONNECTION" - }, - "vhost": "/", - "component": "federation-upstream", - "name": "CEGA-files" - } - ], - "policies": [ - { - "vhost": "/", - "name": "CEGA-files", - "pattern": "files", - "apply-to": "queues", - "definition": { - "federation-upstream": "CEGA-files" - }, - "priority": 0 - }, - { - "vhost": "/", - "name": "CEGA-ids", - "pattern": "stableIDs", - "apply-to": "queues", - "definition": { - "federation-upstream": "CEGA-ids" - }, - "priority": 0 - } - ], - "queues": [ - { - "name": "stableIDs", - "vhost": "/", - "durable": true, - "auto_delete": false, - "arguments": { - } - }, - { - "name": "files", - "vhost": "/", - "durable": true, - "auto_delete": false, - "arguments": { - } - }, - { - "name": "archived", - "vhost": "/", - "durable": true, - "auto_delete": false, - "arguments": { - } - } - ], - "exchanges": [ - { - "name": "cega", - "vhost": "/", - "type": "topic", - "durable": true, - "auto_delete": false, - "internal": false, - "arguments": { - } - }, - { - "name": "lega", - "vhost": "/", - "type": "topic", - "durable": true, - "auto_delete": false, - "internal": false, - "arguments": { - } - } - ], - "bindings": [ - { - "source": "lega", - "vhost": "/", - "destination": "archived", - "destination_type": "queue", - "routing_key": "archived", - "arguments": { - } - } - ] -} diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 80364a2..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: '3.3' - -services: - - mq: - image: egarchive/lega-mq:latest - hostname: mq - ports: - - "5672:5672" - - "15672:15672" - environment: - - MQ_USER=admin - - MQ_PASSWORD_HASH=4tHURqDiZzypw0NTvoHhpn8/MMgONWonWxgRZ4NXgR8nZRBz - - CEGA_CONNECTION diff --git a/entrypoint.sh b/entrypoint.sh index 6e6df95..def1313 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,9 +4,119 @@ [[ -z "${MQ_PASSWORD_HASH}" ]] && echo 'Environment variable MQ_PASSWORD_HASH is empty' 1>&2 && exit 1 [[ -z "${CEGA_CONNECTION}" ]] && echo 'Environment variable CEGA_CONNECTION is empty' 1>&2 && exit 1 -sed -i 's%MQ_USER%'${MQ_USER}'%g' /etc/rabbitmq/definitions.json -sed -i 's%MQ_PASSWORD_HASH%'${MQ_PASSWORD_HASH}'%g' /etc/rabbitmq/definitions.json -sed -i 's%CEGA_CONNECTION%'${CEGA_CONNECTION}'%g' /etc/rabbitmq/definitions.json -sed -i 's%CEGA_CONNECTION%'${CEGA_CONNECTION}'%g' /etc/rabbitmq/advanced.config +cat > /etc/rabbitmq/definitions.json < /etc/rabbitmq/advanced.config <>}, + {queue, <<>>}, + {routing_key, <<"#">>} + ]} + ]}, + {queue, <<>>}, + {prefetch_count, 10} + ]}, + {destination, + [{protocol, amqp091}, + {uris, ["${CEGA_CONNECTION}"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"localega.v1">>}]}]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} + ]}, + {cega_completion, + [{source, + [{protocol, amqp091}, + {uris, ["amqp://"]}, + {declarations, [{'queue.declare', [{exclusive, true}]}, + {'queue.bind', + [{exchange, <<"lega">>}, + {queue, <<>>}, + {routing_key, <<"completed">>} + ]} + ]}, + {queue, <<>>}, + {prefetch_count, 10} + ]}, + {destination, + [{protocol, amqp091}, + {uris, ["amqp://"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"cega">>}, + {routing_key, <<"files.completed">>} + ]} + ]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} + ]} + ]} + ]} +]. +EOF +chown rabbitmq:rabbitmq /etc/rabbitmq/advanced.config +chmod 600 /etc/rabbitmq/advanced.config + exec "$@" From 4ef024bdaa8ca092e72b30a25d022e0ff393647f Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 10 May 2019 09:25:53 +0200 Subject: [PATCH 14/42] Update Docker file Add ca-certificates so that we can validate publicly signed certificates. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1fa7774..8d47869 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,10 +9,10 @@ LABEL org.label-schema.build-date=$BUILD_DATE LABEL org.label-schema.vcs-url="https://github.com/EGA-archive/LocalEGA-mq" LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT -EXPOSE 5672 15672 - VOLUME /var/lib/rabbitmq +RUN apk add --no-cache ca-certificates + RUN rabbitmq-plugins enable --offline rabbitmq_federation rabbitmq_federation_management rabbitmq_shovel rabbitmq_shovel_management COPY entrypoint.sh /usr/local/bin/ega-entrypoint.sh From 46389494e968d97e2f6be9d72c5ad0b612368fec Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 10 May 2019 11:09:52 +0200 Subject: [PATCH 15/42] Parameterize the cert paths --- Dockerfile | 2 -- entrypoint.sh | 17 +++++++++++++++++ rabbitmq.conf | 10 ---------- 3 files changed, 17 insertions(+), 12 deletions(-) delete mode 100644 rabbitmq.conf diff --git a/Dockerfile b/Dockerfile index 8d47869..fa06558 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,8 +19,6 @@ COPY entrypoint.sh /usr/local/bin/ega-entrypoint.sh RUN chmod +x /usr/local/bin/ega-entrypoint.sh -COPY rabbitmq.conf /etc/rabbitmq/rabbitmq.conf - ENTRYPOINT ["/usr/local/bin/ega-entrypoint.sh"] CMD ["rabbitmq-server"] diff --git a/entrypoint.sh b/entrypoint.sh index def1313..ecb26e9 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,6 +4,23 @@ [[ -z "${MQ_PASSWORD_HASH}" ]] && echo 'Environment variable MQ_PASSWORD_HASH is empty' 1>&2 && exit 1 [[ -z "${CEGA_CONNECTION}" ]] && echo 'Environment variable CEGA_CONNECTION is empty' 1>&2 && exit 1 + +cat >> /etc/rabbitmq/rabbitmq.conf < /etc/rabbitmq/definitions.json < Date: Mon, 13 May 2019 17:02:49 +0200 Subject: [PATCH 16/42] Fixing permissions on some files read by rabbitmq user --- entrypoint.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index ecb26e9..90b08b1 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -136,4 +136,11 @@ chown rabbitmq:rabbitmq /etc/rabbitmq/advanced.config chmod 600 /etc/rabbitmq/advanced.config -exec "$@" +# Ownership by 'rabbitmq' +[[ -e "${MQ_CA}" ]] && chown rabbitmq:rabbitmq "${MQ_CA}" +[[ -e "${MQ_SERVER_CERT}" ]] && chown rabbitmq:rabbitmq "${MQ_SERVER_CERT}" +[[ -e "${MQ_SERVER_KEY}" ]] && chown rabbitmq:rabbitmq "${MQ_SERVER_KEY}" +find /var/lib/rabbitmq \! -user rabbitmq -exec chown rabbitmq '{}' + + +# Run as 'rabbitmq' +exec su-exec rabbitmq "$@" From e4eb03101dc38bfbe91f4e613d72ef1397be7f29 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 2 Jul 2019 08:37:49 +0200 Subject: [PATCH 17/42] Run container as rabbitmq user Improve security by not starting the container as root. As long as injected files are placed in the appropriate paths they will still be owned by the rabbitmq user. --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index fa06558..98c16ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,8 @@ COPY entrypoint.sh /usr/local/bin/ega-entrypoint.sh RUN chmod +x /usr/local/bin/ega-entrypoint.sh +USER 100:101 + ENTRYPOINT ["/usr/local/bin/ega-entrypoint.sh"] CMD ["rabbitmq-server"] From 746d5101660c8ae2df82d41da4006b87d20d855a Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 2 Jul 2019 08:41:20 +0200 Subject: [PATCH 18/42] Add more config options to entrypoint script Allow the user to explicitly name the default `vhost` to something other than `/`. Add option to disable peer verification of tls certificates. Remove unnecessary ownership changes of injected files. --- entrypoint.sh | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 90b08b1..094a1f8 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -10,15 +10,15 @@ listeners.ssl.default = 5671 ssl_options.cacertfile = ${MQ_CA:-/etc/rabbitmq/ssl/ca.pem} ssl_options.certfile = ${MQ_SERVER_CERT:-/etc/rabbitmq/ssl/mq-server.pem} ssl_options.keyfile = ${MQ_SERVER_KEY:-/etc/rabbitmq/ssl/mq-server-key.pem} -ssl_options.verify = verify_peer +ssl_options.verify = ${MQ_VERIFY:-verify_peer} ssl_options.fail_if_no_peer_cert = true ssl_options.versions.1 = tlsv1.2 disk_free_limit.absolute = 1GB management.listener.port = 15672 management.load_definitions = /etc/rabbitmq/definitions.json +default_vhost = ${MQ_VHOST:-/} EOF -chown rabbitmq:rabbitmq /etc/rabbitmq/rabbitmq.conf chmod 600 /etc/rabbitmq/rabbitmq.conf cat > /etc/rabbitmq/definitions.json < /etc/rabbitmq/definitions.json < /etc/rabbitmq/advanced.config < /etc/rabbitmq/advanced.config < Date: Tue, 2 Jul 2019 08:41:30 +0200 Subject: [PATCH 19/42] Update readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 5203506..39fe678 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ The following environment variables can be used to configure the broker: | Variable | Description | |---------:|:------------| +| `MQ_VHOST` | Default vhost other than `/` | +| `MQ_VERIFY` | Set to `verify_none` to disable verification of client certificate | | `MQ_USER` | Default user (with admin rights) | | `MQ_PASSWORD_HASH` | Password hash for the above user | | `CEGA_CONNECTION` | DSN URL for the shovels and federated queues with CentralEGA | @@ -28,6 +30,7 @@ services: - "5672:5672" - "15672:15672" environment: + - MQ_VHOST=vhost - MQ_USER=admin - MQ_PASSWORD_HASH=4tHURqDiZzypw0NTvoHhpn8/MMgONWonWxgRZ4NXgR8nZRBz - CEGA_CONNECTION From d7295f7da4887011333b0dd3d2c7d5c8769e2f61 Mon Sep 17 00:00:00 2001 From: Jon Ander Novella Date: Mon, 22 Jul 2019 14:48:59 +0200 Subject: [PATCH 20/42] Update build directive to use cache from nbis repo --- hooks/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/build b/hooks/build index 9867ce3..2e5e17b 100644 --- a/hooks/build +++ b/hooks/build @@ -1,6 +1,6 @@ #!/bin/bash docker build \ - --cache-from egarchive/lega-mq:stable \ + --cache-from nbisweden/ega-mq:m7 \ --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ --build-arg SOURCE_COMMIT=$(git rev-parse --short HEAD) \ --tag $IMAGE_NAME . From df63234338d1052b8ec97bbe10bb2109c8726307 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 7 Apr 2020 12:21:56 +0200 Subject: [PATCH 21/42] Add ENVs to Dockerfile Add default values to some ENVs to specify where the persisted config files are located. --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 98c16ba..2bb4817 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,9 +9,11 @@ LABEL org.label-schema.build-date=$BUILD_DATE LABEL org.label-schema.vcs-url="https://github.com/EGA-archive/LocalEGA-mq" LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT -VOLUME /var/lib/rabbitmq +ENV RABBITMQ_CONFIG_FILE=/var/lib/rabbitmq/rabbitmq +ENV RABBITMQ_ADVANCED_CONFIG_FILE=/var/lib/rabbitmq/advanced +ENV RABBITMQ_LOG_BASE=/var/lib/rabbitmq -RUN apk add --no-cache ca-certificates +RUN apk add --no-cache ca-certificates openssl RUN rabbitmq-plugins enable --offline rabbitmq_federation rabbitmq_federation_management rabbitmq_shovel rabbitmq_shovel_management From 19a5ceb89959f5e55c3d2e2c31890e426f0c46a9 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 7 Apr 2020 12:24:11 +0200 Subject: [PATCH 22/42] Update entrypoint script Remove bash specific tests and ensure that the config files are written to persistent storage. --- entrypoint.sh | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 094a1f8..1e8d026 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,27 +1,37 @@ -#!/bin/bash +#!/bin/sh -[[ -z "${MQ_USER}" ]] && echo 'Environment variable MQ_USER is empty' 1>&2 && exit 1 -[[ -z "${MQ_PASSWORD_HASH}" ]] && echo 'Environment variable MQ_PASSWORD_HASH is empty' 1>&2 && exit 1 -[[ -z "${CEGA_CONNECTION}" ]] && echo 'Environment variable CEGA_CONNECTION is empty' 1>&2 && exit 1 +[ -z "${MQ_USER}" ] && echo 'Environment variable MQ_USER is empty' 1>&2 && exit 1 +[ -z "${MQ_PASSWORD_HASH}" ] && echo 'Environment variable MQ_PASSWORD_HASH is empty' 1>&2 && exit 1 +[ -z "${CEGA_CONNECTION}" ] && echo 'Environment variable CEGA_CONNECTION is empty' 1>&2 && exit 1 +if [ -z "${MQ_SERVER_CERT}" ] || [ -z "${MQ_SERVER_KEY}" ]; then +SSL_SUBJ="/C=SE/ST=Sweden/L=Uppsala/O=NBIS/OU=SysDevs/CN=LocalEGA" +mkdir -p "/var/lib/rabbitmq/ssl" +# Generating the SSL certificate + key +openssl req -x509 -newkey rsa:2048 \ + -keyout "/var/lib/rabbitmq/ssl/mq-server.key" -nodes \ + -out "/var/lib/rabbitmq/ssl/mq-server.pem" -sha256 \ + -days 1000 -subj "${SSL_SUBJ}" && \ + chmod 600 "/var/lib/rabbitmq/ssl/mq-server.*" +fi -cat >> /etc/rabbitmq/rabbitmq.conf <> "/var/lib/rabbitmq/rabbitmq.conf" < /etc/rabbitmq/definitions.json < "/var/lib/rabbitmq/definitions.json" < /etc/rabbitmq/definitions.json < /etc/rabbitmq/advanced.config < "/var/lib/rabbitmq/advanced.config" < /etc/rabbitmq/advanced.config < Date: Wed, 15 Apr 2020 10:55:58 +0200 Subject: [PATCH 23/42] Prepend "/" to vhost name in shovel URI --- entrypoint.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 1e8d026..d155022 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -81,6 +81,9 @@ cat > "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/advanced.config" < "/var/lib/rabbitmq/advanced.config" <>}, @@ -114,7 +117,7 @@ cat > "/var/lib/rabbitmq/advanced.config" <>}, @@ -127,7 +130,7 @@ cat > "/var/lib/rabbitmq/advanced.config" <>}, From 58a001edfb828d17248c3ab6a73753d93e88726a Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 4 May 2020 15:24:35 +0200 Subject: [PATCH 24/42] Update entrypoint script Sets up the MQ for standalone configuration when the ENV `CEGA_CONNECTION` is absent --- entrypoint.sh | 277 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 256 insertions(+), 21 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index d155022..3c746a7 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,7 +2,6 @@ [ -z "${MQ_USER}" ] && echo 'Environment variable MQ_USER is empty' 1>&2 && exit 1 [ -z "${MQ_PASSWORD_HASH}" ] && echo 'Environment variable MQ_PASSWORD_HASH is empty' 1>&2 && exit 1 -[ -z "${CEGA_CONNECTION}" ] && echo 'Environment variable CEGA_CONNECTION is empty' 1>&2 && exit 1 if [ -z "${MQ_SERVER_CERT}" ] || [ -z "${MQ_SERVER_KEY}" ]; then SSL_SUBJ="/C=SE/ST=Sweden/L=Uppsala/O=NBIS/OU=SysDevs/CN=LocalEGA" @@ -31,55 +30,132 @@ EOF chmod 600 "/var/lib/rabbitmq/rabbitmq.conf" +if [ -n "${CEGA_CONNECTION}" ]; then cat > "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/advanced.config" < "/var/lib/rabbitmq/definitions.json" < Date: Tue, 5 May 2020 10:25:09 +0200 Subject: [PATCH 25/42] Fix generated cert permisssions --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 3c746a7..47e0b56 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -11,7 +11,7 @@ openssl req -x509 -newkey rsa:2048 \ -keyout "/var/lib/rabbitmq/ssl/mq-server.key" -nodes \ -out "/var/lib/rabbitmq/ssl/mq-server.pem" -sha256 \ -days 1000 -subj "${SSL_SUBJ}" && \ - chmod 600 "/var/lib/rabbitmq/ssl/mq-server.*" + chmod 600 /var/lib/rabbitmq/ssl/mq-server.* fi cat >> "/var/lib/rabbitmq/rabbitmq.conf" < Date: Tue, 5 May 2020 12:10:33 +0200 Subject: [PATCH 26/42] Rename entrypoint script One further step in migrating from the (l)ega naming schema. --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2bb4817..6a0ad1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,12 +17,12 @@ RUN apk add --no-cache ca-certificates openssl RUN rabbitmq-plugins enable --offline rabbitmq_federation rabbitmq_federation_management rabbitmq_shovel rabbitmq_shovel_management -COPY entrypoint.sh /usr/local/bin/ega-entrypoint.sh +COPY entrypoint.sh /usr/local/bin/docker-entrypoint.sh -RUN chmod +x /usr/local/bin/ega-entrypoint.sh +RUN chmod +x /usr/local/bin/docker-entrypoint.sh USER 100:101 -ENTRYPOINT ["/usr/local/bin/ega-entrypoint.sh"] +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] CMD ["rabbitmq-server"] From 5cd5bd087db64a16a971b09074495c82832ef4c1 Mon Sep 17 00:00:00 2001 From: jonandernovella Date: Wed, 24 Jun 2020 10:11:26 +0200 Subject: [PATCH 27/42] Use files.verified shovel We should publish to this queue whenever we have successfully verified a file. --- entrypoint.sh | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/entrypoint.sh b/entrypoint.sh index 47e0b56..cd11d74 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -215,6 +215,32 @@ cat > "/var/lib/rabbitmq/advanced.config" <>}, + {queue, <<>>}, + {routing_key, <<"verified">>} + ]} + ]}, + {queue, <<>>}, + {prefetch_count, 10} + ]}, + {destination, + [{protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"cega">>}, + {routing_key, <<"files.verified">>} + ]} + ]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} ]} ]} ]} @@ -277,6 +303,13 @@ cat > "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" < Date: Fri, 16 Oct 2020 15:33:47 +0200 Subject: [PATCH 28/42] Update config --- entrypoint.sh | 354 ++++++++++++++++++++++---------------------------- 1 file changed, 155 insertions(+), 199 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index cd11d74..01aa46b 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -56,17 +56,6 @@ cat > "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/advanced.config" <>}, - {queue, <<>>}, - {routing_key, <<"#">>} - ]} - ]}, - {queue, <<>>}, - {prefetch_count, 10} - ]}, - {destination, - [{protocol, amqp091}, - {uris, ["${CEGA_CONNECTION}"]}, - {declarations, []}, - {publish_properties, [{delivery_mode, 2}]}, - {publish_fields, [{exchange, <<"localega.v1">>}]}]}, - {ack_mode, on_confirm}, - {reconnect_delay, 5} - ]}, - {cega_completion, - [{source, - [{protocol, amqp091}, - {uris, ["amqp://${MQ_VHOST:-}"]}, - {declarations, [{'queue.declare', [{exclusive, true}]}, - {'queue.bind', - [{exchange, <<"lega">>}, - {queue, <<>>}, - {routing_key, <<"completed">>} - ]} - ]}, - {queue, <<>>}, - {prefetch_count, 10} - ]}, - {destination, - [{protocol, amqp091}, - {uris, ["amqp://${MQ_VHOST:-}"]}, - {declarations, []}, - {publish_properties, [{delivery_mode, 2}]}, - {publish_fields, [{exchange, <<"cega">>}, - {routing_key, <<"files.completed">>} - ]} - ]}, - {ack_mode, on_confirm}, - {reconnect_delay, 5} + {rabbitmq_shovel, [ + {shovels, [ + {to_cega, [ + {source, [ + {protocol, amqp091}, + {uris,[ "amqp://${MQ_VHOST:-}" ]}, + {declarations, [ + {'queue.declare', [{exclusive, true}]}, + {'queue.bind', [{exchange, <<"to_cega">>}, {queue, <<>>}, {routing_key, <<"#">>}]} + ]}, + {queue, <<>>}, + {prefetch_count, 10} ]}, - {cega_verification, - [{source, - [{protocol, amqp091}, - {uris, ["amqp://${MQ_VHOST:-}"]}, - {declarations, [{'queue.declare', [{exclusive, true}]}, - {'queue.bind', - [{exchange, <<"lega">>}, - {queue, <<>>}, - {routing_key, <<"verified">>} - ]} - ]}, - {queue, <<>>}, - {prefetch_count, 10} - ]}, - {destination, - [{protocol, amqp091}, - {uris, ["amqp://${MQ_VHOST:-}"]}, - {declarations, []}, - {publish_properties, [{delivery_mode, 2}]}, - {publish_fields, [{exchange, <<"cega">>}, - {routing_key, <<"files.verified">>} - ]} - ]}, - {ack_mode, on_confirm}, - {reconnect_delay, 5} + {destination, [ + {protocol, amqp091}, + {uris, ["${CEGA_CONNECTION}"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"localega.v1">>}]} + ]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} + ]}, + {cega_completion, [ + {source, [ + {protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, [{'queue.declare', [{exclusive, true}] }, {'queue.bind', [{exchange, <<"sda">>}, {queue, <<>>}, {routing_key, <<"completed">>}] } ] }, + {queue, <<>>}, + {prefetch_count, 10} + ]}, + {destination, [ + {protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"to_cega">>}, + {routing_key, <<"files.completed">>} + ]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} ]} + ]}, + {cega_error, [ + {source, [ + {protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, [{'queue.declare', [{exclusive, true}] }, {'queue.bind', [{exchange, <<"sda">>}, {queue, <<>>}, {routing_key, <<"error">>}] } ] }, + {queue, <<>>}, + {prefetch_count, 10} + ]}, + {destination, [ + {protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"to_cega">>}, + {routing_key, <<"files.error">>} + ]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} + ]} + ]}, + {cega_inbox, [ + {source, [ + {protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, [{'queue.declare', [{exclusive, true}] }, {'queue.bind', [{exchange, <<"sda">>}, {queue, <<>>}, {routing_key, <<"inbox">>}] } ] }, + {queue, <<>>}, + {prefetch_count, 10} + ]}, + {destination, [ + {protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"to_cega">>}, + {routing_key, <<"files.inbox">>} + ]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} + ]} + ]}, + {cega_verified, [ + {source, [ + {protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, [{'queue.declare', [{exclusive, true}] }, {'queue.bind', [{exchange, <<"sda">>}, {queue, <<>>}, {routing_key, <<"verified">>}] } ] }, + {queue, <<>>}, + {prefetch_count, 10} + ]}, + {destination, [ + {protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"to_cega">>}, + {routing_key, <<"files.verified">>} + ]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} + ]} + ]} ]} - ]} + ]} ]. EOF chmod 600 "/var/lib/rabbitmq/advanced.config" else cat > "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" < Date: Fri, 16 Oct 2020 15:34:26 +0200 Subject: [PATCH 29/42] Reference `neicnordic` not `EGA-Archve` --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6a0ad1c..c6aa1f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ARG SOURCE_COMMIT LABEL maintainer "EGA System Developers" LABEL org.label-schema.schema-version="1.0" LABEL org.label-schema.build-date=$BUILD_DATE -LABEL org.label-schema.vcs-url="https://github.com/EGA-archive/LocalEGA-mq" +LABEL org.label-schema.vcs-url="https://github.com/neicnordic/LocalEGA-mq" LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT ENV RABBITMQ_CONFIG_FILE=/var/lib/rabbitmq/rabbitmq From 36c407e3651c1fc409ab037644a367a72c295e4e Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 19 Oct 2020 09:09:03 +0200 Subject: [PATCH 30/42] Use accessionID instead of stableID --- entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 01aa46b..081141e 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -124,7 +124,7 @@ cat > "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" < Date: Tue, 20 Oct 2020 10:22:54 +0200 Subject: [PATCH 31/42] Add missing error queue --- entrypoint.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/entrypoint.sh b/entrypoint.sh index 081141e..0d57b20 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -318,6 +318,13 @@ cat > "/var/lib/rabbitmq/definitions.json" < Date: Tue, 20 Oct 2020 10:23:07 +0200 Subject: [PATCH 32/42] Add bindings --- entrypoint.sh | 132 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/entrypoint.sh b/entrypoint.sh index 0d57b20..16b05b1 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -157,6 +157,72 @@ cat > "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" < Date: Wed, 21 Oct 2020 11:54:39 +0300 Subject: [PATCH 33/42] add backup queue --- entrypoint.sh | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 16b05b1..e83be1e 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -81,14 +81,21 @@ cat > "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" < Date: Wed, 21 Oct 2020 15:28:15 +0300 Subject: [PATCH 34/42] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 39fe678..3e2d888 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# LocalEGA internal message broker in a docker image +# NeIC SDA internal message broker in a docker image We use [RabbitMQ 3.7.8](https://hub.docker.com/_/rabbitmq) including the management plugins. From 4d61b98350fb57be38096b4da54a57f5b26532ff Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 24 Nov 2020 12:33:08 +0200 Subject: [PATCH 35/42] add missing mappings queue --- entrypoint.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/entrypoint.sh b/entrypoint.sh index e83be1e..ac4bb67 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -230,6 +230,14 @@ cat > "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" < Date: Tue, 24 Nov 2020 12:35:27 +0200 Subject: [PATCH 36/42] stand-alone does not have localega sources --- entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index ac4bb67..4581738 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -525,7 +525,7 @@ cat > "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" < Date: Mon, 29 Nov 2021 10:17:29 +0100 Subject: [PATCH 37/42] bump version & publish dockerhub & github registry --- .github/workflows/tag_and_build.yaml | 71 ++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/tag_and_build.yaml diff --git a/.github/workflows/tag_and_build.yaml b/.github/workflows/tag_and_build.yaml new file mode 100644 index 0000000..ab8d951 --- /dev/null +++ b/.github/workflows/tag_and_build.yaml @@ -0,0 +1,71 @@ +name: Bump version +on: + push: + branches: + - master +jobs: + tag: + name: bump tags + outputs: + part: ${{ steps.bump_tag.outputs.part }} + tag: ${{ steps.bump_tag.outputs.tag }} + new_tag: ${{ steps.bump_tag.outputs.new_tag }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: '2' + - name: Bump version and push tag + id: bump_tag + uses: anothrNick/github-tag-action@1.36.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WITH_V: true + DEFAULT_BUMP: patch + + push_to_registry: + needs: tag + if: needs.tag.outputs.part != '' + name: Push Docker image to Github Container registry + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Log in to the Github Container registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v3 + with: + images: ghcr.io/${{ github.repository }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + push: true + tags: | + ghcr.io/${{ github.repository }}:${{ needs.tag.outputs.tag }} + ghcr.io/${{ github.repository }}:latest + ${{ github.repository }}:${{ needs.tag.outputs.tag }} + ${{ github.repository }}:latest + labels: | + org.opencontainers.image.source=${{ github.event.repository.clone_url }} + org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + org.opencontainers.image.revision=${{ github.sha }} From 410cac42b3b9383cec02c215204a013b7fd38ef1 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 29 Nov 2021 10:58:08 +0100 Subject: [PATCH 38/42] Update mq version to 3.8.16 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c6aa1f4..ae10c90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rabbitmq:3.7.8-management-alpine +FROM rabbitmq:3.8.16-management-alpine ARG BUILD_DATE ARG SOURCE_COMMIT From ff6a22bf33990b337d31274291ac2b6319afa53d Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 29 Nov 2021 11:15:01 +0100 Subject: [PATCH 39/42] Rework entrypoint script to not generate self signed certificates. Adds the NOTLS env to disable SSL completely --- entrypoint.sh | 1122 +++++++++++++++++++++++++------------------------ 1 file changed, 572 insertions(+), 550 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 4581738..edf4be8 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -3,562 +3,584 @@ [ -z "${MQ_USER}" ] && echo 'Environment variable MQ_USER is empty' 1>&2 && exit 1 [ -z "${MQ_PASSWORD_HASH}" ] && echo 'Environment variable MQ_PASSWORD_HASH is empty' 1>&2 && exit 1 -if [ -z "${MQ_SERVER_CERT}" ] || [ -z "${MQ_SERVER_KEY}" ]; then -SSL_SUBJ="/C=SE/ST=Sweden/L=Uppsala/O=NBIS/OU=SysDevs/CN=LocalEGA" -mkdir -p "/var/lib/rabbitmq/ssl" -# Generating the SSL certificate + key -openssl req -x509 -newkey rsa:2048 \ - -keyout "/var/lib/rabbitmq/ssl/mq-server.key" -nodes \ - -out "/var/lib/rabbitmq/ssl/mq-server.pem" -sha256 \ - -days 1000 -subj "${SSL_SUBJ}" && \ - chmod 600 /var/lib/rabbitmq/ssl/mq-server.* -fi +if [ -n "${NOTLS+x}" ]; then + echo "Disabeling TLS" + unset MQ_SERVER_CERT + unset MQ_SERVER_KEY + unset MQ_CA + cat > "/var/lib/rabbitmq/rabbitmq.conf" <<-EOF + listeners.tcp.default = 5672 + disk_free_limit.absolute = 1GB + management.tcp.port = 15672 + management.load_definitions = /var/lib/rabbitmq/definitions.json + default_vhost = ${MQ_VHOST:-/} + EOF +else + if [ -e "${MQ_SERVER_CERT}" ] || [ -e "${MQ_SERVER_KEY}" ]; then + echo "Enabeling TLS" + cat > "/var/lib/rabbitmq/rabbitmq.conf" <<-EOF + listeners.ssl.default = 5671 + ssl_options.certfile = ${MQ_SERVER_CERT} + ssl_options.keyfile = ${MQ_SERVER_KEY} + ssl_options.versions.1 = tlsv1.2 + disk_free_limit.absolute = 1GB + management.ssl.port = 15672 + management.ssl.certfile = ${MQ_SERVER_CERT} + management.ssl.keyfile = ${MQ_SERVER_KEY} + management.load_definitions = /var/lib/rabbitmq/definitions.json + default_vhost = ${MQ_VHOST:-/} + EOF -cat >> "/var/lib/rabbitmq/rabbitmq.conf" <> "/var/lib/rabbitmq/rabbitmq.conf" <<-EOF + ssl_options.verify = verify_peer + ssl_options.fail_if_no_peer_cert = true + ssl_options.cacertfile = ${MQ_CA} + EOF + fi -chmod 600 "/var/lib/rabbitmq/rabbitmq.conf" + else + echo 'No server certificates found, shuting down.' 1>&2 && exit 1 + fi + + chmod 600 "/var/lib/rabbitmq/rabbitmq.conf" +fi if [ -n "${CEGA_CONNECTION}" ]; then -cat > "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" <<-EOF + { + "users": [ + { + "name": "${MQ_USER}", + "password_hash": "${MQ_PASSWORD_HASH}", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "administrator" + } + ], + "vhosts": [ + { + "name": "${MQ_VHOST:-/}" + } + ], + "permissions": [ + { + "user": "${MQ_USER}", + "vhost": "${MQ_VHOST:-/}", + "configure": ".*", + "write": ".*", + "read": ".*" + } + ], + "parameters": [ + { + "name": "CEGA-files", + "vhost": "${MQ_VHOST:-/}", + "component": "federation-upstream", + "value": { + "ack-mode": "on-confirm", + "queue": "v1.files", + "trust-user-id": false, + "uri": "${CEGA_CONNECTION}" + } + } + ], + "policies": [ + { + "vhost": "${MQ_VHOST:-/}", + "name": "CEGA-files", + "pattern": "files", + "apply-to": "queues", + "priority": 0, + "definition": { + "federation-upstream": "CEGA-files" + } + } + ], + "queues": [ + { + "name": "archived", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "backup", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "completed", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "files", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "inbox", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "ingest", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "mappings", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "accessionIDs", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "verified", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + } + ], + "exchanges": [ + { + "name": "to_cega", + "vhost": "${MQ_VHOST:-/}", + "type": "topic", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": {} + }, + { + "name": "sda", + "vhost": "${MQ_VHOST:-/}", + "type": "topic", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": {} + } + ], + "bindings": [ + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "archived", + "routing_key": "archived" + }, + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "accessionIDs", + "routing_key": "accessionIDs" + }, + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "backup", + "routing_key": "backup" + }, + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "completed", + "routing_key": "completed" + }, + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "error", + "routing_key": "error" + }, + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "files", + "routing_key": "files" + }, + { + "source": "localega", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "inbox", + "routing_key": "inbox" + }, + { + "source": "localega", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "ingest", + "routing_key": "ingest" + }, + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "mappings", + "routing_key": "mappings" + }, + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "verified", + "routing_key": "verified" + } + ] + } + EOF -if [ -n "${MQ_VHOST}" ];then -MQ_VHOST="/${MQ_VHOST}" -fi -cat > "/var/lib/rabbitmq/advanced.config" <>}, {queue, <<>>}, {routing_key, <<"#">>}]} - ]}, - {queue, <<>>}, - {prefetch_count, 10} - ]}, - {destination, [ - {protocol, amqp091}, - {uris, ["${CEGA_CONNECTION}"]}, - {declarations, []}, - {publish_properties, [{delivery_mode, 2}]}, - {publish_fields, [{exchange, <<"localega.v1">>}]} - ]}, - {ack_mode, on_confirm}, - {reconnect_delay, 5} - ]}, - {cega_completion, [ - {source, [ - {protocol, amqp091}, - {uris, ["amqp://${MQ_VHOST:-}"]}, - {declarations, [{'queue.declare', [{exclusive, true}] }, {'queue.bind', [{exchange, <<"sda">>}, {queue, <<>>}, {routing_key, <<"completed">>}] } ] }, - {queue, <<>>}, - {prefetch_count, 10} - ]}, - {destination, [ - {protocol, amqp091}, - {uris, ["amqp://${MQ_VHOST:-}"]}, - {declarations, []}, - {publish_properties, [{delivery_mode, 2}]}, - {publish_fields, [{exchange, <<"to_cega">>}, - {routing_key, <<"files.completed">>} - ]}, - {ack_mode, on_confirm}, - {reconnect_delay, 5} - ]} - ]}, - {cega_error, [ - {source, [ - {protocol, amqp091}, - {uris, ["amqp://${MQ_VHOST:-}"]}, - {declarations, [{'queue.declare', [{exclusive, true}] }, {'queue.bind', [{exchange, <<"sda">>}, {queue, <<>>}, {routing_key, <<"error">>}] } ] }, - {queue, <<>>}, - {prefetch_count, 10} - ]}, - {destination, [ - {protocol, amqp091}, - {uris, ["amqp://${MQ_VHOST:-}"]}, - {declarations, []}, - {publish_properties, [{delivery_mode, 2}]}, - {publish_fields, [{exchange, <<"to_cega">>}, - {routing_key, <<"files.error">>} - ]}, - {ack_mode, on_confirm}, - {reconnect_delay, 5} - ]} - ]}, - {cega_inbox, [ - {source, [ - {protocol, amqp091}, - {uris, ["amqp://${MQ_VHOST:-}"]}, - {declarations, [{'queue.declare', [{exclusive, true}] }, {'queue.bind', [{exchange, <<"sda">>}, {queue, <<>>}, {routing_key, <<"inbox">>}] } ] }, - {queue, <<>>}, - {prefetch_count, 10} - ]}, - {destination, [ - {protocol, amqp091}, - {uris, ["amqp://${MQ_VHOST:-}"]}, - {declarations, []}, - {publish_properties, [{delivery_mode, 2}]}, - {publish_fields, [{exchange, <<"to_cega">>}, - {routing_key, <<"files.inbox">>} - ]}, - {ack_mode, on_confirm}, - {reconnect_delay, 5} - ]} - ]}, - {cega_verified, [ - {source, [ - {protocol, amqp091}, - {uris, ["amqp://${MQ_VHOST:-}"]}, - {declarations, [{'queue.declare', [{exclusive, true}] }, {'queue.bind', [{exchange, <<"sda">>}, {queue, <<>>}, {routing_key, <<"verified">>}] } ] }, - {queue, <<>>}, - {prefetch_count, 10} - ]}, - {destination, [ - {protocol, amqp091}, - {uris, ["amqp://${MQ_VHOST:-}"]}, - {declarations, []}, - {publish_properties, [{delivery_mode, 2}]}, - {publish_fields, [{exchange, <<"to_cega">>}, - {routing_key, <<"files.verified">>} - ]}, - {ack_mode, on_confirm}, - {reconnect_delay, 5} - ]} - ]} - ]} - ]} -]. -EOF -chmod 600 "/var/lib/rabbitmq/advanced.config" + if [ -n "${MQ_VHOST}" ];then + MQ_VHOST="/${MQ_VHOST}" + fi + if [ -e "${MQ_SERVER_CERT}" ] && [ -e "${MQ_SERVER_KEY}" ]; then + cat > "/var/lib/rabbitmq/advanced.config" <<-EOF + [ + {rabbit, [ + {tcp_listeners, []} + ]}, + EOF + else + echo "[" > "/var/lib/rabbitmq/advanced.config" + fi + cat >> "/var/lib/rabbitmq/advanced.config" <<-EOF + {rabbitmq_shovel, [ + {shovels, [ + {to_cega, [ + {source, [ + {protocol, amqp091}, + {uris,[ "amqp://${MQ_VHOST:-}" ]}, + {declarations, [ + {'queue.declare', [{exclusive, true}]}, + {'queue.bind', [{exchange, <<"to_cega">>}, {queue, <<>>}, {routing_key, <<"#">>}]} + ]}, + {queue, <<>>}, + {prefetch_count, 10} + ]}, + {destination, [ + {protocol, amqp091}, + {uris, ["${CEGA_CONNECTION}"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"localega.v1">>}]} + ]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} + ]}, + {cega_completion, [ + {source, [ + {protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, [{'queue.declare', [{exclusive, true}] }, {'queue.bind', [{exchange, <<"sda">>}, {queue, <<>>}, {routing_key, <<"completed">>}] } ] }, + {queue, <<>>}, + {prefetch_count, 10} + ]}, + {destination, [ + {protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"to_cega">>}, + {routing_key, <<"files.completed">>} + ]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} + ]} + ]}, + {cega_error, [ + {source, [ + {protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, [{'queue.declare', [{exclusive, true}] }, {'queue.bind', [{exchange, <<"sda">>}, {queue, <<>>}, {routing_key, <<"error">>}] } ] }, + {queue, <<>>}, + {prefetch_count, 10} + ]}, + {destination, [ + {protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"to_cega">>}, + {routing_key, <<"files.error">>} + ]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} + ]} + ]}, + {cega_inbox, [ + {source, [ + {protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, [{'queue.declare', [{exclusive, true}] }, {'queue.bind', [{exchange, <<"sda">>}, {queue, <<>>}, {routing_key, <<"inbox">>}] } ] }, + {queue, <<>>}, + {prefetch_count, 10} + ]}, + {destination, [ + {protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"to_cega">>}, + {routing_key, <<"files.inbox">>} + ]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} + ]} + ]}, + {cega_verified, [ + {source, [ + {protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, [{'queue.declare', [{exclusive, true}] }, {'queue.bind', [{exchange, <<"sda">>}, {queue, <<>>}, {routing_key, <<"verified">>}] } ] }, + {queue, <<>>}, + {prefetch_count, 10} + ]}, + {destination, [ + {protocol, amqp091}, + {uris, ["amqp://${MQ_VHOST:-}"]}, + {declarations, []}, + {publish_properties, [{delivery_mode, 2}]}, + {publish_fields, [{exchange, <<"to_cega">>}, + {routing_key, <<"files.verified">>} + ]}, + {ack_mode, on_confirm}, + {reconnect_delay, 5} + ]} + ]} + ]} + ]} + ]. + EOF + chmod 600 "/var/lib/rabbitmq/advanced.config" else -cat > "/var/lib/rabbitmq/definitions.json" < "/var/lib/rabbitmq/definitions.json" <<-EOF + { + "users": [ + { + "name": "${MQ_USER}", + "password_hash": "${MQ_PASSWORD_HASH}", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "administrator" + } + ], + "vhosts": [ + { + "name": "${MQ_VHOST:-/}" + } + ], + "permissions": [ + { + "user": "${MQ_USER}", + "vhost": "${MQ_VHOST:-/}", + "configure": ".*", + "write": ".*", + "read": ".*" + } + ], + "policies": [], + "queues": [ + { + "name": "archived", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "backup", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "completed", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "error", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "files", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "inbox", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "ingest", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "mappings", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "accessionIDs", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "verified", + "vhost": "${MQ_VHOST:-/}", + "durable": true, + "auto_delete": false, + "arguments": {} + } + ], + "exchanges": [ + { + "name": "sda", + "vhost": "${MQ_VHOST:-/}", + "type": "topic", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": {} + } + ], + "bindings": [ + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "archived", + "routing_key": "archived" + }, + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "accessionIDs", + "routing_key": "accessionIDs" + }, + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "backup", + "routing_key": "backup" + }, + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "completed", + "routing_key": "completed" + }, + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "error", + "routing_key": "error" + }, + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "files", + "routing_key": "files" + }, + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "inbox", + "routing_key": "inbox" + }, + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "ingest", + "routing_key": "ingest" + }, + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "mappings", + "routing_key": "mappings" + }, + { + "source": "sda", + "vhost": "${MQ_VHOST:-/}", + "destination_type": "queue", + "arguments": {}, + "destination": "verified", + "routing_key": "verified" + } + ] + } + EOF fi chmod 600 "/var/lib/rabbitmq/definitions.json" From bce59c29c1660c3fb2c491b4b1387f696f6272a4 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 29 Nov 2021 19:39:29 +0100 Subject: [PATCH 40/42] Update README --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3e2d888..5ac233b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # NeIC SDA internal message broker in a docker image -We use [RabbitMQ 3.7.8](https://hub.docker.com/_/rabbitmq) including the management plugins. +We use [RabbitMQ 3.8.16](https://hub.docker.com/_/rabbitmq) including the management plugins. ## Configuration @@ -13,12 +13,17 @@ The following environment variables can be used to configure the broker: | `MQ_USER` | Default user (with admin rights) | | `MQ_PASSWORD_HASH` | Password hash for the above user | | `CEGA_CONNECTION` | DSN URL for the shovels and federated queues with CentralEGA | +| `MQ_SERVER_CERT` | Path to the server SSL certificate | +| `MQ_SERVER_KEY` | Path to the server SSL key | +| `MQ_CA` | Path to the CA root certificate | +| `MQ_VERIFY` | Require client certificates (`verify_peer` or `verify_none`) +| `NOTLS` | Run the server without SSL | If you want persistent data, you can use a named volume or a bind-mount and make it point to `/var/lib/rabbitmq`. ## Sample Docker Compose definition -``` +```docker-compose version: '3.3' services: @@ -33,6 +38,7 @@ services: - MQ_VHOST=vhost - MQ_USER=admin - MQ_PASSWORD_HASH=4tHURqDiZzypw0NTvoHhpn8/MMgONWonWxgRZ4NXgR8nZRBz + - NOTLS=true - CEGA_CONNECTION ``` From 521e969c9ceb260d3884161cebdbb646fc31ddc9 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 29 Nov 2021 19:53:07 +0100 Subject: [PATCH 41/42] Delete obsolete hooks folder This is not needed anymore since Docker Hub is not building images any longer. --- hooks/build | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 hooks/build diff --git a/hooks/build b/hooks/build deleted file mode 100644 index 2e5e17b..0000000 --- a/hooks/build +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -docker build \ - --cache-from nbisweden/ega-mq:m7 \ - --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - --build-arg SOURCE_COMMIT=$(git rev-parse --short HEAD) \ - --tag $IMAGE_NAME . From e9b6c635dcc6a78c3d09e83a0fccccfb969205af Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 1 Dec 2021 09:55:13 +0100 Subject: [PATCH 42/42] Update tag_and_build.yaml Set fetch depth to 1 --- .github/workflows/tag_and_build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tag_and_build.yaml b/.github/workflows/tag_and_build.yaml index ab8d951..b93dd37 100644 --- a/.github/workflows/tag_and_build.yaml +++ b/.github/workflows/tag_and_build.yaml @@ -14,7 +14,7 @@ jobs: steps: - uses: actions/checkout@v2 with: - fetch-depth: '2' + fetch-depth: '1' - name: Bump version and push tag id: bump_tag uses: anothrNick/github-tag-action@1.36.0