TSD Docker VM Docker Stack file

docker-template.yml template file should be turned into docker-stack.yml file by removing all redundant for this setup microservices, filling in all the required env-vars, injecting all the required secrets and file-system volumes.

In the TSD Docker VM context, we would need the following secrets:

• ega.sec.pem - private master key to decrypt the data
• ega.sec.pass - file with a passphrase for the key above
• client.pem - client SSL certificate to connect to the TSD database
• jwt.pub.pem - public RSA key to validate JWT tokens

Docker secrets are created with the docker secret create command, for example:

sudo docker secret create ega.sec.pem ega-configuration/ega.sec.pem

The complete docker-stack.yml file on a TSD docker VM should look like this (the configuration below is for Go Data-In microservices and Java Data-Out microservice):

version: '3.3'

services:

  ingest:
    image: neicnordic/sda-pipeline:latest
    deploy:
      restart_policy:
        condition: on-failure
        delay: 5s
        window: 120s
    environment:
      - ARCHIVE_TYPE=posix
      - ARCHIVE_LOCATION=/ega/archive
      - BROKER_HOST={{MQ_HOST}}
      - BROKER_PORT=5671
      - BROKER_USER={{PRIVATE_BROKER_USER}}
      - BROKER_PASSWORD={{PRIVATE_BROKER_PASSWORD}}
      - BROKER_VHOST={{PRIVATE_BROKER_VHOST}}
      - BROKER_QUEUE=ingest
      - BROKER_EXCHANGE=sda
      - BROKER_ROUTINGKEY=archived
      - BROKER_ROUTINGERROR=error
      - BROKER_SSL=true
      - BROKER_VERIFYPEER=false
      - BROKER_INSECURESKIPVERIFY=true
      - C4GH_PASSPHRASE={{KEY_PASSWORD}}
      - C4GH_FILEPATH=/etc/ega/ega.sec
      - DB_HOST={{DB_HOST}}
      - DB_PORT=5432
      - DB_USER={{DB_LEGA_IN_USER}}
      - DB_PASSWORD={{DB_LEGA_IN_PASSWORD}}
      - DB_DATABASE=lega
      - DB_SSLMODE=require
      - INBOX_TYPE=posix
      - INBOX_LOCATION=/ega/inbox
      - LOG_LEVEL=debug
    secrets:
      - source: ega.sec.pem
        target: /etc/ega/ega.sec
    volumes:
      - /tmp/tsd:/ega/inbox
      - /tmp/vault:/ega/archive
    command: "sda-ingest"

  verify:
    image: neicnordic/sda-pipeline:latest
    deploy:
      restart_policy:
        condition: on-failure
        delay: 5s
        window: 120s
    environment:
      - ARCHIVE_TYPE=posix
      - ARCHIVE_LOCATION=/ega/archive
      - BROKER_HOST={{MQ_HOST}}
      - BROKER_PORT=5671
      - BROKER_USER={{PRIVATE_BROKER_USER}}
      - BROKER_PASSWORD={{PRIVATE_BROKER_PASSWORD}}
      - BROKER_VHOST={{PRIVATE_BROKER_VHOST}}
      - BROKER_QUEUE=archived
      - BROKER_EXCHANGE=sda
      - BROKER_ROUTINGKEY=verified
      - BROKER_ROUTINGERROR=error
      - BROKER_SSL=true
      - BROKER_VERIFYPEER=false
      - BROKER_INSECURESKIPVERIFY=true
      - C4GH_PASSPHRASE={{KEY_PASSWORD}}
      - C4GH_FILEPATH=/etc/ega/ega.sec
      - DB_HOST={{DB_HOST}}
      - DB_PORT=5432
      - DB_USER={{DB_LEGA_IN_USER}}
      - DB_PASSWORD={{DB_LEGA_IN_PASSWORD}}
      - DB_DATABASE=lega
      - DB_SSLMODE=require
      - LOG_LEVEL=debug
    secrets:
      - source: ega.sec.pem
        target: /etc/ega/ega.sec
    volumes:
      - /tmp/vault:/ega/archive
    command: "sda-verify"

  finalize:
    image: neicnordic/sda-pipeline:latest
    deploy:
      restart_policy:
        condition: on-failure
        delay: 5s
        window: 120s
    environment:
      - BROKER_HOST={{MQ_HOST}}
      - BROKER_PORT=5671
      - BROKER_USER={{PRIVATE_BROKER_USER}}
      - BROKER_PASSWORD={{PRIVATE_BROKER_PASSWORD}}
      - BROKER_VHOST={{PRIVATE_BROKER_VHOST}}
      - BROKER_QUEUE=accessionIDs
      - BROKER_EXCHANGE=sda
      - BROKER_ROUTINGKEY=completed
      - BROKER_ROUTINGERROR=error
      - BROKER_SSL=true
      - BROKER_VERIFYPEER=false
      - BROKER_INSECURESKIPVERIFY=true
      - DB_HOST={{DB_HOST}}
      - DB_PORT=5432
      - DB_USER={{DB_LEGA_IN_USER}}
      - DB_PASSWORD={{DB_LEGA_IN_PASSWORD}}
      - DB_DATABASE=lega
      - DB_SSLMODE=require
      - LOG_LEVEL=debug
    command: "sda-finalize"

  mapper:
    image: neicnordic/sda-pipeline:latest
    deploy:
      restart_policy:
        condition: on-failure
        delay: 5s
        window: 120s
    environment:
      - BROKER_HOST={{MQ_HOST}}
      - BROKER_PORT=5671
      - BROKER_USER={{PRIVATE_BROKER_USER}}
      - BROKER_PASSWORD={{PRIVATE_BROKER_PASSWORD}}
      - BROKER_VHOST={{PRIVATE_BROKER_VHOST}}
      - BROKER_QUEUE=mappings
      - BROKER_EXCHANGE=sda
      - BROKER_ROUTINGERROR=error
      - BROKER_SSL=true
      - BROKER_VERIFYPEER=false
      - BROKER_INSECURESKIPVERIFY=true
      - DB_HOST={{DB_HOST}}
      - DB_PORT=5432
      - DB_USER={{DB_LEGA_OUT_USER}}
      - DB_PASSWORD={{DB_LEGA_OUT_PASSWORD}}
      - DB_DATABASE=lega
      - DB_SSLMODE=require
      - LOG_LEVEL=debug
    command: "sda-mapper"

  doa:
    image: neicnordic/sda-doa:latest
    ports:
      - 80:8080
    deploy:
      restart_policy:
        condition: on-failure
        delay: 5s
        window: 120s
    environment:
      - SSL_MODE=require
      - SSL_ENABLED=false
      - ARCHIVE_PATH={{ARCHIVE_PATH}}
      - DB_INSTANCE={{DB_HOST}}
      - POSTGRES_DB={{DB_DATABASE_NAME}}
      - POSTGRES_PASSWORD={{DB_LEGA_OUT_PASSWORD}}
      - OUTBOX_ENABLED=false
    secrets:
      - source: client.pem
        target: /etc/ega/ssl/client.cert
      - source: jwt.pub.pem
        target: /etc/ega/jwt/passport.pem
      - source: jwt.pub.pem
        target: /etc/ega/jwt/visa.pem
      - source: ega.sec.pem
        target: /etc/ega/crypt4gh/key.pem
      - source: ega.sec.pass
        target: /etc/ega/crypt4gh/key.pass
    volumes:
      - /tmp/vault:/ega/archive

secrets:
  client.pem:
    external: true
  ega.sec.pem:
    external: true
  ega.sec.pass:
    external: true
  jwt.pub.pem:
    external: true

All the template placeholders (those in {{...}}) should be replaced with real values of corresponding env-vars.

Similarly, /tmp/tsd and /tmp/archive file-system location should be replaced with real paths in TSD storage.