prelim elk docker-compose

Believed-basically-working docker-compose.yml. This brings up ELK in a configuration that the security genreally is turned on but the ssl checking is turned off. This gets us JUST far enough that the commands that set the main root password seem to work. This commit is to put a stake in the ground. This was based on a cdrohook config file that was a bit older, so this will need to get merged in with current versions of it. Signed-off-by: Craig P Steffen <[email protected]>
DARPA-CRITICALMAAS · Dec 11, 2024 · b2b84e5 · b2b84e5
1 parent 50172fb
commit b2b84e5
Showing 1 changed file with 328 additions and 0 deletions.
diff --git a/docker-compose.yml_craig b/docker-compose.yml_craig
@@ -0,0 +1,328 @@
+services:
+
+  # ----------------------------------------------------------------------
+  # REVERSE PROXY
+  # ----------------------------------------------------------------------
+
+  setup:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+    user: "0"
+    command: >
+      bash -c '
+        if [ x${ELASTIC_PASSWORD} == x ]; then
+          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
+          exit 1;
+        elif [ x${KIBANA_PASSWORD} == x ]; then
+          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
+          exit 1;
+        fi;
+        if [ ! -f config/certs/ca.zip ]; then
+          echo "Creating CA";
+          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
+          unzip config/certs/ca.zip -d config/certs;
+        fi;
+        if [ ! -f config/certs/certs.zip ]; then
+          echo "Creating certs";
+          echo -ne \
+          "instances:\n"\
+          "  - name: es01\n"\
+          "    dns:\n"\
+          "      - es01\n"\
+          "      - localhost\n"\
+          "    ip:\n"\
+          "      - 127.0.0.1\n"\
+          "  - name: kibana\n"\
+          "    dns:\n"\
+          "      - kibana\n"\
+          "      - localhost\n"\
+          "    ip:\n"\
+          "      - 127.0.0.1\n"\
+          > config/certs/instances.yml;
+          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
+          unzip config/certs/certs.zip -d config/certs;
+         fi;
+         echo "Setting file permissions"
+         chown -R root:root config/certs;
+         find . -type d -exec chmod 750 \{\} \;;
+         find . -type f -exec chmod 640 \{\} \;;
+         echo "Waiting for Elasticsearch availability";
+         until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
+        echo "Setting kibana_system password";
+        until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
+        echo "All done!";
+      '
+    healthcheck:
+      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
+      interval: 1s
+      timeout: 5s
+      retries: 120
+
+  traefik:
+    image: "traefik:v2.11"
+    command:
+      - --log.level=INFO
+      - --api=true
+      - --api.dashboard=true
+      - --api.insecure=true
+      # Entrypoints
+      - --entrypoints.http.address=:80
+      - --entrypoints.http.http.redirections.entryPoint.to=https
+      - --entrypoints.https.address=:443
+      - --entrypoints.https.http.tls.certresolver=myresolver
+      # letsencrypt
+      - --certificatesresolvers.myresolver.acme.email=${TRAEFIK_ACME_EMAIL}
+      - --certificatesresolvers.myresolver.acme.storage=/config/acme.json
+      # uncomment to use testing certs
+      #- --certificatesresolvers.myresolver.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory
+      - --certificatesresolvers.myresolver.acme.httpchallenge=true
+      - --certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=http
+      # Docker setup
+      - --providers.docker=true
+      - --providers.docker.endpoint=unix:///var/run/docker.sock
+      - --providers.docker.exposedbydefault=false
+      - --providers.docker.watch=true
+    restart: "unless-stopped"
+    security_opt:
+      - no-new-privileges:true
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - "traefik:/config"
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+
+  # ----------------------------------------------------------------------
+  # MESSAGE BROKER
+  # ----------------------------------------------------------------------
+  rabbitmq:
+    image: rabbitmq:3.13-management
+    hostname: rabbitmq
+    restart: unless-stopped
+    environment:
+      RABBITMQ_DEFAULT_USER: "${RABBITMQ_USERNAME:-guest}"
+      RABBITMQ_DEFAULT_PASS: "${RABBITMQ_PASSWORD:-guest}"
+    volumes:
+      - rabbitmq:/var/lib/rabbitmq
+      - ./50-criticalmaas.conf:/etc/rabbitmq/conf.d/50-criticalmaas.conf:ro
+
+  # ----------------------------------------------------------------------
+  # CDR HOOK
+  # ----------------------------------------------------------------------
+  cdrhook:
+    image: ncsa/criticalmaas-cdr:latest
+    hostname: cdrhook
+    build: cdrhook
+    restart: unless-stopped
+    depends_on:
+      - rabbitmq
+    environment:
+      CDR_TOKEN: "${CDR_TOKEN}"
+      CDR_KEEP_EVENT: "no"
+      CALLBACK_URL: "https://${SERVER_NAME}/cdr"
+      CALLBACK_SECRET: "${CALLBACK_SECRET}"
+      CALLBACK_USERNAME: "${CALLBACK_USERNAME}"
+      CALLBACK_PASSWORD: "${CALLBACK_PASSWORD}"
+      RABBITMQ_URI: "amqp://${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}@rabbitmq/%2F"
+      PREFIX: ""
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.cdrhook.rule=Host(`${SERVER_NAME}`) && PathPrefix(`/cdr`)"
+    volumes:
+      - cdrhook:/data
+
+  # ----------------------------------------------------------------------
+  # RABBITMQ MONITOR
+  # ----------------------------------------------------------------------
+  monitor:
+    image: ncsa/criticalmaas-monitor:latest
+    hostname: monitor
+    build: monitor
+    restart: unless-stopped
+    depends_on:
+      - rabbitmq
+    environment:
+      RABBITMQ_MGMT_URL: ${RABBITMQ_MGMT_URL}
+      RABBITMQ_USERNAME: ${RABBITMQ_USERNAME}
+      RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD}
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.monitor.rule=Host(`${SERVER_NAME}`) && PathPrefix(`/monitor`)"
+
+  # ----------------------------------------------------------------------
+  # DATA PROCESSING PIPELINE
+  # use one, or more, per model to be executed
+  # ----------------------------------------------------------------------
+  golden_muscat:
+    image: ncsa/criticalmaas-pipeline:latest
+    build: ../uiuc-pipeline
+    runtime: nvidia
+    profiles:
+      - pipeline
+    depends_on:
+      - rabbitmq
+    environment:
+      NVIDIA_VISIBLE_DEVICES: all
+      PREFIX: ""
+    command:
+      - -v 
+      - --data
+      - /data
+      - --log
+      - /logs/logs.latest
+      - --output
+      - /output
+      - --feedback
+      - /feedback
+      - --amqp
+      - "amqp://${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}@rabbitmq/%2F"
+      - --inactive_timeout
+      - "86000"
+      - --model
+      - golden_muscat
+    restart: "unless-stopped"
+    volumes:
+      - "data:/data"
+      - "logs:/logs"
+      - "output:/output"
+      - "feedback:/feedback"
+
+  # ----------------------------------------------------------------------
+  # DOWNLOADER and UPLOADER
+  # ----------------------------------------------------------------------
+  downloader:
+    image: ncsa/criticalmaas-downloader:latest
+    build: uploader
+    restart: "unless-stopped"
+    profiles:
+      - pipeline
+    depends_on:
+      - rabbitmq
+    environment:
+      RABBITMQ_URI: "amqp://${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}@rabbitmq/%2F"
+    volumes:
+      - "data:/data"
+
+  uploader:
+    image: ncsa/criticalmaas-uploader:latest
+    build: uploader
+    restart: "unless-stopped"
+    profiles:
+      - pipeline
+    depends_on:
+      - rabbitmq
+    environment:
+      CDR_TOKEN: "${CDR_TOKEN}"
+      RABBITMQ_URI: "amqp://${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}@rabbitmq/%2F"
+      PREFIX: ""
+    volumes:
+      - "output:/output"
+
+  es01:
+    depends_on:
+      setup:
+        condition: service_healthy
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    labels:
+      co.elastic.logs/module: elasticsearch
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+      - esdata01:/usr/share/elasticsearch/data
+    ports:
+      - ${ES_PORT}:9200
+    environment:
+      - node.name=es01
+      - cluster.name=${CLUSTER_NAME}
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=true
+#      - xpack.security.enabled=false
+#      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.enabled=false
+      - xpack.security.http.ssl.key=certs/es01/es01.key
+      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+#      - xpack.security.transport.ssl.enabled=true
+      - xpack.security.transport.ssl.enabled=false
+      - xpack.security.transport.ssl.key=certs/es01/es01.key
+      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.license.self_generated.type=${LICENSE}
+    mem_limit: ${ES_MEM_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+  kibana:
+    depends_on:
+      es01:
+        condition: service_healthy
+    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
+    labels:
+      co.elastic.logs/module: kibana
+    volumes:
+      - certs:/usr/share/kibana/config/certs
+      - kibanadata:/usr/share/kibana/data
+    ports:
+      - ${KIBANA_PORT}:5601
+    environment:
+      - SERVERNAME=kibana
+      - ELASTICSEARCH_HOSTS=https://es01:9200
+      - ELASTICSEARCH_USERNAME=kibana_system
+      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
+      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
+      - XPACK_SECURITY_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+      - XPACK_REPORTING_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+    mem_limit: ${KB_MEM_LIMIT}
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+#networks:
+#  elk:
+#  default:
+#    name: elastic
+#    driver: bridge
+#    external: false
+
+volumes:
+  traefik:
+  rabbitmq:
+  cdrhook:
+  feedback:
+  data:
+  logs:
+  output:
+  test_data:
+  certs:
+    driver: local
+  esdata01:
+    driver: local
+  kibanadata:
+    driver: local
+  metricbeatdata01:
+    driver: local
+  filebeatdata01:
+    driver: local
+  logstashdata01:
+    driver: local