From b2b84e5d9e4b56581f3f85ad89cfdfce625df3dd Mon Sep 17 00:00:00 2001 From: Craig P Steffen Date: Wed, 11 Dec 2024 13:16:04 -0600 Subject: [PATCH] prelim elk docker-compose Believed-basically-working docker-compose.yml. This brings up ELK in a configuration that the security genreally is turned on but the ssl checking is turned off. This gets us JUST far enough that the commands that set the main root password seem to work. This commit is to put a stake in the ground. This was based on a cdrohook config file that was a bit older, so this will need to get merged in with current versions of it. Signed-off-by: Craig P Steffen --- docker-compose.yml_craig | 328 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 328 insertions(+) create mode 100644 docker-compose.yml_craig diff --git a/docker-compose.yml_craig b/docker-compose.yml_craig new file mode 100644 index 0000000..d5bb221 --- /dev/null +++ b/docker-compose.yml_craig @@ -0,0 +1,328 @@ +services: + + # ---------------------------------------------------------------------- + # REVERSE PROXY + # ---------------------------------------------------------------------- + + setup: + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + volumes: + - certs:/usr/share/elasticsearch/config/certs + user: "0" + command: > + bash -c ' + if [ x${ELASTIC_PASSWORD} == x ]; then + echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; + exit 1; + elif [ x${KIBANA_PASSWORD} == x ]; then + echo "Set the KIBANA_PASSWORD environment variable in the .env file"; + exit 1; + fi; + if [ ! -f config/certs/ca.zip ]; then + echo "Creating CA"; + bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip; + unzip config/certs/ca.zip -d config/certs; + fi; + if [ ! -f config/certs/certs.zip ]; then + echo "Creating certs"; + echo -ne \ + "instances:\n"\ + " - name: es01\n"\ + " dns:\n"\ + " - es01\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + " - name: kibana\n"\ + " dns:\n"\ + " - kibana\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + > config/certs/instances.yml; + bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key; + unzip config/certs/certs.zip -d config/certs; + fi; + echo "Setting file permissions" + chown -R root:root config/certs; + find . -type d -exec chmod 750 \{\} \;; + find . -type f -exec chmod 640 \{\} \;; + echo "Waiting for Elasticsearch availability"; + until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done; + echo "Setting kibana_system password"; + until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; + echo "All done!"; + ' + healthcheck: + test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"] + interval: 1s + timeout: 5s + retries: 120 + + traefik: + image: "traefik:v2.11" + command: + - --log.level=INFO + - --api=true + - --api.dashboard=true + - --api.insecure=true + # Entrypoints + - --entrypoints.http.address=:80 + - --entrypoints.http.http.redirections.entryPoint.to=https + - --entrypoints.https.address=:443 + - --entrypoints.https.http.tls.certresolver=myresolver + # letsencrypt + - --certificatesresolvers.myresolver.acme.email=${TRAEFIK_ACME_EMAIL} + - --certificatesresolvers.myresolver.acme.storage=/config/acme.json + # uncomment to use testing certs + #- --certificatesresolvers.myresolver.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory + - --certificatesresolvers.myresolver.acme.httpchallenge=true + - --certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=http + # Docker setup + - --providers.docker=true + - --providers.docker.endpoint=unix:///var/run/docker.sock + - --providers.docker.exposedbydefault=false + - --providers.docker.watch=true + restart: "unless-stopped" + security_opt: + - no-new-privileges:true + ports: + - "80:80" + - "443:443" + volumes: + - "traefik:/config" + - "/var/run/docker.sock:/var/run/docker.sock:ro" + + # ---------------------------------------------------------------------- + # MESSAGE BROKER + # ---------------------------------------------------------------------- + rabbitmq: + image: rabbitmq:3.13-management + hostname: rabbitmq + restart: unless-stopped + environment: + RABBITMQ_DEFAULT_USER: "${RABBITMQ_USERNAME:-guest}" + RABBITMQ_DEFAULT_PASS: "${RABBITMQ_PASSWORD:-guest}" + volumes: + - rabbitmq:/var/lib/rabbitmq + - ./50-criticalmaas.conf:/etc/rabbitmq/conf.d/50-criticalmaas.conf:ro + + # ---------------------------------------------------------------------- + # CDR HOOK + # ---------------------------------------------------------------------- + cdrhook: + image: ncsa/criticalmaas-cdr:latest + hostname: cdrhook + build: cdrhook + restart: unless-stopped + depends_on: + - rabbitmq + environment: + CDR_TOKEN: "${CDR_TOKEN}" + CDR_KEEP_EVENT: "no" + CALLBACK_URL: "https://${SERVER_NAME}/cdr" + CALLBACK_SECRET: "${CALLBACK_SECRET}" + CALLBACK_USERNAME: "${CALLBACK_USERNAME}" + CALLBACK_PASSWORD: "${CALLBACK_PASSWORD}" + RABBITMQ_URI: "amqp://${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}@rabbitmq/%2F" + PREFIX: "" + labels: + - "traefik.enable=true" + - "traefik.http.routers.cdrhook.rule=Host(`${SERVER_NAME}`) && PathPrefix(`/cdr`)" + volumes: + - cdrhook:/data + + # ---------------------------------------------------------------------- + # RABBITMQ MONITOR + # ---------------------------------------------------------------------- + monitor: + image: ncsa/criticalmaas-monitor:latest + hostname: monitor + build: monitor + restart: unless-stopped + depends_on: + - rabbitmq + environment: + RABBITMQ_MGMT_URL: ${RABBITMQ_MGMT_URL} + RABBITMQ_USERNAME: ${RABBITMQ_USERNAME} + RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD} + labels: + - "traefik.enable=true" + - "traefik.http.routers.monitor.rule=Host(`${SERVER_NAME}`) && PathPrefix(`/monitor`)" + + # ---------------------------------------------------------------------- + # DATA PROCESSING PIPELINE + # use one, or more, per model to be executed + # ---------------------------------------------------------------------- + golden_muscat: + image: ncsa/criticalmaas-pipeline:latest + build: ../uiuc-pipeline + runtime: nvidia + profiles: + - pipeline + depends_on: + - rabbitmq + environment: + NVIDIA_VISIBLE_DEVICES: all + PREFIX: "" + command: + - -v + - --data + - /data + - --log + - /logs/logs.latest + - --output + - /output + - --feedback + - /feedback + - --amqp + - "amqp://${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}@rabbitmq/%2F" + - --inactive_timeout + - "86000" + - --model + - golden_muscat + restart: "unless-stopped" + volumes: + - "data:/data" + - "logs:/logs" + - "output:/output" + - "feedback:/feedback" + + # ---------------------------------------------------------------------- + # DOWNLOADER and UPLOADER + # ---------------------------------------------------------------------- + downloader: + image: ncsa/criticalmaas-downloader:latest + build: uploader + restart: "unless-stopped" + profiles: + - pipeline + depends_on: + - rabbitmq + environment: + RABBITMQ_URI: "amqp://${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}@rabbitmq/%2F" + volumes: + - "data:/data" + + uploader: + image: ncsa/criticalmaas-uploader:latest + build: uploader + restart: "unless-stopped" + profiles: + - pipeline + depends_on: + - rabbitmq + environment: + CDR_TOKEN: "${CDR_TOKEN}" + RABBITMQ_URI: "amqp://${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}@rabbitmq/%2F" + PREFIX: "" + volumes: + - "output:/output" + + es01: + depends_on: + setup: + condition: service_healthy + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + labels: + co.elastic.logs/module: elasticsearch + volumes: + - certs:/usr/share/elasticsearch/config/certs + - esdata01:/usr/share/elasticsearch/data + ports: + - ${ES_PORT}:9200 + environment: + - node.name=es01 + - cluster.name=${CLUSTER_NAME} + - discovery.type=single-node + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - xpack.security.enabled=true +# - xpack.security.enabled=false +# - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.enabled=false + - xpack.security.http.ssl.key=certs/es01/es01.key + - xpack.security.http.ssl.certificate=certs/es01/es01.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt +# - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.enabled=false + - xpack.security.transport.ssl.key=certs/es01/es01.key + - xpack.security.transport.ssl.certificate=certs/es01/es01.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=${LICENSE} + mem_limit: ${ES_MEM_LIMIT} + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", + ] + interval: 10s + timeout: 10s + retries: 120 + + kibana: + depends_on: + es01: + condition: service_healthy + image: docker.elastic.co/kibana/kibana:${STACK_VERSION} + labels: + co.elastic.logs/module: kibana + volumes: + - certs:/usr/share/kibana/config/certs + - kibanadata:/usr/share/kibana/data + ports: + - ${KIBANA_PORT}:5601 + environment: + - SERVERNAME=kibana + - ELASTICSEARCH_HOSTS=https://es01:9200 + - ELASTICSEARCH_USERNAME=kibana_system + - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} + - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt + - XPACK_SECURITY_ENCRYPTIONKEY=${ENCRYPTION_KEY} + - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ENCRYPTION_KEY} + - XPACK_REPORTING_ENCRYPTIONKEY=${ENCRYPTION_KEY} + mem_limit: ${KB_MEM_LIMIT} + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'", + ] + interval: 10s + timeout: 10s + retries: 120 + +#networks: +# elk: +# default: +# name: elastic +# driver: bridge +# external: false + +volumes: + traefik: + rabbitmq: + cdrhook: + feedback: + data: + logs: + output: + test_data: + certs: + driver: local + esdata01: + driver: local + kibanadata: + driver: local + metricbeatdata01: + driver: local + filebeatdata01: + driver: local + logstashdata01: + driver: local