docker-compose.yml

version: '3'
services:
  redis:
    image: redis:alpine
    container_name: redis
    command: redis-server --requirepass ${REDIS_PASSWORD}
    environment:
      - REDIS_REPLICATION_MODE=master
    networks:
      lab-network:
        ipv4_address: 172.23.0.20
    volumes:
      - redis-data:/var/lib/redis
      - ${PWD}/redis/redis.conf:/usr/local/etc/redis/redis.conf

  postgres:
    image: postgres
    container_name: postgres
    environment:
      - POSTGRES_USER=lab
      - POSTGRES_PASSWORD=lab
      - POSTGRES_DB=lab
    networks:
      lab-network:
        ipv4_address: 172.23.0.30
    volumes:
      - postgres-data:/var/lib/postgresql/data
      - postgres-initdbd:/docker-entrypoint-initdb.d
  zookeeper:
    image: bitnami/zookeeper:latest
    container_name: zookeeper
    hostname: zookeeper
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes
      - ZOO_AUTOPURGE_INTERVAL=1
    networks:
      lab-network:
        ipv4_address: 172.23.0.10

  kafka:
    image: bitnami/kafka:latest
    container_name: kafka01
    environment:
      - KAFKA_BROKER_ID=1
      - KAFKA_LISTENERS=PLAINTEXT://:9092
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
    volumes:
      - kafka-data:/bitnami
    networks:
      lab-network:
        ipv4_address: 172.23.0.50
    depends_on:
      - zookeeper

  clickhouse01:
    image: yandex/clickhouse-server:${CLICKHOUSE_VER}
    container_name: clickhouse01
    hostname: clickhouse01
    networks:
      lab-network:
        ipv4_address: 172.23.0.11
    ports:
      - "127.0.0.1:8123:8123"
      - "127.0.0.1:9000:9000"
    ulimits:
      nproc: 65535
      nofile:
        soft: 262144
        hard: 262144
    volumes:
      - clickhouse01-data:/var/lib/clickhouse
      - clickhouse01-config:/etc/clickhouse-server
    depends_on:
      - zookeeper
      - minios3

  clickhouse02:
    image: yandex/clickhouse-server:${CLICKHOUSE_VER}
    container_name: clickhouse02
    hostname: clickhouse02
    networks:
      lab-network:
        ipv4_address: 172.23.0.12
    ulimits:
      nproc: 65535
      nofile:
        soft: 262144
        hard: 262144
    volumes:
      - clickhouse02-data:/var/lib/clickhouse
      - clickhouse02-config:/etc/clickhouse-server
    depends_on:
      - zookeeper
      - minios3

  clickhouse03:
    image: yandex/clickhouse-server:${CLICKHOUSE_VER}
    container_name: clickhouse03
    hostname: clickhouse03
    networks:
      lab-network:
        ipv4_address: 172.23.0.13
    ulimits:
      nproc: 65535
      nofile:
        soft: 262144
        hard: 262144 
    volumes:
      - clickhouse03-data:/var/lib/clickhouse
      - clickhouse03-config:/etc/clickhouse-server
    depends_on:
      - zookeeper
      - minios3

  clickhouse04:
    image: yandex/clickhouse-server:${CLICKHOUSE_VER}
    container_name: clickhouse04
    hostname: clickhouse04
    networks:
      lab-network:
        ipv4_address: 172.23.0.14
    ulimits:
      nproc: 65535
      nofile:
        soft: 262144
        hard: 262144   
    volumes:
      - clickhouse04-data:/var/lib/clickhouse
      - clickhouse04-config:/etc/clickhouse-server
    depends_on:
      - zookeeper
      - minios3
  
  airflow-scheduler:
    image: apache/airflow
    container_name: airflow-scheduler
    entrypoint: ./scripts/scheduler/entrypoint.sh
    restart: on-failure
    depends_on:
      - postgres
    environment:
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
      - AIRFLOW__CORE__EXECUTOR=LocalExecutor
    networks:
      lab-network:
        ipv4_address: 172.23.0.60
    volumes:
      - airflow-dags:/opt/airflow/dags
      - airflow-logs:/opt/airflow/logs
      - airflow-scripts:/opt/airflow/scripts

  airflow-webserver:
    image: apache/airflow
    container_name: airflow-webserver
    entrypoint: ./scripts/webserver/entrypoint.sh
    restart: on-failure
    depends_on:
      - postgres
      - airflow-scheduler
    environment:
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
      - AIRFLOW__CORE__EXECUTOR=LocalExecutor
    networks:
      lab-network:
        ipv4_address: 172.23.0.61

    volumes:
      - airflow-dags:/opt/airflow/dags
      - airflow-logs:/opt/airflow/logs
      - airflow-scripts:/opt/airflow/scripts

  minios3:
    image: minio/minio
    container_name: minios3
    user: "1000:1000"
    command: minio server /data
    volumes:
      - minio-data:/data
    environment:
      MINIO_ACCESS_KEY: lab
      MINIO_SECRET_KEY: lab12345
    networks:
      lab-network:
        ipv4_address: 172.23.0.90 

  sshnode:
    build:
      context: ./ssh_node
      args:
        SSH_PASS: ${SSH_PASS}
        SSH_NODE_INSTALL: ${SSH_NODE_INSTALL}
    image: jumpbox:latest
    container_name: jumpbox
    hostname: jumpbox
    restart: always
    volumes:
      - ${PWD}/ssh_node/sshd_config:/etc/ssh/sshd_config
      - jumpbox-home:/home/${SSH_USER}
    ports:
      - "0.0.0.0:${HOST_SSH_PORT}:22"
    networks:
      lab-network:
        ipv4_address: 172.23.0.40

networks:
  lab-network:
    ipam:
      config:
        - subnet: 172.23.0.0/24

volumes:
  minio-data:
    driver: local
    driver_opts:
      o: bind
      type: none
      device: ${PWD}/minio-data
  postgres-data:
    driver: local
  postgres-initdbd:
    driver: local
    driver_opts:
      o: bind
      type: none
      device: ${PWD}/postgres_initdb.d
  redis-data:
    driver: local
  kafka-data:
    driver: local
  clickhouse01-data:
    driver: local
  clickhouse02-data:
    driver: local
  clickhouse03-data:
    driver: local
  clickhouse04-data:
    driver: local
  jumpbox-home:
    driver: local
    driver_opts:
      o: bind
      type: none
      device: ${PWD}/jumpbox_home
  clickhouse01-config:
    driver: local
    driver_opts:
      o: bind
      type: none
      device: ${PWD}/config/clickhouse01
  clickhouse02-config:
    driver: local
    driver_opts:
      o: bind
      type: none
      device: ${PWD}/config/clickhouse02
  clickhouse03-config:
    driver: local
    driver_opts:
      o: bind
      type: none
      device: ${PWD}/config/clickhouse03
  clickhouse04-config:
    driver: local
    driver_opts:
      o: bind
      type: none
      device: ${PWD}/config/clickhouse04
  airflow-dags:
    driver: local
    driver_opts:
      o: bind
      type: none
      device: ${PWD}/airflow-dags
  airflow-scripts:
    driver: local
    driver_opts:
      o: bind
      type: none
      device: ${PWD}/airflow-scripts
  airflow-logs:
    driver: local
    driver_opts:
      o: bind
      type: none
      device: ${PWD}/airflow-logs