docker-compose.yaml

# Copyright 2022 The AI Flow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
version: '3'
x-aiflow-common:
  &aiflow-common
  image: flinkaiflow/flink-ai-flow-dev:latest
  # build: .
  environment:
    AIFLOW_HOME: /usr/local/aiflow
    AIRFLOW_HOME: /usr/local/airflow
    NOTIFICATION_HOME: /usr/local/notification_service
    AIRFLOW_MYSQL_CONN: mysql://root@mysql/airflow
    AIRFLOW_NOTIFICATION_SERVER_URI: aiflow-server:50052
  depends_on:
    redis:
      condition: service_healthy

services:
  redis:
    image: redis:latest
    ports:
      - 6379:6379
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 30s
      retries: 50
    restart: always

  aiflow-server:
    <<: *aiflow-common
    stdin_open: true
    tty: true
    command: /bin/bash -c "
      init-airflow-with-celery-executor.sh $${AIRFLOW_MYSQL_CONN} $${AIRFLOW_NOTIFICATION_SERVER_URI}
      && /bin/bash"
    cap_add:
      - SYS_PTRACE
    ports:
      - "8080:8080"
      - "50051:50051"
      - "50052:50052"
      - "8000:8000"
    volumes:
      - ./logs/aiflow:/usr/local/aiflow/logs
      - ./logs/airflow:/usr/local/airflow/logs
    restart: always
    depends_on:
      airflow-worker:
        condition: service_healthy
      mysql:
        condition: service_healthy

  mysql:
    image: mysql:8
    environment:
      - MYSQL_ALLOW_EMPTY_PASSWORD=true
      - MYSQL_ROOT_HOST=%
      - MYSQL_DATABASE=airflow
    volumes:
      - ./mysql/conf.d:/etc/mysql/conf.d:ro
      - /dev/urandom:/dev/random
    ports:
      - 13306:3306
    command: ['mysqld', '--character-set-server=utf8mb3',
              '--collation-server=utf8_general_ci']
    healthcheck:
      test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"]
      interval: 10s
      timeout: 10s
      retries: 5
    restart: always

  hadoop-dfs:
    image: pravega/hdfs:2.7.7
    ports:
      - "22022:22"
      - "8020:8020"
      - "50010:50010"
      - "50020:50020"
      - "50070:50070"
      - "50075:50075"
    restart: always

  airflow-worker:
    <<: *aiflow-common
    command: /bin/bash -c "
        init-airflow-with-celery-executor.sh $${AIRFLOW_MYSQL_CONN} $${AIRFLOW_NOTIFICATION_SERVER_URI}
        && airflow celery worker"
    ports:
      - 8793
    scale: 3
    healthcheck:
      test:
        - "CMD-SHELL"
        - 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
      interval: 10s
      timeout: 10s
      retries: 5
    restart: always

  flower:
    <<: *aiflow-common
    command: /bin/bash -c "
        init-airflow-with-celery-executor.sh $${AIRFLOW_MYSQL_CONN} $${AIRFLOW_NOTIFICATION_SERVER_URI}
        && airflow celery flower"
    ports:
      - 5555:5555
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:5555/"]
      interval: 10s
      timeout: 10s
      retries: 5
    restart: always