-
Notifications
You must be signed in to change notification settings - Fork 0
/
docker-compose.yml
124 lines (111 loc) · 3.54 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
# Copyright Broadcom, Inc. All Rights Reserved.
# SPDX-License-Identifier: APACHE-2.0
version: '2'
services:
sftp:
image: lscr.io/linuxserver/openssh-server:latest
volumes:
- ./sftp_stocks_data/:/config/sftp_stocks_data
ports:
- "2222:2222"
environment:
- PUID=1000
- PGID=1000
- TZ=Etc/UTC
- SUDO_ACCESS=true
- PASSWORD_ACCESS=true
- USER_PASSWORD=password
- USER_NAME=abhishek
mysql:
image: bitnami/mysql
ports:
- '3307:3306'
environment:
- MYSQL_ROOT_PASSWORD=root
volumes:
- ./mysql:/opt/bitnami/mysql/data
spark:
image: docker.io/bitnami/spark:3.5
environment:
- SPARK_MODE=master
- SPARK_RPC_AUTHENTICATION_ENABLED=no
- SPARK_RPC_ENCRYPTION_ENABLED=no
- SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
- SPARK_SSL_ENABLED=no
- SPARK_USER=spark
ports:
- '8080:8080'
spark-worker:
image: docker.io/bitnami/spark:3.5
environment:
- SPARK_MODE=worker
- SPARK_MASTER_URL=spark://spark:7077
- SPARK_WORKER_MEMORY=1G
- SPARK_WORKER_CORES=1
- SPARK_RPC_AUTHENTICATION_ENABLED=no
- SPARK_RPC_ENCRYPTION_ENABLED=no
- SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
- SPARK_SSL_ENABLED=no
- SPARK_USER=spark
postgresql:
image: docker.io/bitnami/postgresql:16
volumes:
- './postgresql_data:/bitnami/postgresql'
environment:
- POSTGRESQL_DATABASE=bitnami_airflow
- POSTGRESQL_USERNAME=bn_airflow
- POSTGRESQL_PASSWORD=bitnami1
# ALLOW_EMPTY_PASSWORD is recommended only for development.
- ALLOW_EMPTY_PASSWORD=yes
redis:
image: docker.io/bitnami/redis:7.0
volumes:
- './redis_data:/bitnami'
environment:
# ALLOW_EMPTY_PASSWORD is recommended only for development.
- ALLOW_EMPTY_PASSWORD=yes
airflow-scheduler:
image: docker.io/bitnami/airflow-scheduler:2
environment:
- AIRFLOW_DATABASE_NAME=bitnami_airflow
- AIRFLOW_DATABASE_USERNAME=bn_airflow
- AIRFLOW_DATABASE_PASSWORD=bitnami1
- AIRFLOW_EXECUTOR=CeleryExecutor
- AIRFLOW_WEBSERVER_HOST=airflow
- AIRFLOW_LOAD_EXAMPLES=no
- AIRFLOW_SECRET_KEY=Test
- AIRFLOW__CORE__TEST_CONNECTION=Enabled
volumes:
- ./dags:/opt/bitnami/airflow/dags
airflow-worker:
image: docker.io/bitnami/airflow-worker:2
environment:
- AIRFLOW_DATABASE_NAME=bitnami_airflow
- AIRFLOW_DATABASE_USERNAME=bn_airflow
- AIRFLOW_DATABASE_PASSWORD=bitnami1
- AIRFLOW_EXECUTOR=CeleryExecutor
- AIRFLOW_WEBSERVER_HOST=airflow
- AIRFLOW_LOAD_EXAMPLES=no
- AIRFLOW_SECRET_KEY=Test
- AIRFLOW__CORE__TEST_CONNECTION=Enabled
volumes:
- ./dags:/opt/bitnami/airflow/dags
- ./logs:/opt/bitnami/airflow/logs
- ./stocks_data:/opt/bitnami/stocks_data
airflow:
image: docker.io/bitnami/airflow:2
environment:
- AIRFLOW_DATABASE_NAME=bitnami_airflow
- AIRFLOW_DATABASE_USERNAME=bn_airflow
- AIRFLOW_DATABASE_PASSWORD=bitnami1
- AIRFLOW_PASSWORD=bitnami
- AIRFLOW_USERNAME=user
- AIRFLOW_EXECUTOR=CeleryExecutor
- AIRFLOW_LOAD_EXAMPLES=no
- AIRFLOW_SECRET_KEY=Test
- AIRFLOW__CORE__TEST_CONNECTION=Enabled
ports:
- '8081:8080'
volumes:
- ./logs:/opt/bitnami/airflow/logs