add first version of filebeat: #790, upgrade elasticsearch and kibana…

… version
TrivadisPF · Jan 2, 2024 · 92065a3 · 92065a3
1 parent 34ffed6
commit 92065a3
Show file tree

Hide file tree

Showing 5 changed files with 71 additions and 2 deletions.
diff --git a/documentation/changes.md b/documentation/changes.md
@@ -40,6 +40,7 @@ The Modern Data Platform version 1.16.0 contains the following bug fixes and enh
  * Cribl Stream
  * Cribl Edge
  * Blaze (FHIR Server)
+ * FileBeat
 
 ### Removed Services
 
@@ -87,6 +88,8 @@ The Modern Data Platform version 1.16.0 contains the following bug fixes and enh
  * Update `materialize` to `v0.77.1`
  * Update `cadvisor` to `v0.47.2`
  * Update `portainer` to `2.19.4-alpine`
+ * Update `elasticsearch` to `8.11.3`
+ * Update `kibana` to `8.11.3`
 
 ### Enhancements
 

diff --git a/modern-data-platform-stack/generator-config/stack-config.yml b/modern-data-platform-stack/generator-config/stack-config.yml
@@ -152,7 +152,7 @@ vars:
 
   #elasticsearch
   ELASTICSEARCH_7_version: 7.17.9
-  ELASTICSEARCH_8_version: 8.7.0
+  ELASTICSEARCH_8_version: 8.11.3
 
   # OpenSearch
   OPENSEARCH_version: 2
@@ -201,6 +201,9 @@ vars:
   #FluentD
   FLUENTD_version: v1.16-debian
 
+  #FluentD
+  FILEBEAT_version: 8.11.3
+
   #nodered
   NODERED_version: 2.2.3
 
@@ -289,7 +292,7 @@ vars:
   GRAFANA_version: 10.2.0
   #kibana
   KIBANA_7_version: 7.17.9   # should match elasticsearch version (minor increases should work though)
-  KIBANA_8_version: 8.7.0    # should match elasticsearch version (minor increases should work though)
+  KIBANA_8_version: 8.11.3    # should match elasticsearch version (minor increases should work though)
 
   # Metabase
   METABASE_version: latest

diff --git a/modern-data-platform-stack/generator-config/templates/docker-compose.yml.j2 b/modern-data-platform-stack/generator-config/templates/docker-compose.yml.j2
@@ -196,6 +196,7 @@ networks:
 {% set __CRIBL_version = CRIBL_version | default('latest') -%}
 
 {% set __FLUENTD_version = FLUENTD_version | default('latest') -%}
+{% set __FILEBEAT_version = FILEBEAT_version | default('latest') -%}
 
 {% set __CONDUIT_version = CONDUIT_version | default('latest') -%}
 
@@ -8232,6 +8233,45 @@ services:
     restart: {{container_restart_policy}}
 {% endif %}   {#  FLUENTD_enable  #}
 
+{%if FILEBEAT_enable | default(false) %}
+  #  ================================== FileBeat ========================================== #
+  filebeat:
+    image: docker.elastic.co/beats/filebeat:{{__FILEBEAT_version}}
+    container_name: filebeat
+    hostname: filebeat
+    labels:
+      com.platys.name: 'filebeat'
+    environment:
+      strict.perms: false
+    {% if ELASTICSEARCH_enable | default(false) %}
+      output.elasticsearch.hosts: elasticsearch-1:9200
+      ELASTICSEARCH_HOSTS: elasticsearch-1:9200
+      ELASTICSEARCH_USERNAME: na
+      ELASTICSEARCH_PASSWORD: na
+    {% endif -%}   {#  ELASTICSEARCH_enable #}
+    {% if KIBANA_enable | default(false) %}
+      setup.kibana.host: kibana:5601
+    {% endif -%}   {#  KIBANA_enable #}
+    {%if use_timezone | default(false) %}
+      TZ: {{use_timezone}}
+    {% endif -%}   {#  use_timezone #}
+    volumes:
+      - ./data-transfer:/data-transfer
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    {% if FILEBEAT_conf_file is defined and FILEBEAT_conf_file and FILEBEAT_conf_file | length %}
+      - ./conf/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+    {% endif -%}   {#  FLUENTD_conf_file #}
+    {%if use_timezone | default(false) %}
+      - "./etc/timezone:/etc/timezone:ro"
+      - "./etc/localtime:/etc/localtime:ro"
+    {% endif -%}   {#  use_timezone #}
+    {%if logging_driver is defined and logging_driver and logging_driver in ('fluentd','loki','syslog','splunk') | default(false) %}
+    <<: *logging
+    {% endif -%}   {#  logging_driver is defined ... #}
+    restart: {{container_restart_policy}}
+{% endif %}   {#  FILEBEAT_enable  #}
+
 {%if NODERED_enable | default(false) %}
   #  ================================== Node-RED ========================================== #
   nodered-1:

diff --git a/modern-data-platform-stack/generator-config/vars/config.yml b/modern-data-platform-stack/generator-config/vars/config.yml
@@ -886,6 +886,12 @@
       # one of 'gzip', 'lzo', 'json', 'txt'
       FLUENTD_s3_bucket_file_type: 'json'
 
+      #
+      # ===== FileBeat ========
+      #
+      FILEBEAT_enable: false
+      FILEBEAT_conf_file: ''
+
       #
       # ===== Node Red  ========
       #

diff --git a/modern-data-platform-stack/static-data/conf/filebeat/filebeat.yml.templ b/modern-data-platform-stack/static-data/conf/filebeat/filebeat.yml.templ
@@ -0,0 +1,17 @@
+filebeat.config:
+  modules:
+    path: ${path.config}/modules.d/*.yml
+    reload.enabled: false
+
+filebeat.autodiscover:
+  providers:
+    - type: docker
+      hints.enabled: true
+
+processors:
+- add_cloud_metadata: ~
+
+output.elasticsearch:
+  hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-1:9200}'
+  username: '${ELASTICSEARCH_USERNAME:}'
+  password: '${ELASTICSEARCH_PASSWORD:}'