Init initial

README.md

@@ -0,0 +1,67 @@
+# CENTRALIZAÇÃO DE LOGS DO KUBERNETES COM GRAYLOG + FLUENTD
+
+Neste tutorial será mostrado como centralizar os logs do Kubernetes com Graylog + Fluentd.
+
+Existem outras maneiras talvez mais elegante, mas optamos em enviar os logs (stdout/stderr) dos pods para o Graylog utilizando o [fluentd-kubernetes-daemonset](https://github.com/fluent/fluentd-kubernetes-daemonset). 
+O fluentd(daemonset) irá utilizar as configurações passadas pelo configmap.
+
+
+### Documentação oficial
+
+- [Graylog](https://docs.graylog.org/en/3.2/index.html#)
+- [Fluentd](https://www.fluentd.org/) 
+- [Kubernetes](https://kubernetes.io/)
+
+### Versões utilizadas
+- Graylog >= 3.1 
+- fluentd >= 1.10 
+
+## Use sua configuração
+
+Disponibilizamos dois manifestos de configmap.
+
+> fluent-configmap.yaml --> Responsável pelo arquivo de configurado do fluent.conf
+
+> fluent-kubernetes-configmap.yaml --> Responsável pelo arquivo de configurado do kubernetes.conf
+
+
+Exemplo de como criar um configmap usando como base um arquivo.
+```bash 
+kubectl create configmap fluent-kubernetes-configmap --from-file=kubernetes.conf --namespace=kube-system
+
+kubectl create configmap fluent-configmap --from-file=fluent.conf --namespace=kube-system
+```
+
+
+## Uso   
+
+- Execute os manifestos do configmap. 
+  ```bash
+  kubectl create -f fluent-kubernetes-configmap.yaml -f fluent-configmap.yaml
+  ```
+- Execute o manifesto do rbac.  
+  ```bash
+  kubectl create -f rbac.yaml
+  ```
+- Execute o manifesto do daemonset 
+  ```bash 
+  kubectl create -f fluent-daemonset.yaml
+  ```
+
+
+## Graylog
+
+1. Configure um input gelf. (Protocolo (TCP/UDP) e porta de sua escolha)
+2. No Input gelf criado anteriormente, vá em Manager Extractors
+  ![graylog1](./images/Graylog1.png)
+3. Adicione um extractor
+  ![graylog1](./images/Graylog2.png)
+4. Load a mensagem e crie um json. 
+  ![graylog1](./images/Graylog3.png)
+5. Caso não queira realizar os passos anteriores (1-4) manualmente, disponibilizamos um content-pack na pasta `./files/` para criação do input+extractors.
+    Necessário apenas importar o content pack e instalar!
+6. Exemplo de mensagem 
+  ![graylog1](./images/Graylog4.jpg)
+
+
+

confs/fluent.conf

@@ -0,0 +1,52 @@
+
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb
+
+#@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf"
+#@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf"
+@include kubernetes.conf
+@include conf.d/*.conf
+
+
+# The gelf plugin assumes input in utf-8
+<filter **>
+  @type record_modifier
+  char_encoding utf-8
+</filter>
+
+<match kubernetes.var.log.containers.**fluentd**.log>
+  @type null
+</match>
+
+<match kubernetes.var.log.containers.**kube-system**.log>
+  @type null
+</match>
+
+<match kubernetes.**>
+  @type rewrite_tag_filter
+  <rule>
+    key $['kubernetes']['labels']['app']
+        pattern ^(.+)$
+        tag $1
+      </rule>
+</match>
+
+
+<match **>
+  @type gelf
+  @id out_graylog
+
+  @log_level info
+  include_tag_key true
+  host "#{ENV['FLUENT_GRAYLOG_HOST']}"
+  port "#{ENV['FLUENT_GRAYLOG_PORT']}"
+  protocol "#{ENV['FLUENT_GRAYLOG_PROTOCOL'] || 'udp'}"
+  <buffer>
+    flush_thread_count 8
+    flush_interval 5s
+    chunk_limit_size 8M
+    queue_limit_length 512
+    retry_max_interval 30
+    retry_forever true
+  </buffer>
+</match>

confs/kubernetes.conf

@@ -0,0 +1,36 @@
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb
+
+<label @FLUENT_LOG>
+ <match fluent.**>
+   @type null
+ </match>
+</label>
+
+<source>
+ @type tail
+ @id in_tail_container_logs
+ path /var/log/containers/*.log
+ pos_file /var/log/fluentd-containers.log.pos
+ tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}"
+ exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}"
+ read_from_head true
+ <parse>
+  @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}"
+  time_format %Y-%m-%dT%H:%M:%S.%NZ
+ </parse>
+</source>
+
+<filter kubernetes.**>
+ @type kubernetes_metadata
+ ca_file /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+ bearer_token_file /var/run/secrets/kubernetes.io/serviceaccount/token
+</filter>
+
+<filter access>
+ @type record_transformer
+ <record>
+  hostname "#{ENV['NODENAME']}"
+  host "#{ENV['NODENAME']}"
+  </record>
+</filter>    

files/graylog-gelf_input-content-pack-05062020.json

@@ -0,0 +1,127 @@
+{
+  "v": 1,
+  "id": "8338cc27-ff35-4bf2-b5b2-fea7b1be8b9e",
+  "rev": 1,
+  "name": "gelf_input",
+  "summary": "gelf_input",
+  "description": "",
+  "vendor": "gelf_input",
+  "url": "",
+  "parameters": [],
+  "entities": [
+    {
+      "v": "1",
+      "type": {
+        "name": "input",
+        "version": "1"
+      },
+      "id": "c4760faf-0c7f-4e91-a108-fa6436c14957",
+      "data": {
+        "title": {
+          "@type": "string",
+          "@value": "GELF_UDP_12201"
+        },
+        "configuration": {
+          "recv_buffer_size": {
+            "@type": "integer",
+            "@value": 262144
+          },
+          "port": {
+            "@type": "integer",
+            "@value": 12201
+          },
+          "number_worker_threads": {
+            "@type": "integer",
+            "@value": 4
+          },
+          "bind_address": {
+            "@type": "string",
+            "@value": "0.0.0.0"
+          },
+          "decompress_size_limit": {
+            "@type": "integer",
+            "@value": 8388608
+          }
+        },
+        "static_fields": {},
+        "type": {
+          "@type": "string",
+          "@value": "org.graylog2.inputs.gelf.udp.GELFUDPInput"
+        },
+        "global": {
+          "@type": "boolean",
+          "@value": true
+        },
+        "extractors": [
+          {
+            "target_field": {
+              "@type": "string",
+              "@value": ""
+            },
+            "condition_value": {
+              "@type": "string",
+              "@value": ""
+            },
+            "order": {
+              "@type": "integer",
+              "@value": 0
+            },
+            "converters": [],
+            "configuration": {
+              "list_separator": {
+                "@type": "string",
+                "@value": ", "
+              },
+              "kv_separator": {
+                "@type": "string",
+                "@value": "="
+              },
+              "key_prefix": {
+                "@type": "string",
+                "@value": ""
+              },
+              "key_separator": {
+                "@type": "string",
+                "@value": "_"
+              },
+              "replace_key_whitespace": {
+                "@type": "boolean",
+                "@value": false
+              },
+              "key_whitespace_replacement": {
+                "@type": "string",
+                "@value": "_"
+              }
+            },
+            "source_field": {
+              "@type": "string",
+              "@value": "kubernetes"
+            },
+            "title": {
+              "@type": "string",
+              "@value": "json"
+            },
+            "type": {
+              "@type": "string",
+              "@value": "JSON"
+            },
+            "cursor_strategy": {
+              "@type": "string",
+              "@value": "COPY"
+            },
+            "condition_type": {
+              "@type": "string",
+              "@value": "NONE"
+            }
+          }
+        ]
+      },
+      "constraints": [
+        {
+          "type": "server-version",
+          "version": ">=3.2.4+a407287"
+        }
+      ]
+    }
+  ]
+}

manifests/fluent-configmap.yaml

@@ -0,0 +1,60 @@
+apiVersion: v1
+data:
+  fluent.conf: |2-
+
+    # AUTOMATICALLY GENERATED
+    # DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb
+
+    #@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf"
+    #@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf"
+    @include kubernetes.conf
+    @include conf.d/*.conf
+
+
+    # The gelf plugin assumes input in utf-8
+    <filter **>
+      @type record_modifier
+      char_encoding utf-8
+    </filter>
+
+    <match kubernetes.var.log.containers.**fluentd**.log>
+      @type null
+    </match>
+
+    ## Descomentando esse bloco, voce nao ira receber logs do pods que estão no namespace kube-system 
+    # <match kubernetes.var.log.containers.**kube-system**.log>
+    #   @type null
+    # </match>
+
+    <match kubernetes.**>
+      @type rewrite_tag_filter
+      <rule>
+        key $['kubernetes']['labels']['app']
+            pattern ^(.+)$
+            tag $1
+          </rule>
+    </match>
+
+
+    <match **>
+      @type gelf
+      @id out_graylog
+
+      @log_level info
+      include_tag_key true
+      host "#{ENV['FLUENT_GRAYLOG_HOST']}"
+      port "#{ENV['FLUENT_GRAYLOG_PORT']}"
+      protocol "#{ENV['FLUENT_GRAYLOG_PROTOCOL'] || 'udp'}"
+      <buffer>
+        flush_thread_count 8
+        flush_interval 5s
+        chunk_limit_size 8M
+        queue_limit_length 512
+        retry_max_interval 30
+        retry_forever true
+      </buffer>
+    </match>
+kind: ConfigMap
+metadata:
+  name: fluent-configmap
+  namespace: kube-system

manifests/fluent-kubernetes-configmap.yaml

@@ -0,0 +1,40 @@
+apiVersion: v1
+data:
+  kubernetes.conf: |2- 
+    <label @FLUENT_LOG>
+    <match fluent.**>
+      @type null
+    </match>
+    </label>
+   
+    <source>
+      @type tail
+      @id in_tail_container_logs
+      path /var/log/containers/*.log
+      pos_file /var/log/fluentd-containers.log.pos
+      tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}"
+      exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}"
+      read_from_head true
+      <parse>
+      @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}"
+      time_format %Y-%m-%dT%H:%M:%S.%NZ
+      </parse>
+    </source>
+   
+    <filter kubernetes.**>
+      @type kubernetes_metadata
+      ca_file /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      bearer_token_file /var/run/secrets/kubernetes.io/serviceaccount/token
+    </filter>
+    
+    <filter access>
+      @type record_transformer
+      <record>
+      hostname "#{ENV['NODENAME']}"
+      host "#{ENV['NODENAME']}"
+      </record>
+    </filter>
+kind: ConfigMap
+metadata:
+  name: fluent-kubernetes-configmap
+  namespace: kube-system