[7.11] [logstash] Add support to use pattern files (#883) (#1078)

Co-authored-by: Julien Mailleret <[email protected]> Co-authored-by: Tuan Anh Nguyen <[email protected]>
elastic · Feb 23, 2021 · 67a461d · 67a461d
1 parent d259819
commit 67a461d
Show file tree

Hide file tree

Showing 5 changed files with 60 additions and 0 deletions.
diff --git a/logstash/README.md b/logstash/README.md
@@ -129,6 +129,8 @@ useful for the [http input plugin][], for instance.
 | `logstashConfig`          | Allows you to add any config files in `/usr/share/logstash/config/` such as `logstash.yml` and `log4j2.properties` See [values.yaml][] for an example of the formatting                                                              | `{}`                                  |
 | `logstashJavaOpts`        | Java options for Logstash. This is where you should configure the JVM heap size                                                                                                                                                      | `-Xmx1g -Xms1g`                       |
 | `logstashPipeline`        | Allows you to add any pipeline files in `/usr/share/logstash/pipeline/`                                                                                                                                                              | `{}`                                  |
+| `logstashPatternDir`      | Allows you to define a custom directory to store patten files | `/usr/share/logstash/patterns/` |
+| `logstashPattern`        | Allows you to add any pattern files in `logstashPatternDir` | `{}` |
 | `maxUnavailable`          | The [maxUnavailable][] value for the pod disruption budget. By default this will prevent Kubernetes from having more than 1 unhealthy pod in the node group                                                                          | `1`                                   |
 | `nameOverride`            | Overrides the chart name for resources. If not set the name will default to `.Chart.Name`                                                                                                                                            | `""`                                  |
 | `nodeAffinity`            | Value for the [node affinity settings][]                                                                                                                                                                                             | `{}`                                  |

diff --git a/logstash/templates/configmap-pattern.yaml b/logstash/templates/configmap-pattern.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.logstashPattern }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "logstash.fullname" . }}-pattern
+  labels:
+    app: "{{ template "logstash.fullname" . }}"
+    chart: "{{ .Chart.Name }}"
+    heritage: {{ .Release.Service | quote }}
+    release: {{ .Release.Name | quote }}
+data:
+{{- range $path, $config := .Values.logstashPattern }}
+  {{ $path }}: |
+{{ tpl $config $ | indent 4 -}}
+{{- end -}}
+{{- end -}}
diff --git a/logstash/templates/statefulset.yaml b/logstash/templates/statefulset.yaml
@@ -55,6 +55,9 @@ spec:
         {{- if .Values.logstashPipeline }}
         pipelinechecksum: {{ include (print .Template.BasePath "/configmap-pipeline.yaml") . | sha256sum | trunc 63 }}
         {{- end }}
+        {{- if .Values.logstashPattern }}
+        patternchecksum: {{ include (print .Template.BasePath "/configmap-pattern.yaml") . | sha256sum | trunc 63 }}
+        {{- end }}
         {{- if .Values.secrets }}
         secretschecksum: {{ include (print .Template.BasePath "/secret.yaml") . | sha256sum | trunc 63 }}
         {{- end }}
@@ -127,6 +130,11 @@ spec:
           configMap:
             name: {{ template "logstash.fullname" . }}-pipeline
         {{- end }}
+        {{- if .Values.logstashPattern }}
+        - name: logstashpattern
+          configMap:
+            name: {{ template "logstash.fullname" . }}-pattern
+        {{- end }}
       {{- if .Values.extraVolumes }}
 {{ tpl .Values.extraVolumes . | indent 8 }}
       {{- end }}
@@ -191,6 +199,14 @@ spec:
             mountPath: /usr/share/logstash/pipeline/{{ $path }}
             subPath: {{ $path }}
           {{- end -}}
+          {{- if .Values.logstashPattern }}
+          {{- $logstashPatternDir := .Values.logstashPatternDir -}}
+          {{- range $path, $config := .Values.logstashPattern }}
+          - name: logstashpattern
+            mountPath: {{ $logstashPatternDir }}{{ $path }}
+            subPath: {{ $path }} 
+          {{- end -}}
+          {{- end -}}
         {{- if .Values.extraVolumeMounts }}
 {{ tpl .Values.extraVolumeMounts . | indent 10 }}
         {{- end }}

diff --git a/logstash/tests/logstash_test.py b/logstash/tests/logstash_test.py
@@ -622,6 +622,25 @@ def test_adding_in_pipeline():
     )
 
 
+def test_adding_in_pattern():
+    config = """
+logstashPattern:
+  pattern.conf: |
+    DPKG_VERSION [-+~<>\.0-9a-zA-Z]+
+"""
+    r = helm_template(config)
+    c = r["configmap"][name + "-pattern"]["data"]
+
+    assert "pattern.conf" in c
+
+    assert "DPKG_VERSION [-+~<>\.0-9a-zA-Z]+" in c["pattern.conf"]
+
+    assert (
+        "patternchecksum"
+        in r["statefulset"][name]["spec"]["template"]["metadata"]["annotations"]
+    )
+
+
 def test_priority_class_name():
     config = """
 priorityClassName: ""

diff --git a/logstash/values.yaml b/logstash/values.yaml
@@ -25,6 +25,12 @@ logstashPipeline: {}
 #    }
 #    output { stdout { } }
 
+# Allows you to add any pattern files in your custom pattern dir
+logstashPatternDir: "/usr/share/logstash/patterns/"
+logstashPattern: {}
+#    pattern.conf: |
+#      DPKG_VERSION [-+~<>\.0-9a-zA-Z]+
+
 # Extra environment variables to append to this nodeGroup
 # This will be appended to the current 'env:' key. You can use any of the kubernetes env
 # syntax here