From f9be7c25d01f7a97ea31bd121d1026656db194ca Mon Sep 17 00:00:00 2001 From: dehaocheng Date: Thu, 17 Aug 2023 15:55:28 +0800 Subject: [PATCH] Adjusting the parameters of fluentbit Signed-off-by: dehaocheng --- .../fluentbit/fluentbitconfig-fluentBitConfig.yaml | 1 + roles/common/templates/custom-fluentbit-fluentBit.yaml.j2 | 2 +- roles/ks-auditing/templates/custom-input-auditing.yaml.j2 | 2 +- .../templates/custom-output-elasticsearch-auditing.yaml.j2 | 2 ++ .../templates/custom-output-opensearch-auditing.yaml.j2 | 2 ++ roles/ks-events/templates/custom-input-events.yaml.j2 | 2 +- .../templates/custom-output-elasticsearch-events.yaml.j2 | 2 ++ .../ks-events/templates/custom-output-opensearch-events.yaml.j2 | 2 ++ roles/ks-logging/templates/custom-input-logging.yaml.j2 | 2 +- .../templates/custom-output-elasticsearch-logging.yaml.j2 | 2 ++ .../templates/custom-output-opensearch-logging.yaml.j2 | 2 ++ 11 files changed, 17 insertions(+), 4 deletions(-) diff --git a/roles/common/files/fluentbit-operator/fluentbit/fluentbitconfig-fluentBitConfig.yaml b/roles/common/files/fluentbit-operator/fluentbit/fluentbitconfig-fluentBitConfig.yaml index ff2fda82d..ab2f14285 100644 --- a/roles/common/files/fluentbit-operator/fluentbit/fluentbitconfig-fluentBitConfig.yaml +++ b/roles/common/files/fluentbit-operator/fluentbit/fluentbitconfig-fluentBitConfig.yaml @@ -8,6 +8,7 @@ metadata: spec: service: parsersFile: parsers.conf + httpServer: true inputSelector: matchLabels: logging.kubesphere.io/enabled: "true" diff --git a/roles/common/templates/custom-fluentbit-fluentBit.yaml.j2 b/roles/common/templates/custom-fluentbit-fluentBit.yaml.j2 index 199c1509f..5e30d9cc4 100644 --- a/roles/common/templates/custom-fluentbit-fluentBit.yaml.j2 +++ b/roles/common/templates/custom-fluentbit-fluentBit.yaml.j2 @@ -12,7 +12,7 @@ spec: resources: limits: cpu: 500m - memory: 200Mi + memory: 500Mi requests: cpu: 10m memory: 25Mi diff --git a/roles/ks-auditing/templates/custom-input-auditing.yaml.j2 b/roles/ks-auditing/templates/custom-input-auditing.yaml.j2 index 4aaa07ddd..2ce1c9577 100644 --- a/roles/ks-auditing/templates/custom-input-auditing.yaml.j2 +++ b/roles/ks-auditing/templates/custom-input-auditing.yaml.j2 @@ -10,7 +10,7 @@ spec: tail: db: /fluent-bit/tail/pos-auditing.db dbSync: Normal - memBufLimit: 5MB + memBufLimit: 100MB {% if (logging_container_runtime== 'containerd') or (logging_container_runtime== 'crio') %} parser: cri {% else %} diff --git a/roles/ks-auditing/templates/custom-output-elasticsearch-auditing.yaml.j2 b/roles/ks-auditing/templates/custom-output-elasticsearch-auditing.yaml.j2 index 2e7a9f755..7bed5d982 100644 --- a/roles/ks-auditing/templates/custom-output-elasticsearch-auditing.yaml.j2 +++ b/roles/ks-auditing/templates/custom-output-elasticsearch-auditing.yaml.j2 @@ -9,7 +9,9 @@ metadata: spec: match: kube_auditing es: + bufferSize: 20MB logstashFormat: true + traceError: true generateID: true host: "{% if common.es.externalElasticsearchHost is defined and common.es.externalElasticsearchHost != "" %}{{ common.es.externalElasticsearchHost }}{% else %}elasticsearch-logging-data.kubesphere-logging-system.svc{% endif %}" logstashPrefix: "ks-{{ common.es.elkPrefix }}-auditing" diff --git a/roles/ks-auditing/templates/custom-output-opensearch-auditing.yaml.j2 b/roles/ks-auditing/templates/custom-output-opensearch-auditing.yaml.j2 index df73de3fe..1bea8e3d8 100644 --- a/roles/ks-auditing/templates/custom-output-opensearch-auditing.yaml.j2 +++ b/roles/ks-auditing/templates/custom-output-opensearch-auditing.yaml.j2 @@ -10,6 +10,8 @@ spec: match: kube_auditing opensearch: logstashFormat: true + bufferSize: 20MB + traceError: true host: "{% if common.opensearch.externalOpensearchHost is defined and common.opensearch.externalOpensearchHost != "" %}{{ common.opensearch.externalOpensearchHost }}{% else %}opensearch-cluster-data.kubesphere-logging-system.svc{% endif %}" logstashPrefix: "ks-{{ common.opensearch.opensearchPrefix }}-auditing" port: {% if common.opensearch.externalOpensearchPort is defined and common.opensearch.externalOpensearchPort != "" %}{{ common.opensearch.externalOpensearchPort }}{% else %}9200{% endif %} diff --git a/roles/ks-events/templates/custom-input-events.yaml.j2 b/roles/ks-events/templates/custom-input-events.yaml.j2 index d35a24db6..ede1dbec0 100644 --- a/roles/ks-events/templates/custom-input-events.yaml.j2 +++ b/roles/ks-events/templates/custom-input-events.yaml.j2 @@ -16,7 +16,7 @@ spec: parser: docker {% endif %} refreshIntervalSeconds: 10 - memBufLimit: 5MB + memBufLimit: 100MB skipLongLines: true db: /fluent-bit/tail/pos-events.db dbSync: Normal diff --git a/roles/ks-events/templates/custom-output-elasticsearch-events.yaml.j2 b/roles/ks-events/templates/custom-output-elasticsearch-events.yaml.j2 index f1ae70a5f..8d4848f5f 100644 --- a/roles/ks-events/templates/custom-output-elasticsearch-events.yaml.j2 +++ b/roles/ks-events/templates/custom-output-elasticsearch-events.yaml.j2 @@ -9,6 +9,8 @@ metadata: spec: match: kube_events es: + bufferSize: 20MB + traceError: true logstashFormat: true generateID: true host: "{% if common.es.externalElasticsearchHost is defined and common.es.externalElasticsearchHost != "" %}{{ common.es.externalElasticsearchHost }}{% else %}elasticsearch-logging-data.kubesphere-logging-system.svc{% endif %}" diff --git a/roles/ks-events/templates/custom-output-opensearch-events.yaml.j2 b/roles/ks-events/templates/custom-output-opensearch-events.yaml.j2 index a4ded42b0..a26a3c55b 100644 --- a/roles/ks-events/templates/custom-output-opensearch-events.yaml.j2 +++ b/roles/ks-events/templates/custom-output-opensearch-events.yaml.j2 @@ -9,6 +9,8 @@ metadata: spec: match: kube_events opensearch: + bufferSize: 20MB + traceError: true logstashFormat: true generateID: true host: "{% if common.opensearch.externalOpensearchHost is defined and common.opensearch.externalOpensearchHost != "" %}{{ common.opensearch.externalOpensearchHost }}{% else %}opensearch-cluster-data.kubesphere-logging-system.svc{% endif %}" diff --git a/roles/ks-logging/templates/custom-input-logging.yaml.j2 b/roles/ks-logging/templates/custom-input-logging.yaml.j2 index 2754691c8..cc4e9c667 100644 --- a/roles/ks-logging/templates/custom-input-logging.yaml.j2 +++ b/roles/ks-logging/templates/custom-input-logging.yaml.j2 @@ -16,7 +16,7 @@ spec: parser: docker {% endif %} refreshIntervalSeconds: 10 - memBufLimit: 5MB + memBufLimit: 100MB skipLongLines: true db: /fluent-bit/tail/pos.db dbSync: Normal diff --git a/roles/ks-logging/templates/custom-output-elasticsearch-logging.yaml.j2 b/roles/ks-logging/templates/custom-output-elasticsearch-logging.yaml.j2 index e65108913..cf427e42b 100644 --- a/roles/ks-logging/templates/custom-output-elasticsearch-logging.yaml.j2 +++ b/roles/ks-logging/templates/custom-output-elasticsearch-logging.yaml.j2 @@ -28,6 +28,8 @@ spec: name: "elasticsearch-credentials" {% endif %} {% endif %} + bufferSize: 20MB + traceError: true generateID: true logstashPrefix: "ks-{{ common.es.elkPrefix }}-log" logstashFormat: true diff --git a/roles/ks-logging/templates/custom-output-opensearch-logging.yaml.j2 b/roles/ks-logging/templates/custom-output-opensearch-logging.yaml.j2 index 16ce54ddd..4479fadb7 100644 --- a/roles/ks-logging/templates/custom-output-opensearch-logging.yaml.j2 +++ b/roles/ks-logging/templates/custom-output-opensearch-logging.yaml.j2 @@ -28,6 +28,8 @@ spec: name: "opensearch-credentials" {% endif %} {% endif %} + bufferSize: 20MB + traceError: true logstashPrefix: "ks-{{ common.opensearch.opensearchPrefix }}-logging" logstashFormat: true timeKey: "@timestamp"