diff --git a/charts/fluent-operator/Chart.yaml b/charts/fluent-operator/Chart.yaml index c96c502d..2dce357e 100644 --- a/charts/fluent-operator/Chart.yaml +++ b/charts/fluent-operator/Chart.yaml @@ -23,4 +23,4 @@ dependencies: - name: fluentd-crds repository: "file://charts/fluentd-crds" version: 3.2.0 - condition: fluentd.crdsEnable + condition: fluentd.crdsEnable \ No newline at end of file diff --git a/charts/fluent-operator/README.md b/charts/fluent-operator/README.md index 5a2976f3..73f7fbe0 100644 --- a/charts/fluent-operator/README.md +++ b/charts/fluent-operator/README.md @@ -1,76 +1,35 @@ -# Fluent Operator Helm Chart +# Fluent Operator Helm chart -[Fluent Operator](https://github.com/fluent/fluent-operator) provides great flexibility in building a logging layer based on Fluent Bit and Fluentd. +[Fluent Operator](https://github.com/fluent/fluent-operator/) provides a Kubernetes-native logging pipeline based on Fluent-Bit and Fluentd. -## Deploy Fluent Operator with Helm +## Installation -> Note: For the helm based install, Helm v3.2.1 or higher is needed. - -The Fluent Bit section of the Fluent Operator supports different CRI `docker`, `containerd`, and `CRI-O`. -`containerd` and `CRI-O` use the `CRI Log` format which is different with `docker`, they requires additional parser to parse JSON application logs. You should set different `containerRuntime` depending on your container runtime. - -The default runtime is docker, you can choose other runtimes as follows. - -If your container runtime is `containerd` or `cri-o`, you can set the `containerRuntime` parameter to `containerd` or `crio`. e.g. +To install or upgrade Fluent Operator using Helm: ```shell -helm install fluent-operator --create-namespace -n fluent charts/fluent-operator/ --set containerRuntime=containerd -``` +export FLUENT_OPERATOR_CONTAINER_RUNTIME="containerd" # or "cri-o", "docker" depending on the container runtime being used (see `values.yaml`) -Install through the online chart link: - -```shell -helm install fluent-operator --create-namespace -n fluent https://github.com/fluent/fluent-operator/releases/download/< version >/fluent-operator.tgz +helm repo add fluent https://fluent.github.io/helm-charts +helm upgrade --install fluent-operator fluent/fluent-operator \ + --create-namespace \ + --set containerRuntime=${FLUENT_OPERATOR_CONTAINER_RUNTIME} ``` -> Please replace < version > with a actual version like v1.0.0 +By default, all CRDs required for Fluent Operator will be installed. To prevent `helm install` from installing CRDs, you can set `fluent-bit.crdsEnable` or `fluentd.crdsEnable` to `false`. -Fluent Operator CRDs will be installed by default when running a helm install for the chart. But if the CRD already exists, it will be skipped with a warning. So make sure you install the CRDs by yourself if you upgrade your Fluent Operator version. +## Upgrading -> Note: During the upgrade process, if a CRD was previously created using the create operation, an error will occur during the apply operation. Using apply here allows the CRD to be replaced and created in its entirety in a single operation. +Helm [does not manage the lifecycle of CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), so if the Fluent Operator CRDs already exist, subsequent +chart upgrades will not add or remove CRDs even if they have changed. During upgrades, users should manually update CRDs: -To replace the CRDs pull the current helm chart: ``` wget https://github.com/fluent/fluent-operator/releases/download//fluent-operator.tgz tar -xf fluent-operator.tgz -``` - -To update the CRDs, run the following command: - -```shell kubectl replace -f fluent-operator/crds ``` -## Fluent Operator Walkthrough - -For more info on various use cases of Fluent Operator, you can refer to [Fluent-Operator-Walkthrough](https://github.com/kubesphere-sigs/fluent-operator-walkthrough). - -## Collect Kubernetes logs -This guide provisions a logging pipeline including the Fluent Bit DaemonSet and its log input/filter/output configurations to collect Kubernetes logs including container logs and kubelet logs. - -![logging stack](https://raw.githubusercontent.com/fluent/fluent-operator/master/docs/images/logging-stack.svg) - -> Note that you need a running Elasticsearch v5+ cluster to receive log data before start. **Remember to adjust [output-elasticsearch.yaml](https://github.com/fluent/fluent-operator/blob/master/manifests/logging-stack/output-elasticsearch.yaml) to your own es setup**. Kafka and Fluentd outputs are optional and are turned off by default. - -#### Deploy the Kubernetes logging pipeline with Helm - -You can also deploy the Kubernetes logging pipeline with Helm, just need to set the `Kubernetes` parameter to `true`(default): - -```shell -helm upgrade fluent-operator --create-namespace -n fluent charts/fluent-operator/ --set containerRuntime=docker -``` - -If you want to deploy `fluentd`, just need to set the `fluentd.enable` parameter to `true`.: +## Chart Values -```shell -helm upgrade fluent-operator --create-namespace -n fluent charts/fluent-operator/ --set containerRuntime=docker,fluentd.enable=true -``` - -Within a couple of minutes, you should observe an index available: - -```shell -$ curl localhost:9200/_cat/indices -green open ks-logstash-log-2020.04.26 uwQuoO90TwyigqYRW7MDYQ 1 1 99937 0 31.2mb 31.2mb ``` - -Success! \ No newline at end of file +helm show values fluent/fluent-operator +``` \ No newline at end of file diff --git a/charts/fluent-operator/charts/fluent-bit-crds/Chart.yaml b/charts/fluent-operator/charts/fluent-bit-crds/Chart.yaml index 66f67316..df76be64 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/Chart.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/Chart.yaml @@ -1,14 +1,23 @@ apiVersion: v2 name: fluent-bit-crds -description: A Helm chart delivering fluent-bit controller CRDS -keywords: - - logging - - fluent-bit - - operator +description: A Helm chart delivering fluenbt-bit controller CRDS +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +#type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) version: 3.2.0 -appVersion: "3.2.0" -sources: - - https://github.com/fluent/fluent-operator -maintainers: - - name: wenchajun - email: dehaocheng@kubesphere.io + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "3.1.0" diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml index cdbaaaee..aacf75b0 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml @@ -809,6 +809,10 @@ spec: a file once is rotated in case some pending data is flushed. format: int64 type: integer + skipEmptyLines: + description: Skips empty lines in the log file from any further + processing or output. + type: boolean skipLongLines: description: |- When a monitored file reach it buffer capacity due to a very long line (Buffer_Max_Size), diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml index bea2ab3f..b434232b 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml @@ -289,6 +289,10 @@ spec: logType: description: Name of the event type. type: string + logTypeKey: + description: Set a record key that will populate 'logtype'. If + the key is found, it will have precedence + type: string sharedKey: description: Specify the primary or the secondary client authentication key @@ -1747,7 +1751,6 @@ spec: type: string host: description: IP address or hostname of the target InfluxDB service. - format: ipv6 type: string httpPassword: description: Password for user defined in HTTP_User @@ -1917,8 +1920,8 @@ spec: port: description: TCP port of the target InfluxDB service. format: int32 - maximum: 65536 - minimum: 0 + maximum: 65535 + minimum: 1 type: integer sequenceTag: description: The name of the tag whose value is incremented for diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml index 3342392b..b3b11db9 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml @@ -289,6 +289,10 @@ spec: logType: description: Name of the event type. type: string + logTypeKey: + description: Set a record key that will populate 'logtype'. If + the key is found, it will have precedence + type: string sharedKey: description: Specify the primary or the secondary client authentication key @@ -1747,7 +1751,6 @@ spec: type: string host: description: IP address or hostname of the target InfluxDB service. - format: ipv6 type: string httpPassword: description: Password for user defined in HTTP_User @@ -1917,8 +1920,8 @@ spec: port: description: TCP port of the target InfluxDB service. format: int32 - maximum: 65536 - minimum: 0 + maximum: 65535 + minimum: 1 type: integer sequenceTag: description: The name of the tag whose value is incremented for diff --git a/charts/fluent-operator/charts/fluentd-crds/Chart.yaml b/charts/fluent-operator/charts/fluentd-crds/Chart.yaml index 888f0373..31bf2e16 100644 --- a/charts/fluent-operator/charts/fluentd-crds/Chart.yaml +++ b/charts/fluent-operator/charts/fluentd-crds/Chart.yaml @@ -1,14 +1,23 @@ apiVersion: v2 name: fluentd-crds description: A Helm chart delivering fluentd controller CRDS -keywords: - - logging - - fluentd - - operator +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +#type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) version: 3.2.0 -appVersion: "3.2.0" -sources: - - https://github.com/fluent/fluent-operator -maintainers: - - name: wenchajun - email: dehaocheng@kubesphere.io + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "3.1.0" diff --git a/charts/fluent-operator/templates/fluentbit-fluentBit.yaml b/charts/fluent-operator/templates/fluentbit-fluentBit.yaml index 3523e470..454a4062 100644 --- a/charts/fluent-operator/templates/fluentbit-fluentBit.yaml +++ b/charts/fluent-operator/templates/fluentbit-fluentBit.yaml @@ -113,7 +113,7 @@ spec: {{- end }} {{- if .Values.fluentbit.initContainers }} initContainers: - {{ toYaml .Values.fluentbit.initContainers | indent 4 }} +{{ toYaml .Values.fluentbit.initContainers | indent 4 }} {{- end }} {{- end }} {{- end }} diff --git a/charts/fluent-operator/values.yaml b/charts/fluent-operator/values.yaml index c9ce36c4..7e05ae84 100644 --- a/charts/fluent-operator/values.yaml +++ b/charts/fluent-operator/values.yaml @@ -24,9 +24,9 @@ operator: memory: 64Mi container: repository: "kubesphere/fluent-operator" - tag: "v3.2.0" + tag: "latest" # If set to false, this will disable the creation of ClusterRole, ClusterRoleBinding, - # Deployment, and ServiceAccount resources to avoid conflicts when deploying multiple instances. + # Deployment, and ServiceAccount resources to avoid conflicts when deploying multiple instances. enable: true # nodeSelector configuration for Fluent Operator. Ref: https://kubernetes.io/docs/user-guide/node-selection/ nodeSelector: {} @@ -86,7 +86,7 @@ fluentbit: metricRelabelings: [] image: repository: "ghcr.io/fluent/fluent-operator/fluent-bit" - tag: "3.1.7" + tag: "3.1.8" # fluentbit resources. If you do want to specify resources, adjust them as necessary # You can adjust it based on the log volume. resources: @@ -125,7 +125,7 @@ fluentbit: # List of volumes that can be mounted by containers belonging to the pod. additionalVolumes: [] # Additional rbac rules which will be applied to the fluent-bit clusterrole. Ref: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#rolebinding-and-clusterrolebinding - # NOTE: As fluent-bit is managed by the fluent-operator, fluent-bit can only be granted permissions the operator also has + # NOTE: As fluent-bit is managed by the fluent-operator, fluent-bit can only be granted permissions the operator also has rbacRules: {} # Pod volumes to mount into the container's filesystem. additionalVolumesMounts: [] @@ -203,7 +203,7 @@ fluentbit: # Use storageType as "filesystem" if you want to use filesystem as the buffering mechanism for tail input. storageType: memory pauseOnChunksOverlimit: "off" - # multiline.parser + # multiline.parser # multilineParser: "docker, cri" systemd: enable: true @@ -287,51 +287,51 @@ fluentbit: # See https://docs.fluentbit.io/manual/pipeline/outputs/loki loki: # Switch for generation of fluentbit loki ClusterOutput (and loki basic auth http user and pass secrets if required) - enable: false # Bool - host: 127.0.0.1 # String + enable: false # Bool + host: 127.0.0.1 # String port: 3100 # Int # Either, give http{User,Password},tenantID string values specifying them directly httpUser: myuser httpPassword: mypass tenantID: '' # Or give {http{User,Password},tenantID} as reference to secrets that you have manually installed into your kubernetes cluster - # httpUser: - # valueFrom: - # secretKeyRef: - # key: value - # name: husersecret - # optional: true - # httpPassword: - # valueFrom: - # secretKeyRef: - # key: value - # name: hpasssecret - # optional: true - # tenantID: - # valueFrom: - # secretKeyRef: - # key: value - # name: tenantsecret - # optional: true + #httpUser: + # valueFrom: + # secretKeyRef: + # key: value + # name: husersecret + # optional: true + #httpPassword: + # valueFrom: + # secretKeyRef: + # key: value + # name: hpasssecret + # optional: true + #tenantID: + # valueFrom: + # secretKeyRef: + # key: value + # name: tenantsecret + # optional: true # - # To use bearer token auth instead of http basic auth - # bearerToken: ey.... - # or with existing secret - # bearerToken: - # valueFrom: - # secretKeyRef: - # key: value - # name: bearerTokenSecret - # optional: true - # labels: [] # String list of = - # labelKeys: [] # String list of - # removeKeys: [] # String list of - # labelMapPath: '' # String, path to file, ex /here/it/is - # dropSingleKey: off - # lineFormat: '' # String - # autoKubernetesLabels: on - # tenantIDKey: # String - # tls: {} # *plugins.TLS fluentbit docs + # To use bearer token auth instead of http basic auth + #bearerToken: ey.... + # or with existing secret + #bearerToken: + # valueFrom: + # secretKeyRef: + # key: value + # name: bearerTokenSecret + # optional: true + #labels: [] # String list of = + #labelKeys: [] # String list of + #removeKeys: [] # String list of + #labelMapPath: '' # String, path to file, ex /here/it/is + #dropSingleKey: off + #lineFormat: '' # String + #autoKubernetesLabels: on + #tenantIDKey: # String + #tls: {} # *plugins.TLS fluentbit docs stackdriver: {} # You can configure the stackdriver configuration here @@ -354,7 +354,7 @@ fluentbit: enable: false keyContent: log # emitterMemBufLimit 120 (MB) - emitterMemBufLimit: 120 + emitterMemBufLimit: 120 parsers: - go - python @@ -401,7 +401,7 @@ fluentd: port: 24224 image: repository: "ghcr.io/fluent/fluent-operator/fluentd" - tag: "v1.17.0" + tag: "v1.17.0-4" # Numbers of the Fluentd instance # Applicable when the mode is "collector", and will be ignored when the mode is "agent" replicas: 1