kubernetes tls certificate verification failed

[pandihao]

Bug Report

version info

fluent-bit version:

apiVersion: v1
name: fluent-bit
description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems.
keywords:
  - logging
  - fluent-bit
  - fluentd
version: 0.48.3
appVersion: 3.2.2
icon: https://raw.githubusercontent.com/cncf/artwork/master/projects/fluentd/fluentbit/icon/fluentbit-icon-color.svg
home: https://fluentbit.io/

kubernetes version:

Server Version: version.Info{Major:"1", Minor:"22", GitVersion:"v1.22.14",

fluent-bit config

[Filter]
    Name nest
    Match k8s_event
    Operation lift
    Nested_under metadata
    Add_prefix metadata_ 

---

error report

[2025/01/03 09:21:33] [error] [input:kubernetes_events:kubernetes_events.1] upstream connection initialization error
[2025/01/03 09:21:33] [error] [tls] certificate verification failed, reason: self-signed certificate (X509 code: 18)

[patrick-stephens]

Please follow the issue template.

Do you mean this config is in addition to the default helm chart config? I'm assuming you're using the official OSS helm chart?

The problem seems to be with TLS verification talking to your K8S API, it seems like you are using a self-signed or other custom cert? Have you mounted and provided that?

[pandihao]

@patrick-stephens Thanks

1. use official OSS helm chart
2. container mount serviceaccount info

root@HZxxxxx:~/cloudnativeteam/charts/fluent-bit# kubectl  exec -it netconf-7758f85bb6-xbfsq -- ls /var/run/secrets/kubernetes.io/serviceaccount
ca.crt     namespace  token

3. I use the chart config value content

# Default values for fluent-bit.

# kind -- DaemonSet or Deployment
kind: DaemonSet

# replicaCount -- Only applicable if kind=Deployment
replicaCount: 1

image:
  repository: xxxxxx/eck/fluent/fluent-bit
  # Overrides the image tag whose default is {{ .Chart.AppVersion }}
  # Set to "-" to not use the default value
  tag: 3.2.2
  digest:
  pullPolicy: IfNotPresent

testFramework:
  enabled: true
  namespace:
  image:
    repository: busybox
    pullPolicy: Always
    tag: latest
    digest:

imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""

serviceAccount:
  create: true
  annotations: {}
  name:

rbac:
  create: true
  nodeAccess: true
  eventsAccess: true

# Configure podsecuritypolicy
# Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/
# from Kubernetes 1.25, PSP is deprecated
# See: https://kubernetes.io/blog/2022/08/23/kubernetes-v1-25-release/#pod-security-changes
# We automatically disable PSP if Kubernetes version is 1.25 or higher
podSecurityPolicy:
  create: false
  annotations: {}
  runAsUser:
    rule: RunAsAny
  seLinux:
    # This policy assumes the nodes are using AppArmor rather than SELinux.
    rule: RunAsAny

# OpenShift-specific configuration
openShift:
  enabled: false
  securityContextConstraints:
    # Create SCC for Fluent-bit and allow use it
    create: true
    name: ""
    annotations: {}
    runAsUser:
      type: RunAsAny
    seLinuxContext:
      type: MustRunAs
    # Use existing SCC in cluster, rather then create new one
    existingName: ""

podSecurityContext: {}
#   fsGroup: 2000

hostNetwork: false
dnsPolicy: ClusterFirst

dnsConfig: {}
#   nameservers:
#     - 1.2.3.4
#   searches:
#     - ns1.svc.cluster-domain.example
#     - my.dns.search.suffix
#   options:
#     - name: ndots
#       value: "2"
#     - name: edns0

hostAliases: []
#   - ip: "1.2.3.4"
#     hostnames:
#     - "foo.local"
#     - "bar.local"

securityContext: {}
#   capabilities:
#     drop:
#     - ALL
#   readOnlyRootFilesystem: true
#   runAsNonRoot: true
#   runAsUser: 1000

service:
  type: ClusterIP
  port: 2020
  internalTrafficPolicy:
  loadBalancerClass:
  loadBalancerSourceRanges: []
  labels: {}
  # nodePort: 30020
  # clusterIP: 172.16.10.1
  annotations: {}
  #   prometheus.io/path: "/api/v1/metrics/prometheus"
  #   prometheus.io/port: "2020"
  #   prometheus.io/scrape: "true"
  externalIPs: []
  # externalIPs:
  #  - 2.2.2.2

serviceMonitor:
  enabled: false
  #   namespace: monitoring
  #   interval: 10s
  #   scrapeTimeout: 10s
  #   selector:
  #    prometheus: my-prometheus
  #  ## metric relabel configs to apply to samples before ingestion.
  #  ##
  #  metricRelabelings:
  #    - sourceLabels: [__meta_kubernetes_service_label_cluster]
  #      targetLabel: cluster
  #      regex: (.*)
  #      replacement: ${1}
  #      action: replace
  #  ## relabel configs to apply to samples after ingestion.
  #  ##
  #  relabelings:
  #    - sourceLabels: [__meta_kubernetes_pod_node_name]
  #      separator: ;
  #      regex: ^(.*)$
  #      targetLabel: nodename
  #      replacement: $1
  #      action: replace
  #  scheme: ""
  #  tlsConfig: {}

  ## Bear in mind if you want to collect metrics from a different port
  ## you will need to configure the new ports on the extraPorts property.
  additionalEndpoints: []
  # - port: metrics
  #   path: /metrics
  #   interval: 10s
  #   scrapeTimeout: 10s
  #   scheme: ""
  #   tlsConfig: {}
  #   # metric relabel configs to apply to samples before ingestion.
  #   #
  #   metricRelabelings:
  #     - sourceLabels: [__meta_kubernetes_service_label_cluster]
  #       targetLabel: cluster
  #       regex: (.*)
  #       replacement: ${1}
  #       action: replace
  #   # relabel configs to apply to samples after ingestion.
  #   #
  #   relabelings:
  #     - sourceLabels: [__meta_kubernetes_pod_node_name]
  #       separator: ;
  #       regex: ^(.*)$
  #       targetLabel: nodename
  #       replacement: $1
  #       action: replace

prometheusRule:
  enabled: false
#   namespace: ""
#   additionalLabels: {}
#   rules:
#   - alert: NoOutputBytesProcessed
#     expr: rate(fluentbit_output_proc_bytes_total[5m]) == 0
#     annotations:
#       message: |
#         Fluent Bit instance {{ $labels.instance }}'s output plugin {{ $labels.name }} has not processed any
#         bytes for at least 15 minutes.
#       summary: No Output Bytes Processed
#     for: 15m
#     labels:
#       severity: critical

dashboards:
  enabled: false
  labelKey: grafana_dashboard
  labelValue: 1
  annotations: {}
  namespace: ""
  deterministicUid: false

lifecycle: {}
#   preStop:
#     exec:
#       command: ["/bin/sh", "-c", "sleep 20"]

livenessProbe:
  httpGet:
    path: /
    port: http

readinessProbe:
  httpGet:
    path: /api/v1/health
    port: http

resources: {}
#   limits:
#     cpu: 100m
#     memory: 128Mi
#   requests:
#     cpu: 100m
#     memory: 128Mi

## only available if kind is Deployment
ingress:
  enabled: false
  ingressClassName: ""
  annotations: {}
  #  kubernetes.io/ingress.class: nginx
  #  kubernetes.io/tls-acme: "true"
  hosts: []
  # - host: fluent-bit.example.tld
  extraHosts: []
  # - host: fluent-bit-extra.example.tld
  ## specify extraPort number
  #   port: 5170
  tls: []
  #  - secretName: fluent-bit-example-tld
  #    hosts:
  #      - fluent-bit.example.tld

## only available if kind is Deployment
autoscaling:
  vpa:
    enabled: false

    annotations: {}

    # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory
    controlledResources: []

    # Define the max allowed resources for the pod
    maxAllowed: {}
    # cpu: 200m
    # memory: 100Mi
    # Define the min allowed resources for the pod
    minAllowed: {}
    # cpu: 200m
    # memory: 100Mi

    updatePolicy:
      # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates
      # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto".
      updateMode: Auto

  enabled: false
  minReplicas: 1
  maxReplicas: 3
  targetCPUUtilizationPercentage: 75
  #  targetMemoryUtilizationPercentage: 75
  ## see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics
  customRules: []
  #     - type: Pods
  #       pods:
  #         metric:
  #           name: packets-per-second
  #         target:
  #           type: AverageValue
  #           averageValue: 1k
  ## see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior
  behavior: {}
#      scaleDown:
#        policies:
#          - type: Pods
#            value: 4
#            periodSeconds: 60
#          - type: Percent
#            value: 10
#            periodSeconds: 60

## only available if kind is Deployment
podDisruptionBudget:
  enabled: false
  annotations: {}
  maxUnavailable: "30%"

nodeSelector: {}

tolerations:
  - key: "node-role.kubernetes.io/master"
    operator: "Exists"
    effect: "NoSchedule"

affinity: {}

labels: {}

annotations: {}

podAnnotations: {}

podLabels: {}

## How long (in seconds) a pods needs to be stable before progressing the deployment
##
minReadySeconds:

## How long (in seconds) a pod may take to exit (useful with lifecycle hooks to ensure lb deregistration is done)
##
terminationGracePeriodSeconds:

priorityClassName: ""

env:
- name: HOST_IP
  valueFrom:
    fieldRef:
      fieldPath: status.hostIP
- name: CLUSTER
  value: pro01
- name: CLUSTER_ENV
  value: pro
#  - name: FOO
#    value: "bar"

# The envWithTpl array below has the same usage as "env", but is using the tpl function to support templatable string.
# This can be useful when you want to pass dynamic values to the Chart using the helm argument "--set <variable>=<value>"
# https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function
envWithTpl: []
#  - name: FOO_2
#    value: "{{ .Values.foo2 }}"
#
# foo2: bar2

envFrom: []

# This supports either a structured array or a templatable string
extraContainers: []

# Array mode
# extraContainers:
#   - name: do-something
#     image: busybox
#     command: ['do', 'something']

# String mode
# extraContainers: |-
#   - name: do-something
#     image: bitnami/kubectl:{{ .Capabilities.KubeVersion.Major }}.{{ .Capabilities.KubeVersion.Minor }}
#     command: ['kubectl', 'version']

flush: 1

metricsPort: 2020

extraPorts: []
#   - port: 5170
#     containerPort: 5170
#     protocol: TCP
#     name: tcp
#     nodePort: 30517

extraVolumes: []

extraVolumeMounts: []

updateStrategy: {}
#   type: RollingUpdate
#   rollingUpdate:
#     maxUnavailable: 1

# Make use of a pre-defined configmap instead of the one templated here
existingConfigMap: ""

networkPolicy:
  enabled: false
#   ingress:
#     from: []

luaScripts: {}


## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/configuration-file
config:
  service: |
    [SERVICE]
        Daemon Off
        Flush {{ .Values.flush }}
        Log_Level {{ .Values.logLevel }}
        Parsers_File /fluent-bit/etc/parsers.conf
        Parsers_File /fluent-bit/etc/conf/custom_parsers.conf
        HTTP_Server On
        HTTP_Listen 0.0.0.0
        HTTP_Port {{ .Values.metricsPort }}
        Health_Check On

  ## https://docs.fluentbit.io/manual/pipeline/inputs
  inputs: |

    [INPUT]
        Name            kubernetes_events
        Tag             k8s_event
        kube_url        https://kubernetes.default.svc
        
    [INPUT]
        Name systemd
        Tag service.data
        Path /var/log/journal
        Systemd_Filter _SYSTEMD_UNIT=kubelet.service
        Systemd_Filter _SYSTEMD_UNIT=docker.service
        Systemd_Filter _SYSTEMD_UNIT=containerd.service
        Systemd_Filter _SYSTEMD_UNIT=kube-controller-manager.service
        Systemd_Filter _SYSTEMD_UNIT=etcd.service
        Systemd_Filter _SYSTEMD_UNIT=kube-scheduler.service
        Systemd_Filter _SYSTEMD_UNIT=kube-apiserver.service
        Systemd_Filter _SYSTEMD_UNIT=kube-proxy.service
        Read_From_Tail On

  ## https://docs.fluentbit.io/manual/pipeline/filters
  filters: |
    [FILTER]
        Name kubernetes
        Match kube.*
        Kube_Tag_Prefix  kube.
        Regex_Parser  custom-tag
        Merge_Log On
        Keep_Log Off
        K8S-Logging.Parser On
        K8S-Logging.Exclude On
        tls.verify Off

    [Filter]
        Name nest
        Match_regex kube.*
        Operation lift
        Nested_under kubernetes
        Add_prefix kubernetes_

    [Filter]
        Name    modify
        Match_regex kube.*
        Remove    stream
        Remove    event
        Remove    kubernetes_pod_id
        Remove    kubernetes_container_hash
        Remove    kubernetes_annotations
        #Remove    kubernetes_labels
        Remove    kubernetes_container_image
        Remove    kubernetes_docker_id
        Rename    kubernetes_host kubernetes_host_ip
        Add    cluster ${CLUSTER}
        Add    env ${CLUSTER_ENV}
        
    [Filter]
        Name    nest
        Match_regex kube.*
        Operation    nest
        Wildcard    kubernetes_*
        Nest_under    kubernetes
        Remove_prefix    kubernetes_


    [Filter]
        Name    modify
        Match   service.*
        Remove  _CMDLINE
        Remove  _CAP_EFFECTIVE
        Remove  _SYSTEMD_INVOCATION_ID
        Remove  _SYSTEMD_CGROUP
        Remove  _SELINUX_CONTEXT
        Remove  _STREAM_ID
        Remove  _TRANSPORT
        Remove  PRIORITY
        Remove  _EXE
        Remove  _MACHINE_ID
        Remove  SYSLOG_FACILITY
        Remove  _BOOT_ID
        Remove  _UID
        Add    host_ip ${HOST_IP}
        Add    cluster ${CLUSTER}
        Add    env ${CLUSTER_ENV}

        #[Filter]
        # Name    grep
        #Match   service.*
        #Exclude MESSAGE  .*Exec process.*  

    [Filter]
        Name nest
        Match k8s_event
        Operation lift
        Nested_under metadata
        Add_prefix metadata_

    [Filter]
        Name    modify
        Match     k8s_event
        Remove    metadata_managedFields
        Remove    metadata_selfLink
        Remove    metadata_resourceVersion
        Remove    involvedObject
        Add       cluster ${CLUSTER}
        Add       env ${CLUSTER_ENV}


    [Filter]
        Name    nest
        Match    k8s_event
        Operation    nest
        Wildcard    metadata_*
        Nest_under  metadata
        Remove_prefix   metadata_

  ## https://docs.fluentbit.io/manual/pipeline/outputs
  outputs: |
    [OUTPUT]
        Name kafka
        Match k8s_event
        Brokers 
        Topics k8s_pro_event_topic
        Timestamp_Key _time_
        Timestamp_Format  iso8601
        Retry_Limit       false
        rdkafka.log.connection.close false
        rdkafka.request.required.acks 1
 
    [OUTPUT]
        Name kafka
        Match service.*
        Brokers 
        Topics k8s_services_topic
        Timestamp_Key _time_
        Timestamp_Format  iso8601
        Retry_Limit       false
        rdkafka.log.connection.close false
        rdkafka.request.required.acks 1



        # [OUTPUT]
        #Name        stdout
        #Match       ebpf

        #    [OUTPUT]
        # Name kafka
        # Match host.*
        #Topics k8s_syslog_mgmt_topic
        #Timestamp_Key _time_
        #Timestamp_Format  iso8601
        #Retry_Limit       false
        #rdkafka.log.connection.close false
        #rdkafka.request.required.acks 1

  ## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/upstream-servers
  ## This configuration is deprecated, please use `extraFiles` instead.
  upstream: {}

  ## https://docs.fluentbit.io/manual/pipeline/parsers
  customParsers: |
    [PARSER]
        Name docker_no_time
        Format json
        Time_Keep Off
        Time_Key time
        Time_Format %Y-%m-%dT%H:%M:%S.%L

    [PARSER]
        Name    custom-tag
        Format  regex
        Regex   ^(?<namespace_name>[^_]+)\.(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)\.(?<container_name>.+)\.(?<container_id>[a-z0-9]{64})

  # This allows adding more files with arbitrary filenames to /fluent-bit/etc/conf by providing key/value pairs.
  # The key becomes the filename, the value becomes the file content.
  extraFiles: {}
#     upstream.conf: |
#       [UPSTREAM]
#           upstream1
#
#       [NODE]
#           name       node-1
#           host       127.0.0.1
#           port       43000
#     example.conf: |
#       [OUTPUT]
#           Name example
#           Match foo.*
#           Host bar

# The config volume is mounted by default, either to the existingConfigMap value, or the default of "fluent-bit.fullname"
volumeMounts:
  - name: config
    mountPath: /fluent-bit/etc/conf

daemonSetVolumes:
  - name: varlog
    hostPath:
      path: /var/log
  - name: journallog
    hostPath:
      path: /var/log/journal
  - name: varlibdockercontainers
    hostPath:
      path: /var/lib/docker/containers
  - name: etcmachineid
    hostPath:
      path: /etc/machine-id
      type: File

daemonSetVolumeMounts:
  - name: varlog
    mountPath: /var/log
  - name: journallog
    mountPath: /var/log/journal
  - name: varlibdockercontainers
    mountPath: /var/lib/docker/containers
    readOnly: true
  - name: etcmachineid
    mountPath: /etc/machine-id
    readOnly: true

command:
  - /fluent-bit/bin/fluent-bit

args:
  - --workdir=/fluent-bit/etc
  - --config=/fluent-bit/etc/conf/fluent-bit.conf

# This supports either a structured array or a templatable string
initContainers: []

# Array mode
# initContainers:
#   - name: do-something
#     image: bitnami/kubectl:1.22
#     command: ['kubectl', 'version']

# String mode
# initContainers: |-
#   - name: do-something
#     image: bitnami/kubectl:{{ .Capabilities.KubeVersion.Major }}.{{ .Capabilities.KubeVersion.Minor }}
#     command: ['kubectl', 'version']

logLevel: info

hotReload:
  enabled: false
  image:
    repository: eck/fluent/configmap-reload
    tag: v0.14.0
    digest:
    pullPolicy: IfNotPresent
  resources: `{}`