diff --git a/.archive/filebeat/config/filebeat.yml b/.archive/filebeat/config/filebeat.yml deleted file mode 100644 index 439653183..000000000 --- a/.archive/filebeat/config/filebeat.yml +++ /dev/null @@ -1,36 +0,0 @@ -filebeat.config.inputs: - enabled: true - path: ${path.config}/inputs.d/*.yml - reload.enabled: true - reload.period: 10m - -filebeat.config.modules: - path: ${path.config}/modules.d/*.yml - reload.enabled: true - reload.period: 10m - -output.redis: - hosts: ${REDIS_HOSTS} - key: '${REDIS_KEY_NAME}' # default: bm_hl - db: 4 - #keys: '${REDIS_KEY_NAME}_%{[agent.hostname]}' - - #ssl: - # enabled: ${REDIS_ENABLE_SSL} - # verification_mode: ${REDIS_SSL_VERIFY} - # certificate: "/etc/filebeat/ssl/filebeat.pem" - # key: "/etc/filebeat/ssl/filebeat.key" - # certificate_authorities: /etc/filebeat/ssl/filebeat.ca - # supported:protocols: [ TLSv1.0, TLSv1.1, TLSv1.2,SSLv3 ] - - -logging.level: debug -logging.to_files: true -logging.to_syslog: false -logging.files: - path: /var/log/beats - name: filebeat.log - keepfiles: 7 - permissions: 0644 - -filebeat.max_procs: 1 diff --git a/.archive/filebeat/ds_filebeat.yaml b/.archive/filebeat/ds_filebeat.yaml deleted file mode 100644 index eaaa52612..000000000 --- a/.archive/filebeat/ds_filebeat.yaml +++ /dev/null @@ -1,118 +0,0 @@ ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: filebeat - namespace: logging - labels: - k8s-app: filebeat -spec: - selector: - matchLabels: - k8s-app: filebeat - template: - metadata: - labels: - k8s-app: filebeat - spec: - serviceAccountName: filebeat - terminationGracePeriodSeconds: 30 - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: filebeat - image: docker.elastic.co/beats/filebeat:8.6.1 - args: [ -# "-c", "/etc/filebeat.yml", - "-e", - ] - env: - - name: ELASTICSEARCH_HOST - value: elk-es-http.logging.svc - - name: ELASTICSEARCH_PORT - value: "9200" - - name: REDIS_HOSTS - value: "['redis-master.datahub.svc:6379']" - - name: LOG_TYPE - value: k3s_cluster - - name: CLUSTER_NAME - value: k3s_prod - - name: REDIS_KEY_NAME - value: "k3s_prod" - - name: BUFFER_FILE - value: "/var/buffer/filebeat/filebeat_spool.dat" - - name: BUFFER_FILE_SIZE - value: "1024MiB" - - name: BUFFER_PAGE_SIZE - value: "4KiB" - - name: BUFFER_FLUSH_SIZE - value: "10MiB" - - name: BUFFER_FLUSH_TIMEOUT - value: "1s" - - name: BUFFER_FLUSH_EVENTS - value: "50000" - - name: REDIS_ENABLE_SSL - value: "false" - - name: REDIS_SSL_VERIFY - value: "none" - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - runAsUser: 0 - resources: - limits: - memory: 200Mi - requests: - cpu: 15m - memory: 50Mi - volumeMounts: - - name: filebeatcfg-volume - mountPath: /usr/share/filebeat/filebeat.yml - subPath: filebeat.yml - readOnly: true - - name: filebeatinputs-volume - mountPath: /usr/share/filebeat/inputs.d - readOnly: true - #- name: filebeat-modules-volume - # mountPath: /usr/share/filebeat/modules.d - # readOnly: true - - name: data - mountPath: /usr/share/filebeat/data - - name: varlibdockercontainers - mountPath: /var/lib/docker/containers - readOnly: true - - name: varlog - mountPath: /var/log - readOnly: true - #- name: redis-certs - # mountPath: "/etc/filebeat/ssl/" - volumes: - - name: filebeatcfg-volume - secret: - secretName: filebeatcfg - - name: filebeatinputs-volume - secret: - secretName: filebeatinputs - #- name: filebeat-modules-volume - # configMap: - # defaultMode: 0640 - # name: filebeat-modules - - name: varlibdockercontainers - hostPath: - path: /var/lib/docker/containers - - name: varlog - hostPath: - path: /var/log - # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart - - name: data - hostPath: - # When filebeat runs as non-root user, this directory needs to be writable by group (g+w). - path: /var/lib/filebeat-data - type: DirectoryOrCreate - #- name: redis-certs - # secret: - # secretName: redis-certs - # defaultMode: 256 - diff --git a/.archive/filebeat/inputs.d/container.yml b/.archive/filebeat/inputs.d/container.yml deleted file mode 100644 index ab23168ee..000000000 --- a/.archive/filebeat/inputs.d/container.yml +++ /dev/null @@ -1,19 +0,0 @@ -- type: container - paths: - #- /var/log/containers/event*.log - - /var/log/containers/*.log - exclude_files: ['\.gz$'] - tail_files: true - - processors: - - add_kubernetes_metadata: - in_cluster: true - #host: ${NODE_NAME} - matchers: - - logs_path: - logs_path: "/var/log/containers/" - fields: - k3s_cluster_name: ${CLUSTER_NAME} - log_type: ${LOG_TYPE} - fields_under_root: true - diff --git a/.archive/filebeat/inputs.d/docker.yml b/.archive/filebeat/inputs.d/docker.yml deleted file mode 100644 index 88778b25a..000000000 --- a/.archive/filebeat/inputs.d/docker.yml +++ /dev/null @@ -1,17 +0,0 @@ -- type: docker - containers.ids: - - "*" - processors: - - add_kubernetes_metadata: - in_cluster: true - - add_docker_metadata: - host: "unix:///var/run/docker.sock" - json.message_key: log - json.keys_under_root: true -# json.add_error_key: true - json.ignore_decoding_error: true - fields_under_root: true - ignore_older: 24h - tail_files: true - fields: - fileset.module: docker diff --git a/.archive/filebeat/kustomization.yaml b/.archive/filebeat/kustomization.yaml deleted file mode 100644 index 60f4a3125..000000000 --- a/.archive/filebeat/kustomization.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -metadata: - name: filbeat - namespace: logging - labels: - app: "filebeat" -resources: - #- redis-certs-sops.yaml TODO: add this not using SSL for now - - ds_filebeat.yaml - - sa_filebeat.yaml - -generatorOptions: - disableNameSuffixHash: true - labels: - namespace: logging - -secretGenerator: -- name: filebeatcfg - namespace: logging - files: - - config/filebeat.yml - -- name: filebeatinputs - namespace: logging - files: - - ./inputs.d/container.yml - - ./inputs.d/docker.yml - - diff --git a/.archive/filebeat/modules.d/traefik.yaml.enabled b/.archive/filebeat/modules.d/traefik.yaml.enabled deleted file mode 100644 index 16ec37f97..000000000 --- a/.archive/filebeat/modules.d/traefik.yaml.enabled +++ /dev/null @@ -1,8 +0,0 @@ -- module: traefik - # Access logs - access: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/.archive/filebeat/sa_filebeat.yaml b/.archive/filebeat/sa_filebeat.yaml deleted file mode 100644 index 678a7824e..000000000 --- a/.archive/filebeat/sa_filebeat.yaml +++ /dev/null @@ -1,100 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: filebeat -subjects: -- kind: ServiceAccount - name: filebeat - namespace: logging -roleRef: - kind: ClusterRole - name: filebeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: filebeat - namespace: logging -subjects: - - kind: ServiceAccount - name: filebeat - namespace: logging -roleRef: - kind: Role - name: filebeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: filebeat-kubeadm-config - namespace: logging -subjects: - - kind: ServiceAccount - name: filebeat - namespace: logging -roleRef: - kind: Role - name: filebeat-kubeadm-config - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: filebeat - labels: - k8s-app: filebeat -rules: -- apiGroups: [""] # "" indicates the core API group - resources: - - namespaces - - pods - - nodes - verbs: - - get - - watch - - list -- apiGroups: ["apps"] - resources: - - replicasets - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: filebeat - # should be the namespace where filebeat is running - namespace: logging - labels: - k8s-app: filebeat -rules: - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: ["get", "create", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: filebeat-kubeadm-config - namespace: logging - labels: - k8s-app: filebeat -rules: - - apiGroups: [""] - resources: - - configmaps - resourceNames: - - kubeadm-config - verbs: ["get"] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: filebeat - namespace: logging - labels: - k8s-app: filebeat diff --git a/kubernetes/apps/dev/demo-service/ks.yaml b/kubernetes/apps/dev/demo-service/ks.yaml index 230c59474..9af95b314 100644 --- a/kubernetes/apps/dev/demo-service/ks.yaml +++ b/kubernetes/apps/dev/demo-service/ks.yaml @@ -9,7 +9,7 @@ metadata: spec: dependsOn: - name: cluster-apps-kube-prometheus-stack - path: ./kubernetes/apps/default/demo-service/app + path: ./kubernetes/apps/dev/demo-service/app prune: true sourceRef: kind: GitRepository diff --git a/kubernetes/apps/observability/elastic/elastic-exporter/app/dp_elasticsearch_exporter.yaml b/kubernetes/apps/observability/elastic/elastic-exporter/app/dp_elasticsearch_exporter.yaml deleted file mode 100644 index 68357681d..000000000 --- a/kubernetes/apps/observability/elastic/elastic-exporter/app/dp_elasticsearch_exporter.yaml +++ /dev/null @@ -1,110 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: prometheus-elasticsearch-exporter - labels: - app: prometheus-elasticsearch-exporter -spec: - replicas: 1 - selector: - matchLabels: - app: prometheus-elasticsearch-exporter - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - type: RollingUpdate - template: - metadata: - labels: - app: prometheus-elasticsearch-exporter - spec: - serviceAccountName: default - restartPolicy: Always - securityContext: - runAsNonRoot: true - runAsUser: 1000 - containers: - - name: exporter - env: - - name: ES_USERNAME - value: elastic - - name: ES_PASSWORD - valueFrom: - secretKeyRef: - key: elastic - name: elk-es-elastic-user - image: "quay.io/prometheuscommunity/elasticsearch-exporter:v1.6.0" - imagePullPolicy: IfNotPresent - command: ["elasticsearch_exporter", - "--log.format=logfmt", - "--log.level=info", - "--es.uri=https://elk-es-http.logging.svc:9200", - "--es.all", - "--es.indices", - "--es.indices_settings", - "--es.indices_mappings", - "--es.aliases", - "--es.shards", - "--es.snapshots", - "--es.data_stream", - "--es.timeout=30s", - "--es.ssl-skip-verify", - "--es.ca=/usr/share/elasticsearch-exporter/config/certs/ca.crt", - "--web.listen-address=:9108", - "--web.telemetry-path=/metrics" - ] - securityContext: - capabilities: - drop: - - SETPCAP - - MKNOD - - AUDIT_WRITE - - CHOWN - - NET_RAW - - DAC_OVERRIDE - - FOWNER - - FSETID - - KILL - - SETGID - - SETUID - - NET_BIND_SERVICE - - SYS_CHROOT - - SETFCAP - readOnlyRootFilesystem: true - ports: - - containerPort: 9108 - name: http - livenessProbe: - httpGet: - path: /healthz - port: http - initialDelaySeconds: 5 - timeoutSeconds: 5 - periodSeconds: 5 - readinessProbe: - httpGet: - path: /healthz - port: http - initialDelaySeconds: 1 - timeoutSeconds: 5 - periodSeconds: 5 - resources: - requests: - cpu: 11m - memory: 20Mi - limits: - #cpu: - memory: 100Mi - lifecycle: - preStop: - exec: - command: ["/bin/bash", "-c", "sleep 20"] - volumeMounts: - - name: ca-certs - mountPath: /usr/share/elasticsearch-exporter/config/certs - volumes: - - name: ca-certs - secret: - secretName: elk-es-http-certs-public diff --git a/kubernetes/apps/observability/elastic/elastic-exporter/app/kustomization.yaml b/kubernetes/apps/observability/elastic/elastic-exporter/app/kustomization.yaml deleted file mode 100644 index 019558b31..000000000 --- a/kubernetes/apps/observability/elastic/elastic-exporter/app/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: observability -resources: - - svc_elasticsearch-exporter.yaml - - svcmon-elasticsearch-exporter.yaml - - dp_elasticsearch_exporter.yaml diff --git a/kubernetes/apps/observability/elastic/elastic-exporter/app/svc_elasticsearch-exporter.yaml b/kubernetes/apps/observability/elastic/elastic-exporter/app/svc_elasticsearch-exporter.yaml deleted file mode 100644 index 85b528a97..000000000 --- a/kubernetes/apps/observability/elastic/elastic-exporter/app/svc_elasticsearch-exporter.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -kind: Service -apiVersion: v1 -metadata: - name: prometheus-elasticsearch-exporter - labels: - app: prometheus-elasticsearch-exporter -spec: - type: ClusterIP - ports: - - name: http - port: 9108 - protocol: TCP - targetPort: 9108 - selector: - app: prometheus-elasticsearch-exporter \ No newline at end of file diff --git a/kubernetes/apps/observability/elastic/elastic-exporter/app/svcmon-elasticsearch-exporter.yaml b/kubernetes/apps/observability/elastic/elastic-exporter/app/svcmon-elasticsearch-exporter.yaml deleted file mode 100644 index 64563bca0..000000000 --- a/kubernetes/apps/observability/elastic/elastic-exporter/app/svcmon-elasticsearch-exporter.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: prometheus-elasticsearch-exporter-monitor - labels: - app: prometheus-elasticsearch-exporter -spec: - namespaceSelector: - matchNames: - - observability - selector: - matchLabels: - app: prometheus-elasticsearch-exporter - endpoints: - - port: http diff --git a/kubernetes/apps/observability/elastic/elastic-exporter/ks.yaml b/kubernetes/apps/observability/elastic/elastic-exporter/ks.yaml deleted file mode 100644 index b2edc1f85..000000000 --- a/kubernetes/apps/observability/elastic/elastic-exporter/ks.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1beta2 -kind: Kustomization -metadata: - name: cluster-apps-elastic-exporter - namespace: flux-system - labels: - substitution.flux.home.arpa/enabled: "true" -spec: - path: ./kubernetes/apps/observability/elastic-exporter/app - dependsOn: - - name: cluster-apps-elastic - prune: true - sourceRef: - kind: GitRepository - name: home-kubernetes - interval: 30m - retryInterval: 1m - timeout: 5m diff --git a/kubernetes/apps/observability/elastic/elastic-operator/app/helmrelease.yaml b/kubernetes/apps/observability/elastic/elastic-operator/app/helmrelease.yaml deleted file mode 100644 index 24d2e35f2..000000000 --- a/kubernetes/apps/observability/elastic/elastic-operator/app/helmrelease.yaml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# https://github.com/elastic/cloud-on-k8s/blob/main/deploy/eck-operator/values.yaml -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: eck-operator - namespace: observability -spec: - interval: 5m - chart: - spec: - chart: ./deploy/eck-operator - sourceRef: - kind: GitRepository - name: eck-operator - namespace: flux-system - interval: 1m - values: - resources: - requests: - cpu: 16m - memory: 99Mi - limits: - #cpu: - memory: 500Mi diff --git a/kubernetes/apps/observability/elastic/elastic-operator/app/kustomization.yaml b/kubernetes/apps/observability/elastic/elastic-operator/app/kustomization.yaml deleted file mode 100644 index 4fd210252..000000000 --- a/kubernetes/apps/observability/elastic/elastic-operator/app/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: observability -resources: - - helmrelease.yaml diff --git a/kubernetes/apps/observability/elastic/elastic-operator/ks.yaml b/kubernetes/apps/observability/elastic/elastic-operator/ks.yaml deleted file mode 100644 index cb1e17b1d..000000000 --- a/kubernetes/apps/observability/elastic/elastic-operator/ks.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1beta2 -kind: Kustomization -metadata: - name: cluster-apps-elastic-operator - namespace: flux-system - labels: - substitution.flux.home.arpa/enabled: "true" -spec: - dependsOn: - - name: cluster-apps-nginx-external - - name: cluster-apps-external-secrets-stores - path: ./kubernetes/apps/observability/elastic-operator/app - prune: true - sourceRef: - kind: GitRepository - name: home-kubernetes - interval: 30m - retryInterval: 1m - timeout: 5m