-
Notifications
You must be signed in to change notification settings - Fork 1
/
kubelog.yaml
371 lines (371 loc) · 8.78 KB
/
kubelog.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
apiVersion: v1
kind: ConfigMap
metadata:
name: kubemon-loki-alert-config
labels:
app.kubernetes.io/name: kubemon
app.kubernetes.io/component: prometheus
data:
rules.yml: |-
"groups":
- name: apps-rules
rules:
- alert: AppsErrorlog
expr: sum by (pod,namespace)(count_over_time({pod!=""}|~ "[Ee]rror"[1m])) >= 1
for: 1m
labels:
severity: warning
category: logs
annotations:
title: "Application error in logs"
messages: "Found error in POD ({{ $labels.pod }}) in namespace ({{ $labels.namespace }}) is above 1 (current value: {{ $value }})"
- alert: NoSuchHostError
expr: sum by (pod,namespace)(count_over_time({pod!=""}|~ "no such host"[1m])) >= 1
for: 1m
labels:
severity: warning
category: logs
annotations:
title: "No Such Host Error"
messages: "No Such Host error found in POD ({{ $labels.pod }}) in namespace ({{ $labels.namespace }}) is above 1 (current value: {{ $value }})"
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
app: loki
name: loki
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
app: fluent-bit-loki
name: loki-fluent-bit-loki
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
app: loki
name: loki
rules:
- apiGroups:
- extensions
resourceNames:
- loki
resources:
- podsecuritypolicies
verbs:
- use
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app: loki
name: loki
namespace: logging
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: loki
subjects:
- kind: ServiceAccount
name: loki
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app: fluent-bit-loki
name: loki-fluent-bit-loki-clusterrole
rules:
- apiGroups:
- ""
resources:
- namespaces
- pods
verbs:
- get
- watch
- list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app: fluent-bit-loki
name: loki-fluent-bit-loki-clusterrolebinding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: loki-fluent-bit-loki-clusterrole
subjects:
- kind: ServiceAccount
name: loki-fluent-bit-loki
namespace: logging
---
apiVersion: v1
kind: ConfigMap
metadata:
labels:
app: fluent-bit-loki
name: loki-fluent-bit-loki
data:
fluent-bit.conf: |-
[SERVICE]
Flush 1
Daemon Off
Log_Level warn
Parsers_File parsers.conf
[INPUT]
Name tail
Tag kube.*
Path /var/log/containers/*.log
Exclude_Path /var/log/containers/*_monitoring_*.log,/var/log/containers/*_logging_*.log,/var/log/containers/*_kube-node-lease_*.log,/var/log/containers/*_kube-public_*.log
Parser docker
DB /run/fluent-bit/flb_kube.db
Mem_Buf_Limit 5MB
[FILTER]
Name kubernetes
Match kube.*
Kube_URL https://kubernetes.default.svc:443
Merge_Log On
[Output]
Name grafana-loki
Match *
Url ${LOKI_CEN_URL}
Labels {cluster="kube-one", job="fluent-bit"}
RemoveKeys kubernetes,stream
LabelMapPath /fluent-bit/etc/labelmap.json
LineFormat json
LogLevel warn
[Output]
Name grafana-loki
Match *
Url http://loki:3100/api/prom/push
Labels {job="fluent-bit"}
RemoveKeys kubernetes,stream
LabelMapPath /fluent-bit/etc/labelmap.json
LineFormat json
LogLevel warn
labelmap.json: |-
{
"kubernetes": {
"container_name": "container",
"host": "node",
"labels": {
"app": "app",
"release": "release"
},
"namespace_name": "namespace",
"pod_name": "instance"
},
"stream": "stream"
}
parsers.conf: |-
[PARSER]
Name docker
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
labels:
app: loki
name: loki
spec:
podManagementPolicy: OrderedReady
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: loki
serviceName: loki-headless
template:
metadata:
labels:
app: loki
spec:
containers:
- args:
- -config.file=/etc/loki/loki.yaml
#image: docker.io/grafana/loki:2.0.0-amd64
image: docker.io/grafana/loki:2.8.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /ready
port: http-metrics
scheme: HTTP
initialDelaySeconds: 45
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: loki
ports:
- containerPort: 3100
name: http-metrics
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /ready
port: http-metrics
scheme: HTTP
initialDelaySeconds: 45
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
securityContext:
procMount: Default
readOnlyRootFilesystem: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /etc/loki
name: config
- mountPath: /etc/loki/rules
name: alert
- mountPath: /data
name: storage
- mountPath: /temp
name: temp
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext:
fsGroup: 10001
runAsGroup: 10001
runAsNonRoot: true
runAsUser: 10001
serviceAccount: loki
serviceAccountName: loki
terminationGracePeriodSeconds: 30
volumes:
- name: config
secret:
defaultMode: 420
secretName: loki
- name: alert
configMap:
name: kubemon-loki-alert-config
- emptyDir: {}
name: storage
- emptyDir: {}
name: temp
updateStrategy:
type: RollingUpdate
---
apiVersion: v1
kind: Service
metadata:
labels:
app: loki
name: loki
spec:
ports:
- name: http-metrics
port: 3100
protocol: TCP
targetPort: http-metrics
selector:
app: loki
sessionAffinity: None
type: ClusterIP
---
apiVersion: v1
kind: Service
metadata:
labels:
app: loki
name: loki-headless
spec:
clusterIP: None
ports:
- name: http-metrics
port: 3100
protocol: TCP
targetPort: http-metrics
selector:
app: loki
sessionAffinity: None
type: ClusterIP
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
labels:
app: fluent-bit-loki
name: loki-fluent-bit-loki
spec:
selector:
matchLabels:
app: fluent-bit-loki
template:
metadata:
labels:
app: fluent-bit-loki
spec:
containers:
#- image: grafana/fluent-bit-plugin-loki:0.1
- image: docker.io/grafana/fluent-bit-plugin-loki:latest
imagePullPolicy: IfNotPresent
name: fluent-bit-loki
env:
- name: LOKI_CEN_URL
value: "http://loki.172.31.11.185.nip.io/api/prom/push"
ports:
- containerPort: 2020
name: http-metrics
protocol: TCP
resources:
limits:
memory: 100Mi
requests:
cpu: 100m
memory: 100Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /fluent-bit/etc
name: config
- mountPath: /run/fluent-bit
name: run
- mountPath: /var/log
name: varlog
- mountPath: /var/lib/docker/containers
name: varlibdockercontainers
readOnly: true
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
serviceAccount: loki-fluent-bit-loki
serviceAccountName: loki-fluent-bit-loki
terminationGracePeriodSeconds: 10
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
volumes:
- configMap:
defaultMode: 420
name: loki-fluent-bit-loki
name: config
- hostPath:
path: /run/fluent-bit
type: ""
name: run
- hostPath:
path: /var/log
type: ""
name: varlog
- hostPath:
path: /var/lib/docker/containers
type: ""
name: varlibdockercontainers
updateStrategy:
type: RollingUpdate