monitoring deployment revision

infra/hack/kubernetes/README.md

@@ -1,33 +1,40 @@
 ### notes
 
-#### dashboard login duration
-
-fix dashboard login ttl by adding `'--token-ttl=0'` to args in the kubernetes-dashboard container template
+#### deploying
+in `infra/hack/kubernetes/monitoring` run
+```
+./deploy-monitoring.sh
+```
 
-#### monitoring
-in `infra/hack/kubernetes/prometheus`
+#### accessing services
+retrieve grafana admin username and password
+```
+echo "Admin User:" $(kubectl get secret -n monitoring grafana-admin -o jsonpath="{.data.admin-user}" | base64 --decode) && \
+echo "Admin Password:" $(kubectl get secret -n monitoring grafana-admin -o jsonpath="{.data.admin-password}" | base64 --decode)
 ```
-go get -u github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb
-go get github.com/google/go-jsonnet/cmd/jsonnet
-go get github.com/brancz/gojsontoyaml
 
-jb init
-./build.sh
-kubectl create -f manifests/setup
-kubectl create -f manifests
+add an entry to /etc/hosts
+```
+<node ip> strims.monitoring.local 
 ```
+replacing node ip with the ip address of the node that's hosting the ingress
+
+#
+access grafana at http://strims.monitoring.local/grafana/
+
+#
+access prometheus at http://strims.monitoring.local/prometheus
+
+
+#### dashboard login duration
+
+fix dashboard login ttl by adding `'--token-ttl=0'` to args in the kubernetes-dashboard container template
 
 #### dashboards
 ##### kubernetes
 ```
 kube proxy
 ```
 http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/
-##### grafana
-```
-kubectl --namespace monitoring port-forward svc/grafana 3000
-```
-##### prometheus
-```
-kubectl --namespace monitoring port-forward svc/prometheus-k8s 9090
-```
+
+

infra/hack/kubernetes/monitoring/deploy_monitoring.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+
+kubectl create namespace monitoring
+
+#add repos
+sudo helm repo add stable https://kubernetes-charts.storage.googleapis.com
+sudo helm repo add loki https://grafana.github.io/loki/charts
+sudo helm repo update
+
+
+#generate grafana credentials
+export grafana_user=$(pwgen 30 1 -s) 
+export grafana_password=$(pwgen 30 1 -s)
+
+< grafana/grafana-admin.yaml envsubst | kubectl apply -f -
+
+unset grafana_user
+unset grafana_password
+
+#install
+#https://github.com/helm/charts/tree/master/stable/prometheus-operator
+sudo helm install prometheus-operator -n monitoring stable/prometheus-operator -f prometheus/prom-operator-values.yaml
+
+#https://github.com/grafana/loki/tree/master/production/helm/loki
+sudo helm install loki -n monitoring -f loki/loki-values.yaml loki/loki
+
+#https://github.com/grafana/loki/tree/master/production/helm/promtail
+sudo helm install loki-promtail -n monitoring --set 'loki.serviceName=loki' loki/promtail
+
+
+
+

infra/hack/kubernetes/monitoring/grafana/grafana-admin.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: grafana-admin
+  namespace: monitoring
+type: Opaque
+stringData:
+  admin-user: ${grafana_user}
+  admin-password: ${grafana_password}
+
+
+
+
+
+

infra/hack/kubernetes/monitoring/loki/loki-values.yaml

@@ -0,0 +1,38 @@
+replicas: 1
+
+resources: {
+  "requests":
+  {"memory":"128Mi", "cpu":"100m"},
+  "limits":
+  {"memory":"1024Mi", "cpu":"500m"}
+  }
+
+persistence:
+  enabled: true
+  storageClassName: local-path
+  size: 10Gi
+
+config:
+    schema_config:
+      configs:
+      - from: 2018-04-15
+        store: boltdb
+        object_store: filesystem
+        schema: v9
+        index:
+          prefix: index_
+          period: 168h
+
+    table_manager:
+      retention_deletes_enabled: true
+      retention_period: 168h #7 day retention period
+
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: kubernetes.io/hostname
+          operator: In
+          values:
+          - controller      

infra/hack/kubernetes/monitoring/prometheus/prom-operator-values.yaml

@@ -0,0 +1,89 @@
+alertmanager:
+  enabled: false
+
+kube-state-metrics:
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: kubernetes.io/hostname
+            operator: In
+            values:
+            - controller
+
+prometheusOperator:
+  affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+          - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+              - controller
+
+prometheus:
+  prometheusSpec:
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+          - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+              - controller
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          storageClassName: local-path
+          accessModes: ["ReadWriteOnce"]
+          resources:
+            requests:
+              storage: 10Gi #storage requests don't actually do anything when using storage provisioned on a node locally
+    routePrefix: /prometheus
+    externalUrl: 'http://strims.monitoring.local/prometheus'
+  ingress:
+    enabled: true
+    hosts:
+      - strims.monitoring.local
+    path: "/prometheus"
+
+grafana:
+  admin:
+    existingSecret: grafana-admin
+
+  persistence:
+    storageClassName: local-path
+    enabled: true
+    size: 10Gi
+
+  ingress:
+    hosts:
+      - strims.monitoring.local
+    enabled: true
+    annotations:
+      nginx.ingress.kubernetes.io/rewrite-target: /$1
+    path: "/grafana/(.*)"
+
+  grafana.ini:
+    server:
+      root_url: http://strims.monitoring.local/grafana/
+      serve_from_sub_path: true
+      domain: strims.monitoring.local
+  additionalDataSources:
+  - name: Loki
+    type: loki
+    url: http://loki:3100/
+    access: proxy
+
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: kubernetes.io/hostname
+            operator: In
+            values:
+            - controller