WIP: Run on kubernetes

README.md

@@ -44,9 +44,11 @@ This application comprises a number of different processes:
 ## Prerequisite(s)
 
 The following package(s) are required;
-	- `sbt`.
 
-Install them with your package manager:
+	- `sbt`;
+    - and `ingress-nginx`.
+
+Install with your package manager:
 
 ```
 brew update
@@ -58,6 +60,32 @@ apt update
 apt install sbt 
 ```
 
+### Ingress Nginx
+
+Our setup uses an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) with Nginx to define reroutes to the webshop and the
+API (aka service) used by the webshop. The following steps 
+are required:
+
+For all deployments:
+
+```
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/mandatory.yaml
+```
+
+Provider specific:
+
+**For Docker for mac**
+```
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/cloud-generic.yaml
+```
+
+**For minikube**
+```
+minikube addons enable ingress
+```
+
+For more info, see [here](https://github.com/kubernetes/ingress-nginx/blob/master/docs/deploy/index.md#installation-guide).
+
 ## Running with Docker
 
 The easiest way to get started is with Docker Compose.

k8s/clickstream-analyzer-deployment.yaml

@@ -0,0 +1,33 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: clickstream-analyzer
+  labels:
+    app: clickstream-analyzer
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: clickstream-analyzer
+  template:
+    metadata:
+      labels:
+        app: clickstream-analyzer
+    spec:
+      containers:
+      - env:
+        - name: KAFKA_BROKERS
+          value: kafka:9092
+        - name: KAFKA_TOPIC
+          value: divolte
+        - name: SPARK_JOB_CLASS
+          value: sparkjob.SparkEventsPerMinute
+        image: shop_clickstream-analyzer
+        name: clickstream-analyzer
+        imagePullPolicy: Never # always local
+        ports:
+        - containerPort: 7077
+        - containerPort: 4040
+        - containerPort: 18080
+      hostname: clickstream-analyzer
+      restartPolicy: Always

k8s/clickstream-analyzer-service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: clickstream-analyzer
+  labels:
+    app: clickstream-analyzer
+spec:
+  ports:
+  - name: "7077"
+    port: 7077
+    targetPort: 7077
+  - name: "4040"
+    port: 4040
+    targetPort: 4040
+  - name: "18080"
+    port: 18080
+    targetPort: 18080
+  selector:
+    app: clickstream-analyzer

k8s/clickstream-collector-deployment.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: clickstream-collector
+  labels:
+    app: clickstream-collector
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: clickstream-collector
+  template:
+    metadata:
+      labels:
+        app: clickstream-collector
+    spec:
+      containers:
+      - env:
+        - name: DIVOLTE_CONF_DIR
+          value: /etc/shop/divolte
+        - name: DIVOLTE_KAFKA_BROKER_LIST
+          value: kafka:9092
+        image: shop_clickstream-collector
+        name: clickstream-collector
+        imagePullPolicy: Never
+        ports:
+        - containerPort: 8290
+      hostname: clickstream-collector
+      restartPolicy: Always
+      subdomain: divolte-divolte

k8s/clickstream-collector-service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: clickstream-collector
+  labels:
+    app: clickstream-collector
+spec:
+  ports:
+  - name: "8290"
+    port: 8290
+    targetPort: 8290
+  selector:
+    app: clickstream-collector

k8s/elastic-deployment.yaml

@@ -0,0 +1,38 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: elastic
+  labels:
+    app: elastic
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: elastic
+  template:
+    metadata:
+      labels:
+        app: elastic
+    spec:
+      containers:
+      - env:
+        - name: ES_JAVA_OPTS
+          value: -Xms512m -Xmx512m
+        - name: cluster.name
+          value: elasticsearch
+        - name: discovery.type
+          value: single-node
+        - name: network.bind_host
+          value: 0.0.0.0
+        - name: network.publish_host
+          value: elastic
+        - name: xpack.security.enabled
+          value: "false"
+        image: docker.elastic.co/elasticsearch/elasticsearch:6.4.0
+        name: elastic
+        ports:
+        - containerPort: 9200
+        - containerPort: 9300
+      hostname: elastic
+      restartPolicy: Always
+      subdomain: divolte-divolte

k8s/elastic-service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: elastic
+  labels:
+    app: elastic
+spec:
+  ports:
+  - name: "9200"
+    port: 9200
+    targetPort: 9200
+  - name: "9300"
+    port: 9300
+    targetPort: 9300
+  selector:
+    app: elastic

k8s/ingress.yaml

@@ -0,0 +1,18 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+    name: nginx-ingress
+    annotations:
+      ingress.kubernetes.io/rewrite-target: /
+spec:
+    rules:
+      - http:
+          paths:
+            - path: /
+              backend:
+                serviceName: shop-webapp
+                servicePort: 9011
+            - path: /api
+              backend:
+                serviceName: shop-api
+                servicePort: 8080

k8s/kafka-deployment.yaml

@@ -0,0 +1,42 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kafka
+  labels:
+    app: kafka
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kafka
+  template:
+    metadata:
+      labels:
+        app: kafka
+    spec:
+      containers:
+      - env:
+        - name: ADVERTISED_HOST
+          value: kafka
+        - name: ADVERTISED_LISTENERS
+          value: PLAINTEXT://kafka:9092,INTERNAL://localhost:9093
+        - name: AUTO_CREATE_TOPICS
+          value: "false"
+        - name: INTER_BROKER
+          value: INTERNAL
+        - name: KAFKA_CREATE_TOPICS
+          value: divolte:4:1
+        - name: LISTENERS
+          value: PLAINTEXT://0.0.0.0:9092,INTERNAL://0.0.0.0:9093
+        - name: LOG_RETENTION_HOURS
+          value: "1"
+        - name: SECURITY_PROTOCOL_MAP
+          value: PLAINTEXT:PLAINTEXT,INTERNAL:PLAINTEXT
+        image: krisgeus/docker-kafka
+        name: kafka
+        ports:
+        - containerPort: 9092
+        - containerPort: 2181
+      hostname: kafka
+      restartPolicy: Always
+      subdomain: divolte-divolte

k8s/kafka-service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kafka
+  labels:
+    app: kafka
+spec:
+  ports:
+  - name: "9092"
+    port: 9092
+    targetPort: 9092
+  - name: "2182"
+    port: 2181
+    targetPort: 2181
+  selector:
+    app: kafka

k8s/kibana-deployment.yaml

@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kibana
+  labels:
+    app: kibana
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kibana
+  template:
+    metadata:
+      labels:
+        app: kibana
+    spec:
+      containers:
+      - env:
+        - name: ELASTICSEARCH_URL
+          value: http://elastic:9200
+        - name: LOGGING_QUIET
+          value: "true"
+        - name: SERVER_NAME
+          value: kibana.divolte-divolte
+        image: docker.elastic.co/kibana/kibana:6.4.0
+        name: kibana
+        ports:
+        - containerPort: 5601
+      hostname: kibana
+      restartPolicy: Always
+      subdomain: divolte-divolte

k8s/kibana-service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kibana
+  labels:
+    app: kibana
+spec:
+  ports:
+  - name: "5601"
+    port: 5601
+    targetPort: 5601
+  selector:
+    app: kibana

k8s/put-categories-deployment.yaml

@@ -0,0 +1,27 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: put-categories
+spec:
+  template:
+    spec:
+      containers:
+        - name: put-categories
+          image: python:3.6
+          workingDir: /shop
+          volumeMounts:
+            - mountPath: /shop
+              name: divolte-shop
+          command: ["bash"]
+          args: ["-c", "pip install requests && python catalog-builder/put-categories.py --api-base-url http://shop-api:8080/api data/categories/animals.json data/categories/architecture.json data/categories/cars.json data/categories/cities.json data/categories/flowers.json data/categories/landscape.json data/categories/nautical.json"]
+      restartPolicy: Never
+      volumes:
+        - name: divolte-shop
+          hostPath:
+            path: /Users/bas/code/gdd/shop
+            type: Directory
+      initContainers:
+        - name: wait-for-shop-api
+          image: busybox:1.28
+          command: ['sh', '-c', 'sleep 30']
+  backoffLimit: 4

k8s/shop-api-deployment.yaml

@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: shop-api
+  labels:
+    app: shop-api
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: shop-api
+  template:
+    metadata:
+      labels:
+        app: shop-api
+    spec:
+      containers:
+      - env:
+        - name: ELASTICSEARCH_CLUSTER_NAME
+          value: elasticsearch
+        - name: ELASTICSEARCH_HOSTS
+          value: '["elastic"]'
+        - name: JAVA_OPTS
+          value: -Xms512m -Xmx512m
+        image: shop_shop-api
+        name: shop-api
+        imagePullPolicy: Never
+        ports:
+        - containerPort: 8080
+        - containerPort: 8081
+        livenessProbe:
+          httpGet:
+            path: /api/status/ping
+            port: 8080
+          initialDelaySeconds: 3
+          periodSeconds: 3
+      hostname: shop-api
+      restartPolicy: Always
+      subdomain: divolte-divolte