From d480ec0c3559a5474ab4cbfece063975d4c48e52 Mon Sep 17 00:00:00 2001 From: Bas van de Lustgraaf Date: Tue, 24 Sep 2019 13:43:51 +0200 Subject: [PATCH 01/32] Initial setup for kubernetes using `kompose convert` and some manual fixes --- k8s/clickstream-analyzer-deployment.yaml | 40 ++++++++++++++++++ k8s/clickstream-analyzer-service.yaml | 25 ++++++++++++ k8s/clickstream-collector-deployment.yaml | 37 +++++++++++++++++ k8s/clickstream-collector-service.yaml | 19 +++++++++ k8s/elastic-deployment.yaml | 45 +++++++++++++++++++++ k8s/elastic-service.yaml | 22 ++++++++++ k8s/kafka-deployment.yaml | 49 +++++++++++++++++++++++ k8s/kafka-service.yaml | 22 ++++++++++ k8s/kibana-deployment.yaml | 38 ++++++++++++++++++ k8s/kibana-service.yaml | 19 +++++++++ k8s/shop-api-deployment.yaml | 40 ++++++++++++++++++ k8s/shop-api-service.yaml | 22 ++++++++++ k8s/shop-webapp-deployment.yaml | 43 ++++++++++++++++++++ k8s/shop-webapp-service.yaml | 19 +++++++++ k8s/toppick-bandit-deployment.yaml | 39 ++++++++++++++++++ k8s/toppick-bandit-service.yaml | 19 +++++++++ k8s/toppick-consumer-deployment.yaml | 39 ++++++++++++++++++ k8s/toppick-redis-deployment.yaml | 31 ++++++++++++++ k8s/toppick-redis-service.yaml | 19 +++++++++ 19 files changed, 587 insertions(+) create mode 100644 k8s/clickstream-analyzer-deployment.yaml create mode 100644 k8s/clickstream-analyzer-service.yaml create mode 100644 k8s/clickstream-collector-deployment.yaml create mode 100644 k8s/clickstream-collector-service.yaml create mode 100644 k8s/elastic-deployment.yaml create mode 100644 k8s/elastic-service.yaml create mode 100644 k8s/kafka-deployment.yaml create mode 100644 k8s/kafka-service.yaml create mode 100644 k8s/kibana-deployment.yaml create mode 100644 k8s/kibana-service.yaml create mode 100644 k8s/shop-api-deployment.yaml create mode 100644 k8s/shop-api-service.yaml create mode 100644 k8s/shop-webapp-deployment.yaml create mode 100644 k8s/shop-webapp-service.yaml create mode 100644 k8s/toppick-bandit-deployment.yaml create mode 100644 k8s/toppick-bandit-service.yaml create mode 100644 k8s/toppick-consumer-deployment.yaml create mode 100644 k8s/toppick-redis-deployment.yaml create mode 100644 k8s/toppick-redis-service.yaml diff --git a/k8s/clickstream-analyzer-deployment.yaml b/k8s/clickstream-analyzer-deployment.yaml new file mode 100644 index 0000000..1a05a24 --- /dev/null +++ b/k8s/clickstream-analyzer-deployment.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: clickstream-analyzer + name: clickstream-analyzer +spec: + replicas: 1 + selector: + matchLabels: + app: clickstream-analyzer + strategy: {} + template: + metadata: + labels: + app: clickstream-analyzer + spec: + containers: + - env: + - name: KAFKA_BROKERS + value: kafka:9092 + - name: KAFKA_TOPIC + value: divolte + - name: SPARK_JOB_CLASS + value: sparkjob.SparkEventsPerMinute + image: shop_clickstream-analyzer + name: clickstream-analyzer + imagePullPolicy: Never + ports: + - containerPort: 7077 + - containerPort: 4040 + - containerPort: 18080 + resources: {} + hostname: clickstream-analyzer + restartPolicy: Always +status: {} diff --git a/k8s/clickstream-analyzer-service.yaml b/k8s/clickstream-analyzer-service.yaml new file mode 100644 index 0000000..436a9cf --- /dev/null +++ b/k8s/clickstream-analyzer-service.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: clickstream-analyzer + name: clickstream-analyzer +spec: + ports: + - name: "7077" + port: 7077 + targetPort: 7077 + - name: "4040" + port: 4040 + targetPort: 4040 + - name: "18080" + port: 18080 + targetPort: 18080 + selector: + io.kompose.service: clickstream-analyzer +status: + loadBalancer: {} diff --git a/k8s/clickstream-collector-deployment.yaml b/k8s/clickstream-collector-deployment.yaml new file mode 100644 index 0000000..cc2203e --- /dev/null +++ b/k8s/clickstream-collector-deployment.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: clickstream-collector + name: clickstream-collector +spec: + replicas: 1 + selector: + matchLabels: + app: clickstream-collector + strategy: {} + template: + metadata: + labels: + app: clickstream-collector + spec: + containers: + - env: + - name: DIVOLTE_CONF_DIR + value: /etc/shop/divolte + - name: DIVOLTE_KAFKA_BROKER_LIST + value: kafka:9092 + image: shop_clickstream-collector + name: clickstream-collector + imagePullPolicy: Never + ports: + - containerPort: 8290 + resources: {} + hostname: clickstream-collector + restartPolicy: Always + subdomain: divolte-divolte +status: {} diff --git a/k8s/clickstream-collector-service.yaml b/k8s/clickstream-collector-service.yaml new file mode 100644 index 0000000..e791430 --- /dev/null +++ b/k8s/clickstream-collector-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: clickstream-collector + name: clickstream-collector +spec: + ports: + - name: "8290" + port: 8290 + targetPort: 8290 + selector: + io.kompose.service: clickstream-collector +status: + loadBalancer: {} diff --git a/k8s/elastic-deployment.yaml b/k8s/elastic-deployment.yaml new file mode 100644 index 0000000..4e0d1bb --- /dev/null +++ b/k8s/elastic-deployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: elastic + name: elastic +spec: + replicas: 1 + selector: + matchLabels: + app: elastic + strategy: {} + template: + metadata: + labels: + app: elastic + spec: + containers: + - env: + - name: ES_JAVA_OPTS + value: -Xms512m -Xmx512m + - name: cluster.name + value: elasticsearch + - name: discovery.type + value: single-node + - name: network.bind_host + value: 0.0.0.0 + - name: network.publish_host + value: elastic + - name: xpack.security.enabled + value: "false" + image: docker.elastic.co/elasticsearch/elasticsearch:6.4.0 + name: elastic + ports: + - containerPort: 9200 + - containerPort: 9300 + resources: {} + hostname: elastic + restartPolicy: Always + subdomain: divolte-divolte +status: {} diff --git a/k8s/elastic-service.yaml b/k8s/elastic-service.yaml new file mode 100644 index 0000000..73930a0 --- /dev/null +++ b/k8s/elastic-service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: elastic + name: elastic +spec: + ports: + - name: "9200" + port: 9200 + targetPort: 9200 + - name: "9300" + port: 9300 + targetPort: 9300 + selector: + io.kompose.service: elastic +status: + loadBalancer: {} diff --git a/k8s/kafka-deployment.yaml b/k8s/kafka-deployment.yaml new file mode 100644 index 0000000..6779f8e --- /dev/null +++ b/k8s/kafka-deployment.yaml @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: kafka + name: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: kafka + strategy: {} + template: + metadata: + labels: + app: kafka + spec: + containers: + - env: + - name: ADVERTISED_HOST + value: kafka + - name: ADVERTISED_LISTENERS + value: PLAINTEXT://kafka:9092,INTERNAL://localhost:9093 + - name: AUTO_CREATE_TOPICS + value: "false" + - name: INTER_BROKER + value: INTERNAL + - name: KAFKA_CREATE_TOPICS + value: divolte:4:1 + - name: LISTENERS + value: PLAINTEXT://0.0.0.0:9092,INTERNAL://0.0.0.0:9093 + - name: LOG_RETENTION_HOURS + value: "1" + - name: SECURITY_PROTOCOL_MAP + value: PLAINTEXT:PLAINTEXT,INTERNAL:PLAINTEXT + image: krisgeus/docker-kafka + name: kafka + ports: + - containerPort: 9092 + - containerPort: 2181 + resources: {} + hostname: kafka + restartPolicy: Always + subdomain: divolte-divolte +status: {} diff --git a/k8s/kafka-service.yaml b/k8s/kafka-service.yaml new file mode 100644 index 0000000..308ba72 --- /dev/null +++ b/k8s/kafka-service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: kafka + name: kafka +spec: + ports: + - name: "9092" + port: 9092 + targetPort: 9092 + - name: "2181" + port: 2181 + targetPort: 2181 + selector: + io.kompose.service: kafka +status: + loadBalancer: {} diff --git a/k8s/kibana-deployment.yaml b/k8s/kibana-deployment.yaml new file mode 100644 index 0000000..e10f23c --- /dev/null +++ b/k8s/kibana-deployment.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: kibana + name: kibana +spec: + replicas: 1 + selector: + matchLabels: + app: kibana + strategy: {} + template: + metadata: + labels: + app: kibana + spec: + containers: + - env: + - name: ELASTICSEARCH_URL + value: http://elastic:9200 + - name: LOGGING_QUIET + value: "true" + - name: SERVER_NAME + value: kibana.divolte_divolte.io + image: docker.elastic.co/kibana/kibana:6.4.0 + name: kibana + ports: + - containerPort: 5601 + resources: {} + hostname: kibana + restartPolicy: Always + subdomain: divolte-divolte +status: {} diff --git a/k8s/kibana-service.yaml b/k8s/kibana-service.yaml new file mode 100644 index 0000000..47eb121 --- /dev/null +++ b/k8s/kibana-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: kibana + name: kibana +spec: + ports: + - name: "5601" + port: 5601 + targetPort: 5601 + selector: + io.kompose.service: kibana +status: + loadBalancer: {} diff --git a/k8s/shop-api-deployment.yaml b/k8s/shop-api-deployment.yaml new file mode 100644 index 0000000..823b1cd --- /dev/null +++ b/k8s/shop-api-deployment.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: shop-api + name: shop-api +spec: + replicas: 1 + selector: + matchLabels: + app: shop-api + strategy: {} + template: + metadata: + labels: + app: shop-api + spec: + containers: + - env: + - name: ELASTICSEARCH_CLUSTER_NAME + value: elasticsearch + - name: ELASTICSEARCH_HOSTS + value: '["elastic"]' + - name: JAVA_OPTS + value: -Xms512m -Xmx512m + image: shop_shop-api + name: shop-api + imagePullPolicy: Never + ports: + - containerPort: 8080 + - containerPort: 8081 + resources: {} + hostname: shop-api + restartPolicy: Always + subdomain: divolte-divolte +status: {} diff --git a/k8s/shop-api-service.yaml b/k8s/shop-api-service.yaml new file mode 100644 index 0000000..edc18df --- /dev/null +++ b/k8s/shop-api-service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: shop-api + name: shop-api +spec: + ports: + - name: "8080" + port: 8080 + targetPort: 8080 + - name: "8081" + port: 8081 + targetPort: 8081 + selector: + io.kompose.service: shop-api +status: + loadBalancer: {} diff --git a/k8s/shop-webapp-deployment.yaml b/k8s/shop-webapp-deployment.yaml new file mode 100644 index 0000000..1de19c2 --- /dev/null +++ b/k8s/shop-webapp-deployment.yaml @@ -0,0 +1,43 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: shop-webapp + name: shop-webapp +spec: + replicas: 1 + selector: + matchLabels: + app: shop-webapp + strategy: {} + template: + metadata: + labels: + app: shop-webapp + spec: + containers: + - env: + - name: SHOP_API_URL + value: http://shop-api:8080/api + - name: SHOP_BANDIT_URL + value: http://toppick-bandit:8989/item + - name: SHOP_DIVOLTE_URL + value: //localhost:8290/divolte.js + - name: SHOP_WEBAPP_PORT + value: "9011" + - name: SHOP_WEBAPP_SECRET + value: foo + image: shop_shop-webapp + name: shop-webapp + imagePullPolicy: Never + ports: + - containerPort: 9011 + resources: {} + hostname: shop-webapp + restartPolicy: Always + subdomain: divolte-divolte +status: {} diff --git a/k8s/shop-webapp-service.yaml b/k8s/shop-webapp-service.yaml new file mode 100644 index 0000000..08ffa76 --- /dev/null +++ b/k8s/shop-webapp-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: shop-webapp + name: shop-webapp +spec: + ports: + - name: "9011" + port: 9011 + targetPort: 9011 + selector: + io.kompose.service: shop-webapp +status: + loadBalancer: {} diff --git a/k8s/toppick-bandit-deployment.yaml b/k8s/toppick-bandit-deployment.yaml new file mode 100644 index 0000000..28a54b5 --- /dev/null +++ b/k8s/toppick-bandit-deployment.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: toppick-bandit + name: toppick-bandit +spec: + replicas: 1 + selector: + matchLabels: + app: toppick-bandit + strategy: {} + template: + metadata: + labels: + app: toppick-bandit + spec: + containers: + - env: + - name: BIND_HOST + value: 0.0.0.0 + - name: BIND_PORT + value: "8989" + - name: REDIS_HOST_PORT + value: toppick-redis:6379 + image: shop_toppick-bandit + name: toppick-bandit + imagePullPolicy: Never + ports: + - containerPort: 8989 + resources: {} + hostname: toppick-bandit + restartPolicy: Always + subdomain: divolte-divolte +status: {} diff --git a/k8s/toppick-bandit-service.yaml b/k8s/toppick-bandit-service.yaml new file mode 100644 index 0000000..75eb658 --- /dev/null +++ b/k8s/toppick-bandit-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: toppick-bandit + name: toppick-bandit +spec: + ports: + - name: "8989" + port: 8989 + targetPort: 8989 + selector: + io.kompose.service: toppick-bandit +status: + loadBalancer: {} diff --git a/k8s/toppick-consumer-deployment.yaml b/k8s/toppick-consumer-deployment.yaml new file mode 100644 index 0000000..63fe209 --- /dev/null +++ b/k8s/toppick-consumer-deployment.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: toppick-consumer + name: toppick-consumer +spec: + replicas: 1 + selector: + matchLabels: + app: toppick-consumer + strategy: {} + template: + metadata: + labels: + app: toppick-consumer + spec: + containers: + - env: + - name: ELASTIC_HOST_PORT + value: elastic:9200 + - name: KAFKA_BROKERS + value: kafka:9092 + - name: KAFKA_TOPIC + value: divolte + - name: REDIS_HOST_PORT + value: toppick-redis:6379 + image: shop_toppick-consumer + name: toppick-consumer + imagePullPolicy: Never + resources: {} + hostname: toppick-consumer + restartPolicy: Always + subdomain: divolte-divolte +status: {} diff --git a/k8s/toppick-redis-deployment.yaml b/k8s/toppick-redis-deployment.yaml new file mode 100644 index 0000000..2474116 --- /dev/null +++ b/k8s/toppick-redis-deployment.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: toppick-redis + name: toppick-redis +spec: + replicas: 1 + selector: + matchLabels: + app: toppick-redis + strategy: {} + template: + metadata: + labels: + app: toppick-redis + spec: + containers: + - image: redis:alpine + name: toppick-redis + ports: + - containerPort: 6379 + resources: {} + hostname: toppick-redis + restartPolicy: Always + subdomain: divolte-divolte +status: {} diff --git a/k8s/toppick-redis-service.yaml b/k8s/toppick-redis-service.yaml new file mode 100644 index 0000000..178cbb2 --- /dev/null +++ b/k8s/toppick-redis-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.18.0 () + creationTimestamp: null + labels: + io.kompose.service: toppick-redis + name: toppick-redis +spec: + ports: + - name: "6379" + port: 6379 + targetPort: 6379 + selector: + io.kompose.service: toppick-redis +status: + loadBalancer: {} From 58502df04802e653706e738d9c88c6465327e403 Mon Sep 17 00:00:00 2001 From: Bas van de Lustgraaf Date: Tue, 24 Sep 2019 15:00:54 +0200 Subject: [PATCH 02/32] Fix the key in `labels` to get the network between pods going --- k8s/clickstream-analyzer-deployment.yaml | 2 +- k8s/clickstream-analyzer-service.yaml | 4 ++-- k8s/clickstream-collector-deployment.yaml | 2 +- k8s/clickstream-collector-service.yaml | 4 ++-- k8s/elastic-deployment.yaml | 2 +- k8s/elastic-service.yaml | 4 ++-- k8s/kafka-deployment.yaml | 2 +- k8s/kafka-service.yaml | 4 ++-- k8s/kibana-deployment.yaml | 4 ++-- k8s/kibana-service.yaml | 4 ++-- k8s/shop-api-deployment.yaml | 2 +- k8s/shop-api-service.yaml | 4 ++-- k8s/shop-webapp-deployment.yaml | 2 +- k8s/shop-webapp-service.yaml | 4 ++-- k8s/toppick-bandit-deployment.yaml | 2 +- k8s/toppick-bandit-service.yaml | 4 ++-- k8s/toppick-consumer-deployment.yaml | 2 +- k8s/toppick-redis-deployment.yaml | 2 +- k8s/toppick-redis-service.yaml | 4 ++-- 19 files changed, 29 insertions(+), 29 deletions(-) diff --git a/k8s/clickstream-analyzer-deployment.yaml b/k8s/clickstream-analyzer-deployment.yaml index 1a05a24..58964bb 100644 --- a/k8s/clickstream-analyzer-deployment.yaml +++ b/k8s/clickstream-analyzer-deployment.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: clickstream-analyzer + app: clickstream-analyzer name: clickstream-analyzer spec: replicas: 1 diff --git a/k8s/clickstream-analyzer-service.yaml b/k8s/clickstream-analyzer-service.yaml index 436a9cf..e0fe30e 100644 --- a/k8s/clickstream-analyzer-service.yaml +++ b/k8s/clickstream-analyzer-service.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: clickstream-analyzer + app: clickstream-analyzer name: clickstream-analyzer spec: ports: @@ -20,6 +20,6 @@ spec: port: 18080 targetPort: 18080 selector: - io.kompose.service: clickstream-analyzer + app: clickstream-analyzer status: loadBalancer: {} diff --git a/k8s/clickstream-collector-deployment.yaml b/k8s/clickstream-collector-deployment.yaml index cc2203e..42e2283 100644 --- a/k8s/clickstream-collector-deployment.yaml +++ b/k8s/clickstream-collector-deployment.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: clickstream-collector + app: clickstream-collector name: clickstream-collector spec: replicas: 1 diff --git a/k8s/clickstream-collector-service.yaml b/k8s/clickstream-collector-service.yaml index e791430..7af0207 100644 --- a/k8s/clickstream-collector-service.yaml +++ b/k8s/clickstream-collector-service.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: clickstream-collector + app: clickstream-collector name: clickstream-collector spec: ports: @@ -14,6 +14,6 @@ spec: port: 8290 targetPort: 8290 selector: - io.kompose.service: clickstream-collector + app: clickstream-collector status: loadBalancer: {} diff --git a/k8s/elastic-deployment.yaml b/k8s/elastic-deployment.yaml index 4e0d1bb..acd7edf 100644 --- a/k8s/elastic-deployment.yaml +++ b/k8s/elastic-deployment.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: elastic + app: elastic name: elastic spec: replicas: 1 diff --git a/k8s/elastic-service.yaml b/k8s/elastic-service.yaml index 73930a0..61375f4 100644 --- a/k8s/elastic-service.yaml +++ b/k8s/elastic-service.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: elastic + app: elastic name: elastic spec: ports: @@ -17,6 +17,6 @@ spec: port: 9300 targetPort: 9300 selector: - io.kompose.service: elastic + app: elastic status: loadBalancer: {} diff --git a/k8s/kafka-deployment.yaml b/k8s/kafka-deployment.yaml index 6779f8e..9faf965 100644 --- a/k8s/kafka-deployment.yaml +++ b/k8s/kafka-deployment.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: kafka + app: kafka name: kafka spec: replicas: 1 diff --git a/k8s/kafka-service.yaml b/k8s/kafka-service.yaml index 308ba72..c9a73a0 100644 --- a/k8s/kafka-service.yaml +++ b/k8s/kafka-service.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: kafka + app: kafka name: kafka spec: ports: @@ -17,6 +17,6 @@ spec: port: 2181 targetPort: 2181 selector: - io.kompose.service: kafka + app: kafka status: loadBalancer: {} diff --git a/k8s/kibana-deployment.yaml b/k8s/kibana-deployment.yaml index e10f23c..aff8904 100644 --- a/k8s/kibana-deployment.yaml +++ b/k8s/kibana-deployment.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: kibana + app: kibana name: kibana spec: replicas: 1 @@ -26,7 +26,7 @@ spec: - name: LOGGING_QUIET value: "true" - name: SERVER_NAME - value: kibana.divolte_divolte.io + value: kibana.divolte-divolte image: docker.elastic.co/kibana/kibana:6.4.0 name: kibana ports: diff --git a/k8s/kibana-service.yaml b/k8s/kibana-service.yaml index 47eb121..d4bb3af 100644 --- a/k8s/kibana-service.yaml +++ b/k8s/kibana-service.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: kibana + app: kibana name: kibana spec: ports: @@ -14,6 +14,6 @@ spec: port: 5601 targetPort: 5601 selector: - io.kompose.service: kibana + app: kibana status: loadBalancer: {} diff --git a/k8s/shop-api-deployment.yaml b/k8s/shop-api-deployment.yaml index 823b1cd..1a830af 100644 --- a/k8s/shop-api-deployment.yaml +++ b/k8s/shop-api-deployment.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: shop-api + app: shop-api name: shop-api spec: replicas: 1 diff --git a/k8s/shop-api-service.yaml b/k8s/shop-api-service.yaml index edc18df..ecb4c10 100644 --- a/k8s/shop-api-service.yaml +++ b/k8s/shop-api-service.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: shop-api + app: shop-api name: shop-api spec: ports: @@ -17,6 +17,6 @@ spec: port: 8081 targetPort: 8081 selector: - io.kompose.service: shop-api + app: shop-api status: loadBalancer: {} diff --git a/k8s/shop-webapp-deployment.yaml b/k8s/shop-webapp-deployment.yaml index 1de19c2..b44bc17 100644 --- a/k8s/shop-webapp-deployment.yaml +++ b/k8s/shop-webapp-deployment.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: shop-webapp + app: shop-webapp name: shop-webapp spec: replicas: 1 diff --git a/k8s/shop-webapp-service.yaml b/k8s/shop-webapp-service.yaml index 08ffa76..35f0ef9 100644 --- a/k8s/shop-webapp-service.yaml +++ b/k8s/shop-webapp-service.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: shop-webapp + app: shop-webapp name: shop-webapp spec: ports: @@ -14,6 +14,6 @@ spec: port: 9011 targetPort: 9011 selector: - io.kompose.service: shop-webapp + app: shop-webapp status: loadBalancer: {} diff --git a/k8s/toppick-bandit-deployment.yaml b/k8s/toppick-bandit-deployment.yaml index 28a54b5..a9101aa 100644 --- a/k8s/toppick-bandit-deployment.yaml +++ b/k8s/toppick-bandit-deployment.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: toppick-bandit + app: toppick-bandit name: toppick-bandit spec: replicas: 1 diff --git a/k8s/toppick-bandit-service.yaml b/k8s/toppick-bandit-service.yaml index 75eb658..90b7ffc 100644 --- a/k8s/toppick-bandit-service.yaml +++ b/k8s/toppick-bandit-service.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: toppick-bandit + app: toppick-bandit name: toppick-bandit spec: ports: @@ -14,6 +14,6 @@ spec: port: 8989 targetPort: 8989 selector: - io.kompose.service: toppick-bandit + app: toppick-bandit status: loadBalancer: {} diff --git a/k8s/toppick-consumer-deployment.yaml b/k8s/toppick-consumer-deployment.yaml index 63fe209..9cfabdf 100644 --- a/k8s/toppick-consumer-deployment.yaml +++ b/k8s/toppick-consumer-deployment.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: toppick-consumer + app: toppick-consumer name: toppick-consumer spec: replicas: 1 diff --git a/k8s/toppick-redis-deployment.yaml b/k8s/toppick-redis-deployment.yaml index 2474116..562de8e 100644 --- a/k8s/toppick-redis-deployment.yaml +++ b/k8s/toppick-redis-deployment.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: toppick-redis + app: toppick-redis name: toppick-redis spec: replicas: 1 diff --git a/k8s/toppick-redis-service.yaml b/k8s/toppick-redis-service.yaml index 178cbb2..e43c975 100644 --- a/k8s/toppick-redis-service.yaml +++ b/k8s/toppick-redis-service.yaml @@ -6,7 +6,7 @@ metadata: kompose.version: 1.18.0 () creationTimestamp: null labels: - io.kompose.service: toppick-redis + app: toppick-redis name: toppick-redis spec: ports: @@ -14,6 +14,6 @@ spec: port: 6379 targetPort: 6379 selector: - io.kompose.service: toppick-redis + app: toppick-redis status: loadBalancer: {} From b3b4e4a4b340ac8fc35c5936b7a6f53caa9d1993 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Tue, 24 Sep 2019 15:23:08 +0200 Subject: [PATCH 03/32] Change web app service type to be LoadBalancer --- k8s/shop-webapp-service.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s/shop-webapp-service.yaml b/k8s/shop-webapp-service.yaml index 35f0ef9..d6acadb 100644 --- a/k8s/shop-webapp-service.yaml +++ b/k8s/shop-webapp-service.yaml @@ -15,5 +15,6 @@ spec: targetPort: 9011 selector: app: shop-webapp + type: LoadBalancer status: loadBalancer: {} From 38dd8d379f8a578cba5363443a45e4ea685b381c Mon Sep 17 00:00:00 2001 From: Bas Beelen Date: Tue, 24 Sep 2019 15:43:10 +0200 Subject: [PATCH 04/32] Make shop-api also of type loadbalancer to expose it on localhost --- k8s/shop-api-service.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s/shop-api-service.yaml b/k8s/shop-api-service.yaml index ecb4c10..663786d 100644 --- a/k8s/shop-api-service.yaml +++ b/k8s/shop-api-service.yaml @@ -18,5 +18,6 @@ spec: targetPort: 8081 selector: app: shop-api + type: LoadBalancer status: loadBalancer: {} From 3ff4214cedece1def20d069f1fa89ab40bf91836 Mon Sep 17 00:00:00 2001 From: Bas van de Lustgraaf Date: Tue, 24 Sep 2019 16:18:20 +0200 Subject: [PATCH 05/32] Start --- k8s/put-categories-deployment.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 k8s/put-categories-deployment.yaml diff --git a/k8s/put-categories-deployment.yaml b/k8s/put-categories-deployment.yaml new file mode 100644 index 0000000..372408b --- /dev/null +++ b/k8s/put-categories-deployment.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: put-categories +spec: + template: + spec: + containers: + - name: put-categories + image: python:3.6 + workingDir: /shop + volumeMounts: + - mountPath: /shop + name: divolte-shop + command: ["ls"] + restartPolicy: Never + volumes: + - name: divolte-shop + hostPath: + # directory location on host + path: /Users/bas/code/gdd/shop + # this field is optional + type: Directory + backoffLimit: 4 \ No newline at end of file From faa2216a16f7dc43e11b4506f368d8bc016f3f22 Mon Sep 17 00:00:00 2001 From: Bas van de Lustgraaf Date: Tue, 24 Sep 2019 16:20:46 +0200 Subject: [PATCH 06/32] First attempt to load shop content using Job pod, not working --- k8s/put-categories-deployment.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/k8s/put-categories-deployment.yaml b/k8s/put-categories-deployment.yaml index 372408b..b81a3cf 100644 --- a/k8s/put-categories-deployment.yaml +++ b/k8s/put-categories-deployment.yaml @@ -12,7 +12,14 @@ spec: volumeMounts: - mountPath: /shop name: divolte-shop - command: ["ls"] + command: ["bash -c 'pip install requests && python catalog-builder/put-categories.py \ + data/categories/animals.json \ + data/categories/architecture.json \ + data/categories/cars.json \ + data/categories/cities.json \ + data/categories/flowers.json \ + data/categories/landscape.json \ + data/categories/nautical.json'"] restartPolicy: Never volumes: - name: divolte-shop From 9c1e9f0588bb8a5255e45c0d85686f85a842e792 Mon Sep 17 00:00:00 2001 From: Bas van de Lustgraaf Date: Tue, 24 Sep 2019 16:51:44 +0200 Subject: [PATCH 07/32] Loading the shop content using job pod --- k8s/put-categories-deployment.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/k8s/put-categories-deployment.yaml b/k8s/put-categories-deployment.yaml index b81a3cf..16068d3 100644 --- a/k8s/put-categories-deployment.yaml +++ b/k8s/put-categories-deployment.yaml @@ -12,14 +12,8 @@ spec: volumeMounts: - mountPath: /shop name: divolte-shop - command: ["bash -c 'pip install requests && python catalog-builder/put-categories.py \ - data/categories/animals.json \ - data/categories/architecture.json \ - data/categories/cars.json \ - data/categories/cities.json \ - data/categories/flowers.json \ - data/categories/landscape.json \ - data/categories/nautical.json'"] + command: ["bash"] + args: ["-c", "pip install requests && python catalog-builder/put-categories.py --api-base-url http://shop-api:8080/api data/categories/animals.json data/categories/architecture.json data/categories/cars.json data/categories/cities.json data/categories/flowers.json data/categories/landscape.json data/categories/nautical.json"] restartPolicy: Never volumes: - name: divolte-shop From 8b405c7eff991fe916e904eb25031e3f6b57ea98 Mon Sep 17 00:00:00 2001 From: Bas van de Lustgraaf Date: Wed, 25 Sep 2019 13:29:49 +0200 Subject: [PATCH 08/32] Back the bare minimum config --- k8s/clickstream-analyzer-deployment.yaml | 11 ++--------- k8s/clickstream-analyzer-service.yaml | 8 +------- k8s/clickstream-collector-deployment.yaml | 9 +-------- k8s/clickstream-collector-service.yaml | 8 +------- k8s/elastic-deployment.yaml | 9 +-------- k8s/elastic-service.yaml | 8 +------- k8s/kafka-deployment.yaml | 9 +-------- k8s/kafka-service.yaml | 10 ++-------- k8s/kibana-deployment.yaml | 9 +-------- k8s/kibana-service.yaml | 8 +------- k8s/put-categories-deployment.yaml | 2 -- k8s/shop-api-deployment.yaml | 9 +-------- k8s/shop-api-service.yaml | 9 ++------- k8s/shop-webapp-deployment.yaml | 9 +-------- k8s/shop-webapp-service.yaml | 8 +------- k8s/toppick-bandit-deployment.yaml | 9 +-------- k8s/toppick-bandit-service.yaml | 8 +------- k8s/toppick-consumer-deployment.yaml | 9 +-------- k8s/toppick-redis-deployment.yaml | 9 +-------- k8s/toppick-redis-service.yaml | 8 +------- 20 files changed, 22 insertions(+), 147 deletions(-) diff --git a/k8s/clickstream-analyzer-deployment.yaml b/k8s/clickstream-analyzer-deployment.yaml index 58964bb..ac29023 100644 --- a/k8s/clickstream-analyzer-deployment.yaml +++ b/k8s/clickstream-analyzer-deployment.yaml @@ -1,19 +1,14 @@ apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: clickstream-analyzer labels: app: clickstream-analyzer - name: clickstream-analyzer spec: replicas: 1 selector: matchLabels: app: clickstream-analyzer - strategy: {} template: metadata: labels: @@ -29,12 +24,10 @@ spec: value: sparkjob.SparkEventsPerMinute image: shop_clickstream-analyzer name: clickstream-analyzer - imagePullPolicy: Never + imagePullPolicy: Never # always local ports: - containerPort: 7077 - containerPort: 4040 - containerPort: 18080 - resources: {} hostname: clickstream-analyzer restartPolicy: Always -status: {} diff --git a/k8s/clickstream-analyzer-service.yaml b/k8s/clickstream-analyzer-service.yaml index e0fe30e..0e0d98f 100644 --- a/k8s/clickstream-analyzer-service.yaml +++ b/k8s/clickstream-analyzer-service.yaml @@ -1,13 +1,9 @@ apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: clickstream-analyzer labels: app: clickstream-analyzer - name: clickstream-analyzer spec: ports: - name: "7077" @@ -21,5 +17,3 @@ spec: targetPort: 18080 selector: app: clickstream-analyzer -status: - loadBalancer: {} diff --git a/k8s/clickstream-collector-deployment.yaml b/k8s/clickstream-collector-deployment.yaml index 42e2283..10cc96b 100644 --- a/k8s/clickstream-collector-deployment.yaml +++ b/k8s/clickstream-collector-deployment.yaml @@ -1,19 +1,14 @@ apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: clickstream-collector labels: app: clickstream-collector - name: clickstream-collector spec: replicas: 1 selector: matchLabels: app: clickstream-collector - strategy: {} template: metadata: labels: @@ -30,8 +25,6 @@ spec: imagePullPolicy: Never ports: - containerPort: 8290 - resources: {} hostname: clickstream-collector restartPolicy: Always subdomain: divolte-divolte -status: {} diff --git a/k8s/clickstream-collector-service.yaml b/k8s/clickstream-collector-service.yaml index 7af0207..3131154 100644 --- a/k8s/clickstream-collector-service.yaml +++ b/k8s/clickstream-collector-service.yaml @@ -1,13 +1,9 @@ apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: clickstream-collector labels: app: clickstream-collector - name: clickstream-collector spec: ports: - name: "8290" @@ -15,5 +11,3 @@ spec: targetPort: 8290 selector: app: clickstream-collector -status: - loadBalancer: {} diff --git a/k8s/elastic-deployment.yaml b/k8s/elastic-deployment.yaml index acd7edf..1c221e9 100644 --- a/k8s/elastic-deployment.yaml +++ b/k8s/elastic-deployment.yaml @@ -1,19 +1,14 @@ apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: elastic labels: app: elastic - name: elastic spec: replicas: 1 selector: matchLabels: app: elastic - strategy: {} template: metadata: labels: @@ -38,8 +33,6 @@ spec: ports: - containerPort: 9200 - containerPort: 9300 - resources: {} hostname: elastic restartPolicy: Always subdomain: divolte-divolte -status: {} diff --git a/k8s/elastic-service.yaml b/k8s/elastic-service.yaml index 61375f4..a1edd21 100644 --- a/k8s/elastic-service.yaml +++ b/k8s/elastic-service.yaml @@ -1,13 +1,9 @@ apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: elastic labels: app: elastic - name: elastic spec: ports: - name: "9200" @@ -18,5 +14,3 @@ spec: targetPort: 9300 selector: app: elastic -status: - loadBalancer: {} diff --git a/k8s/kafka-deployment.yaml b/k8s/kafka-deployment.yaml index 9faf965..3565e39 100644 --- a/k8s/kafka-deployment.yaml +++ b/k8s/kafka-deployment.yaml @@ -1,19 +1,14 @@ apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: kafka labels: app: kafka - name: kafka spec: replicas: 1 selector: matchLabels: app: kafka - strategy: {} template: metadata: labels: @@ -42,8 +37,6 @@ spec: ports: - containerPort: 9092 - containerPort: 2181 - resources: {} hostname: kafka restartPolicy: Always subdomain: divolte-divolte -status: {} diff --git a/k8s/kafka-service.yaml b/k8s/kafka-service.yaml index c9a73a0..7228eaa 100644 --- a/k8s/kafka-service.yaml +++ b/k8s/kafka-service.yaml @@ -1,22 +1,16 @@ apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: kafka labels: app: kafka - name: kafka spec: ports: - name: "9092" port: 9092 targetPort: 9092 - - name: "2181" + - name: "2182" port: 2181 targetPort: 2181 selector: app: kafka -status: - loadBalancer: {} diff --git a/k8s/kibana-deployment.yaml b/k8s/kibana-deployment.yaml index aff8904..434b239 100644 --- a/k8s/kibana-deployment.yaml +++ b/k8s/kibana-deployment.yaml @@ -1,19 +1,14 @@ apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: kibana labels: app: kibana - name: kibana spec: replicas: 1 selector: matchLabels: app: kibana - strategy: {} template: metadata: labels: @@ -31,8 +26,6 @@ spec: name: kibana ports: - containerPort: 5601 - resources: {} hostname: kibana restartPolicy: Always subdomain: divolte-divolte -status: {} diff --git a/k8s/kibana-service.yaml b/k8s/kibana-service.yaml index d4bb3af..68c1430 100644 --- a/k8s/kibana-service.yaml +++ b/k8s/kibana-service.yaml @@ -1,13 +1,9 @@ apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: kibana labels: app: kibana - name: kibana spec: ports: - name: "5601" @@ -15,5 +11,3 @@ spec: targetPort: 5601 selector: app: kibana -status: - loadBalancer: {} diff --git a/k8s/put-categories-deployment.yaml b/k8s/put-categories-deployment.yaml index 16068d3..4a20639 100644 --- a/k8s/put-categories-deployment.yaml +++ b/k8s/put-categories-deployment.yaml @@ -18,8 +18,6 @@ spec: volumes: - name: divolte-shop hostPath: - # directory location on host path: /Users/bas/code/gdd/shop - # this field is optional type: Directory backoffLimit: 4 \ No newline at end of file diff --git a/k8s/shop-api-deployment.yaml b/k8s/shop-api-deployment.yaml index 1a830af..fa2be92 100644 --- a/k8s/shop-api-deployment.yaml +++ b/k8s/shop-api-deployment.yaml @@ -1,19 +1,14 @@ apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: shop-api labels: app: shop-api - name: shop-api spec: replicas: 1 selector: matchLabels: app: shop-api - strategy: {} template: metadata: labels: @@ -33,8 +28,6 @@ spec: ports: - containerPort: 8080 - containerPort: 8081 - resources: {} hostname: shop-api restartPolicy: Always subdomain: divolte-divolte -status: {} diff --git a/k8s/shop-api-service.yaml b/k8s/shop-api-service.yaml index 663786d..58a12e9 100644 --- a/k8s/shop-api-service.yaml +++ b/k8s/shop-api-service.yaml @@ -1,13 +1,9 @@ apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: shop-api labels: app: shop-api - name: shop-api spec: ports: - name: "8080" @@ -19,5 +15,4 @@ spec: selector: app: shop-api type: LoadBalancer -status: - loadBalancer: {} + diff --git a/k8s/shop-webapp-deployment.yaml b/k8s/shop-webapp-deployment.yaml index b44bc17..97a27c5 100644 --- a/k8s/shop-webapp-deployment.yaml +++ b/k8s/shop-webapp-deployment.yaml @@ -1,19 +1,14 @@ apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: shop-webapp labels: app: shop-webapp - name: shop-webapp spec: replicas: 1 selector: matchLabels: app: shop-webapp - strategy: {} template: metadata: labels: @@ -36,8 +31,6 @@ spec: imagePullPolicy: Never ports: - containerPort: 9011 - resources: {} hostname: shop-webapp restartPolicy: Always subdomain: divolte-divolte -status: {} diff --git a/k8s/shop-webapp-service.yaml b/k8s/shop-webapp-service.yaml index d6acadb..5b071de 100644 --- a/k8s/shop-webapp-service.yaml +++ b/k8s/shop-webapp-service.yaml @@ -1,13 +1,9 @@ apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: shop-webapp labels: app: shop-webapp - name: shop-webapp spec: ports: - name: "9011" @@ -16,5 +12,3 @@ spec: selector: app: shop-webapp type: LoadBalancer -status: - loadBalancer: {} diff --git a/k8s/toppick-bandit-deployment.yaml b/k8s/toppick-bandit-deployment.yaml index a9101aa..4ddb9c1 100644 --- a/k8s/toppick-bandit-deployment.yaml +++ b/k8s/toppick-bandit-deployment.yaml @@ -1,19 +1,14 @@ apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: toppick-bandit labels: app: toppick-bandit - name: toppick-bandit spec: replicas: 1 selector: matchLabels: app: toppick-bandit - strategy: {} template: metadata: labels: @@ -32,8 +27,6 @@ spec: imagePullPolicy: Never ports: - containerPort: 8989 - resources: {} hostname: toppick-bandit restartPolicy: Always subdomain: divolte-divolte -status: {} diff --git a/k8s/toppick-bandit-service.yaml b/k8s/toppick-bandit-service.yaml index 90b7ffc..6ff6c6b 100644 --- a/k8s/toppick-bandit-service.yaml +++ b/k8s/toppick-bandit-service.yaml @@ -1,13 +1,9 @@ apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: toppick-bandit labels: app: toppick-bandit - name: toppick-bandit spec: ports: - name: "8989" @@ -15,5 +11,3 @@ spec: targetPort: 8989 selector: app: toppick-bandit -status: - loadBalancer: {} diff --git a/k8s/toppick-consumer-deployment.yaml b/k8s/toppick-consumer-deployment.yaml index 9cfabdf..e94b64c 100644 --- a/k8s/toppick-consumer-deployment.yaml +++ b/k8s/toppick-consumer-deployment.yaml @@ -1,19 +1,14 @@ apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: toppick-consumer labels: app: toppick-consumer - name: toppick-consumer spec: replicas: 1 selector: matchLabels: app: toppick-consumer - strategy: {} template: metadata: labels: @@ -32,8 +27,6 @@ spec: image: shop_toppick-consumer name: toppick-consumer imagePullPolicy: Never - resources: {} hostname: toppick-consumer restartPolicy: Always subdomain: divolte-divolte -status: {} diff --git a/k8s/toppick-redis-deployment.yaml b/k8s/toppick-redis-deployment.yaml index 562de8e..797ada2 100644 --- a/k8s/toppick-redis-deployment.yaml +++ b/k8s/toppick-redis-deployment.yaml @@ -1,19 +1,14 @@ apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: toppick-redis labels: app: toppick-redis - name: toppick-redis spec: replicas: 1 selector: matchLabels: app: toppick-redis - strategy: {} template: metadata: labels: @@ -24,8 +19,6 @@ spec: name: toppick-redis ports: - containerPort: 6379 - resources: {} hostname: toppick-redis restartPolicy: Always subdomain: divolte-divolte -status: {} diff --git a/k8s/toppick-redis-service.yaml b/k8s/toppick-redis-service.yaml index e43c975..02c92bd 100644 --- a/k8s/toppick-redis-service.yaml +++ b/k8s/toppick-redis-service.yaml @@ -1,13 +1,9 @@ apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.18.0 () - creationTimestamp: null + name: toppick-redis labels: app: toppick-redis - name: toppick-redis spec: ports: - name: "6379" @@ -15,5 +11,3 @@ spec: targetPort: 6379 selector: app: toppick-redis -status: - loadBalancer: {} From 7b413168e1fd1e59cbc2c749f300075d7add63a9 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Wed, 25 Sep 2019 14:29:58 +0200 Subject: [PATCH 09/32] Change shop API service to ClusterIP --- k8s/shop-api-service.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/k8s/shop-api-service.yaml b/k8s/shop-api-service.yaml index 58a12e9..c3c06ba 100644 --- a/k8s/shop-api-service.yaml +++ b/k8s/shop-api-service.yaml @@ -14,5 +14,3 @@ spec: targetPort: 8081 selector: app: shop-api - type: LoadBalancer - From 8b89bb32566f6cd78883ce0e88e985e100c56fd5 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Wed, 25 Sep 2019 14:30:18 +0200 Subject: [PATCH 10/32] Change shop webapp service to ClusterIP --- k8s/shop-webapp-service.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/k8s/shop-webapp-service.yaml b/k8s/shop-webapp-service.yaml index 5b071de..fa591dd 100644 --- a/k8s/shop-webapp-service.yaml +++ b/k8s/shop-webapp-service.yaml @@ -11,4 +11,3 @@ spec: targetPort: 9011 selector: app: shop-webapp - type: LoadBalancer From 1be73e4f8d837825d7e2dded9f8cbdb3df593272 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Wed, 25 Sep 2019 14:30:34 +0200 Subject: [PATCH 11/32] Add ingress yaml --- k8s/ingress.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 k8s/ingress.yaml diff --git a/k8s/ingress.yaml b/k8s/ingress.yaml new file mode 100644 index 0000000..b6482d9 --- /dev/null +++ b/k8s/ingress.yaml @@ -0,0 +1,18 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: nginx-ingress + annotations: + ingress.kubernetes.io/rewrite-target: / +spec: + rules: + - http: + paths: + - path: / + backend: + serviceName: shop-webapp + servicePort: 9011 + - path: /api + backend: + serviceName: shop-api + servicePort: 8080 From 601b8080dc7f4b95c39ac16f1fd6bf313029d92a Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Wed, 25 Sep 2019 14:52:11 +0200 Subject: [PATCH 12/32] Change path to webshop to be index --- k8s/ingress.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/ingress.yaml b/k8s/ingress.yaml index b6482d9..82684be 100644 --- a/k8s/ingress.yaml +++ b/k8s/ingress.yaml @@ -8,7 +8,7 @@ spec: rules: - http: paths: - - path: / + - path: /shop backend: serviceName: shop-webapp servicePort: 9011 From 966daac7b71d7594070113d738028a829b6d8ec6 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Wed, 25 Sep 2019 14:55:28 +0200 Subject: [PATCH 13/32] Add set-up for Ingress nginx to README --- README.md | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 827638a..fb5adf0 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,11 @@ This application comprises a number of different processes: ## Prerequisite(s) The following package(s) are required; - - `sbt`. -Install them with your package manager: + - `sbt`; + - and `ingress-nginx`. + +Install with your package manager: ``` brew update @@ -58,6 +60,32 @@ apt update apt install sbt ``` +### Ingress Nginx + +Our setup uses an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) with Nginx to define reroutes to the webshop and the +API (aka service) used by the webshop. The following steps +are required: + +For all deployments: + +``` +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/mandatory.yaml +``` + +Provider specific: + +**For Docker for mac** +``` +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/cloud-generic.yaml +``` + +**For minikube** +``` +minikube addons enable ingress +``` + +For more info, see [here](https://github.com/kubernetes/ingress-nginx/blob/master/docs/deploy/index.md#installation-guide). + ## Running with Docker The easiest way to get started is with Docker Compose. From cc058ec8de2a5f692ff5e7296e70c970f3341c79 Mon Sep 17 00:00:00 2001 From: Bas van de Lustgraaf Date: Wed, 25 Sep 2019 15:15:34 +0200 Subject: [PATCH 14/32] Add initContainers to delay the start of the put-categories pods. --- k8s/put-categories-deployment.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/k8s/put-categories-deployment.yaml b/k8s/put-categories-deployment.yaml index 4a20639..d704e24 100644 --- a/k8s/put-categories-deployment.yaml +++ b/k8s/put-categories-deployment.yaml @@ -20,4 +20,8 @@ spec: hostPath: path: /Users/bas/code/gdd/shop type: Directory + initContainers: + - name: wait-for-shop-api + image: busybox:1.28 + command: ['sh', '-c', 'sleep 30'] backoffLimit: 4 \ No newline at end of file From 49d91b9691349f6951e2dbdb26c836034765ef99 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Wed, 25 Sep 2019 15:18:22 +0200 Subject: [PATCH 15/32] Bug fix: change index path was not committed correctly --- k8s/ingress.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/ingress.yaml b/k8s/ingress.yaml index 82684be..b6482d9 100644 --- a/k8s/ingress.yaml +++ b/k8s/ingress.yaml @@ -8,7 +8,7 @@ spec: rules: - http: paths: - - path: /shop + - path: / backend: serviceName: shop-webapp servicePort: 9011 From 32dc2cc5b972c9541caed64d881c18e381fb6ff8 Mon Sep 17 00:00:00 2001 From: Anibal Kolker Date: Wed, 25 Sep 2019 15:33:09 +0200 Subject: [PATCH 16/32] add Liveness probe for API with own route --- k8s/shop-api-deployment.yaml | 6 ++++ .../src/main/java/io/divolte/shop/Main.java | 7 ++--- .../shop/catalog/APIStatusResource.java | 31 +++++++++++++++++++ 3 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 service/src/main/java/io/divolte/shop/catalog/APIStatusResource.java diff --git a/k8s/shop-api-deployment.yaml b/k8s/shop-api-deployment.yaml index fa2be92..4d9acfc 100644 --- a/k8s/shop-api-deployment.yaml +++ b/k8s/shop-api-deployment.yaml @@ -28,6 +28,12 @@ spec: ports: - containerPort: 8080 - containerPort: 8081 + livenessProbe: + httpGet: + path: /api/status/ping + port: 8080 + initialDelaySeconds: 3 + periodSeconds: 3 hostname: shop-api restartPolicy: Always subdomain: divolte-divolte diff --git a/service/src/main/java/io/divolte/shop/Main.java b/service/src/main/java/io/divolte/shop/Main.java index 0896ca5..5291b29 100644 --- a/service/src/main/java/io/divolte/shop/Main.java +++ b/service/src/main/java/io/divolte/shop/Main.java @@ -1,10 +1,6 @@ package io.divolte.shop; -import io.divolte.shop.catalog.BasketResource; -import io.divolte.shop.catalog.CatalogCategoryResource; -import io.divolte.shop.catalog.DataAccess; -import io.divolte.shop.catalog.CatalogItemResource; -import io.divolte.shop.catalog.CheckoutResource; +import io.divolte.shop.catalog.*; import io.dropwizard.Application; import io.dropwizard.setup.Bootstrap; import io.dropwizard.setup.Environment; @@ -44,6 +40,7 @@ public void run(final ServiceConfiguration configuration, final Environment envi environment.jersey().register(new CatalogCategoryResource(client)); environment.jersey().register(new BasketResource(client)); environment.jersey().register(new CheckoutResource()); + environment.jersey().register(new APIStatusResource()); environment.healthChecks().register("ElasticSearch", new ElasticSearchHealthCheck(client)); } diff --git a/service/src/main/java/io/divolte/shop/catalog/APIStatusResource.java b/service/src/main/java/io/divolte/shop/catalog/APIStatusResource.java new file mode 100644 index 0000000..6a41d9a --- /dev/null +++ b/service/src/main/java/io/divolte/shop/catalog/APIStatusResource.java @@ -0,0 +1,31 @@ +package io.divolte.shop.catalog; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; + + +@Path("/api/status") +public class APIStatusResource { + + public APIStatusResource() { + + } + + /** + Purpose is to determine whether the api is still alive and running + */ + @Path("/ping") + @GET + @Produces(MediaType.TEXT_PLAIN) + public Response getStatus() { + Response.ResponseBuilder builder = Response.status(Status.ACCEPTED); + + return builder.entity("pong").build(); + + } + +} From 8cdee5db48a17be372a585c7f83cf9ebda34637f Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Thu, 26 Sep 2019 13:38:09 +0200 Subject: [PATCH 17/32] Change init container to check if shop-api service is alive --- k8s/put-categories-deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/put-categories-deployment.yaml b/k8s/put-categories-deployment.yaml index d704e24..3478ef0 100644 --- a/k8s/put-categories-deployment.yaml +++ b/k8s/put-categories-deployment.yaml @@ -21,7 +21,7 @@ spec: path: /Users/bas/code/gdd/shop type: Directory initContainers: - - name: wait-for-shop-api + - name: wait-for-shop-api-service image: busybox:1.28 - command: ['sh', '-c', 'sleep 30'] - backoffLimit: 4 \ No newline at end of file + command: ['sh', '-c', 'until nslookup shop-api; do echo waiting for shop api; sleep 10; done;'] + backoffLimit: 4 From dc839471010c8b971c02f85c26f7dea491a4f525 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Thu, 26 Sep 2019 13:38:29 +0200 Subject: [PATCH 18/32] Add init containers which checks if API is alive --- k8s/put-categories-deployment.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/k8s/put-categories-deployment.yaml b/k8s/put-categories-deployment.yaml index 3478ef0..63becd7 100644 --- a/k8s/put-categories-deployment.yaml +++ b/k8s/put-categories-deployment.yaml @@ -24,4 +24,7 @@ spec: - name: wait-for-shop-api-service image: busybox:1.28 command: ['sh', '-c', 'until nslookup shop-api; do echo waiting for shop api; sleep 10; done;'] + - name: wait-for-shop-api-ping + image: busybox:1.28 + command: ['sh', '-c', 'until wget -q --spider "http://shop-api:8080/api/status/ping"; do echo waiting for API ping; sleep 10; done;'] backoffLimit: 4 From 9f0c10654fa780c1e3cc1a094d18c7b18c189206 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Thu, 26 Sep 2019 13:54:56 +0200 Subject: [PATCH 19/32] Solve flake8 issues --- catalog-builder/put-categories.py | 44 ++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/catalog-builder/put-categories.py b/catalog-builder/put-categories.py index e76b008..e39931e 100644 --- a/catalog-builder/put-categories.py +++ b/catalog-builder/put-categories.py @@ -27,10 +27,11 @@ def make_item(json, filename): 'flickr_url': size['url'], 'img_source': size['source'] } - for name,size in json['sizes'].items() + for name, size in json['sizes'].items() } } + def merge_items(*input): items = list(filter(lambda x: x, input)) @@ -43,10 +44,12 @@ def merge_items(*input): result = {} result.update(items[0]) for item in items[1:]: - result['categories'] = list(set(result['categories'] + item['categories'])) + result['categories'] = list( + set(result['categories'] + item['categories'])) return result + def with_price(item, min_favs, max_favs): r = (max_favs - min_favs + 1) / 2.0 result = { @@ -56,19 +59,19 @@ def with_price(item, min_favs, max_favs): return result + def main(args): print('Items from:\n\t%s' % '\n\t'.join(args.files)) item_list = reduce( - lambda x,y: x+y, - [[ - make_item(json.loads(line), fn) - for line in open(fn).readlines()] - for fn in args.files]) + lambda x, y: x+y, + [[make_item(json.loads(line), fn) for line in open(fn).readlines()] + for fn in args.files]) min_favs = min([item['favs'] for item in item_list]) max_favs = max([item['favs'] for item in item_list]) - priced_item_list = [with_price(item, min_favs, max_favs) for item in item_list] + priced_item_list = [ + with_price(item, min_favs, max_favs) for item in item_list] items = {} for item in priced_item_list: @@ -76,7 +79,10 @@ def main(args): responses = Counter() for item in items.values(): - r = requests.put(args.api_base_url + '/catalog/item', data=json.dumps(item), headers={'Content-Type': 'application/json'}) + r = requests.put( + args.api_base_url + '/catalog/item', + data=json.dumps(item), + headers={'Content-Type': 'application/json'}) print('PUT %s, %d' % (item['id'], r.status_code)) if r.status_code != 200: print(json.dumps(item)) @@ -84,15 +90,29 @@ def main(args): print('\nSummary:') for status, count in responses.items(): - print('\t%d times status code %d' % (count,status)) + print('\t%d times status code %d' % (count, status)) + def parse_args(): - parser = argparse.ArgumentParser(description='Take downloaded category JSON files and push them into the Catalog API.') - parser.add_argument('--api-base-url', '-u', metavar='API_BASE_URL', type=str, default='http://localhost:8080/api', help='The base URL of the Catalog API.') + parser = argparse.ArgumentParser( + description=( + 'Take downloaded category JSON files and push them into the ' + 'Catalog API.' + ) + ) + parser.add_argument( + '--api-base-url', + '-u', + metavar='API_BASE_URL', + type=str, + default='http://localhost:8080/api', + help='The base URL of the Catalog API.' + ) parser.add_argument('files', metavar='JSON_FILE', type=str, nargs='+') return parser.parse_args() + if __name__ == '__main__': args = parse_args() main(args) From 76fed7bbf58cffa5f26b29ce36cad72ddec19910 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Thu, 26 Sep 2019 13:55:30 +0200 Subject: [PATCH 20/32] Change input order according to PEP8 --- catalog-builder/put-categories.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalog-builder/put-categories.py b/catalog-builder/put-categories.py index e39931e..c911771 100644 --- a/catalog-builder/put-categories.py +++ b/catalog-builder/put-categories.py @@ -1,11 +1,11 @@ import argparse -import requests import json import os - from functools import reduce from collections import Counter +import requests + def make_item(json, filename): return { From 4393bf435c22ce10d929f71a6da67c1cbf06f700 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Thu, 26 Sep 2019 13:59:12 +0200 Subject: [PATCH 21/32] Put json requests in separate function --- catalog-builder/put-categories.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/catalog-builder/put-categories.py b/catalog-builder/put-categories.py index c911771..bbbdf0a 100644 --- a/catalog-builder/put-categories.py +++ b/catalog-builder/put-categories.py @@ -60,6 +60,11 @@ def with_price(item, min_favs, max_favs): return result +def _requests_put_json(url, data): + headers = {'Content-Type': 'application/json'} + return requests.put(url, data=data, headers=headers) + + def main(args): print('Items from:\n\t%s' % '\n\t'.join(args.files)) @@ -79,10 +84,10 @@ def main(args): responses = Counter() for item in items.values(): - r = requests.put( + r = _requests_put_json( args.api_base_url + '/catalog/item', - data=json.dumps(item), - headers={'Content-Type': 'application/json'}) + data=json.dump(item) + ) print('PUT %s, %d' % (item['id'], r.status_code)) if r.status_code != 200: print(json.dumps(item)) From 292fe7d0d86907fcccb5424228b744f8280776c2 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Thu, 26 Sep 2019 14:05:59 +0200 Subject: [PATCH 22/32] Retry json put if 10 times ConnectionError Sometimes a connection fails once. Then uploading the data will fail. Add a retry so that the uploading continues if simply a request failed. --- catalog-builder/put-categories.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/catalog-builder/put-categories.py b/catalog-builder/put-categories.py index bbbdf0a..e4321c4 100644 --- a/catalog-builder/put-categories.py +++ b/catalog-builder/put-categories.py @@ -5,6 +5,8 @@ from collections import Counter import requests +from requests.exceptions import ConnectionError +from retrying import retry def make_item(json, filename): @@ -60,6 +62,9 @@ def with_price(item, min_favs, max_favs): return result +@retry(stop_max_attempt_number=10, + wait_fixed=1000, # wait 1 sec + retry_on_exception=lambda e: isinstance(e, ConnectionError)) def _requests_put_json(url, data): headers = {'Content-Type': 'application/json'} return requests.put(url, data=data, headers=headers) From 2a5d2f01b0132a555ed4bfaea4ad1e9a01a08790 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Thu, 26 Sep 2019 14:13:24 +0200 Subject: [PATCH 23/32] Bug fix: json.dump --> json.dumps --- catalog-builder/put-categories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog-builder/put-categories.py b/catalog-builder/put-categories.py index e4321c4..8ace401 100644 --- a/catalog-builder/put-categories.py +++ b/catalog-builder/put-categories.py @@ -91,7 +91,7 @@ def main(args): for item in items.values(): r = _requests_put_json( args.api_base_url + '/catalog/item', - data=json.dump(item) + data=json.dumps(item) ) print('PUT %s, %d' % (item['id'], r.status_code)) if r.status_code != 200: From 34385369fd0d0b3c8305983cc78386edc64ac297 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Thu, 26 Sep 2019 14:14:04 +0200 Subject: [PATCH 24/32] Add retrying to be pip installed --- README.md | 2 +- k8s/put-categories-deployment.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fb5adf0..19b688a 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ docker run -it --rm --volume $PWD:/divolte-shop \ --workdir /divolte-shop \ --network host \ python:3.6 \ - bash -c 'pip install requests && python catalog-builder/put-categories.py \ + bash -c 'pip install requests retrying && python catalog-builder/put-categories.py \ data/categories/animals.json \ data/categories/architecture.json \ data/categories/cars.json \ diff --git a/k8s/put-categories-deployment.yaml b/k8s/put-categories-deployment.yaml index 63becd7..95bdc27 100644 --- a/k8s/put-categories-deployment.yaml +++ b/k8s/put-categories-deployment.yaml @@ -13,12 +13,12 @@ spec: - mountPath: /shop name: divolte-shop command: ["bash"] - args: ["-c", "pip install requests && python catalog-builder/put-categories.py --api-base-url http://shop-api:8080/api data/categories/animals.json data/categories/architecture.json data/categories/cars.json data/categories/cities.json data/categories/flowers.json data/categories/landscape.json data/categories/nautical.json"] + args: ["-c", "pip install requests retrying && python catalog-builder/put-categories.py --api-base-url http://shop-api:8080/api data/categories/animals.json data/categories/architecture.json data/categories/cars.json data/categories/cities.json data/categories/flowers.json data/categories/landscape.json data/categories/nautical.json"] restartPolicy: Never volumes: - name: divolte-shop hostPath: - path: /Users/bas/code/gdd/shop + path: /Users/cor/github/divolte/shop type: Directory initContainers: - name: wait-for-shop-api-service From 3e01af4cdb774a932f4a05fee07aaae94a7fceb7 Mon Sep 17 00:00:00 2001 From: Alexander Bij Date: Thu, 26 Sep 2019 14:51:57 +0200 Subject: [PATCH 25/32] Add utility script to build the jars, without the need of sbt/gradle. --- README.md | 27 ++++++++++++++++++++++----- build-jars.sh | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 5 deletions(-) create mode 100755 build-jars.sh diff --git a/README.md b/README.md index 19b688a..48306d3 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ This application comprises a number of different processes: - a **kafka** topic - A **top-pick-service** that receives data from Kafka and continuously selects popular products, using... - a **redis** store to tally all the clicks +- A **spark** streaming consumer of the events to calculate metrics like: nr of events per type per 2 minutes ```text + @@ -92,7 +93,7 @@ The easiest way to get started is with Docker Compose. Make sure you have Docker running locally. You can download a proper version at the [Docker Store][ds]. -We are running 6 containers, and the default size is not large enough. Boost the ram of Docker to at least 4GB, +We are running a couple of containers, and the default size is not large enough. Boost the ram of Docker to at least 4GB, otherwise you will run into startup problems (Exit with code: 137) when running the `docker-compose up` command. We are going to build these containers locally: @@ -109,6 +110,20 @@ We will use these public containers: [ds]:https://store.docker.com/ +### Building Project Jars (required) + +Build **jars** from the source-code, before you can wrap them into docker containers for: +- service +- spark-container + +```bash +# You can use the utility script, build inside a dockers containers: +bash build-jars.sh + +# Or setup the tools (sbt) yourself and run: +service/gradlew -p service build +cd spark-container/streaming && sbt assembly +``` ### Running with docker compose @@ -118,12 +133,14 @@ through [localhost:9011](http://localhost:9011/). > These ports should be available: 9011, 8080, 8081, 9200, 9300, 8290, 9092, 2181, 6379, 8989 ```bash -service/gradlew -p service build && (cd spark-container/streaming && sbt assembly) && docker-compose up -d --build +# note: make sure the jar's have been build + +docker-compose up -d --build ``` -#### Download new products +#### Download new products (optional) -Optionally you can download new image-data from flickr. +You can download new image-data from flickr. These are stored in `data/categories` folder. > Note: you need to fill in your own Flickr Api key, and wait a very long time...! @@ -157,6 +174,6 @@ docker run -it --rm --volume $PWD:/divolte-shop \ data/categories/cities.json \ data/categories/flowers.json \ data/categories/landscape.json \ - data/categories/nautical.json' + data/categories/nautical.json' ``` Go to [localhost:9011](http://localhost:9011/). diff --git a/build-jars.sh b/build-jars.sh new file mode 100755 index 0000000..7c4170d --- /dev/null +++ b/build-jars.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Utility script to build the project code into jars using public containers +# instead of installing tools locally like SBT, gradle. +# +# Just requires docker. + +set -e # exit on error +set -u # Need to bind specified variables + + +# Create local folders for java dependecies (jars) +mkdir -p ~/.ivy2 && mkdir -p ~/.m2 && mkdir -p ~/.sbt + +echo '[1/2] Build "spark-streaming" fat jar using SBT ...' +docker run --rm \ + -v ~/.m2:/root/.m2 \ + -v ~/.ivy2:/root/.ivy2 \ + -v ~/.sbt:/root/.sbt \ + -v "${PWD}/spark-container/streaming":/app \ + -w /app \ + mozilla/sbt sbt assembly + +echo '[2/2] Build "service" fat jar using Gradle...' +docker run --rm \ + -u gradle \ + -v ~/.m2:/root/.m2 \ + -v ~/.ivy2:/root/.ivy2 \ + -v "${PWD}/service":/home/gradle/project \ + -w /home/gradle/project \ + gradle:4.6.0-jdk8-alpine gradle build + +echo '' +echo 'Done building the jars for the projects' From ffa2b942eb9a67dd333a6cf222881ec353915d16 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Thu, 26 Sep 2019 14:56:24 +0200 Subject: [PATCH 26/32] Add comment explaining ttlSecondsAfterFinished --- k8s/put-categories-deployment.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/k8s/put-categories-deployment.yaml b/k8s/put-categories-deployment.yaml index 95bdc27..f0b6a5f 100644 --- a/k8s/put-categories-deployment.yaml +++ b/k8s/put-categories-deployment.yaml @@ -5,6 +5,9 @@ metadata: spec: template: spec: + # Cleans job after it has completed only works when + # '--feture-gate="TTLAfterFinished=True"' is set in kubernetes config + ttlSecondsAfterFinished: 100 containers: - name: put-categories image: python:3.6 From be74dc2502a7e3fc545779e865697e7707e570ce Mon Sep 17 00:00:00 2001 From: Anibal Kolker Date: Thu, 26 Sep 2019 15:28:32 +0200 Subject: [PATCH 27/32] Make catalog-builder an image to load data into the shop --- catalog-builder/Dockerfile | 22 ++++++++++++++++++ docker-compose.yml | 8 +++++++ k8s/put-categories-deployment.yaml | 37 ++++++++++++++++++------------ 3 files changed, 52 insertions(+), 15 deletions(-) create mode 100644 catalog-builder/Dockerfile diff --git a/catalog-builder/Dockerfile b/catalog-builder/Dockerfile new file mode 100644 index 0000000..0f81849 --- /dev/null +++ b/catalog-builder/Dockerfile @@ -0,0 +1,22 @@ +# Data population image for initial data +# +# +FROM python:3.6 + +RUN mkdir -p /usr/local/shop/data +WORKDIR /usr/local/shop + +COPY catalog-builder/ ./catalog-builder/ +COPY data/categories/ ./data/categories/ + + +RUN pip install requests +CMD ["sh", "-c", "python catalog-builder/put-categories.py --api-base-url=$SHOP_API_URL \ + data/categories/animals.json \ + data/categories/architecture.json \ + data/categories/cars.json \ + data/categories/cities.json \ + data/categories/flowers.json \ + data/categories/landscape.json \ + data/categories/nautical.json" \ +] diff --git a/docker-compose.yml b/docker-compose.yml index 195c544..53d4eb3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -194,6 +194,14 @@ services: - kafka:kafka networks: - divolte.io + + catalog-builder: + build: + context: . + dockerfile: ./catalog-builder/Dockerfile + container_name: catalog-builder + + # divolte-druid: # # image: fokkodriesprong/docker-druid # image: shop/divolte-druid diff --git a/k8s/put-categories-deployment.yaml b/k8s/put-categories-deployment.yaml index d704e24..0e5058b 100644 --- a/k8s/put-categories-deployment.yaml +++ b/k8s/put-categories-deployment.yaml @@ -6,22 +6,29 @@ spec: template: spec: containers: - - name: put-categories - image: python:3.6 - workingDir: /shop - volumeMounts: - - mountPath: /shop - name: divolte-shop - command: ["bash"] - args: ["-c", "pip install requests && python catalog-builder/put-categories.py --api-base-url http://shop-api:8080/api data/categories/animals.json data/categories/architecture.json data/categories/cars.json data/categories/cities.json data/categories/flowers.json data/categories/landscape.json data/categories/nautical.json"] + - env: + - name: SHOP_API_URL + value: "http://shop-api:8080/api" + + name: put-categories + image: shop_catalog-builder:latest + imagePullPolicy: Never + #workingDir: /shop + #volumeMounts: + # - mountPath: /shop + # name: divolte-shop + # command: ["bash"] + # args: ["-c", "pip install requests && python catalog-builder/put-categories.py --api-base-url http://shop-api:8080/api data/categories/animals.json data/categories/architecture.json data/categories/cars.json data/categories/cities.json data/categories/flowers.json data/categories/landscape.json data/categories/nautical.json"] + restartPolicy: Never - volumes: - - name: divolte-shop - hostPath: - path: /Users/bas/code/gdd/shop - type: Directory + + # volumes: + # - name: divolte-shop + # hostPath: + # path: /Users/bas/code/gdd/shop + # type: Directory initContainers: - name: wait-for-shop-api image: busybox:1.28 - command: ['sh', '-c', 'sleep 30'] - backoffLimit: 4 \ No newline at end of file + command: ['sh', '-c', 'sleep 300'] + backoffLimit: 4 From b8ff7a17896571e34587cfa07e33deb7a7d7ac90 Mon Sep 17 00:00:00 2001 From: Anibal Kolker Date: Thu, 26 Sep 2019 15:31:37 +0200 Subject: [PATCH 28/32] Clean up deployment file --- k8s/put-categories-deployment.yaml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/k8s/put-categories-deployment.yaml b/k8s/put-categories-deployment.yaml index 5a7e609..63bbc88 100644 --- a/k8s/put-categories-deployment.yaml +++ b/k8s/put-categories-deployment.yaml @@ -9,24 +9,10 @@ spec: - env: - name: SHOP_API_URL value: "http://shop-api:8080/api" - name: put-categories image: shop_catalog-builder:latest imagePullPolicy: Never - #workingDir: /shop - #volumeMounts: - # - mountPath: /shop - # name: divolte-shop - # command: ["bash"] - # args: ["-c", "pip install requests && python catalog-builder/put-categories.py --api-base-url http://shop-api:8080/api data/categories/animals.json data/categories/architecture.json data/categories/cars.json data/categories/cities.json data/categories/flowers.json data/categories/landscape.json data/categories/nautical.json"] - restartPolicy: Never - - # volumes: - # - name: divolte-shop - # hostPath: - # path: /Users/bas/code/gdd/shop - # type: Directory initContainers: - name: wait-for-shop-api-service image: busybox:1.28 From b4831d5b3f77440360194e6d4e1f0eb272447ba8 Mon Sep 17 00:00:00 2001 From: Alexander Bij Date: Thu, 26 Sep 2019 15:50:37 +0200 Subject: [PATCH 29/32] Make sure the docker-compose also works. --- docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 53d4eb3..1604fe4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -200,6 +200,10 @@ services: context: . dockerfile: ./catalog-builder/Dockerfile container_name: catalog-builder + environment: + SHOP_API_URL: "http://shop-api:8080/api" + depends_on: + - shop-api # divolte-druid: From 30814780b053b9baa52d058f27611ede22b1622c Mon Sep 17 00:00:00 2001 From: Alexander Bij Date: Thu, 26 Sep 2019 15:52:16 +0200 Subject: [PATCH 30/32] Fix typo --- k8s/put-categories-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/put-categories-deployment.yaml b/k8s/put-categories-deployment.yaml index 4cca240..26a9d89 100644 --- a/k8s/put-categories-deployment.yaml +++ b/k8s/put-categories-deployment.yaml @@ -6,7 +6,7 @@ spec: template: spec: # Cleans job after it has completed only works when - # '--feture-gate="TTLAfterFinished=True"' is set in kubernetes config + # '--feature-gate="TTLAfterFinished=True"' is set in kubernetes config ttlSecondsAfterFinished: 100 containers: - env: From 89277735bdc713c638b81dacd9632734f88be1a5 Mon Sep 17 00:00:00 2001 From: Alexander Bij Date: Thu, 26 Sep 2019 16:04:04 +0200 Subject: [PATCH 31/32] Uniform the docker context of catalog builder. --- README.md | 37 +++++++++---------- catalog-builder/Dockerfile | 24 +++++------- .../categories/animals.json | 0 .../categories/architecture.json | 0 .../categories/cars.json | 0 .../categories/cities.json | 0 .../categories/flowers.json | 0 .../categories/landscape.json | 0 .../categories/nautical.json | 0 catalog-builder/download-category.py | 4 +- catalog-builder/requirements.txt | 1 + docker-compose.yml | 3 +- install_without_docker.md | 16 ++++---- 13 files changed, 41 insertions(+), 44 deletions(-) rename {data => catalog-builder}/categories/animals.json (100%) rename {data => catalog-builder}/categories/architecture.json (100%) rename {data => catalog-builder}/categories/cars.json (100%) rename {data => catalog-builder}/categories/cities.json (100%) rename {data => catalog-builder}/categories/flowers.json (100%) rename {data => catalog-builder}/categories/landscape.json (100%) rename {data => catalog-builder}/categories/nautical.json (100%) diff --git a/README.md b/README.md index 48306d3..5b9ecdd 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ service/gradlew -p service build cd spark-container/streaming && sbt assembly ``` -### Running with docker compose +### (Option 1) Running with docker compose When you have the containers up and running you can access the webshop through [localhost:9011](http://localhost:9011/). @@ -138,9 +138,25 @@ through [localhost:9011](http://localhost:9011/). docker-compose up -d --build ``` +### (Option 2) Running with Kubernetes + +```bash +# note: make sure the jar's have been build + +# Create docker images +docker-compose build + +# Register all services +kubectl apply -f k8s +``` + +### Initial Data + +> Note: Data will be loaded automatically, after all services are started. + #### Download new products (optional) -You can download new image-data from flickr. These are stored in `data/categories` folder. +You can download new image-data from flickr. These are stored in `catalog-builder/categories` folder. > Note: you need to fill in your own Flickr Api key, and wait a very long time...! @@ -159,21 +175,4 @@ docker run --rm -it \ --key ${FLICKR_API_KEY}" ``` -#### Loading products -The first time you start the docker composition, you have to load the product catalog, like this: - -```text -docker run -it --rm --volume $PWD:/divolte-shop \ - --workdir /divolte-shop \ - --network host \ - python:3.6 \ - bash -c 'pip install requests retrying && python catalog-builder/put-categories.py \ - data/categories/animals.json \ - data/categories/architecture.json \ - data/categories/cars.json \ - data/categories/cities.json \ - data/categories/flowers.json \ - data/categories/landscape.json \ - data/categories/nautical.json' -``` Go to [localhost:9011](http://localhost:9011/). diff --git a/catalog-builder/Dockerfile b/catalog-builder/Dockerfile index 0f81849..c555497 100644 --- a/catalog-builder/Dockerfile +++ b/catalog-builder/Dockerfile @@ -3,20 +3,16 @@ # FROM python:3.6 -RUN mkdir -p /usr/local/shop/data -WORKDIR /usr/local/shop +# Copy all stuff from this dir into the default working directory +COPY . . -COPY catalog-builder/ ./catalog-builder/ -COPY data/categories/ ./data/categories/ - - -RUN pip install requests +RUN pip install -r requirements.txt CMD ["sh", "-c", "python catalog-builder/put-categories.py --api-base-url=$SHOP_API_URL \ - data/categories/animals.json \ - data/categories/architecture.json \ - data/categories/cars.json \ - data/categories/cities.json \ - data/categories/flowers.json \ - data/categories/landscape.json \ - data/categories/nautical.json" \ + categories/animals.json \ + categories/architecture.json \ + categories/cars.json \ + categories/cities.json \ + categories/flowers.json \ + categories/landscape.json \ + categories/nautical.json" \ ] diff --git a/data/categories/animals.json b/catalog-builder/categories/animals.json similarity index 100% rename from data/categories/animals.json rename to catalog-builder/categories/animals.json diff --git a/data/categories/architecture.json b/catalog-builder/categories/architecture.json similarity index 100% rename from data/categories/architecture.json rename to catalog-builder/categories/architecture.json diff --git a/data/categories/cars.json b/catalog-builder/categories/cars.json similarity index 100% rename from data/categories/cars.json rename to catalog-builder/categories/cars.json diff --git a/data/categories/cities.json b/catalog-builder/categories/cities.json similarity index 100% rename from data/categories/cities.json rename to catalog-builder/categories/cities.json diff --git a/data/categories/flowers.json b/catalog-builder/categories/flowers.json similarity index 100% rename from data/categories/flowers.json rename to catalog-builder/categories/flowers.json diff --git a/data/categories/landscape.json b/catalog-builder/categories/landscape.json similarity index 100% rename from data/categories/landscape.json rename to catalog-builder/categories/landscape.json diff --git a/data/categories/nautical.json b/catalog-builder/categories/nautical.json similarity index 100% rename from data/categories/nautical.json rename to catalog-builder/categories/nautical.json diff --git a/catalog-builder/download-category.py b/catalog-builder/download-category.py index c9c8234..9d22a2e 100644 --- a/catalog-builder/download-category.py +++ b/catalog-builder/download-category.py @@ -147,7 +147,9 @@ def parse_args(): parser = argparse.ArgumentParser( description='Create a JSON description of a set of images retrieved from the Flickr API based on keyword searches.') parser.add_argument('--key', '-k', metavar='API_KEY', type=str, required=True, help='The Flickr API key to use.') - parser.add_argument('--dest', '-d', metavar='DEST', type=str, help='The output directory to store the jsons', default="data/categories") + parser.add_argument('--dest', '-d', metavar='DEST', type=str, help='The output directory to ' + 'store the jsons', + default="catalog-builder/categories") parser.add_argument('--searches', '-s', metavar='SEARCH_KEYWORD', type=str, required=True, default='categories.yml', help='Yaml file containing the categories and keywords to search for.') parser.add_argument('--fav-threshold', '-t', metavar='FAVOURITES_THRESHOLD', type=int, default=30, diff --git a/catalog-builder/requirements.txt b/catalog-builder/requirements.txt index c2e8e98..5c4bb7d 100644 --- a/catalog-builder/requirements.txt +++ b/catalog-builder/requirements.txt @@ -1,3 +1,4 @@ requests xmltodict pyyaml +retrying diff --git a/docker-compose.yml b/docker-compose.yml index 1604fe4..90403b1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -197,8 +197,7 @@ services: catalog-builder: build: - context: . - dockerfile: ./catalog-builder/Dockerfile + context: ./catalog-builder container_name: catalog-builder environment: SHOP_API_URL: "http://shop-api:8080/api" diff --git a/install_without_docker.md b/install_without_docker.md index 04ac0e3..78c24b8 100644 --- a/install_without_docker.md +++ b/install_without_docker.md @@ -56,13 +56,13 @@ To load the data from the data/catalog directory into elasticsearch the `catalog ``` python put-catagories.py \ - ../data/categories/animals.json \ - ../data/categories/architecture.json \ - ../data/categories/cars.json \ - ../data/categories/cities.json \ - ../data/categories/flowers.json \ - ../data/categories/landscape.json \ - ../data/categories/nautical.json + categories/animals.json \ + categories/architecture.json \ + categories/cars.json \ + categories/cities.json \ + categories/flowers.json \ + categories/landscape.json \ + categories/nautical.json ``` ## Running the webapp @@ -89,4 +89,4 @@ python consumer.py --schema ../divolte/schema/src/main/resources/ShopEventRecord --client toppick-client --group toppick-group python bandit_service.py -``` \ No newline at end of file +``` From 97224107083edfcca0ad0718f95eda45d2e32f62 Mon Sep 17 00:00:00 2001 From: Alexander Bij Date: Wed, 29 Jan 2020 09:11:18 +0100 Subject: [PATCH 32/32] Small readme update --- README.md | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 5b9ecdd..21376c7 100644 --- a/README.md +++ b/README.md @@ -42,26 +42,7 @@ This application comprises a number of different processes: + ``` -## Prerequisite(s) - -The following package(s) are required; - - - `sbt`; - - and `ingress-nginx`. - -Install with your package manager: - -``` -brew update -brew install sbt -``` - -``` -apt update -apt install sbt -``` - -### Ingress Nginx +## Kubernetes: Ingress Nginx Our setup uses an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) with Nginx to define reroutes to the webshop and the API (aka service) used by the webshop. The following steps @@ -87,7 +68,7 @@ minikube addons enable ingress For more info, see [here](https://github.com/kubernetes/ingress-nginx/blob/master/docs/deploy/index.md#installation-guide). -## Running with Docker +## Using docker containers The easiest way to get started is with Docker Compose. @@ -102,7 +83,6 @@ We are going to build these containers locally: - shop/docker-shop-service - shop/docker-shop-webapp -We will use these public containers: - redis - docker.elastic.co/elasticsearch/elasticsearch