e2e: add separate docker-compose (#419)

* Renamed `docker/e2e.yml` to `docker/e2e-old.yml` * Created `docker/e2e.yml` which resembles `local-dev-dual-comparison.yml`, but has only `e2e` tables/Kibana data views.
QuesmaOrg · Jul 2, 2024 · 869e965 · 869e965
1 parent bdca706
commit 869e965
Show file tree

Hide file tree

Showing 6 changed files with 270 additions and 20 deletions.
diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
@@ -75,7 +75,7 @@ jobs:
 
       - name: Build and start docker-compose
         timeout-minutes: 10
-        run: docker-compose -f docker/e2e.yml up -d
+        run: docker-compose -f docker/e2e-old.yml up -d
 
       - name: Wait until Elasticsearch is up
         timeout-minutes: 6

diff --git a/docker/e2e-old.yml b/docker/e2e-old.yml
@@ -0,0 +1,56 @@
+version: "3.7"
+services:
+  quesma:
+    build: ../quesma
+    image: quesma:latest
+    environment:
+      - QUESMA_CONFIG_FILE=/mnt/ci-config.yaml
+      - QUESMA_elasticsearch_url=http://elasticsearch:9200
+      - QUESMA_port=8080
+      - QUESMA_logging_path=/var/quesma/logs
+      - QUESMA_clickhouse_url=clickhouse://clickhouse:9000
+      - QUESMA_logging_fileLogging=false
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_healthy
+    ports:
+      - "9999:9999"
+      - "8080:8080"
+      - "9200:8080"
+    volumes:
+      - ./quesma/config/ci-config.yaml:/mnt/ci-config.yaml
+    restart: unless-stopped
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.1
+    container_name: elasticsearch
+    environment:
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+      - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
+    ports:
+      - "9201:9200"
+      - "9300:9300"
+    healthcheck:
+      test: curl -s http://elasticsearch:9200 >/dev/null || exit 1
+      start_period: 1m
+      interval: 1s
+      timeout: 1s
+    deploy:
+      resources:
+        limits:
+          memory: 4G
+  clickhouse:
+    image: clickhouse/clickhouse-server:23.12.2.59-alpine
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    volumes:
+      - ./clickhouse/additional_config.xml:/etc/clickhouse-server/users.d/additional_config.xml
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://clickhouse:8123/ping || exit 1
+      interval: 1s
+      timeout: 1s
+      start_period: 1m
+
diff --git a/docker/e2e.yml b/docker/e2e.yml
@@ -1,56 +1,204 @@
-version: "3.7"
 services:
   quesma:
     build: ../quesma
     image: quesma:latest
+    env_file:
+      # We require .env file to be present
+      # and contain QUESMA_licenseKey with your personal license key
+      - .env
     environment:
-      - QUESMA_CONFIG_FILE=/mnt/ci-config.yaml
-      - QUESMA_elasticsearch_url=http://elasticsearch:9200
+      - QUESMA_elasticsearch_url=http://elasticsearch-quesma:9200
       - QUESMA_port=8080
       - QUESMA_logging_path=/var/quesma/logs
-      - QUESMA_clickhouse_url=clickhouse://clickhouse:9000
-      - QUESMA_logging_fileLogging=false
+      - QUESMA_mode=dual-write-query-clickhouse
+      - QUESMA_CONFIG_FILE=/config/e2e.yaml
+      - QUESMA_logging_fileLogging=true
     depends_on:
-      clickhouse:
-        condition: service_healthy
-      elasticsearch:
+      clean-clickhouse:
+        condition: service_completed_successfully
+      elasticsearch-quesma:
         condition: service_healthy
     ports:
       - "9999:9999"
       - "8080:8080"
-      - "9200:8080"
     volumes:
-      - ./quesma/config/ci-config.yaml:/mnt/ci-config.yaml
+      - ./quesma/logs/:/var/quesma/logs
+      - ./quesma/config:/config # mount local config for indices
+    deploy:
+      resources:
+        limits:
+          memory: 512M
     restart: unless-stopped
-  elasticsearch:
+  elasticsearch-quesma:
     image: docker.elastic.co/elasticsearch/elasticsearch:8.11.1
-    container_name: elasticsearch
+    container_name: elasticsearch-quesma
     environment:
       - discovery.type=single-node
       - xpack.security.enabled=false
-      - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
+      - "ES_JAVA_OPTS=-Xmx2G"
     ports:
       - "9201:9200"
       - "9300:9300"
     healthcheck:
-      test: curl -s http://elasticsearch:9200 >/dev/null || exit 1
+      test: curl -s http://elasticsearch-quesma:9200 >/dev/null || exit 1
       start_period: 1m
       interval: 1s
       timeout: 1s
     deploy:
       resources:
         limits:
           memory: 4G
-  clickhouse:
+  elasticsearch_direct:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.1
+    container_name: elasticsearch_direct
+    environment:
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+      - "ES_JAVA_OPTS=-Xmx2G"
+    ports:
+      - "9301:9300"
+    healthcheck:
+      test: curl -s http://elasticsearch_direct:9200 >/dev/null || exit 1
+      start_period: 1m
+      interval: 1s
+      timeout: 1s
+    deploy:
+      resources:
+        limits:
+          memory: 4G
+  kibana:
+    image: docker.elastic.co/kibana/kibana:8.11.1
+    environment:
+      ELASTICSEARCH_HOSTS: '["http://mitmproxy:8080"]'
+      XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'QUESMAQUESMAQUESMAQUESMAQUESMAQUESMAQUESMAQUESMA' # Just to get rid of annoying ERROR in logs
+    #      LOGGING_ROOT_LEVEL: 'debug'
+    depends_on:
+      clean-mitmproxy:
+        condition: service_completed_successfully
+      quesma:
+        condition: service_healthy
+      elasticsearch-quesma:
+        condition: service_healthy
+    ports:
+      - "5601:5601"
+    restart: unless-stopped
+    healthcheck:
+      test: "curl -s http://kibana:5601/api/status >/dev/null || exit 1"
+      start_period: 2m
+      interval: 1s
+      timeout: 1s
+  mitmproxy_direct:
+    image: mitmproxy/mitmproxy:10.1.5
+    tty: true
+    ports:
+      - "9202:8080"
+      - "8082:8081"
+    command: >
+      mitmweb --set termlog_verbosity=warn --no-web-open-browser --web-host 0.0.0.0 --mode reverse:http://elasticsearch_direct:9200/
+    # -s /var/mitmproxy/request.py
+    # Uncomment above, if you would like to log requests using mitmproxy/request.py
+    #   The files will be available in mitmproxy/{query,requests}/*.txt
+    depends_on:
+      elasticsearch_direct:
+        condition: service_healthy
+    restart: unless-stopped
+    healthcheck:
+      # No curl/wget, going old school
+      test: timeout 10s bash -c ':> /dev/tcp/127.0.0.1/8080' || exit 1
+      interval: 1s
+      start_period: 1m
+      timeout: 1s
+  kibana_direct:
+    image: docker.elastic.co/kibana/kibana:8.11.1
+    environment:
+      ELASTICSEARCH_HOSTS: '["http://mitmproxy_direct:8080"]'
+      XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'QUESMAQUESMAQUESMAQUESMAQUESMAQUESMAQUESMAQUESMA' # Just to get rid of annoying ERROR in logs
+    #      LOGGING_ROOT_LEVEL: 'debug'
+    depends_on:
+      clean-mitmproxy:
+        condition: service_completed_successfully
+      elasticsearch_direct:
+        condition: service_healthy
+    ports:
+      - "5602:5601"
+    restart: unless-stopped
+    healthcheck:
+      test: "curl -s http://kibana_direct:5601/api/status >/dev/null || exit 1"
+      start_period: 2m
+      interval: 1s
+      timeout: 1s
+  kibana-sidecar:
+    image: docker.elastic.co/kibana/kibana:8.11.1
+    restart: "no"
+    depends_on:
+      kibana:
+        condition: service_healthy
+    volumes:
+      - ./kibana/:/local_mount
+    command: ["/bin/bash", "-c", "/local_mount/add_e2e_data_views.sh http://kibana:5601"]
+  kibana-sidecar_direct:
+    image: docker.elastic.co/kibana/kibana:8.11.1
+    restart: "no"
+    depends_on:
+      kibana_direct:
+        condition: service_healthy
+    volumes:
+      - ./kibana/:/local_mount
+    command: ["/bin/bash", "-c", "/local_mount/add_e2e_data_views.sh http://kibana_direct:5601"]
+  clickhouse: # user: 'default', no password
     image: clickhouse/clickhouse-server:23.12.2.59-alpine
     ports:
       - "8123:8123"
       - "9000:9000"
-    volumes:
-      - ./clickhouse/additional_config.xml:/etc/clickhouse-server/users.d/additional_config.xml
     healthcheck:
       test: wget --no-verbose --tries=1 --spider http://clickhouse:8123/ping || exit 1
       interval: 1s
       timeout: 1s
       start_period: 1m
-
+  mitmproxy:
+    image: mitmproxy/mitmproxy:10.1.5
+    tty: true
+    ports:
+      - "9200:8080"
+      - "8081:8081"
+    command: >
+      mitmweb --set termlog_verbosity=warn --no-web-open-browser --web-host 0.0.0.0 --mode reverse:http://quesma:8080/
+    # -s /var/mitmproxy/request.py
+    # Uncomment above, if you would like to log requests using mitmproxy/request.py
+    #   The files will be available in mitmproxy/{query,requests}/*.txt
+    depends_on:
+      quesma:
+        condition: service_healthy
+    restart: unless-stopped
+    volumes:
+      - ./mitmproxy:/var/mitmproxy
+    healthcheck:
+      # No curl/wget, going old school
+      test: timeout 10s bash -c ':> /dev/tcp/127.0.0.1/8080' || exit 1
+      interval: 1s
+      start_period: 1m
+      timeout: 1s
+  clean-clickhouse:
+    build: clean-clickhouse
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+    restart: "no"
+    volumes:
+      - ./mitmproxy:/var/mitmproxy
+  clean-mitmproxy:
+    build: clean-mitmproxy
+    depends_on:
+      mitmproxy:
+        condition: service_healthy
+    restart: "no"
+    volumes:
+      - ./mitmproxy:/var/mitmproxy
+  e2e-data-generator:
+    build: e2e-data-generator
+    depends_on:
+      clean-clickhouse:
+        condition: service_completed_successfully
+      elasticsearch_direct:
+        condition: service_healthy
+    restart: "no"
diff --git a/docker/kibana/add_e2e_data_views.sh b/docker/kibana/add_e2e_data_views.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+# Adds sample data to Kibana
+cd "$(dirname "$0")"
+source lib.sh
+
+wait_until_available
+
+add_view() {
+  local view_name=$1
+
+  echo -n "Adding data view $view_name"
+  do_silent_http_post "api/data_views/data_view" "{
+    \"data_view\": {
+      \"name\":  \"$view_name\",
+      \"title\": \"$view_name\",
+      \"id\":    \"$view_name\",
+      \"timeFieldName\": \"timestamp\",
+      \"allowNoIndex\": true
+    },
+    \"override\": true
+  }"
+  echo ""
+}
+
+add_view "e2e_table_1"
+echo -e "\nData views added."
diff --git a/docker/kibana/lib.sh b/docker/kibana/lib.sh
@@ -53,7 +53,7 @@ do_silent_http_post() {
     -H 'Content-Type: application/json' \
     -d "$body"
 }
--
+
 add_sample_dataset() {
     local sample_data=$1
     START_TIME=$(date +%s)

diff --git a/docker/quesma/config/e2e.yaml b/docker/quesma/config/e2e.yaml
@@ -0,0 +1,20 @@
+mode: "dual-write-query-clickhouse"
+port: 8080  # public tcp port to listen for incoming traffic
+elasticsearch:
+  #url: "http://localhost:9200"
+  call: false
+  adminUrl: "http://localhost:5601"
+clickhouse:
+  url: "clickhouse://clickhouse:9000"
+  adminUrl: "http://localhost:8123/play"
+ingestStatistics: true
+internalTelemetryUrl: "https://api.quesma.com/phone-home"
+logging:
+  path: "logs"
+  level: "info"
+  remoteUrl: "https://api.quesma.com/phone-home"
+  disableFileLogging: false
+indexes:
+  e2e_table_1:
+    enabled: true
+    fullTextFields: [ "meme" ]