Clean up configuration files used in CI (#752)

The main motivation of this PR is that `local-dev.yaml` from kibana-sample-data example is not actively used in any CI tests. That leaves it prone to bit-rot, which is problematic since this is Quesma's main example. Solve the issue by using kibana-sample-data's `local-dev.yaml` in CI (smoke test). Additionally remove unused `ci/e2e.yml` and rename (actively used) `ci/e2e-old.yml`.
QuesmaOrg · Sep 11, 2024 · 0b9707c · 0b9707c
1 parent d18aa67
commit 0b9707c
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 226 deletions.
diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
@@ -76,7 +76,7 @@ jobs:
 
       - name: Build and start docker compose
         timeout-minutes: 10
-        run: docker compose -f ci/e2e-old.yml up -d
+        run: docker compose -f ci/e2e.yml up -d
 
       - name: Wait until Elasticsearch is up
         timeout-minutes: 6

diff --git a/ci/ci.yml b/ci/ci.yml
@@ -18,7 +18,7 @@ services:
       - "9200:8080"
     volumes:
       - ./quesma/logs/:/var/quesma/logs
-      - ./quesma/config/ci-config.yaml:/mnt/ci-config.yaml
+      - ../examples/kibana-sample-data/quesma/config/local-dev.yaml:/mnt/ci-config.yaml
     restart: unless-stopped
   log-generator:
     build: log-generator

diff --git a/ci/e2e-old.yml b/ci/e2e-old.yml
diff --git a/ci/e2e.yml b/ci/e2e.yml
@@ -1,204 +1,58 @@
+version: "3.7"
 services:
   quesma:
     build: ../quesma
     image: quesma:latest
-    env_file:
-      # We require .env file to be present
-      # and contain QUESMA_licenseKey with your personal license key
-      - .env
     environment:
-      - QUESMA_elasticsearch_url=http://elasticsearch-quesma:9200
+      - QUESMA_CONFIG_FILE=/mnt/ci-config.yaml
+      - QUESMA_elasticsearch_url=http://elasticsearch:9200
       - QUESMA_port=8080
       - QUESMA_logging_path=/var/quesma/logs
-      - QUESMA_mode=dual-write-query-clickhouse
-      - QUESMA_CONFIG_FILE=/config/e2e.yaml
-      - QUESMA_logging_fileLogging=true
+      - QUESMA_installationId=old-e2e-test-run
+      - QUESMA_connectors_my-clickhouse-connector_type=clickhouse-os
+      - QUESMA_clickhouse_url=clickhouse://clickhouse:9000
+      - QUESMA_logging_fileLogging=false
     depends_on:
-      clean-clickhouse:
-        condition: service_completed_successfully
-      elasticsearch-quesma:
+      clickhouse:
+        condition: service_healthy
+      elasticsearch:
         condition: service_healthy
     ports:
       - "9999:9999"
       - "8080:8080"
+      - "9200:8080"
     volumes:
-      - ./quesma/logs/:/var/quesma/logs
-      - ./quesma/config:/config # mount local config for indices
-    deploy:
-      resources:
-        limits:
-          memory: 512M
+      - ./quesma/config/ci-config.yaml:/mnt/ci-config.yaml
     restart: unless-stopped
-  elasticsearch-quesma:
+  elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:8.11.1
-    container_name: elasticsearch-quesma
+    container_name: elasticsearch
     environment:
       - discovery.type=single-node
       - xpack.security.enabled=false
-      - "ES_JAVA_OPTS=-Xmx2G"
+      - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
     ports:
       - "9201:9200"
       - "9300:9300"
     healthcheck:
-      test: curl -s http://elasticsearch-quesma:9200 >/dev/null || exit 1
+      test: curl -s http://elasticsearch:9200 >/dev/null || exit 1
       start_period: 1m
       interval: 1s
       timeout: 1s
     deploy:
       resources:
         limits:
           memory: 4G
-  elasticsearch_direct:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.1
-    container_name: elasticsearch_direct
-    environment:
-      - discovery.type=single-node
-      - xpack.security.enabled=false
-      - "ES_JAVA_OPTS=-Xmx2G"
-    ports:
-      - "9301:9300"
-    healthcheck:
-      test: curl -s http://elasticsearch_direct:9200 >/dev/null || exit 1
-      start_period: 1m
-      interval: 1s
-      timeout: 1s
-    deploy:
-      resources:
-        limits:
-          memory: 4G
-  kibana:
-    image: docker.elastic.co/kibana/kibana:8.11.1
-    environment:
-      ELASTICSEARCH_HOSTS: '["http://mitmproxy:8080"]'
-      XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'QUESMAQUESMAQUESMAQUESMAQUESMAQUESMAQUESMAQUESMA' # Just to get rid of annoying ERROR in logs
-    #      LOGGING_ROOT_LEVEL: 'debug'
-    depends_on:
-      clean-mitmproxy:
-        condition: service_completed_successfully
-      quesma:
-        condition: service_healthy
-      elasticsearch-quesma:
-        condition: service_healthy
-    ports:
-      - "5601:5601"
-    restart: unless-stopped
-    healthcheck:
-      test: "curl -s http://kibana:5601/api/status >/dev/null || exit 1"
-      start_period: 2m
-      interval: 1s
-      timeout: 1s
-  mitmproxy_direct:
-    image: mitmproxy/mitmproxy:10.1.5
-    tty: true
-    ports:
-      - "9202:8080"
-      - "8082:8081"
-    command: >
-      mitmweb --set termlog_verbosity=warn --no-web-open-browser --web-host 0.0.0.0 --mode reverse:http://elasticsearch_direct:9200/
-    # -s /var/mitmproxy/request.py
-    # Uncomment above, if you would like to log requests using mitmproxy/request.py
-    #   The files will be available in mitmproxy/{query,requests}/*.txt
-    depends_on:
-      elasticsearch_direct:
-        condition: service_healthy
-    restart: unless-stopped
-    healthcheck:
-      # No curl/wget, going old school
-      test: timeout 10s bash -c ':> /dev/tcp/127.0.0.1/8080' || exit 1
-      interval: 1s
-      start_period: 1m
-      timeout: 1s
-  kibana_direct:
-    image: docker.elastic.co/kibana/kibana:8.11.1
-    environment:
-      ELASTICSEARCH_HOSTS: '["http://mitmproxy_direct:8080"]'
-      XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'QUESMAQUESMAQUESMAQUESMAQUESMAQUESMAQUESMAQUESMA' # Just to get rid of annoying ERROR in logs
-    #      LOGGING_ROOT_LEVEL: 'debug'
-    depends_on:
-      clean-mitmproxy:
-        condition: service_completed_successfully
-      elasticsearch_direct:
-        condition: service_healthy
-    ports:
-      - "5602:5601"
-    restart: unless-stopped
-    healthcheck:
-      test: "curl -s http://kibana_direct:5601/api/status >/dev/null || exit 1"
-      start_period: 2m
-      interval: 1s
-      timeout: 1s
-  kibana-sidecar:
-    image: docker.elastic.co/kibana/kibana:8.11.1
-    restart: "no"
-    depends_on:
-      kibana:
-        condition: service_healthy
-    volumes:
-      - ./kibana/:/local_mount
-    command: ["/bin/bash", "-c", "/local_mount/add_e2e_data_views.sh http://kibana:5601"]
-  kibana-sidecar_direct:
-    image: docker.elastic.co/kibana/kibana:8.11.1
-    restart: "no"
-    depends_on:
-      kibana_direct:
-        condition: service_healthy
-    volumes:
-      - ./kibana/:/local_mount
-    command: ["/bin/bash", "-c", "/local_mount/add_e2e_data_views.sh http://kibana_direct:5601"]
-  clickhouse: # user: 'default', no password
+  clickhouse:
     image: clickhouse/clickhouse-server:23.12.2.59-alpine
     ports:
       - "8123:8123"
       - "9000:9000"
+    volumes:
+      - ./clickhouse/additional_config.xml:/etc/clickhouse-server/users.d/additional_config.xml
     healthcheck:
       test: wget --no-verbose --tries=1 --spider http://clickhouse:8123/ping || exit 1
       interval: 1s
       timeout: 1s
       start_period: 1m
-  mitmproxy:
-    image: mitmproxy/mitmproxy:10.1.5
-    tty: true
-    ports:
-      - "9200:8080"
-      - "8081:8081"
-    command: >
-      mitmweb --set termlog_verbosity=warn --no-web-open-browser --web-host 0.0.0.0 --mode reverse:http://quesma:8080/
-    # -s /var/mitmproxy/request.py
-    # Uncomment above, if you would like to log requests using mitmproxy/request.py
-    #   The files will be available in mitmproxy/{query,requests}/*.txt
-    depends_on:
-      quesma:
-        condition: service_healthy
-    restart: unless-stopped
-    volumes:
-      - ./mitmproxy:/var/mitmproxy
-    healthcheck:
-      # No curl/wget, going old school
-      test: timeout 10s bash -c ':> /dev/tcp/127.0.0.1/8080' || exit 1
-      interval: 1s
-      start_period: 1m
-      timeout: 1s
-  clean-clickhouse:
-    build: clean-clickhouse
-    depends_on:
-      clickhouse:
-        condition: service_healthy
-    restart: "no"
-    volumes:
-      - ./mitmproxy:/var/mitmproxy
-  clean-mitmproxy:
-    build: clean-mitmproxy
-    depends_on:
-      mitmproxy:
-        condition: service_healthy
-    restart: "no"
-    volumes:
-      - ./mitmproxy:/var/mitmproxy
-  e2e-data-generator:
-    build: e2e-data-generator
-    depends_on:
-      clean-clickhouse:
-        condition: service_completed_successfully
-      elasticsearch_direct:
-        condition: service_healthy
-    restart: "no"
+