diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 13aca05bd..8fcbf2e50 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -76,7 +76,7 @@ jobs: - name: Build and start docker compose timeout-minutes: 10 - run: docker compose -f ci/e2e-old.yml up -d + run: docker compose -f ci/e2e.yml up -d - name: Wait until Elasticsearch is up timeout-minutes: 6 diff --git a/ci/ci.yml b/ci/ci.yml index 2857c2393..60bce6fd6 100644 --- a/ci/ci.yml +++ b/ci/ci.yml @@ -18,7 +18,7 @@ services: - "9200:8080" volumes: - ./quesma/logs/:/var/quesma/logs - - ./quesma/config/ci-config.yaml:/mnt/ci-config.yaml + - ../examples/kibana-sample-data/quesma/config/local-dev.yaml:/mnt/ci-config.yaml restart: unless-stopped log-generator: build: log-generator diff --git a/ci/e2e-old.yml b/ci/e2e-old.yml deleted file mode 100644 index 1b16af6c4..000000000 --- a/ci/e2e-old.yml +++ /dev/null @@ -1,58 +0,0 @@ -version: "3.7" -services: - quesma: - build: ../quesma - image: quesma:latest - environment: - - QUESMA_CONFIG_FILE=/mnt/ci-config.yaml - - QUESMA_elasticsearch_url=http://elasticsearch:9200 - - QUESMA_port=8080 - - QUESMA_logging_path=/var/quesma/logs - - QUESMA_installationId=old-e2e-test-run - - QUESMA_connectors_my-clickhouse-connector_type=clickhouse-os - - QUESMA_clickhouse_url=clickhouse://clickhouse:9000 - - QUESMA_logging_fileLogging=false - depends_on: - clickhouse: - condition: service_healthy - elasticsearch: - condition: service_healthy - ports: - - "9999:9999" - - "8080:8080" - - "9200:8080" - volumes: - - ./quesma/config/ci-config.yaml:/mnt/ci-config.yaml - restart: unless-stopped - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.11.1 - container_name: elasticsearch - environment: - - discovery.type=single-node - - xpack.security.enabled=false - - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m" - ports: - - "9201:9200" - - "9300:9300" - healthcheck: - test: curl -s http://elasticsearch:9200 >/dev/null || exit 1 - start_period: 1m - interval: 1s - timeout: 1s - deploy: - resources: - limits: - memory: 4G - clickhouse: - image: clickhouse/clickhouse-server:23.12.2.59-alpine - ports: - - "8123:8123" - - "9000:9000" - volumes: - - ./clickhouse/additional_config.xml:/etc/clickhouse-server/users.d/additional_config.xml - healthcheck: - test: wget --no-verbose --tries=1 --spider http://clickhouse:8123/ping || exit 1 - interval: 1s - timeout: 1s - start_period: 1m - diff --git a/ci/e2e.yml b/ci/e2e.yml index ee4434381..1b16af6c4 100644 --- a/ci/e2e.yml +++ b/ci/e2e.yml @@ -1,46 +1,41 @@ +version: "3.7" services: quesma: build: ../quesma image: quesma:latest - env_file: - # We require .env file to be present - # and contain QUESMA_licenseKey with your personal license key - - .env environment: - - QUESMA_elasticsearch_url=http://elasticsearch-quesma:9200 + - QUESMA_CONFIG_FILE=/mnt/ci-config.yaml + - QUESMA_elasticsearch_url=http://elasticsearch:9200 - QUESMA_port=8080 - QUESMA_logging_path=/var/quesma/logs - - QUESMA_mode=dual-write-query-clickhouse - - QUESMA_CONFIG_FILE=/config/e2e.yaml - - QUESMA_logging_fileLogging=true + - QUESMA_installationId=old-e2e-test-run + - QUESMA_connectors_my-clickhouse-connector_type=clickhouse-os + - QUESMA_clickhouse_url=clickhouse://clickhouse:9000 + - QUESMA_logging_fileLogging=false depends_on: - clean-clickhouse: - condition: service_completed_successfully - elasticsearch-quesma: + clickhouse: + condition: service_healthy + elasticsearch: condition: service_healthy ports: - "9999:9999" - "8080:8080" + - "9200:8080" volumes: - - ./quesma/logs/:/var/quesma/logs - - ./quesma/config:/config # mount local config for indices - deploy: - resources: - limits: - memory: 512M + - ./quesma/config/ci-config.yaml:/mnt/ci-config.yaml restart: unless-stopped - elasticsearch-quesma: + elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:8.11.1 - container_name: elasticsearch-quesma + container_name: elasticsearch environment: - discovery.type=single-node - xpack.security.enabled=false - - "ES_JAVA_OPTS=-Xmx2G" + - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m" ports: - "9201:9200" - "9300:9300" healthcheck: - test: curl -s http://elasticsearch-quesma:9200 >/dev/null || exit 1 + test: curl -s http://elasticsearch:9200 >/dev/null || exit 1 start_period: 1m interval: 1s timeout: 1s @@ -48,157 +43,16 @@ services: resources: limits: memory: 4G - elasticsearch_direct: - image: docker.elastic.co/elasticsearch/elasticsearch:8.11.1 - container_name: elasticsearch_direct - environment: - - discovery.type=single-node - - xpack.security.enabled=false - - "ES_JAVA_OPTS=-Xmx2G" - ports: - - "9301:9300" - healthcheck: - test: curl -s http://elasticsearch_direct:9200 >/dev/null || exit 1 - start_period: 1m - interval: 1s - timeout: 1s - deploy: - resources: - limits: - memory: 4G - kibana: - image: docker.elastic.co/kibana/kibana:8.11.1 - environment: - ELASTICSEARCH_HOSTS: '["http://mitmproxy:8080"]' - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'QUESMAQUESMAQUESMAQUESMAQUESMAQUESMAQUESMAQUESMA' # Just to get rid of annoying ERROR in logs - # LOGGING_ROOT_LEVEL: 'debug' - depends_on: - clean-mitmproxy: - condition: service_completed_successfully - quesma: - condition: service_healthy - elasticsearch-quesma: - condition: service_healthy - ports: - - "5601:5601" - restart: unless-stopped - healthcheck: - test: "curl -s http://kibana:5601/api/status >/dev/null || exit 1" - start_period: 2m - interval: 1s - timeout: 1s - mitmproxy_direct: - image: mitmproxy/mitmproxy:10.1.5 - tty: true - ports: - - "9202:8080" - - "8082:8081" - command: > - mitmweb --set termlog_verbosity=warn --no-web-open-browser --web-host 0.0.0.0 --mode reverse:http://elasticsearch_direct:9200/ - # -s /var/mitmproxy/request.py - # Uncomment above, if you would like to log requests using mitmproxy/request.py - # The files will be available in mitmproxy/{query,requests}/*.txt - depends_on: - elasticsearch_direct: - condition: service_healthy - restart: unless-stopped - healthcheck: - # No curl/wget, going old school - test: timeout 10s bash -c ':> /dev/tcp/127.0.0.1/8080' || exit 1 - interval: 1s - start_period: 1m - timeout: 1s - kibana_direct: - image: docker.elastic.co/kibana/kibana:8.11.1 - environment: - ELASTICSEARCH_HOSTS: '["http://mitmproxy_direct:8080"]' - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'QUESMAQUESMAQUESMAQUESMAQUESMAQUESMAQUESMAQUESMA' # Just to get rid of annoying ERROR in logs - # LOGGING_ROOT_LEVEL: 'debug' - depends_on: - clean-mitmproxy: - condition: service_completed_successfully - elasticsearch_direct: - condition: service_healthy - ports: - - "5602:5601" - restart: unless-stopped - healthcheck: - test: "curl -s http://kibana_direct:5601/api/status >/dev/null || exit 1" - start_period: 2m - interval: 1s - timeout: 1s - kibana-sidecar: - image: docker.elastic.co/kibana/kibana:8.11.1 - restart: "no" - depends_on: - kibana: - condition: service_healthy - volumes: - - ./kibana/:/local_mount - command: ["/bin/bash", "-c", "/local_mount/add_e2e_data_views.sh http://kibana:5601"] - kibana-sidecar_direct: - image: docker.elastic.co/kibana/kibana:8.11.1 - restart: "no" - depends_on: - kibana_direct: - condition: service_healthy - volumes: - - ./kibana/:/local_mount - command: ["/bin/bash", "-c", "/local_mount/add_e2e_data_views.sh http://kibana_direct:5601"] - clickhouse: # user: 'default', no password + clickhouse: image: clickhouse/clickhouse-server:23.12.2.59-alpine ports: - "8123:8123" - "9000:9000" + volumes: + - ./clickhouse/additional_config.xml:/etc/clickhouse-server/users.d/additional_config.xml healthcheck: test: wget --no-verbose --tries=1 --spider http://clickhouse:8123/ping || exit 1 interval: 1s timeout: 1s start_period: 1m - mitmproxy: - image: mitmproxy/mitmproxy:10.1.5 - tty: true - ports: - - "9200:8080" - - "8081:8081" - command: > - mitmweb --set termlog_verbosity=warn --no-web-open-browser --web-host 0.0.0.0 --mode reverse:http://quesma:8080/ - # -s /var/mitmproxy/request.py - # Uncomment above, if you would like to log requests using mitmproxy/request.py - # The files will be available in mitmproxy/{query,requests}/*.txt - depends_on: - quesma: - condition: service_healthy - restart: unless-stopped - volumes: - - ./mitmproxy:/var/mitmproxy - healthcheck: - # No curl/wget, going old school - test: timeout 10s bash -c ':> /dev/tcp/127.0.0.1/8080' || exit 1 - interval: 1s - start_period: 1m - timeout: 1s - clean-clickhouse: - build: clean-clickhouse - depends_on: - clickhouse: - condition: service_healthy - restart: "no" - volumes: - - ./mitmproxy:/var/mitmproxy - clean-mitmproxy: - build: clean-mitmproxy - depends_on: - mitmproxy: - condition: service_healthy - restart: "no" - volumes: - - ./mitmproxy:/var/mitmproxy - e2e-data-generator: - build: e2e-data-generator - depends_on: - clean-clickhouse: - condition: service_completed_successfully - elasticsearch_direct: - condition: service_healthy - restart: "no" \ No newline at end of file +