Several small changes during debugging (#278)

Series of minor fixes:
- accept file names as argument of our scripts `bin/down.sh
docker/local-dev-dual-comparison.yml`
- fix `bin/rebuild.sh`
- add a second `mitmproxy` so I can compare outputs
- minor fix

bin/lib.sh

@@ -3,7 +3,10 @@
 DEFAULT_COMPOSE_FILE="docker/local-dev.yml"
 : "${QUESMA_COMPOSE_FILE:=$DEFAULT_COMPOSE_FILE}"
 if [ ! -z "$1" ]; then
-  QUESMA_COMPOSE_FILE="docker/$1.yml"
+  QUESMA_COMPOSE_FILE="$1" # it can be file na,e
+  if [ ! -f "$QUESMA_COMPOSE_FILE" ]; then # orname
+    QUESMA_COMPOSE_FILE="docker/$1.yml"
+  fi
   echo "QUESMA_COMPOSE_FILE set to '$QUESMA_COMPOSE_FILE' by argument '$1'"
 else
   if [ "$QUESMA_COMPOSE_FILE" != "$DEFAULT_COMPOSE_FILE" ]; then

bin/rebuild.sh

@@ -5,6 +5,6 @@ set -e
 cd "$(dirname "$0/")/.."
 source bin/lib.sh
 
-docker compose -f "$QUESMA_COMPOSE_FILE" build "$QUESMA_COMPOSE_FILE" build --build-arg QUESMA_BUILD_DATE="$QUESMA_BUILD_DATE" --build-arg QUESMA_VERSION="$QUESMA_VERSION" --build-arg QUESMA_BUILD_SHA="$QUESMA_BUILD_SHA" quesma && \
+docker compose -f "$QUESMA_COMPOSE_FILE" build  --build-arg QUESMA_BUILD_DATE="$QUESMA_BUILD_DATE" --build-arg QUESMA_VERSION="$QUESMA_VERSION" --build-arg QUESMA_BUILD_SHA="$QUESMA_BUILD_SHA" quesma && \
   docker compose -f "$QUESMA_COMPOSE_FILE" stop quesma && \
   docker compose -f "$QUESMA_COMPOSE_FILE" up -d --no-deps quesma

docker/local-dev-dual-comparison.yml

@@ -56,7 +56,6 @@ services:
       - xpack.security.enabled=false
       - "ES_JAVA_OPTS=-Xmx2G"
     ports:
-      - "9202:9200"
       - "9301:9300"
     healthcheck:
       test: curl -s http://elasticsearch_direct:9200 >/dev/null || exit 1
@@ -88,10 +87,31 @@ services:
       start_period: 2m
       interval: 1s
       timeout: 1s
+  mitmproxy_direct:
+    image: mitmproxy/mitmproxy:10.1.5
+    tty: true
+    ports:
+      - "9202:8080"
+      - "8082:8081"
+    command: >
+      mitmweb --set termlog_verbosity=warn --no-web-open-browser --web-host 0.0.0.0 --mode reverse:http://elasticsearch_direct:9200/
+    # -s /var/mitmproxy/request.py
+    # Uncomment above, if you would like to log requests using mitmproxy/request.py
+    #   The files will be available in mitmproxy/{query,requests}/*.txt
+    depends_on:
+      elasticsearch_direct:
+        condition: service_healthy
+    restart: unless-stopped
+    healthcheck:
+      # No curl/wget, going old school
+      test: timeout 10s bash -c ':> /dev/tcp/127.0.0.1/8080' || exit 1
+      interval: 1s
+      start_period: 1m
+      timeout: 1s
   kibana_direct:
     image: docker.elastic.co/kibana/kibana:8.11.1
     environment:
-      ELASTICSEARCH_HOSTS: '["http://elasticsearch_direct:9200"]'
+      ELASTICSEARCH_HOSTS: '["http://mitmproxy_direct:8080"]'
       XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'QUESMAQUESMAQUESMAQUESMAQUESMAQUESMAQUESMAQUESMA' # Just to get rid of annoying ERROR in logs
     #      LOGGING_ROOT_LEVEL: 'debug'
     depends_on:

docker/quesma/config/hydrolix-demo.yml

@@ -15,6 +15,12 @@ indexes:
         source: "@timestamp"
         target: "reqTimeSec"
     fullTextFields: [ "city", "reqHost", "reqMethod", "reqPath",  "UA", "state", "rspContentType" ]
+    mappings:
+      "cliIP": "ip"
+    static-schema:
+      fields:
+        "cliIP":
+          type: "ip"
 
   siem:
     enabled: true