Merge pull request #795 from IATI/upgrade-es-into-docker

Elasticsearch v6 -> 7, into Docker & more RAM

.github/workflows/workflow.yml

@@ -113,7 +113,7 @@ jobs:
             az vm create \
             --resource-group "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
             --name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
-            --size Standard_B2s --public-ip-sku Standard \
+            --size Standard_B2ms --public-ip-sku Standard \
             --image "Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:latest" \
             --ssh-key-values ${{ secrets.DEV_PUB_KEYS }} && \
             export NEW_IP=$(az vm list-ip-addresses --name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" --resource-group "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" --query [].virtualMachine.network[].publicIpAddresses[][].ipAddress --output tsv) && \
@@ -186,12 +186,27 @@ jobs:
             --name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
             --command-id RunShellScript \
             --scripts "\
+              echo 'vm.max_map_count=262144' >> /etc/sysctl.conf && \
+              sysctl -p && \
+              mkdir -p /home/elasticsearch/data && \
+              chown 1000:0 /home/elasticsearch/data && \
+              docker network create iati-standard-website && \
               docker login -u '${{ env.REGISTRY_USERNAME }}' -p '${{ env.REGISTRY_PASSWORD }}' ${{ env.REGISTRY_LOGIN_SERVER }} && \
               docker pull '${{ env.REGISTRY_LOGIN_SERVER }}/${{env.STAGE}}-${{env.NAME}}:${{ env.TAG }}' && \
+              docker run --restart always --name elasticsearch -d \
+              -e ES_SETTING_XPACK_SECURITY_ENABLED=false \
+              -e ES_SETTING_DISCOVERY_TYPE=single-node \
+              --network iati-standard-website \
+              --ulimit nofile=65535:65535 \
+              -e 'bootstrap.memory_lock=true' --ulimit memlock=-1:-1 \
+              -v '/home/elasticsearch/data:/usr/share/elasticsearch/data' \
+              'docker.elastic.co/elasticsearch/elasticsearch:7.17.25' && \
               docker run --restart always --name website -d -p 5000:5000 \
               --log-driver 'json-file' \
               --log-opt max-size=100m \
               --log-opt max-file=3 \
+              --network iati-standard-website \
+              --link elasticsearch:elasticsearch \
               -e DJANGO_SETTINGS_MODULE='iati.settings.dev_public' \
               -e SECRET_KEY='${{ secrets.DEV_SECRET_KEY }}' \
               -e DATABASE_NAME='${{ secrets.DEV_DATABASE_NAME }}' \
@@ -327,7 +342,7 @@ jobs:
             az vm create \
             --resource-group "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
             --name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
-            --size Standard_B2s --public-ip-sku Standard \
+            --size Standard_B2ms --public-ip-sku Standard \
             --image "Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:latest" \
             --ssh-key-values ${{ secrets.DEV_PUB_KEYS }} && \
             export NEW_IP=$(az vm list-ip-addresses --name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" --resource-group "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" --query [].virtualMachine.network[].publicIpAddresses[][].ipAddress --output tsv) && \
@@ -400,12 +415,27 @@ jobs:
             --name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
             --command-id RunShellScript \
             --scripts "\
+              echo 'vm.max_map_count=262144' >> /etc/sysctl.conf && \
+              sysctl -p && \
+              mkdir -p /home/elasticsearch/data && \
+              chown 1000:0 /home/elasticsearch/data && \
+              docker network create iati-standard-website && \
               docker login -u '${{ env.REGISTRY_USERNAME }}' -p '${{ env.REGISTRY_PASSWORD }}' ${{ env.REGISTRY_LOGIN_SERVER }} && \
               docker pull '${{ env.REGISTRY_LOGIN_SERVER }}/${{env.STAGE}}-${{env.NAME}}:${{ env.TAG }}' && \
+              docker run --restart always --name elasticsearch -d \
+              -e ES_SETTING_XPACK_SECURITY_ENABLED=false \
+              -e ES_SETTING_DISCOVERY_TYPE=single-node \
+              --network iati-standard-website \
+              --ulimit nofile=65535:65535 \
+              -e 'bootstrap.memory_lock=true' --ulimit memlock=-1:-1 \
+              -v '/home/elasticsearch/data:/usr/share/elasticsearch/data' \
+              'docker.elastic.co/elasticsearch/elasticsearch:7.17.25' && \
               docker run --restart always --name website -d -p 5000:5000 \
               --log-driver 'json-file' \
               --log-opt max-size=100m \
               --log-opt max-file=3 \
+              --network iati-standard-website \
+              --link elasticsearch:elasticsearch \
               -e DJANGO_SETTINGS_MODULE='iati.settings.production' \
               -e SECRET_KEY='${{ secrets.PROD_SECRET_KEY }}' \
               -e DATABASE_NAME='${{ secrets.PROD_DATABASE_NAME }}' \

Dockerfile

@@ -6,7 +6,7 @@ ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONIOENCODING utf_8
 
 RUN apk update
-RUN apk add --no-cache bash
+RUN apk add --no-cache bash curl
 
 # Init engine
 
@@ -40,52 +40,6 @@ RUN apk add --no-cache jpeg-dev zlib-dev
 RUN apk add --no-cache postgresql-dev
 RUN apk add --no-cache libmemcached-dev zlib-dev
 
-# Elasticsearch from https://github.com/blacktop/docker-elasticsearch-alpine/blob/master/6.8/Dockerfile
-
-RUN apk add --no-cache openjdk8-jre su-exec
-
-ENV VERSION 6.8.23
-ENV DOWNLOAD_URL "https://artifacts.elastic.co/downloads/elasticsearch"
-ENV ES_TARBAL "${DOWNLOAD_URL}/elasticsearch-oss-${VERSION}.tar.gz"
-# ENV EXPECTED_SHA_URL "${DOWNLOAD_URL}/elasticsearch-oss-${VERSION}.tar.gz.sha512"
-ENV ES_TARBALL_SHA "14dbb2809b06499373c3ec5035d829d62255c2c93103618fbfe3d7d03cecf8847f654e83c78f765f23224126ff18ed713b959857e8ecf435c475b11bcd143d3f"
-RUN apk add --no-cache -t .build-deps wget ca-certificates gnupg openssl \
-  && set -ex \
-  && cd /tmp \
-  && echo "===> Install Elasticsearch..." \
-  && wget --progress=bar:force -O elasticsearch.tar.gz "$ES_TARBAL"; \
-  if [ "$ES_TARBALL_SHA" ]; then \
-  echo "$ES_TARBALL_SHA *elasticsearch.tar.gz" | sha512sum -c -; \
-  fi; \
-  tar -xf elasticsearch.tar.gz \
-  && ls -lah \
-  && mv elasticsearch-$VERSION /usr/share/elasticsearch \
-  && adduser -D -h /usr/share/elasticsearch elasticsearch \
-  && echo "===> Creating Elasticsearch Paths..." \
-  && for path in \
-  /usr/share/elasticsearch/data \
-  /usr/share/elasticsearch/logs \
-  /usr/share/elasticsearch/config \
-  /usr/share/elasticsearch/config/scripts \
-  /usr/share/elasticsearch/tmp \
-  /usr/share/elasticsearch/plugins \
-  ; do \
-  mkdir -p "$path"; \
-  chown -R elasticsearch:elasticsearch "$path"; \
-  done \
-  && rm -rf /tmp/* \
-  && apk del --purge .build-deps
-
-COPY config/elastic/elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml
-COPY config/elastic/log4j2.properties /usr/share/elasticsearch/config/log4j2.properties
-RUN chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/config
-
-RUN mkdir -p /var/log/messages
-RUN apk add logrotate
-COPY config/elastic/logrotate /etc/logrotate.d/elasticsearch
-RUN chmod 644 /etc/logrotate.d/elasticsearch
-COPY config/elastic/elasticsearch.service /etc/init.d/elasticsearch.service
-
 # Web app dependencies
 
 RUN mkdir -p /usr/src/app

Dockerfile_deploy

@@ -30,6 +30,6 @@ COPY manage.py /usr/src/app/manage.py
 COPY pytest.ini /usr/src/app/pytest.init
 COPY setup.cfg /usr/src/app/setup.cfg
 
-ENV ELASTICSEARCH_URL=http://localhost:9200
+ENV ELASTICSEARCH_URL=http://elasticsearch:9200
 ENV GUNICORN_WORKERS=5
 ENV COMPRESS_ENABLED='True'

README.md

@@ -19,6 +19,12 @@ This repository hosts the new IATI website based on Django and Wagtail CMS. A Po
 
 - Set a SECRET_KEY
 
+A limit on your kernel must be increased. There are ways to do this permanently, but to do so temporarily:
+
+```
+sudo sysctl -w vm.max_map_count=262144 
+```
+
 Build the project. The following will build linked `web` and `postgres` containers.
 
 ```

docker-compose.dev.yml

@@ -6,23 +6,39 @@ services:
       - POSTGRES_HOST_AUTH_METHOD=trust
     ports:
       - 5432:5432
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.25
+    environment:
+      - ES_SETTING_XPACK_SECURITY_ENABLED=false
+      - ES_SETTING_DISCOVERY_TYPE=single-node
+    ports:
+      - 9200:9200
+
   web:
     build:
       context: .
     environment:
       - DATABASE_URL=postgres://postgres:@postgres:5432/postgres
       - SECRET_KEY=enter-a-long-unguessable-string-here
-      - ELASTICSEARCH_URL=http://localhost:9200
+      - ELASTICSEARCH_URL=http://elasticsearch:9200
       - DJANGO_SETTINGS_MODULE=iati.settings.dev
       - GUNICORN_WORKERS=1
       - GITHUB_TOKEN
       - RECAPTCHA_PUBLIC_KEY
       - RECAPTCHA_PRIVATE_KEY
       - DEBUG_SERVER
       - COMPRESS_ENABLED
+    # I'm not sure why these links are needed; I thought Docker defaults would just do this for us but without it
+    # the Web container can't see the others.
     links:
-      - postgres
+      - postgres:postgres
+      - elasticsearch:elasticsearch
     ports:
       - 5000:5000
     volumes:
       - ./:/usr/src/app
+
+networks:
+  default:
+    name: iati-standard-website

entrypoint.sh

@@ -13,12 +13,10 @@ else
   done
 fi
 
-if [[ -z "${ELASTICSEARCH_URL}" ]]; then
-  >&2 echo "Skipping Elasticsearch"
-else
-  >&2 echo "Starting Elasticsearch"
-  rc-service elasticsearch.service start
-fi
+until curl --output /dev/null --silent --head --fail ${ELASTICSEARCH_URL}; do
+    >&2 echo "Elasticsearch is unavailable - sleeping"
+    sleep 10
+done
 
 if [[ -z "${DEBUG_SERVER}" ]]; then
   gunicorn iati.wsgi:application --bind 0.0.0.0:5000 --workers $GUNICORN_WORKERS >> /var/log/gunicorn/gunicorn.log 2>&1 &

iati/settings/base.py

@@ -554,7 +554,7 @@
 # Search settings
 WAGTAILSEARCH_BACKENDS = {
     'default': {
-        'BACKEND': 'wagtail.search.backends.elasticsearch6',
+        'BACKEND': 'wagtail.search.backends.elasticsearch7',
         'URLS': [os.getenv('ELASTICSEARCH_URL', 'http://localhost:9200')],
         'INDEX': 'iati',
     },

requirements.in

@@ -14,7 +14,7 @@ django-recaptcha3 @ git+https://github.com/bartsanchez/django-recaptcha3.git@313
 django-storages[azure]
 django-widget-tweaks
 Django>=4.2,<4.3
-elasticsearch>=6.8,<7
+elasticsearch>=7,<8
 gunicorn
 opencensus-ext-azure
 opencensus-ext-django

requirements.txt

@@ -29,7 +29,9 @@ bleach==6.1.0
 cachetools==5.4.0
     # via google-auth
 certifi==2024.7.4
-    # via requests
+    # via
+    #   elasticsearch
+    #   requests
 cffi==1.17.0
     # via
     #   cryptography
@@ -107,7 +109,7 @@ djangorestframework==3.15.2
     # via wagtail
 draftjs-exporter==2.1.7
     # via wagtail
-elasticsearch==6.8.2
+elasticsearch==7.9.1
     # via -r requirements.in
 et-xmlfile==1.1.0
     # via openpyxl

requirements_dev.txt

@@ -49,6 +49,7 @@ cachetools==5.4.0
 certifi==2024.7.4
     # via
     #   -r requirements.txt
+    #   elasticsearch
     #   requests
     #   selenium
 cffi==1.17.0
@@ -159,7 +160,7 @@ draftjs-exporter==2.1.7
     # via
     #   -r requirements.txt
     #   wagtail
-elasticsearch==6.8.2
+elasticsearch==7.9.1
     # via -r requirements.txt
 et-xmlfile==1.1.0
     # via