diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 92ed4c3ce..4c13e08be 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -118,10 +118,10 @@ jobs: --ssh-key-values ${{ secrets.DEV_PUB_KEYS }} && \ export NEW_IP=$(az vm list-ip-addresses --name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" --resource-group "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" --query [].virtualMachine.network[].publicIpAddresses[][].ipAddress --output tsv) && \ echo "NEW_IP=$NEW_IP" >> $GITHUB_ENV && \ - az postgres server firewall-rule update \ + az postgres flexible-server firewall-rule update \ --resource-group "${{ secrets.DEV_PSQL_RESOURCE_GROUP }}" \ - --server-name "${{ secrets.DEV_PSQL_NAME }}" \ - --name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \ + --name "${{ secrets.DEV_PSQL_NAME }}" \ + --rule-name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \ --start-ip-address $NEW_IP \ --end-ip-address $NEW_IP && \ az network nsg rule create \ @@ -186,20 +186,12 @@ jobs: --name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \ --command-id RunShellScript \ --scripts "\ - docker network create iati-standard-website && \ docker login -u '${{ env.REGISTRY_USERNAME }}' -p '${{ env.REGISTRY_PASSWORD }}' ${{ env.REGISTRY_LOGIN_SERVER }} && \ docker pull '${{ env.REGISTRY_LOGIN_SERVER }}/${{env.STAGE}}-${{env.NAME}}:${{ env.TAG }}' && \ - docker run --restart always --name elasticsearch -d \ - -e ES_SETTING_XPACK_SECURITY_ENABLED=False \ - -e ES_SETTING_DISCOVERY_TYPE=single-node \ - --network iati-standard-website \ - 'docker.elastic.co/elasticsearch/elasticsearch:8.15.3' && \ docker run --restart always --name website -d -p 5000:5000 \ --log-driver 'json-file' \ --log-opt max-size=100m \ --log-opt max-file=3 \ - --network iati-standard-website \ - --link elasticsearch:elasticsearch \ -e DJANGO_SETTINGS_MODULE='iati.settings.dev_public' \ -e SECRET_KEY='${{ secrets.DEV_SECRET_KEY }}' \ -e DATABASE_NAME='${{ secrets.DEV_DATABASE_NAME }}' \ @@ -340,10 +332,10 @@ jobs: --ssh-key-values ${{ secrets.DEV_PUB_KEYS }} && \ export NEW_IP=$(az vm list-ip-addresses --name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" --resource-group "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" --query [].virtualMachine.network[].publicIpAddresses[][].ipAddress --output tsv) && \ echo "NEW_IP=$NEW_IP" >> $GITHUB_ENV && \ - az postgres server firewall-rule update \ + az postgres flexible-server firewall-rule update \ --resource-group "${{ secrets.PROD_PSQL_RESOURCE_GROUP }}" \ - --server-name "${{ secrets.PROD_PSQL_NAME }}" \ - --name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \ + --name "${{ secrets.PROD_PSQL_NAME }}" \ + --rule-name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \ --start-ip-address $NEW_IP \ --end-ip-address $NEW_IP && \ az network nsg rule create \ @@ -408,20 +400,12 @@ jobs: --name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \ --command-id RunShellScript \ --scripts "\ - docker network create iati-standard-website && \ docker login -u '${{ env.REGISTRY_USERNAME }}' -p '${{ env.REGISTRY_PASSWORD }}' ${{ env.REGISTRY_LOGIN_SERVER }} && \ docker pull '${{ env.REGISTRY_LOGIN_SERVER }}/${{env.STAGE}}-${{env.NAME}}:${{ env.TAG }}' && \ - docker run --restart always --name elasticsearch -d \ - -e ES_SETTING_XPACK_SECURITY_ENABLED=False \ - -e ES_SETTING_DISCOVERY_TYPE=single-node \ - --network iati-standard-website \ - 'docker.elastic.co/elasticsearch/elasticsearch:8.15.3' && \ docker run --restart always --name website -d -p 5000:5000 \ --log-driver 'json-file' \ --log-opt max-size=100m \ --log-opt max-file=3 \ - --network iati-standard-website \ - --link elasticsearch:elasticsearch \ -e DJANGO_SETTINGS_MODULE='iati.settings.production' \ -e SECRET_KEY='${{ secrets.PROD_SECRET_KEY }}' \ -e DATABASE_NAME='${{ secrets.PROD_DATABASE_NAME }}' \ diff --git a/Dockerfile b/Dockerfile index 3ab3a3f19..2bab6f60b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONIOENCODING utf_8 RUN apk update -RUN apk add --no-cache bash curl +RUN apk add --no-cache bash # Init engine @@ -40,6 +40,52 @@ RUN apk add --no-cache jpeg-dev zlib-dev RUN apk add --no-cache postgresql-dev RUN apk add --no-cache libmemcached-dev zlib-dev +# Elasticsearch from https://github.com/blacktop/docker-elasticsearch-alpine/blob/master/6.8/Dockerfile + +RUN apk add --no-cache openjdk8-jre su-exec + +ENV VERSION 6.8.23 +ENV DOWNLOAD_URL "https://artifacts.elastic.co/downloads/elasticsearch" +ENV ES_TARBAL "${DOWNLOAD_URL}/elasticsearch-oss-${VERSION}.tar.gz" +# ENV EXPECTED_SHA_URL "${DOWNLOAD_URL}/elasticsearch-oss-${VERSION}.tar.gz.sha512" +ENV ES_TARBALL_SHA "14dbb2809b06499373c3ec5035d829d62255c2c93103618fbfe3d7d03cecf8847f654e83c78f765f23224126ff18ed713b959857e8ecf435c475b11bcd143d3f" +RUN apk add --no-cache -t .build-deps wget ca-certificates gnupg openssl \ + && set -ex \ + && cd /tmp \ + && echo "===> Install Elasticsearch..." \ + && wget --progress=bar:force -O elasticsearch.tar.gz "$ES_TARBAL"; \ + if [ "$ES_TARBALL_SHA" ]; then \ + echo "$ES_TARBALL_SHA *elasticsearch.tar.gz" | sha512sum -c -; \ + fi; \ + tar -xf elasticsearch.tar.gz \ + && ls -lah \ + && mv elasticsearch-$VERSION /usr/share/elasticsearch \ + && adduser -D -h /usr/share/elasticsearch elasticsearch \ + && echo "===> Creating Elasticsearch Paths..." \ + && for path in \ + /usr/share/elasticsearch/data \ + /usr/share/elasticsearch/logs \ + /usr/share/elasticsearch/config \ + /usr/share/elasticsearch/config/scripts \ + /usr/share/elasticsearch/tmp \ + /usr/share/elasticsearch/plugins \ + ; do \ + mkdir -p "$path"; \ + chown -R elasticsearch:elasticsearch "$path"; \ + done \ + && rm -rf /tmp/* \ + && apk del --purge .build-deps + +COPY config/elastic/elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml +COPY config/elastic/log4j2.properties /usr/share/elasticsearch/config/log4j2.properties +RUN chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/config + +RUN mkdir -p /var/log/messages +RUN apk add logrotate +COPY config/elastic/logrotate /etc/logrotate.d/elasticsearch +RUN chmod 644 /etc/logrotate.d/elasticsearch +COPY config/elastic/elasticsearch.service /etc/init.d/elasticsearch.service + # Web app dependencies RUN mkdir -p /usr/src/app diff --git a/Dockerfile_deploy b/Dockerfile_deploy index bd4ec864d..625ba2601 100644 --- a/Dockerfile_deploy +++ b/Dockerfile_deploy @@ -30,6 +30,6 @@ COPY manage.py /usr/src/app/manage.py COPY pytest.ini /usr/src/app/pytest.init COPY setup.cfg /usr/src/app/setup.cfg -ENV ELASTICSEARCH_URL=http://elasticsearch:9200 +ENV ELASTICSEARCH_URL=http://localhost:9200 ENV GUNICORN_WORKERS=5 ENV COMPRESS_ENABLED='True' diff --git a/README.md b/README.md index d7a39b0d1..e69fd7881 100644 --- a/README.md +++ b/README.md @@ -19,12 +19,6 @@ This repository hosts the new IATI website based on Django and Wagtail CMS. A Po - Set a SECRET_KEY -A limit on your kernel must be increased. There are ways to do this permanently, but to do so temporarily: - -``` -sudo sysctl -w vm.max_map_count=262144 -``` - Build the project. The following will build linked `web` and `postgres` containers. ``` diff --git a/config/elastic/elasticsearch.service b/config/elastic/elasticsearch.service new file mode 100755 index 000000000..61854c371 --- /dev/null +++ b/config/elastic/elasticsearch.service @@ -0,0 +1,58 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: elasticsearch +# Required-Start: $all +# Required-Stop: $all +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Starts elasticsearch +# Description: Starts elasticsearch using start-stop-daemon +### END INIT INFO + +ES_HOME=/usr/share/elasticsearch +JAVA_HOME=/usr/lib/jvm/default-jvm/ +ES_MIN_MEM=256m +ES_MAX_MEM=1g +ES_JAVA_OPTS="-Xms256m -Xmx1g" +DAEMON=$ES_HOME/bin/elasticsearch +NAME=elasticsearch +DESC=elasticsearch +PID_FILE=$ES_HOME/$NAME.pid +DAEMON_OPTS="--pidfile $PID_FILE" + +test -x $DAEMON || exit 0 + +set -e + +case "$1" in + start) + echo -n "Starting $DESC: " + if start-stop-daemon --start --background --user elasticsearch --pidfile $PID_FILE --exec $DAEMON -- $DAEMON_OPTS + then + echo "started." + else + echo "failed." + fi + ;; + stop) + echo -n "Stopping $DESC: " + if start-stop-daemon --stop --user elasticsearch --pidfile $PID_FILE + then + echo "stopped." + else + echo "failed." + fi + ;; + restart|force-reload) + ${0} stop + sleep 0.5 + ${0} start + ;; + *) + N=/etc/init.d/$NAME + echo "Usage: $N {start|stop|restart|force-reload}" >&2 + exit 1 + ;; +esac + +exit 0 diff --git a/config/elastic/elasticsearch.yml b/config/elastic/elasticsearch.yml new file mode 100644 index 000000000..b56b4e97a --- /dev/null +++ b/config/elastic/elasticsearch.yml @@ -0,0 +1,3 @@ +cluster.name: "docker-cluster" +network.host: 0.0.0.0 +discovery.type: single-node diff --git a/config/elastic/log4j2.properties b/config/elastic/log4j2.properties new file mode 100644 index 000000000..9ad290ad8 --- /dev/null +++ b/config/elastic/log4j2.properties @@ -0,0 +1,9 @@ +status = error + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console diff --git a/config/elastic/logrotate b/config/elastic/logrotate new file mode 100644 index 000000000..5c96e06d8 --- /dev/null +++ b/config/elastic/logrotate @@ -0,0 +1,15 @@ +/var/log/elasticsearch/*.log { + daily + rotate 50 + size 50M + copytruncate + compress + delaycompress + missingok + notifempty + create 644 elasticsearch elasticsearch +} +/var/log/gunicorn/*.log { + size 10M + copytruncate +} \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 75de23020..0ecd3b604 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -6,22 +6,13 @@ services: - POSTGRES_HOST_AUTH_METHOD=trust ports: - 5432:5432 - - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.15.3 - environment: - - ES_SETTING_XPACK_SECURITY_ENABLED=False - - ES_SETTING_DISCOVERY_TYPE=single-node - ports: - - 9200:9200 - web: build: context: . environment: - DATABASE_URL=postgres://postgres:@postgres:5432/postgres - SECRET_KEY=enter-a-long-unguessable-string-here - - ELASTICSEARCH_URL=http://elasticsearch:9200 + - ELASTICSEARCH_URL=http://localhost:9200 - DJANGO_SETTINGS_MODULE=iati.settings.dev - GUNICORN_WORKERS=1 - GITHUB_TOKEN @@ -29,16 +20,9 @@ services: - RECAPTCHA_PRIVATE_KEY - DEBUG_SERVER - COMPRESS_ENABLED - # I'm not sure why these links are needed; I thought Docker defaults would just do this for us but without it - # the Web container can't see the others. links: - - postgres:postgres - - elasticsearch:elasticsearch + - postgres ports: - 5000:5000 volumes: - ./:/usr/src/app - -networks: - default: - name: iati-standard-website diff --git a/entrypoint.sh b/entrypoint.sh index 74076a973..c628b560b 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -13,10 +13,12 @@ else done fi -until curl --output /dev/null --silent --head --fail ${ELASTICSEARCH_URL}; do - >&2 echo "Elasticsearch is unavailable - sleeping" - sleep 10 -done +if [[ -z "${ELASTICSEARCH_URL}" ]]; then + >&2 echo "Skipping Elasticsearch" +else + >&2 echo "Starting Elasticsearch" + rc-service elasticsearch.service start +fi if [[ -z "${DEBUG_SERVER}" ]]; then gunicorn iati.wsgi:application --bind 0.0.0.0:5000 --workers $GUNICORN_WORKERS >> /var/log/gunicorn/gunicorn.log 2>&1 & diff --git a/iati/settings/base.py b/iati/settings/base.py index 2fc327783..ce1e4af3f 100644 --- a/iati/settings/base.py +++ b/iati/settings/base.py @@ -554,7 +554,7 @@ # Search settings WAGTAILSEARCH_BACKENDS = { 'default': { - 'BACKEND': 'wagtail.search.backends.elasticsearch8', + 'BACKEND': 'wagtail.search.backends.elasticsearch6', 'URLS': [os.getenv('ELASTICSEARCH_URL', 'http://localhost:9200')], 'INDEX': 'iati', }, diff --git a/requirements.in b/requirements.in index 0754a81c6..0163ce8b9 100644 --- a/requirements.in +++ b/requirements.in @@ -14,7 +14,7 @@ django-recaptcha3 @ git+https://github.com/bartsanchez/django-recaptcha3.git@313 django-storages[azure] django-widget-tweaks Django>=4.2,<4.3 -elasticsearch>=8,<9 +elasticsearch>=6.8,<7 gunicorn opencensus-ext-azure opencensus-ext-django diff --git a/requirements.txt b/requirements.txt index 967eb32d8..ae3774bc2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,9 +29,7 @@ bleach==6.1.0 cachetools==5.4.0 # via google-auth certifi==2024.7.4 - # via - # elastic-transport - # requests + # via requests cffi==1.17.0 # via # cryptography @@ -109,9 +107,7 @@ djangorestframework==3.15.2 # via wagtail draftjs-exporter==2.1.7 # via wagtail -elastic-transport==8.15.1 - # via elasticsearch -elasticsearch==8.15.1 +elasticsearch==6.8.2 # via -r requirements.in et-xmlfile==1.1.0 # via openpyxl @@ -240,7 +236,7 @@ tzdata==2024.1 # via -r requirements.in urllib3==2.2.2 # via - # elastic-transport + # elasticsearch # pygithub # requests wagtail==5.2.6 diff --git a/requirements_dev.txt b/requirements_dev.txt index 570154890..ec760b291 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -49,7 +49,6 @@ cachetools==5.4.0 certifi==2024.7.4 # via # -r requirements.txt - # elastic-transport # requests # selenium cffi==1.17.0 @@ -160,11 +159,7 @@ draftjs-exporter==2.1.7 # via # -r requirements.txt # wagtail -elastic-transport==8.15.1 - # via - # -r requirements.txt - # elasticsearch -elasticsearch==8.15.1 +elasticsearch==6.8.2 # via -r requirements.txt et-xmlfile==1.1.0 # via @@ -444,7 +439,7 @@ tzdata==2024.1 urllib3[socks]==2.2.2 # via # -r requirements.txt - # elastic-transport + # elasticsearch # pygithub # pytest-splinter # requests