Merge branch 'milesmcc:master' into master

.devcontainer/Dockerfile

@@ -0,0 +1,15 @@
+FROM mcr.microsoft.com/devcontainers/python:1-3.11-bullseye
+
+ENV PYTHONUNBUFFERED 1
+
+# [Optional] If your requirements rarely change, uncomment this section to add them to the image.
+# COPY requirements.txt /tmp/pip-tmp/
+# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
+#    && rm -rf /tmp/pip-tmp
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>
+
+
+

.devcontainer/devcontainer.json

@@ -0,0 +1,29 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/postgres
+{
+	"name": "Python 3 & PostgreSQL",
+	"dockerComposeFile": "docker-compose.yml",
+	"service": "app",
+	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+	"features": {
+		"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {},
+		"ghcr.io/devcontainers/features/node:1": {},
+		"ghcr.io/devcontainers-contrib/features/poetry:2": {}
+	}
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// This can be used to network with other containers or the host.
+	// "forwardPorts": [5000, 5432],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "pip install --user -r requirements.txt",
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,35 @@
+version: '3.8'
+
+services:
+  app:
+    build:
+      context: ..
+      dockerfile: .devcontainer/Dockerfile
+
+    volumes:
+      - ../..:/workspaces:cached
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: sleep infinity
+
+    # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function.
+    network_mode: service:db
+
+    # Use "forwardPorts" in **devcontainer.json** to forward an app port locally.
+    # (Adding the "ports" property to this file will not forward from a Codespace.)
+
+  db:
+    image: postgres:latest
+    restart: unless-stopped
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_DB: postgres
+      POSTGRES_PASSWORD: postgres
+
+    # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally.
+    # (Adding the "ports" property to this file will not forward from a Codespace.)
+
+volumes:
+  postgres-data:

.github/workflows/build-docker-edge.yml

@@ -9,40 +9,50 @@ jobs:
   publish_to_docker_hub:
     runs-on: ubuntu-latest
     steps:
+      - name: Create Docker Metadata
+        id: metadata
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            milesmcc/shynet
+            ghcr.io/milesmcc/shynet
+          tags:
+            type=edge
+
       - name: Set swap space
         uses: pierotofy/set-swap-space@master
         with:
           swap-size-gb: 5
 
       - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Prepare tags
-        id: prep
-        run: |
-          DOCKER_IMAGE=milesmcc/shynet
-          TAGS="${DOCKER_IMAGE}:edge"
-          echo ::set-output name=tags::${TAGS}
-          
+        uses: actions/checkout@v4
+
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        
+        uses: docker/setup-qemu-action@v3
+
       - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-
+        uses: docker/setup-buildx-action@v3
+
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_TOKEN }}
 
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Build and push advanced image
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: ./Dockerfile
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: ${{ steps.prep.outputs.tags }}
+          tags: ${{ steps.metadata.outputs.tags }}
+          labels: ${{ steps.metadata.outputs.labels }}

.github/workflows/build-docker-manual.yml

@@ -9,40 +9,50 @@ jobs:
   publish_to_docker_hub:
     runs-on: ubuntu-latest
     steps:
+      - name: Create Docker Metadata
+        id: metadata
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            milesmcc/shynet
+            ghcr.io/milesmcc/shynet
+          tags:
+            type=raw,value=${{ github.event.inputs.tag }}
+
       - name: Set swap space
         uses: pierotofy/set-swap-space@master
         with:
           swap-size-gb: 5
 
       - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Prepare tags
-        id: prep
-        run: |
-          DOCKER_IMAGE=milesmcc/shynet
-          TAGS="${DOCKER_IMAGE}:${{ github.event.inputs.tag }}"
-          echo ::set-output name=tags::${TAGS}
-          
+        uses: actions/checkout@v4
+
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        
+        uses: docker/setup-qemu-action@v3
+
       - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-
+        uses: docker/setup-buildx-action@v3
+
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_TOKEN }}
 
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Build and push advanced image
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: ./Dockerfile
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: ${{ steps.prep.outputs.tags }}
+          tags: ${{ steps.metadata.outputs.tags }}
+          labels: ${{ steps.metadata.outputs.labels }}

.github/workflows/build-docker-release.yml

@@ -9,41 +9,52 @@ jobs:
   publish_to_docker_hub:
     runs-on: ubuntu-latest
     steps:
+      # https://github.com/docker/metadata-action/tree/v4/#typeref
+      - name: Create Docker Metadata
+        id: metadata
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            milesmcc/shynet
+            ghcr.io/milesmcc/shynet
+          tags:
+            type=raw,value=latest
+            type=ref,event=tag
+
       - name: Set swap space
         uses: pierotofy/set-swap-space@master
         with:
           swap-size-gb: 5
 
       - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Prepare tags
-        id: prep
-        run: |
-          DOCKER_IMAGE=milesmcc/shynet
-          VERSION=${GITHUB_REF#refs/tags/}
-          TAGS="${DOCKER_IMAGE}:${VERSION},${DOCKER_IMAGE}:latest"
-          echo ::set-output name=tags::${TAGS}
-          
+        uses: actions/checkout@v4
+
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_TOKEN }}
 
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Build and push advanced image
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: ./Dockerfile
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: ${{ steps.prep.outputs.tags }}
+          tags: ${{ steps.metadata.outputs.tags }}
+          labels: ${{ steps.metadata.outputs.labels }}

.github/workflows/run-tests.yml

@@ -27,11 +27,12 @@ jobs:
       - name: Run image
         uses: abatilo/[email protected]
         with:
-          poetry-version: 1.1.6
+          poetry-version: 1.2.2
+      - name: Preinstall dependencies (temporary)
+        run: poetry run pip install "Cython<3.0" "pyyaml==5.4.1" "django-allauth==0.45.0" --no-build-isolation
       - name: Install dependencies 
         run: poetry install
-
       - name: Django Testing project
         run: |
           cp TEMPLATE.env .env
-          poetry run ./shynet/manage.py test
+          poetry run ./shynet/manage.py test

Dockerfile

@@ -7,36 +7,49 @@ WORKDIR /usr/src/shynet
 ARG GF_UID="500"
 ARG GF_GID="500"
 RUN apk update && \
-	apk add gettext curl bash npm libffi-dev rust cargo
+	apk add --no-cache gettext bash npm postgresql-libs && \
+	test "$(arch)" != "x86_64" && apk add libffi-dev rust cargo || echo "amd64 build, skipping Rust installation"
 	# libffi-dev and rust are used for the cryptography package,
 	# which we indirectly rely on. Necessary for aarch64 support.
 
+# MaxMind scans GitHub for exposed license keys and deactivates them. This
+# (encoded) license key is intened to be public; it is not configured with any
+# billing, and can only access MaxMind's public databases. These databases used
+# to be available for download without authentication, but they are now auth
+# gated. It is very important that the Shynet community have a simple,
+# easily-pullable Docker image with all "batteries included." As a result, we
+# intentionally "expose" this API key to the community. The "fix" is for MaxMind
+# to offer these free, public datasets in a way that doesn't require an API key.
+ARG MAXMIND_LICENSE_KEY_BASE64="Z2tySDgwX1htSEtmS3d4cDB1SnlMWTdmZ1hMMTQxNzRTQ2o5X21taw=="
+
+RUN echo $MAXMIND_LICENSE_KEY_BASE64 > .mmdb_key
+
 # Collect GeoIP Database
-RUN curl -m 180 "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-ASN&license_key=kKG1ebhL3iWVd0iv&suffix=tar.gz" | tar -xvz -C /tmp && \
-	curl -m 180 "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=kKG1ebhL3iWVd0iv&suffix=tar.gz" | tar -xvz -C /tmp && \
+COPY assets/GeoLite2-ASN_20191224.tar.gz GeoLite2-ASN.tar.gz
+COPY assets/GeoLite2-City_20191224.tar.gz GeoLite2-City.tar.gz
+RUN apk add --no-cache curl && \
+	tar -xvz -C /tmp < GeoLite2-ASN.tar.gz && \
+	tar -xvz -C /tmp < GeoLite2-City.tar.gz && \
 	mv /tmp/GeoLite2*/*.mmdb /etc && \
-	apk del curl
+	rm GeoLite2-ASN.tar.gz GeoLite2-City.tar.gz && \
+	apk --purge del curl
 
 # Move dependency files
 COPY poetry.lock pyproject.toml ./
 COPY package.json package-lock.json ../
 # Django expects node_modules to be in its parent directory.
 
-# Install more dependencies
-RUN apk add --no-cache postgresql-libs && \
-	apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev && \
+# Install more dependencies and cleanup build dependencies afterwards
+RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev libressl-dev libffi-dev && \
 	npm i -P --prefix .. && \
-	pip install poetry==1.2.2
-
-# Install Python dependencies
-RUN poetry config virtualenvs.create false && \
-    poetry install --no-dev --no-interaction --no-ansi
+	pip install poetry==1.2.2 && \
+	poetry config virtualenvs.create false && \
+	poetry run pip install "Cython<3.0" "pyyaml==5.4.1" "django-allauth==0.45.0" --no-build-isolation && \
+	poetry install --no-dev --no-interaction --no-ansi && \
+	apk --purge del .build-deps
 
-# Cleanup dependencies & setup user group
-RUN apk --purge del .build-deps && \
-	rm -rf /var/lib/apt/lists/* && \
-	rm /var/cache/apk/* && \
-	addgroup --system -g $GF_GID appgroup && \
+# Setup user group
+RUN addgroup --system -g $GF_GID appgroup && \
 	adduser appuser --system --uid $GF_UID -G appgroup && \
 	mkdir -p /var/local/shynet/db/ && \
 	chown -R appuser:appgroup /var/local/shynet

GUIDE.md

@@ -20,7 +20,7 @@
 
 **If you install Shynet, you should strongly consider enabling notifications when new versions are released.** You can do this under the "Watch" tab on GitHub (above). This will ensure that you are notified when new versions are available, some of which may be security updates. (Shynet will never automatically update itself.)
 
-> **When you do update, read the release notes!** These will tell you if you need to make changes to your deployment. (E.g., Shynet 0.13.0 requires additional configuration.)
+> **When you do update, read the release notes!** These will tell you if you need to make changes to your deployment. (E.g., Shynet 0.13.1 requires additional configuration.)
 
 ## Installation
 

TEMPLATE.env

@@ -102,7 +102,7 @@ AGGRESSIVE_HASH_SALTING=True
 #  - https://www.openstreetmap.org/?mlat=$LATITUDE&mlon=$LONGITUDE (default)
 #  - https://www.google.com/maps/search/?api=1&query=$LATITUDE,$LONGITUDE
 #  - https://www.mapquest.com/near-$LATITUDE,$LONGITUDE
-LOCATION_URL=https://www.openstreetmap.org/?mlat=$LATITUDE&mlon=$LONGITUDE
+LOCATION_URL=https://www.openstreetmap.org/?mlat=$$LATITUDE&mlon=$$LONGITUDE
 
 # How many services should be displayed on dashboard page?
 # Set to big number if you don't want pagination at all.

assets/README.md

@@ -0,0 +1,3 @@
+This file contains GeoIP databases accurate as of 2019. We'd use newer databases, but MaxMind gates their free GeoIP databases behind a license key citing (an overly strict interpretation of) global data privacy regulation.
+
+These files are the most recent version licensed under Creative Commons, pulled from the Internet Archive. For more information, see https://forum.matomo.org/t/maxmind-is-changing-access-to-free-geolite2-databases/35439/2.