Merge pull request #884 from bunkerity/dev

Merge branch "dev" into branch "ui"

.github/workflows/codeql.yml

@@ -35,12 +35,12 @@ jobs:
           python -m pip install --no-cache-dir --require-hashes -r src/common/db/requirements.txt
           echo "CODEQL_PYTHON=$(which python)" >> $GITHUB_ENV
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12
+        uses: github/codeql-action/init@0b21cf2492b6b02c465a3e5d7c473717ad7721ba # v3.23.1
         with:
           languages: ${{ matrix.language }}
           config-file: ./.github/codeql.yml
           setup-python-dependencies: false
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12
+        uses: github/codeql-action/analyze@0b21cf2492b6b02c465a3e5d7c473717ad7721ba # v3.23.1
         with:
           category: "/language:${{matrix.language}}"

.github/workflows/container-build.yml

@@ -56,6 +56,8 @@ jobs:
           echo "$SSH_KEY" > ~/.ssh/id_rsa_arm
           chmod 600 ~/.ssh/id_rsa_arm
           echo "$SSH_CONFIG" | sed "s/SSH_IP/$SSH_IP/g" > ~/.ssh/config
+          echo "ServerAliveInterval 60" >> ~/.ssh/config
+          echo "ServerAliveCountMax 10" >> ~/.ssh/config
         env:
           SSH_KEY: ${{ secrets.ARM_SSH_KEY }}
           SSH_IP: ${{ secrets.ARM_SSH_IP }}
@@ -84,7 +86,7 @@ jobs:
       # Compute metadata
       - name: Extract metadata
         id: meta
-        uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v5.4.0
+        uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v5.5.0
         with:
           images: bunkerity/${{ inputs.IMAGE }}
       # Build cached image

.github/workflows/create-arm.yml

@@ -46,7 +46,7 @@ jobs:
           default-organization-id: ${{ secrets.SCW_DEFAULT_ORGANIZATION_ID }}
       - name: Extract ARM type
         run: |
-          TYPE=$(echo "$JSON" | jq '.servers | with_entries(select(.key | contains("AMP"))) | with_entries(select(.value.availability != "shortage")) | keys[] | select(. | test("^AMP2-C[0-9]+$")) | sub("AMP2-C"; "") | tonumber' | sort -n | tail -n 1 | xargs -I {} echo "AMP2-C{}")
+          TYPE=$(echo "$JSON" | jq '.servers | with_entries(select(.key | contains("COPARM1-"))) | with_entries(select(.value.availability != "shortage")) | keys[] | select(. | test("^COPARM1-[0-9]+C-[0-9]+G$"))' | sed 's/"//g' | cut -d '-' -f 2,3 | sort -g | tail -n 1 | xargs -I {} echo "COPARM1-{}")
           echo "Type is $TYPE"
           echo "TYPE=$TYPE" >> "$GITHUB_ENV"
         env:
@@ -81,6 +81,6 @@ jobs:
           SSH_IP: ${{ fromJson(steps.scw.outputs.json).public_ip.address }}
           SSH_CONFIG: ${{ secrets.ARM_SSH_CONFIG }}
       - name: Install Docker
-        run: ssh root@$SSH_IP "curl -fsSL https://test.docker.com -o test-docker.sh ; sh test-docker.sh"
+        run: ssh root@$SSH_IP "curl -fsSL https://test.docker.com -o test-docker.sh ; sh test-docker.sh ; echo 'ClientAliveInterval 60' >> /etc/ssh/sshd_config ; echo 'ClientAliveCountMax 0' >> /etc/ssh/sshd_config ; systemctl restart ssh"
         env:
           SSH_IP: ${{ fromJson(steps.scw.outputs.json).public_ip.address }}

.github/workflows/dev.yml

@@ -13,6 +13,7 @@ jobs:
       contents: read
       packages: write
     strategy:
+      fail-fast: false
       matrix:
         image: [bunkerweb, scheduler, autoconf, ui]
         include:
@@ -84,7 +85,6 @@ jobs:
 
   # Core tests
   prepare-tests-core:
-    needs: [build-containers, build-packages]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -96,7 +96,7 @@ jobs:
     outputs:
       tests: ${{ steps.set-matrix.outputs.tests }}
   tests-core:
-    needs: prepare-tests-core
+    needs: [build-containers, prepare-tests-core]
     strategy:
       fail-fast: false
       matrix:
@@ -106,7 +106,7 @@ jobs:
       TEST: ${{ matrix.test }}
       RELEASE: dev
   tests-core-linux:
-    needs: prepare-tests-core
+    needs: [build-packages, prepare-tests-core]
     strategy:
       fail-fast: false
       matrix:

.github/workflows/doc-to-pdf.yml

@@ -18,8 +18,8 @@ jobs:
         uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
         with:
           python-version: "3.10"
-      - name: Install doc requirements
-        run: pip install --no-cache-dir --require-hashes -r docs/requirements.txt
+      - name: Install doc dependencies
+        run: pip install --no-cache-dir --require-hashes -r docs/requirements.txt && sudo apt install -y libcairo2-dev libfreetype6-dev libffi-dev libjpeg-dev libpng-dev libz-dev
       - name: Install chromium
         run: sudo apt install chromium-browser
       - name: Install node
@@ -32,7 +32,7 @@ jobs:
         run: mkdocs serve & sleep 10
       - name: Run pdf script
         run: node docs/misc/pdf.js http://localhost:8000/print_page/ BunkerWeb_documentation_v${{ inputs.VERSION }}.pdf 'BunkerWeb documentation v${{ inputs.VERSION }}'
-      - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      - uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0
         with:
           name: BunkerWeb_documentation_v${{ inputs.VERSION }}.pdf
           path: BunkerWeb_documentation_v${{ inputs.VERSION }}.pdf

.github/workflows/linux-build.yml

@@ -65,6 +65,8 @@ jobs:
           echo "$SSH_KEY" > ~/.ssh/id_rsa_arm
           chmod 600 ~/.ssh/id_rsa_arm
           echo "$SSH_CONFIG" | sed "s/SSH_IP/$SSH_IP/g" > ~/.ssh/config
+          echo "ServerAliveInterval 60" >> ~/.ssh/config
+          echo "ServerAliveCountMax 10" >> ~/.ssh/config
         env:
           SSH_KEY: ${{ secrets.ARM_SSH_KEY }}
           SSH_IP: ${{ secrets.ARM_SSH_IP }}
@@ -127,15 +129,15 @@ jobs:
           scp -r root@arm:/root/package-${{ inputs.LINUX }} ./package-${{ inputs.LINUX }}
         env:
           LARCH: ${{ env.LARCH }}
-      - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      - uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0
         with:
           name: package-${{ inputs.LINUX }}-${{ env.LARCH }}
           path: package-${{ inputs.LINUX }}/*.${{ inputs.PACKAGE }}
       # Build test image
       - name: Extract metadata
         if: inputs.TEST == true
         id: meta
-        uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v5.4.0
+        uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v5.5.0
         with:
           images: ghcr.io/bunkerity/${{ inputs.LINUX }}-tests:${{ inputs.RELEASE }}
       - name: Build test image

.github/workflows/push-doc.yml

@@ -32,8 +32,8 @@ jobs:
       - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
         with:
           python-version: "3.10"
-      - name: Install doc requirements
-        run: pip install --no-cache-dir --require-hashes -r docs/requirements.txt
+      - name: Install doc dependencies
+        run: pip install --no-cache-dir --require-hashes -r docs/requirements.txt && sudo apt install -y libcairo2-dev libfreetype6-dev libffi-dev libjpeg-dev libpng-dev libz-dev
       - name: Push doc
         run: mike deploy --update-aliases --push --alias-type=copy ${{ inputs.VERSION }} ${{ inputs.ALIAS }}
       - name: Set default doc

.github/workflows/push-docker.yml

@@ -51,6 +51,8 @@ jobs:
           echo "$SSH_KEY" > ~/.ssh/id_rsa_arm
           chmod 600 ~/.ssh/id_rsa_arm
           echo "$SSH_CONFIG" | sed "s/SSH_IP/$SSH_IP/g" > ~/.ssh/config
+          echo "ServerAliveInterval 60" >> ~/.ssh/config
+          echo "ServerAliveCountMax 10" >> ~/.ssh/config
         env:
           SSH_KEY: ${{ secrets.ARM_SSH_KEY }}
           SSH_IP: ${{ secrets.ARM_SSH_IP }}
@@ -63,7 +65,7 @@ jobs:
       # Compute metadata
       - name: Extract metadata
         id: meta
-        uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v5.4.0
+        uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v5.5.0
         with:
           images: bunkerity/${{ inputs.IMAGE }}
       # Build and push

.github/workflows/push-github.yml

@@ -19,7 +19,7 @@ jobs:
       # Get PDF doc
       - name: Get documentation
         if: inputs.VERSION != 'testing'
-        uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0
+        uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
         with:
           name: BunkerWeb_documentation_v${{ inputs.VERSION }}.pdf
       # Create tag

.github/workflows/push-packagecloud.yml

@@ -42,18 +42,18 @@ jobs:
       - name: Check out repository code
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: Install ruby
-        uses: ruby/setup-ruby@360dc864d5da99d54fcb8e9148c14a84b90d3e88 # v1.165.1
+        uses: ruby/setup-ruby@5daca165445f0ae10478593083f72ca2625e241d # v1.169.0
         with:
           ruby-version: "3.0"
       - name: Install packagecloud
         run: gem install package_cloud
       # Download packages
-      - uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0
+      - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
         if: inputs.LINUX != 'el'
         with:
           name: package-${{ inputs.LINUX }}-${{ inputs.PACKAGE_ARCH }}
           path: /tmp/${{ inputs.LINUX }}
-      - uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0
+      - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
         if: inputs.LINUX == 'el'
         with:
           name: package-rhel-${{ inputs.PACKAGE_ARCH }}

.github/workflows/scorecards-analysis.yml

@@ -25,6 +25,6 @@ jobs:
           results_format: sarif
           publish_results: true
       - name: "Upload SARIF results to code scanning"
-        uses: github/codeql-action/upload-sarif@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12
+        uses: github/codeql-action/upload-sarif@0b21cf2492b6b02c465a3e5d7c473717ad7721ba # v3.23.1
         with:
           sarif_file: results.sarif

.github/workflows/staging-create-infra.yml

@@ -55,7 +55,7 @@ jobs:
         if: always()
         env:
           SECRET_KEY: ${{ secrets.SECRET_KEY }}
-      - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      - uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0
         if: always()
         with:
           name: tf-${{ inputs.TYPE }}

.github/workflows/staging-delete-infra.yml

@@ -23,7 +23,7 @@ jobs:
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: Install terraform
         uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3.0.0
-      - uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0
+      - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
         with:
           name: tf-${{ inputs.TYPE }}
           path: /tmp

.github/workflows/staging-tests.yml

@@ -43,7 +43,7 @@ jobs:
         if: inputs.TYPE == 'swarm'
       - name: Install test dependencies
         run: pip3 install --no-cache-dir --require-hashes --no-deps -r tests/requirements.txt
-      - uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0
+      - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
         with:
           name: tf-k8s
           path: /tmp

.github/workflows/staging.yml

@@ -85,7 +85,6 @@ jobs:
       SECRET_KEY: ${{ secrets.SECRET_KEY }}
       K8S_IP: ${{ secrets.K8S_IP }}
   prepare-tests-core:
-    needs: [codeql, build-containers, build-packages]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -130,7 +129,7 @@ jobs:
       RUNS_ON: ${{ matrix.runs_on }}
     secrets: inherit
   tests-core:
-    needs: prepare-tests-core
+    needs: [build-containers, prepare-tests-core]
     strategy:
       fail-fast: false
       matrix:
@@ -140,7 +139,7 @@ jobs:
       TEST: ${{ matrix.test }}
       RELEASE: testing
   tests-core-linux:
-    needs: prepare-tests-core
+    needs: [build-packages, prepare-tests-core]
     strategy:
       fail-fast: false
       matrix:

.github/workflows/test-core-linux.yml

@@ -97,7 +97,7 @@ jobs:
         run: |
           export MAKEFLAGS="-j $(nproc)"
           pip install --no-cache-dir --ignore-installed --require-hashes -r src/deps/requirements-deps.txt
-          MAKEFLAGS="-j $(nproc)" find tests/core -name "requirements.txt" -exec pip install --no-cache-dir --require-hashes --no-deps -r {} \;
-          cd ./tests/core/${{ inputs.TEST }}
+          cd tests/core/${{ inputs.TEST }}
+          find . -name "requirements.txt" -exec pip install --no-cache-dir --require-hashes --no-deps -r {} \;
           sudo truncate -s 0 /var/log/bunkerweb/error.log
           ./test.sh "linux"

.github/workflows/tests-ui-linux.yml

@@ -66,14 +66,13 @@ jobs:
       - name: Fix version without a starting number
         if: inputs.RELEASE == 'testing' || inputs.RELEASE == 'dev' || inputs.RELEASE == 'ui'
         run: echo "force-bad-version" | sudo tee -a /etc/dpkg/dpkg.cfg
-      - name: Install BunkerWeb
-        run: sudo apt install -fy /tmp/bunkerweb.deb
       - name: Edit configuration files
         run: |
           # Misc
           echo "127.0.0.1 www.example.com" | sudo tee -a /etc/hosts
           echo "127.0.0.1 app1.example.com" | sudo tee -a /etc/hosts
           # BunkerWeb
+          sudo mkdir -p /etc/bunkerweb
           echo "SERVER_NAME=" | sudo tee /etc/bunkerweb/variables.env
           echo "HTTP_PORT=80" | sudo tee -a /etc/bunkerweb/variables.env
           echo 'DNS_RESOLVERS=9.9.9.9 8.8.8.8 8.8.4.4' | sudo tee -a /etc/bunkerweb/variables.env
@@ -92,6 +91,8 @@ jobs:
 
           sudo chown nginx:nginx /etc/bunkerweb/variables.env /etc/bunkerweb/ui.env
           sudo chmod 777 /etc/bunkerweb/variables.env /etc/bunkerweb/ui.env
+      - name: Install BunkerWeb
+        run: sudo apt install -fy /tmp/bunkerweb.deb
       - name: Run tests
         run: |
           export MAKEFLAGS="-j $(nproc)"

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
-exclude: (^LICENSE.md$|^src/VERSION$|^env/|^src/(bw/misc/root-ca.pem$|deps/src/|common/core/modsecurity/files|ui/static/js/(editor/|utils/purify/|tsparticles\.bundle\.min\.js))|\.(svg|drawio|patch\d?|ascii|tf|tftpl|key)$)
+exclude: (^LICENSE.md$|^src/VERSION$|^env/|^src/(bw/misc/root-ca.pem$|deps/src/|common/core/modsecurity/files|ui/static/(js/(editor/|utils/purify/|tsparticles\.bundle\.min\.js)|css/dashboard\.css))|\.(svg|drawio|patch\d?|ascii|tf|tftpl|key)$)
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: c4a0b883114b00d8d76b479c820ce7950211c99b # frozen: v4.5.0
@@ -30,7 +30,7 @@ repos:
         name: Prettier Code Formatter
 
   - repo: https://github.com/JohnnyMorganz/StyLua
-    rev: f9afc7f33bc19f7708fbc1d7eea0606e0d41080a # frozen: v0.19.1
+    rev: 84c370104d6a8d1eef00c80a3ebd42f7033aaaad # frozen: v0.20.0
     hooks:
       - id: stylua-github
         exclude: ^src/(bw/lua/middleclass.lua|common/core/antibot/captcha.lua)$
@@ -50,7 +50,7 @@ repos:
         args: ["--max-line-length=250", "--ignore=E266,E402,E722,W503"]
 
   - repo: https://github.com/dosisod/refurb
-    rev: a7c461fcfaa2ca3248d489cdf7fed8e2d4fd8520 # frozen: v1.26.0
+    rev: a295cee6d188f5797aefe5d7cf77a353ed48ea93 # frozen: v1.27.0
     hooks:
       - id: refurb
         name: Refurb Python Refactoring Tool
@@ -62,7 +62,7 @@ repos:
       - id: codespell
         name: Codespell Spell Checker
         exclude: (^src/(ui/templates|common/core/.+/files|bw/loading)/.+.html|modsecurity-rules.conf.*)$
-        entry: codespell --ignore-regex="(tabEl|Widgits)" --skip src/ui/static/js/utils/flatpickr.js,CHANGELOG.md
+        entry: codespell --ignore-regex="(tabEl|Widgits)" --skip src/ui/static/js/utils/flatpickr.js,src/ui/static/css/style.css,CHANGELOG.md
         language: python
         types: [text]
 

.trivyignore

@@ -0,0 +1 @@
+CVE-2023-6129

CHANGELOG.md

@@ -1,23 +1,33 @@
 # Changelog
 
-## v1.5.5 - YYYY/MM/DD
+## v1.5.6 - YYYY/MM/DD
 
-- [BUGFIX] Fix issues with the database when upgrading from version 1.5.3 and 1.5.4 to the most recent version
+- [MISC] Updated Linux base images in Dockerfiles
+- [DEPS] Updated stream-lua-nginx-module to v0.0.14
+- [DEPS] Updated lua-nginx-module version to v0.10.26
+- [DEPS] Updated libmaxminddb version to v1.9.1
+- [DEPS] Updated lua-resty-core to v0.1.28
+- [DEPS] Updated zlib version to v1.3.1
+
+## v1.5.5 - 2024/01/12
+
+- [BUGFIX] Fix issues with the database when upgrading from one version to a newer one
 - [BUGFIX] Fix ModSecurity-nginx to make it work with brotli
 - [BUGFIX] Remove certbot renew delay causing errors on k8s
 - [BUGFIX] Fix missing custom modsec files when BW instances change
 - [BUGFIX] Fix inconsistency on config changes when using Redis
+- [BUGFIX] Fix web UI not working when using / URL
 - [FEATURE] Add Anonymous reporting feature
 - [FEATURE] Add support for fallback Referrer-Policies
-- [FEATURE] Add profile page to web ui and the possibility to activate the 2FA
+- [FEATURE] Add 2FA support to web UI
+- [FEATURE] Add username and password management to web UI
 - [FEATURE] Add setting REVERSE_PROXY_INCLUDES to manually add "include" directives in the reverse proxies
 - [FEATURE] Add support for Redis Sentinel
 - [FEATURE] Add support for tls in Ingress definition
 - [MISC] Fallback to default HTTPS certificate to prevent errors
 - [MISC] Various internal improvements in LUA code
 - [MISC] Check nginx configuration before reload
 - [MISC] Updated Python Docker image to 3.12.1-alpine3.18 in Dockerfiles
-- [MISC] Switch gunicorn worker_class back to gevent in web UI
 - [DEPS] Updated ModSecurity to v3.0.11
 
 ## v1.5.4 - 2023/12/04

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-	<img alt="BunkerWeb logo" src="https://github.com/bunkerity/bunkerweb/raw/v1.5.5/misc/logo.png" />
+	<img alt="BunkerWeb logo" src="https://github.com/bunkerity/bunkerweb/raw/v1.5.5/misc/logo.png" height=100 width=350 />
 </p>
 
 <p align="center">

docs/package.json

@@ -0,0 +1,5 @@
+{
+    "dependencies": {
+      "puppeteer": "^21.3.6"
+    }
+}

docs/requirements.in

@@ -1,5 +1,5 @@
 mike==2.0.0
 mkdocs==1.5.3
-mkdocs-material[imaging]==9.5.3
+mkdocs-material[imaging]==9.5.4
 mkdocs-print-site-plugin==2.3.6
 pytablewriter==1.2.0