Merge pull request #21 from heviat/oidc

Update from upstream

.github/workflows/multiarch.yml

@@ -2,7 +2,7 @@ name: start-linux-multiarch
 on:
   pull_request:
     branches:
-      - main
+      - master
       - oidc
   merge_group:
 
@@ -24,7 +24,7 @@ jobs:
         MAILU_VERSION: ${{ env.MAILU_VERSION }}
         PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }}
         DOCKER_ORG: ${{ env.DOCKER_ORG_DERIVED }}
-        BRANCH: ${{ env.BRANCH }}
+        BRANCH: ${{ env.BRANCH}}
         DEPLOY: ${{ env.DEPLOY }}
         RELEASE: ${{ env.RELEASE }}
     steps:
@@ -35,24 +35,13 @@ jobs:
       - name: Extract branch name
         shell: bash
         run: |
-          echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
-        #For branch TESTING, we set the image tag to pr-xxxx
-      - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION and DEPLOY/RELEASE for branch testing
-        if: env.BRANCH == 'testing'
+          echo "BRANCH=${GITHUB_REF_NAME}" >> $GITHUB_ENV
+      - name: Derive MAILU_VERSION and DEPLOY/RELEASE for development branches
         shell: bash
         env:
-          COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
+          BRANCH: ${{ github.ref_name }}
         run: |
-          echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV
-          echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV
-          echo "DOCKER_ORG_DERIVED=${{ env.DOCKER_ORG }}" >> $GITHUB_ENV
-          echo "DEPLOY=true" >> $GITHUB_ENV
-          echo "RELEASE=false" >> $GITHUB_ENV
-      - name: Derive MAILU_VERSION and DEPLOY/RELEASE for other branches than testing
-        if: env.BRANCH != 'testing'
-        shell: bash
-        run: |
-          echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV
+          echo "MAILU_VERSION="${BRANCH////-} >> $GITHUB_ENV
           echo "DOCKER_ORG_DERIVED=${{ env.DOCKER_ORG }}" >> $GITHUB_ENV
           echo "DEPLOY=true" >> $GITHUB_ENV
           echo "RELEASE=false" >> $GITHUB_ENV
@@ -94,13 +83,15 @@ jobs:
       - derive-variables
     uses: ./.github/workflows/build_test_deploy.yml
     with:
-      architecture: 'linux/amd64,linux/arm64/v8,linux/arm/v7'
+      # linux/arm64/v8,linux/arm/v7 will be added when GitHub hosted arm64 runners are available by the end of 2024
+      # https://github.blog/news-insights/product-news/arm64-on-github-actions-powering-faster-more-efficient-build-systems/
+      architecture: 'linux/amd64'
       mailu_version: ${{needs.derive-variables.outputs.MAILU_VERSION}}
       pinned_mailu_version: ${{needs.derive-variables.outputs.PINNED_MAILU_VERSION}}
       docker_org: ${{needs.derive-variables.outputs.DOCKER_ORG}}
       branch: ${{needs.derive-variables.outputs.BRANCH}}
-      deploy: ${{needs.derive-variables.outputs.DEPLOY}}
-      release: ${{needs.derive-variables.outputs.RELEASE}}
+      deploy: ${{needs.derive-variables.outputs.DEPLOY == 'true'}}
+      release: ${{needs.derive-variables.outputs.RELEASE == 'true'}}
     secrets: inherit
 
  # This job is watched by bors. It only complets if building,testing and deploy worked.

.gitignore

@@ -9,6 +9,7 @@ pip-selfcheck.json
 /docs/lib*
 /docs/bin
 /docs/include
+/docs/contributors/mailu-network-diagram.svg
 /docs/_build
 /.env
 /.venv

core/admin/Dockerfile

@@ -23,7 +23,7 @@ RUN set -euxo pipefail \
 RUN echo $VERSION >/version
 
 #EXPOSE 8080/tcp
-HEALTHCHECK CMD curl -skfLo /dev/null http://localhost:8080/ping
+HEALTHCHECK CMD curl -m3 -skfLo /dev/null http://localhost:8080/ping
 
 VOLUME ["/data","/dkim"]
 

core/admin/assets/assets/app.js

@@ -21,6 +21,9 @@ function sha1(string) {
 }
 
 function hibpCheck(pwd) {
+    if (pwd === null || pwd === undefined || pwd.length === 0) {
+	    return;
+    }
     // We hash the pwd first
     sha1(pwd).then(function(hash){
         // We send the first 5 chars of the hash to hibp's API

core/admin/mailu/internal/nginx.py

@@ -18,7 +18,11 @@
         "sieve": "AuthFailed"
     }),
     "encryption": ("Must issue a STARTTLS command first", {
-        "smtp": "530 5.7.0"
+        "imap": "PRIVACYREQUIRED",
+        "smtp": "530 5.7.0",
+        "submission": "530 5.7.0",
+        "pop3": "-ERR Authentication canceled.",
+        "sieve": "ENCRYPT-NEEDED"
     }),
     "ratelimit": ("Temporary authentication failure (rate-limit)", {
         "imap": "LIMIT",
@@ -68,7 +72,7 @@ def handle_authentication(headers):
     # Incoming mail, no authentication
     if method in ['', 'none'] and protocol in ['smtp', 'lmtp']:
         server, port = get_server(protocol, False)
-        if app.config["INBOUND_TLS_ENFORCE"]:
+        if app.config["INBOUND_TLS_ENFORCE"] and protocol == 'smtp':
             if "Auth-SSL" in headers and headers["Auth-SSL"] == "on":
                 return {
                     "Auth-Status": "OK",
@@ -91,20 +95,14 @@ def handle_authentication(headers):
     # Authenticated user
     elif method in ['plain', 'login']:
         is_valid_user = False
-        # According to RFC2616 section 3.7.1 and PEP 3333, HTTP headers should
-        # be ASCII and are generally considered ISO8859-1. However when passing
-        # the password, nginx does not transcode the input UTF string, thus
-        # we need to manually decode.
-        raw_user_email = urllib.parse.unquote(headers["Auth-User"])
-        raw_password = urllib.parse.unquote(headers["Auth-Pass"])
         user_email = 'invalid'
         password = 'invalid'
         try:
-            user_email = raw_user_email.encode("iso8859-1").decode("utf8")
-            password = raw_password.encode("iso8859-1").decode("utf8")
+            user_email = urllib.parse.unquote(headers["Auth-User"])
+            password = urllib.parse.unquote(headers["Auth-Pass"])
             ip = urllib.parse.unquote(headers["Client-Ip"])
         except:
-            app.logger.warn(f'Received undecodable user/password from nginx: {raw_user_email!r}/{raw_password!r}')
+            app.logger.warn(f'Received undecodable user/password from front: {headers.get("Auth-User", "")!r}')
         else:
             try:
                 user = models.User.query.get(user_email) if '@' in user_email else None

core/admin/mailu/internal/views/auth.py

@@ -29,7 +29,6 @@ def nginx_authentication():
         response.headers['Auth-Status'] = status
         response.headers['Auth-Error-Code'] = code
         return response
-    raw_password = urllib.parse.unquote(headers['Auth-Pass']) if 'Auth-Pass' in headers else ''
     headers = nginx.handle_authentication(flask.request.headers)
     response = flask.Response()
     for key, value in headers.items():
@@ -50,14 +49,8 @@ def nginx_authentication():
         if not is_port_25:
             utils.limiter.exempt_ip_from_ratelimits(client_ip)
     elif is_valid_user:
-        password = None
-        try:
-            password = raw_password.encode("iso8859-1").decode("utf8")
-        except:
-            app.logger.warn(f'Received undecodable password for {username} from nginx: {raw_password!r}')
-            utils.limiter.rate_limit_user(username, client_ip, password=None)
-        else:
-            utils.limiter.rate_limit_user(username, client_ip, password=password)
+        password = urllib.parse.unquote(headers.get('Auth-Pass', ''))
+        utils.limiter.rate_limit_user(username, client_ip, password=password)
     elif not is_from_webmail:
         utils.limiter.rate_limit_ip(client_ip, username)
     return response

core/admin/mailu/models.py

@@ -279,7 +279,7 @@ def dns_autoconfig(self):
             f'_{proto}._tcp.{self.name}. 600 IN SRV {prio} 1 {port} {hostname}.' if port in ports else f'_{proto}._tcp.{self.name}. 600 IN SRV 0 0 0 .'
             for proto, port, prio
             in protocols
-        ]+[f'autoconfig.{self.name}. 600 IN CNAME {hostname}.']
+        ]+[f'autoconfig.{self.name}. 600 IN CNAME {hostname}.', f'autodiscover.{self.name}. 600 IN CNAME {hostname}.']
 
     @cached_property
     def dns_tlsa(self):
@@ -680,7 +680,7 @@ def set_password(self, password, raw=False, keep_sessions=None):
 set() containing the sessions to keep
         """
         self.password = password if raw else User.get_password_context().hash(password)
-        if keep_sessions is not True:
+        if keep_sessions is not True and self.email is not None:
             utils.MailuSessionExtension.prune_sessions(uid=self.email, keep=keep_sessions)
 
     def get_managed_domains(self):

core/admin/mailu/ui/views/users.py

@@ -24,7 +24,7 @@ def user_create(domain_name):
             flask.url_for('.user_list', domain_name=domain.name))
     form = forms.UserForm()
     form.pw.validators = [wtforms.validators.DataRequired()]
-    form.quota_bytes.default = app.config['DEFAULT_QUOTA']
+    form.quota_bytes.default = int(app.config['DEFAULT_QUOTA'])
     if domain.max_quota_bytes:
         form.quota_bytes.validators = [
             wtforms.validators.NumberRange(max=domain.max_quota_bytes)]
@@ -93,12 +93,12 @@ def user_settings(user_email):
     form = forms.UserSettingsForm(obj=user)
     utils.formatCSVField(form.forward_destination)
     if form.validate_on_submit():
-        if form.forward_enabled.data and (form.forward_destination.data in ['', None] or type(form.forward_destination.data) is list):
+        user.forward_enabled = bool(flask.request.form.get('forward_enabled', False))
+        if user.forward_enabled and not form.forward_destination.data:
             flask.flash('Destination email address is missing', 'error')
-            user.forward_enabled = True
-            return flask.render_template('user/settings.html', form=form, user=user)
-        if form.forward_enabled.data:
-            form.forward_destination.data = form.forward_destination.data.replace(" ","").split(",")
+            return flask.redirect(
+                flask.url_for('.user_settings', user_email=user_email))
+        form.forward_destination.data = form.forward_destination.data.replace(" ","").split(",")
         form.populate_obj(user)
         models.db.session.commit()
         form.forward_destination.data = ", ".join(form.forward_destination.data)
@@ -107,8 +107,9 @@ def user_settings(user_email):
             return flask.redirect(
                 flask.url_for('.user_list', domain_name=user.domain.name))
     elif form.is_submitted() and not form.validate():
-        user.forward_enabled = form.forward_enabled.data
-        return flask.render_template('user/settings.html', form=form, user=user)
+            flask.flash('Error validating the form', 'error')
+            return flask.redirect(
+                flask.url_for('.user_settings', user_email=user_email))
     return flask.render_template('user/settings.html', form=form, user=user)
 
 def _process_password_change(form, user_email):

core/admin/mailu/utils.py

@@ -698,6 +698,7 @@ def isBadOrPwned(form):
 
 def formatCSVField(field):
     if not field.data:
+        field.data = ''
         return
     if isinstance(field.data,str):
         data = field.data.replace(" ","").split(",")

core/admin/start.py

@@ -64,7 +64,7 @@ def test_DNS():
 
 cmdline = [
     "gunicorn",
-    "--threads", f"{os.cpu_count()}",
+    "--threads", os.environ.get('CPU_COUNT', '1'),
     # If SUBNET6 is defined, gunicorn must listen on IPv6 as well as IPv4
     "-b", f"{'[::]' if os.environ.get('SUBNET6') else '0.0.0.0'}:8080",
     "--logger-class mailu.Logger",

core/base/Dockerfile

@@ -3,7 +3,7 @@
 # base system image (intermediate)
 # Note when updating the alpine tag, first manually run the workflow .github/workflows/mirror.yml.
 # Just run the workflow with the tag that must be synchronised.
-ARG DISTRO=ghcr.io/mailu/alpine:3.20
+ARG DISTRO=ghcr.io/mailu/alpine:3.20.3
 FROM $DISTRO as system
 
 ENV TZ=Etc/UTC LANG=C.UTF-8

core/base/libs/socrate/socrate/system.py

@@ -31,30 +31,29 @@ def _coerce_value(value):
 
 class LogFilter(object):
     def __init__(self, stream, re_patterns):
-        self.stream = stream
-        if isinstance(re_patterns, list):
-            self.pattern = re.compile('|'.join([fr'(?:{pattern})' for pattern in re_patterns]))
-        elif isinstance(re_patterns, str):
-            self.pattern = re.compile(re_patterns)
-        else:
-            self.pattern = re_patterns
-        self.found = False
+        self.stream  = stream
+        self.pattern = re.compile(b'|'.join([b''.join([b'(?:', pattern, b')']) for pattern in re_patterns]))
+        self.buffer  = b''
 
     def __getattr__(self, attr_name):
         return getattr(self.stream, attr_name)
 
     def write(self, data):
-        if data == '\n' and self.found:
-            self.found = False
-        else:
-            if not self.pattern.search(data):
-                self.stream.write(data)
+        if type(data) is str:
+            data = data.encode('utf-8')
+        self.buffer += data
+        while b'\n' in self.buffer:
+            line, cr, rest = self.buffer.partition(b'\n')
+            if not self.pattern.search(line):
+                self.stream.buffer.write(line)
+                self.stream.buffer.write(cr)
                 self.stream.flush()
-            else:
-                # caught bad pattern
-                self.found = True
+            self.buffer = rest
 
     def flush(self):
+        # write out buffer on flush even if it's not a complete line
+        if self.buffer and not self.pattern.search(self.buffer):
+            self.stream.buffer.write(self.buffer)
         self.stream.flush()
 
 def _is_compatible_with_hardened_malloc():
@@ -100,16 +99,49 @@ def set_env(required_secrets=[], log_filters=[]):
     for secret in required_secrets:
         os.environ[f'{secret}_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray(secret, 'utf-8'), 'sha256').hexdigest()
 
-    os.system('find /run -xdev -type f -name \*.pid -print -delete')
+    os.system(r'find /run -xdev -type f -name \*.pid -print -delete')
 
     return {
             key: _coerce_value(os.environ.get(key, value))
             for key, value in os.environ.items()
            }
 
 def clean_env():
-    """ remove all secret keys """
+    """ remove all secret keys, normalize PROXY_PROTOCOL """
     [os.environ.pop(key, None) for key in os.environ.keys() if key.endswith("_KEY")]
+    # Configure PROXY_PROTOCOL
+    PROTO_MAIL=['25', '110', '995', '143', '993', '587', '465', '4190']
+    PROTO_ALL_BUT_HTTP=PROTO_MAIL.copy()
+    PROTO_ALL_BUT_HTTP.extend(['443'])
+    PROTO_ALL=PROTO_ALL_BUT_HTTP.copy()
+    PROTO_ALL.extend(['80'])
+    for item in os.environ.get('PROXY_PROTOCOL', '').split(','):
+        if item.isdigit():
+            os.environ[f'PROXY_PROTOCOL_{item}']='True'
+        elif item == 'mail':
+            for p in PROTO_MAIL: os.environ[f'PROXY_PROTOCOL_{p}']='True'
+        elif item == 'all-but-http':
+            for p in PROTO_ALL_BUT_HTTP: os.environ[f'PROXY_PROTOCOL_{p}']='True'
+        elif item == 'all':
+            for p in PROTO_ALL: os.environ[f'PROXY_PROTOCOL_{p}']='True'
+        elif item == '':
+            pass
+        else:
+            log.error(f'Not sure what to do with {item} in PROXY_PROTOCOL ({args.get("PROXY_PROTOCOL")})')
+
+    PORTS_REQUIRING_TLS=['443', '465', '993', '995']
+    ALL_PORTS='25,80,443,465,993,995,4190'
+    for item in os.environ.get('PORTS', ALL_PORTS).split(','):
+        if item in PORTS_REQUIRING_TLS and os.environ.get('TLS_FLAVOR','') == 'notls':
+            continue
+        os.environ[f'PORT_{item}']='True'
+
+    if os.environ.get('TLS_FLAVOR', '') != 'notls':
+        for item in os.environ.get('TLS', ALL_PORTS).split(','):
+            if item in PORTS_REQUIRING_TLS:
+                os.environ[f'TLS_{item}']='True'
+    if 'CPU_COUNT' not in os.environ:
+        os.environ['CPU_COUNT'] = str(os.cpu_count())
 
 def drop_privs_to(username='mailu'):
     pwnam = getpwnam(username)
@@ -127,7 +159,7 @@ def forward_text_lines(src, dst):
 
 # runs a process and passes its standard/error output to the standard/error output of the current python script
 def run_process_and_forward_output(cmd):
-    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
     stdout_thread = threading.Thread(target=forward_text_lines, args=(process.stdout, sys.stdout))
     stdout_thread.daemon = True
@@ -137,4 +169,7 @@ def run_process_and_forward_output(cmd):
     stderr_thread.daemon = True
     stderr_thread.start()
 
-    process.wait()
+    rc = process.wait()
+    sys.stdout.flush()
+    sys.stderr.flush()
+    return rc

core/dovecot/conf/dovecot.conf

@@ -137,12 +137,22 @@ service imap-login {
   inet_listener imap {
     port = 143
   }
+  service_count = 0
+  client_limit = 25000
+  process_min_avail = {{ CPU_COUNT }}
+  process_limit = {{ CPU_COUNT }}
+  vsz_limit = 256M
 }
 
 service pop3-login {
   inet_listener pop3 {
     port = 110
   }
+  service_count = 0
+  client_limit = 25000
+  process_min_avail = {{ CPU_COUNT }}
+  process_limit = {{ CPU_COUNT }}
+  vsz_limit = 256M
 }
 
 ###############
@@ -166,6 +176,11 @@ service managesieve-login {
   inet_listener sieve {
     port = 4190
   }
+  service_count = 0
+  client_limit = 25000
+  process_min_avail = {{ CPU_COUNT }}
+  process_limit = {{ CPU_COUNT }}
+  vsz_limit = 256M
 }
 
 protocol sieve {