Merge pull request #4126 from wireapp/release_2024-07-08_09_28

Release 2024-07-08 - (expected chart version 5.4.0)

.github/workflows/ci.yml

@@ -14,8 +14,8 @@ jobs:
      - uses: actions/checkout@v2
        with:
          submodules: true
-     - uses: cachix/install-nix-action@v20
-     - uses: cachix/cachix-action@v12
+     - uses: cachix/install-nix-action@v27
+     - uses: cachix/cachix-action@v15
        with:
          name: wire-server
          signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'

.hlint.yaml

@@ -24,7 +24,7 @@
 - error: { name: Use shutdown, lhs: runSettings, rhs: runSettingsWithShutdown }
 - ignore: { name: Use shutdown, within: [
     Network.Wai.Utilities.Server,  # this is the implementation 'runSettingsWithShutdown'
-    Federator.Response,            # this is just a naming conincidence
+    Federator.Interpreter,         # this is just a naming coincidence
     Cannon.Run                     # we do something similar, but not identical here by hand
   ] }
 

.ormolu

@@ -1,5 +1,15 @@
-infixr 10 .=
+module Imports exports Prelude
+infixl 9 .=
+infixl 9 .:
+infixr 4 ?~
+infixr 4 .~
+infixl 1 &
 infix 4 ===
 infix 4 =/=
 infixr 3 !!!
 infixr 3 <!!
+infixr 3 &&~
+infixr 2 ||~
+infix 4 <$$>
+infix 4 <$$$>
+infixl 1 `bindResponse`

CHANGELOG.md

@@ -1,3 +1,154 @@
+# [2024-07-08] (Chart Release 5.4.0)
+
+## Release notes
+
+
+* Phone registration and login is not supported anymore. All API endpoints dealing with phone numbers and phone activation codes now fail with a 400 error. Brig options related to phone number support have now been deleted, namely:
+   - `setTwilio`
+   - `setNexmo`
+   - `setAllowlistPhonePrefixes`. (#4045)
+
+
+## API changes
+
+
+* Internal API endpoints related to phone numbers have been removed.
+
+  In brig:
+  - `iGetPhonePrefix`
+  - `iDeletePhonePrefix`
+  - `iPostPhonePrefix`.
+
+  In stern:
+  - `get-users-by-phone`
+  - `put-phone`. (#4045)
+
+
+## Features
+
+
+* charts/coturn: support putting coturn into 'drain' mode when terminating pods, denying new incoming client connections. This speeds up graceful coturn restarts significantly. (#4098)
+
+* Set SFT usernames's `shared` field according to team settings (#4117)
+
+* Updated the `mlsE2EId` feature config with two additional fields `crlProxy` and `useProxyOnMobile` (#4051)
+
+* reject MLS messages for future epochs (#4110)
+
+* Introduce more configuration options to the `coturn` helm chart (#4083)
+
+* Update email templates to v1.0.121. (#4064)
+
+* Support connecting to RabbitMQ over TLS. See "Configure RabbitMQ" section in the documentation for details. (#4094)
+
+* Support connecting to Redis over TLS
+
+  It can be enabled by setting these options on the wire-server helm chart:
+
+  ```yaml
+  gundeck:
+    config:
+      redis:
+        enableTls: true
+
+        # When custom CAs are required, one of these must be set:
+        tlsCa: <PEM encoded CA certificates>
+        tlsCaSecretRef:
+          name: <Name of the secret>
+          key: <Key in the secret containing pem encoded CA Cert>
+
+        # When TLS needs to be used without verification:
+        insecureSkipVerifyTls: true
+  ```
+  (#4016)
+
+
+## Bug fixes and other updates
+
+
+* fixed stern endpoint `/i/users/meta-info` (#4101)
+
+* Log password reset errors instead of propagating them (#4114)
+
+* Log request ids in brig. (#4086)
+
+* Do not set update origin "scim" in public brig api. (#4072)
+
+* Disabling legalhold before user's approval doesn't result in an error  (#4104)
+
+* Make scim-delete-user idempotent. Hide information about existing users (make delete idempotent) (#4120)
+
+* Expose /providers/assets via nginz (#4082)
+
+* federator: Expect a client certificate to be the certificate chain
+
+  Without this openssl doesn't forward to whole chain causing mTLS to not succeed. (#4089)
+
+* Only resend proposals once after external commit (#4103)
+
+* gundeck: Better tolerance for redis-cluster restarts (#4084)
+
+* GHC does not support repeated --with-rtsopts options, and it simply applies the last one. This means many of the baked-in options were actually not being passed, including -N for some of the services and -T for cannon. (#4118)
+
+* Ensure that a Request ID is logged whenever unexpected errors are caught in any service (#4059)
+
+* charts/coturn: use allowed dir to write PID file (#4098)
+
+* Make pending LH requests (with no LH devices listening yet) not throw LH policy errors.  This helps eg. in cases where a LH request is issued to the wrong user by accident, and the user can clear up the mistake. (#4056)
+
+
+## Documentation
+
+
+* Adjust documentation for migrated helm charts (#4058)
+
+
+## Internal changes
+
+
+* Adapt EJPD data to current requirements. (#3945)
+
+* Port team feature tests to the `integration` package (#4063)
+
+* Ported flaky legalhold test to the new integration test suite (#4057)
+
+* Added profile update operations to the user subsystem. (#4046)
+
+* Introduce authentication subsystem with password reset. (#4086)
+
+* update nixpkgs and hence GHC version as well as some other tooling. (#4071)
+
+* nginz: Added `allowlisted_fqdn_origins` to `nginx_conf` value (#4087)
+
+* Add weeder for dead code elimination. (#4088)
+
+* Introduce email subsystem (#4111)
+
+* replace cabal.project.local template and update cabal.project (#4119)
+
+* Add HTTP proxy in the local setup for elasticsearch in federation-v0. This makes it possible to use a single elasticsearch instance for both the main backends and federation-v0. (#4062)
+
+* federator: Add metrics for garbage collections and unexpected errors that were caught (#4085)
+
+* federator: Simplify polysemy setup to make it similar to other services so the
+  interpreter is only used for hoisting the servant application and not explicitly
+  inside handler of an endpoint (#4059)
+
+* Added prometheus enable and datacenter size variables for k8ssandra-test-cluster helm chart. (#4011)
+
+* Make `Handle` type abstract to guarantee it always contains *valid* Handles. (#4076)
+
+* metrics-core: Delete `Data.Metrics` in favour of defining metrics closer to where they are being emitted (#4085)
+
+* add more metadata into the meta attribute of all nix derivations produced locally (#4069)
+
+* Do not log anything when warp kills a worker thread. (#4112)
+
+* Introduce VerificationCodSubsystem (#4121)
+
+* add tests for bots that use self-signed certs and add documentation on why we cannot test the bots to work with PKI (#4027)
+
+
 # [2024-05-21] (Chart Release 5.3.0)
 
 ## API changes

Makefile

@@ -73,8 +73,7 @@ clean-hint:
 
 .PHONY: cabal.project.local
 cabal.project.local:
-	echo "optimization: False" > ./cabal.project.local
-	./hack/bin/cabal-project-local-template.sh "ghc-options: -O0" >> ./cabal.project.local
+	cp ./hack/bin/cabal.project.local.template ./cabal.project.local
 
 # Usage: make c package=brig test=1
 .PHONY: c
@@ -127,11 +126,8 @@ devtest:
 	ghcid --command 'cabal repl integration' --test='Testlib.Run.mainI []'
 
 .PHONY: sanitize-pr
-sanitize-pr:
-	./hack/bin/generate-local-nix-packages.sh
-	make formatf
-	make hlint-inplace-pr
-	make hlint-check-pr  # sometimes inplace has been observed not to do its job very well.
+sanitize-pr: 
+	make lint-all-shallow
 	make git-add-cassandra-schema
 	@git diff-files --quiet -- || ( echo "There are unstaged changes, please take a look, consider committing them, and try again."; exit 1 )
 	@git diff-index --quiet --cached HEAD -- || ( echo "There are staged changes, please take a look, consider committing them, and try again."; exit 1 )
@@ -155,7 +151,25 @@ ghcid:
 
 # Used by CI
 .PHONY: lint-all
-lint-all: formatc hlint-check-all check-local-nix-derivations treefmt-check
+lint-all: formatc hlint-check-all lint-common
+
+# For use by local devs.
+#
+# This is not safe for CI because files not changed on the branch may
+# have been pushed to develop, or caused by merging develop into the
+# branch implicitly on github.
+#
+# The extra 'hlint-check-pr' has been witnessed to be necessary due to
+# some bu in `hlint-inplace-pr`.  Details got lost in history.
+.PHONY: lint-all-shallow
+lint-all-shallow: formatf hlint-inplace-pr hlint-check-pr lint-common
+
+.PHONY: lint-common
+lint-common: check-local-nix-derivations treefmt-check # weeder (does not work on CI yet)
+
+.PHONY: weeder
+weeder:
+	weeder -N
 
 .PHONY: hlint-check-all
 hlint-check-all:

cabal.project

@@ -19,7 +19,6 @@ packages:
   , libs/metrics-core/
   , libs/metrics-wai/
   , libs/polysemy-wire-zoo/
-  , libs/ropes/
   , libs/schema-profunctor/
   , libs/sodium-crypto-sign/
   , libs/ssl-util/
@@ -62,117 +61,7 @@ packages:
 tests: True
 benchmarks: True
 
-package assets
-    ghc-options: -Werror
-package auto-whitelist
-    ghc-options: -Werror
-package background-worker
-    ghc-options: -Werror
-package bilge
-    ghc-options: -Werror
-package brig
-    ghc-options: -Werror
-package brig-types
-    ghc-options: -Werror
-package cannon
-    ghc-options: -Werror
-package cargohold
-    ghc-options: -Werror
-package cargohold-types
-    ghc-options: -Werror
-package cassandra-util
-    ghc-options: -Werror
-package deriving-swagger2
-    ghc-options: -Werror
-package dns-util
-    ghc-options: -Werror
-package extended
-    ghc-options: -Werror
-package federator
-    ghc-options: -Werror
-package find-undead
-    ghc-options: -Werror
-package galley
-    ghc-options: -Werror
-package galley-types
-    ghc-options: -Werror
-package gundeck
-    ghc-options: -Werror
-package gundeck-types
-    ghc-options: -Werror
-package hscim
-    ghc-options: -Werror
-package http2-manager
-    ghc-options: -Werror
-package inconsistencies
-    ghc-options: -Werror
-package integration
-    ghc-options: -Werror
-package imports
-    ghc-options: -Werror
-package jwt-tools
-    ghc-options: -Werror
-package metrics-core
-    ghc-options: -Werror
-package metrics-wai
-    ghc-options: -Werror
-package migrate-sso-feature-flag
-    ghc-options: -Werror
-package mlsstats
-    ghc-options: -Werror
-package move-team
-    ghc-options: -Werror
-package polysemy-wire-zoo
-    ghc-options: -Werror
-package proxy
-    ghc-options: -Werror
-package mlsstats
-    ghc-options: -Werror
-package phone-users
-    ghc-options: -Werror
-package rabbitmq-consumer
-    ghc-options: -Werror
-package repair-handles
-    ghc-options: -Werror
-package rex
-    ghc-options: -Werror
-package ropes
-    ghc-options: -Werror
-package schema-profunctor
-    ghc-options: -Werror
-package service-backfill
-    ghc-options: -Werror
-package sodium-crypto-sign
-    ghc-options: -Werror
-package spar
-    ghc-options: -Werror
-package ssl-util
-    ghc-options: -Werror
-package stern
-    ghc-options: -Werror
-package tasty-cannon
-    ghc-options: -Werror
-package test-stats
-    ghc-options: -Werror
-package types-common
-    ghc-options: -Werror
-package types-common-aws
-    ghc-options: -Werror
-package types-common-journal
-    ghc-options: -Werror
-package wai-utilities
-    ghc-options: -Werror
-package wire-api
-    ghc-options: -Werror
-package wire-api-federation
-    ghc-options: -Werror
-package wire-message-proto-lens
-    ghc-options: -Werror
-package wire-subsystems
-    ghc-options: -Werror
-package zauth
-    ghc-options: -Werror
-package fedcalls
+program-options
     ghc-options: -Werror
 
 -- NOTE:

cassandra-schema.cql

@@ -1205,9 +1205,11 @@ CREATE TABLE galley_test.team_features (
     mls_default_ciphersuite int,
     mls_default_protocol int,
     mls_e2eid_acme_discovery_url blob,
+    mls_e2eid_crl_proxy blob,
     mls_e2eid_grace_period int,
     mls_e2eid_lock_status int,
     mls_e2eid_status int,
+    mls_e2eid_use_proxy_on_mobile boolean,
     mls_e2eid_ver_exp timestamp,
     mls_lock_status int,
     mls_migration_finalise_regardless_after timestamp,