Merge branch 'develop' into mls

.gitignore

@@ -20,6 +20,7 @@ target
 .#*
 *#*#
 .*.sw[a-z]
+**/__pycache__
 .cabal-sandbox
 ID
 cabal.sandbox.config
@@ -113,5 +114,4 @@ result-*
 
 services/nginz/third_party/headers-more-nginx-module
 services/nginz/third_party/nginx-module-vts
-
-**/__pycache__
+services/.integration

.hlint.yaml

@@ -1,5 +1,5 @@
 # We need quasi quotes support.
-- arguments: [ -XQuasiQuotes, --color ]
+- arguments: [ -XQuasiQuotes, -XOverloadedRecordDot, --color ]
 
 # Used to enforce ormolu styling. Can be revisited if we change formatters.
 - ignore: { name: Redundant $ }

Makefile

@@ -7,13 +7,13 @@ DOCKER_TAG            ?= $(USER)
 # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything)
 HELM_SEMVER           ?= 0.0.42
 # The list of helm charts needed on internal kubernetes testing environments
-CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana sftd restund coturn
+CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster rabbitmq fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana sftd restund coturn
 # The list of helm charts to publish on S3
 # FUTUREWORK: after we "inline local subcharts",
 # (e.g. move charts/brig to charts/wire-server/brig)
 # this list could be generated from the folder names under ./charts/ like so:
 # CHARTS_RELEASE := $(shell find charts/ -maxdepth 1 -type d | xargs -n 1 basename | grep -v charts)
-CHARTS_RELEASE := wire-server redis-ephemeral redis-cluster databases-ephemeral	\
+CHARTS_RELEASE := wire-server redis-ephemeral redis-cluster rabbitmq databases-ephemeral	\
 fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice		\
 calling-test demo-smtp elasticsearch-curator elasticsearch-external				\
 elasticsearch-ephemeral minio-external cassandra-external						\
@@ -84,18 +84,65 @@ ifeq ($(test), 1)
 endif
 	./hack/bin/cabal-install-artefacts.sh $(package)
 
-# ci here doesn't refer to continuous integration, but to cabal-integration
-# Usage: make ci package=brig test=1
-# If you want to pass arguments to the test-suite call the script directly.
+# ci here doesn't refer to continuous integration, but to cabal-run-integration.sh
+# Usage: make ci                        - build & run all tests
+#        make ci package=brig           - build brig & run "brig-integration" and "integration"
+#        make ci package=brig suite=old - build brig & run "brig-integration"
+#        make ci package=brig suite=new - build brig & run "integration"
+#        make ci package=integration    - build & run "integration"
+#
+# You can pass environment variables to all the suites, like so
+# TASTY_PATTERN=".."  make ci package=brig
+#
+# If you want to pass arguments to the test-suite call cabal-run-integration.sh directly.
 .PHONY: ci
 ci: c db-migrate
-	./hack/bin/cabal-run-integration.sh $(package)
+ifeq ("$(package)", "all")
+    ifneq ("$(suite)", "new")
+		echo ./hack/bin/cabal-run-integration.sh all
+    endif
+    ifneq ("$(suite)", "old")
+		make c package=integration
+		echo ./hack/bin/cabal-run-integration.sh integration
+    endif
+else
+  ifeq ("$(package)", "integration")
+	./hack/bin/cabal-run-integration.sh integration
+  else
+    ifeq ("$(suite)", "old")
+		./hack/bin/cabal-run-integration.sh $(package)
+    else
+      ifeq ("$(suite)", "new")
+		make c package=integration
+		./hack/bin/cabal-run-integration.sh integration
+      else
+		make c package=integration
+		./hack/bin/cabal-run-integration.sh $(package)
+		./hack/bin/cabal-run-integration.sh integration
+      endif
+    endif
+  endif
+endif
+
+# Compile and run services
+# Usage: make crun `OR` make crun package=galley
+.PHONY: cr
+cr: c db-migrate
+	./services/run-services
+
+# Run integration from new test suite
+# Usage: make devtest
+# Usage: TEST_INCLUDE=test1,test2 make devtest
+.PHONY: devtest
+devtest:
+	ghcid --command 'cabal repl integration' --test='Testlib.Run.mainI []'
 
 .PHONY: sanitize-pr
 sanitize-pr:
 	./hack/bin/generate-local-nix-packages.sh
 	make formatf
 	make hlint-inplace-pr
+	make hlint-check-pr  # sometimes inplace has been observed not to do its job very well.
 	make git-add-cassandra-schema
 	@git diff-files --quiet -- || ( echo "There are unstaged changes, please take a look, consider committing them, and try again."; exit 1 )
 	@git diff-index --quiet --cached HEAD -- || ( echo "There are staged changes, please take a look, consider committing them, and try again."; exit 1 )

cabal.project

@@ -1,5 +1,6 @@
 packages:
-    libs/api-bot/
+    integration
+  , libs/api-bot/
   , libs/api-client/
   , libs/bilge/
   , libs/brig-types/

cassandra-schema.cql

@@ -137,6 +137,8 @@ CREATE TABLE galley_test.team_features (
     mls_allowed_ciphersuites set<int>,
     mls_default_ciphersuite int,
     mls_default_protocol int,
+    mls_e2eid_acme_discovery_url blob,
+    mls_e2eid_grace_period int,
     mls_e2eid_lock_status int,
     mls_e2eid_status int,
     mls_e2eid_ver_exp timestamp,

changelog.d/0-release-notes/helm-tag-rename

@@ -0,0 +1,33 @@
+A few helm values related to federation have been renamed, no action is required if federation was disabled. 
+If federation was enabled these values must be renamed in the wire-server chart:
+- tags.federator -> tags.federation
+- brig.enableFederator -> brig.enableFederation
+- galley.enableFederator -> galley.enableFederation
+- cargohold.enableFederator -> galley.enableFederation
+
+So, an old config which looked like this:
+
+```yaml
+tags:
+  federator: true
+brig:
+  enableFederator: true
+galley:
+  enableFederator: true
+cargohold:
+  enableFederator: true
+```
+
+would now look like this:
+
+```yaml
+tags:
+  federation: true
+brig:
+  enableFederation: true
+galley:
+  enableFederation: true
+cargohold:
+  enableFederation: true
+```
+

changelog.d/2-features/pr-3222

@@ -0,0 +1 @@
+Nonce base 64 encoding is now unpadded

changelog.d/2-features/pr-3237

@@ -0,0 +1 @@
+`MlsE2EIdConfig` does now contain an ACME discovery URL and `verificationExpiration` is now a duration. (#3237, #3244)

changelog.d/3-bug-fixes/coturn-bum

@@ -0,0 +1 @@
+Bump coturn default image to upstream coturn 4.6.2 + custom Wire code including a bugfix for a bug that resulted in unstable operation during higher load.

changelog.d/3-bug-fixes/pr-3255

@@ -0,0 +1 @@
+Get the correct domain for DPoP access token generation

changelog.d/3-bug-fixes/pr-3263

@@ -0,0 +1 @@
+Correct http host is passed to proxy request

changelog.d/3-bug-fixes/pr-3267

@@ -0,0 +1 @@
+Use backend domain for DPoP access token request

changelog.d/3-bug-fixes/pr-3269

@@ -0,0 +1 @@
+The DPoP access token is now base64 encoded (once)

changelog.d/4-docs/document-randomness-sources

@@ -0,0 +1 @@
+Document crypto library dependencies and sources of randomness

changelog.d/4-docs/grepinclude

@@ -0,0 +1 @@
+Add 'grepinclude' sphinx directive to document with some code snippets.

changelog.d/4-docs/swagger-ui-toc

@@ -0,0 +1 @@
+Make /api/swagger{-ui,.json} TOC html pages to all versions

changelog.d/5-internal/coturn-verbose

@@ -0,0 +1 @@
+By default, the coturn helm chart will no longer log verbosely. This can be enabled if desired.

changelog.d/5-internal/pr-3222

@@ -0,0 +1 @@
+End-to-end test for creating a DPoP access token for the E2EID client certificate enrollment

changelog.d/5-internal/pr-3239

@@ -0,0 +1 @@
+More integration tests for stern/backoffice

changelog.d/5-internal/pr-3240

@@ -0,0 +1 @@
+Fixed eventually function in test for potentially less flakiness

changelog.d/5-internal/searchcontacts-permission

@@ -0,0 +1 @@
+Introduce SearchContacts permission

charts/background-worker/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+description: Backend notification pusheer
+name: background-worker
+version: 0.0.42

charts/background-worker/README.md

@@ -0,0 +1,5 @@
+Note that background-worker depends on some provisioned storage, namely:
+
+- rabbitmq
+
+These are dealt with independently from this chart.

charts/background-worker/templates/configmap.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "background-worker"
+  labels:
+    app: background-worker
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  {{- with .Values.config }}
+  background-worker.yaml: |
+    logFormat: {{ .logFormat }}
+    logLevel: {{ .logLevel }}
+
+    federatorInternal:
+      host: federator
+      port: 8080
+    rabbitmq:
+{{toYaml .rabbitmq | indent 6 }}
+    remoteDomains:
+{{toYaml .remoteDomains | indent 6 }}
+  {{- end }}

charts/background-worker/templates/deployment.yaml

@@ -0,0 +1,59 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: background-worker
+  labels:
+    app: background-worker
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  # TODO(elland): Review this
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 0
+      maxSurge: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: background-worker
+  template:
+    metadata:
+      labels:
+        app: background-worker
+        release: {{ .Release.Name }}
+      annotations:
+        # An annotation of the configmap checksum ensures changes to the configmap cause a redeployment upon `helm upgrade`
+        checksum/configmap: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum }}
+        checksum/secret: {{ include (print .Template.BasePath "/secret.yaml") . | sha256sum }}
+        fluentbit.io/parser: json
+    spec:
+      serviceAccountName: {{ .Values.serviceAccount.name }}
+      volumes:
+        - name: "background-worker-config"
+          configMap:
+            name: "background-worker"
+        - name: "background-worker-secrets"
+          secret:
+            secretName: "background-worker"
+      containers:
+        - name: background-worker
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ default "" .Values.imagePullPolicy | quote }}
+          volumeMounts:
+          - name: "background-worker-config"
+            mountPath: "/etc/wire/background-worker/conf"
+          env:
+          - name: RABBITMQ_USERNAME
+            valueFrom:
+              secretKeyRef:
+                name: background-worker
+                key: rabbitmqUsername
+          - name: RABBITMQ_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: background-worker
+                key: rabbitmqPassword
+          resources:
+{{ toYaml .Values.resources | indent 12 }}

charts/background-worker/templates/secret.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: background-worker
+  labels:
+    app: background-worker
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+type: Opaque
+data:
+  {{/* for_helm_linting is necessary only since the 'with' block below does not throw an error upon an empty .Values.secrets */}}
+  for_helm_linting: {{ required "No .secrets found in configuration. Did you forget to helm <command> -f path/to/secrets.yaml ?" .Values.secrets | quote | b64enc | quote }}
+
+  {{- with .Values.secrets }}
+  rabbitmqUsername: {{ .rabbitmq.username | b64enc | quote }}
+  rabbitmqPassword: {{ .rabbitmq.password | b64enc | quote }}
+  {{- end }}

charts/background-worker/templates/serviceaccount.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.serviceAccount.name }}
+  labels:
+    app: brig
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }}
+{{- end }}

charts/background-worker/values.yaml

@@ -0,0 +1,35 @@
+replicaCount: 1
+image:
+  repository: quay.io/wire/background-worker
+  tag: do-not-use
+# FUTUREWORK: Review these values when we have some experience
+resources:
+  requests:
+    memory: "200Mi"
+    cpu: "100m"
+  limits:
+    memory: "512Mi"
+# TODO(elland): Create issue for a metrics endpoint
+# metrics:
+#   serviceMonitor:
+#     enabled: false
+config:
+  # TODO(elland): Proper logging
+  logLevel: Info
+  logFormat: StructuredJSON
+  rabbitmq:
+    host: rabbitmq
+    port: 5672
+    vHost: /
+  remoteDomains: []
+
+serviceAccount:
+  # When setting this to 'false', either make sure that a service account named
+  # 'background-worker' exists or change the 'name' field to 'default'
+  create: true
+  name: background-worker
+  annotations: {}
+  automountServiceAccountToken: true
+
+
+secrets: {}

charts/brig/templates/configmap.yaml

@@ -48,7 +48,7 @@ data:
       host: gundeck
       port: 8080
 
-    {{- if .enableFederator }}
+    {{- if .enableFederation }}
     # TODO remove this
     federator:
       host: federator
@@ -57,6 +57,8 @@ data:
     federatorInternal:
       host: federator
       port: 8080
+    rabbitmq:
+{{ toYaml .rabbitmq | indent 6}}
     {{- end }}
 
     {{- with .aws }}

charts/brig/templates/deployment.yaml

@@ -130,7 +130,19 @@ spec:
           - name: NO_PROXY
             value: {{ join "," .noProxyList | quote }}
           {{- end }}
-          {{- end }} 
+          {{- end }}
+          {{- if .Values.config.enableFederation }}
+          - name: RABBITMQ_USERNAME
+            valueFrom:
+              secretKeyRef:
+                name: brig
+                key: rabbitmqUsername
+          - name: RABBITMQ_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: brig
+                key: rabbitmqPassword
+          {{- end }}
           ports:
             - containerPort: {{ .Values.service.internalPort }}
           startupProbe: