Merge pull request #1722 from wireapp/release_2021_08_27

Release 20201 08 027

.github/pull_request_template.md

@@ -4,7 +4,7 @@
  - [ ] The **PR description** provides context as to why the change should occur and what the code contributes to that effect. This could also be a link to a JIRA ticket or a Github issue, if there is one.
  - [ ] If end-points have been added or changed: the **endpoint / config-flag checklist** (see Wire-employee only backend [wiki page](https://github.com/zinfra/backend-wiki/wiki/Checklists)) has been followed.
  - [ ] If a schema migration has been added, I ran **`make git-add-cassandra-schema`** to update the cassandra schema documentation.
- - [ ] Section *Unreleased* of **CHANGELOG.md** contains the following bits of information:
+ - [ ] Section *Unreleased* of **CHANGELOG-draft.md** contains the following bits of information:
    - [ ] A line with the title and number of the PR in one or more suitable sub-sections.
    - [ ] If /a: measures to be taken by instance operators.
    - [ ] If /a: list of cassandra migrations.

CHANGELOG-draft.md

@@ -0,0 +1,22 @@
+THIS FILE ACCUMULATES THE RELEASE NOTES FOR THE UPCOMING RELEASE.
+
+<!--
+
+# [2021-xx-xx]
+
+[please put all changes that only affect federation into this section to unclutter the rest of the release notes.]
+[if something is both an API change and a feature, please mention it twice (you can abbreviate the second mention and add "see above").]
+
+## Release Notes
+
+## API Changes
+
+## Features
+
+## Bug fixes and other updates
+
+## Documentation
+
+## Internal changes
+
+-->

CHANGELOG.md

@@ -1,41 +1,42 @@
-<!--
 
-# [2021-xx-xx]
+<!-- if you're not the release manager, do your edits to changelog in CHANGELOG-draft.md -->
 
-[please put all changes that only affect federation into this section to unclutter the rest of the release notes.]
-[if something is both an API change and a feature, please mention it twice (you can abbreviate the second mention and add "see above").]
+# [2021-08-27]
 
 ## Release Notes
 
 ## API Changes
 
+* Deprecate `DELETE /conversations/:cnv/members/:usr` (#1697)
+* Add `DELETE /conversations/:cnv/members/:domain/:usr` (#1697)
+
 ## Features
 
 ## Bug fixes and other updates
 
-## Documentation
-
-## Internal changes
-
--->
+* Fix case sensitivity in schema parser in hscim library (#1714)
+* [helm charts] resolve a rate-limiting issue when using certificate-manager alongside wire-server and nginx-ingress-services helm charts (#1715)
 
+## Documentation
 
-# [unreleased]
-
-[please put all changes that only affect federation into this section to unclutter the rest of the release notes.]
-[if something is both an API change and a feature, please mention it twice (you can abbreviate the second mention and add "see above").]
-
-## Release Notes
-
-## API Changes
+* Improve Swagger for `DELETE /conversations/:cnv/members/:usr` (#1697)
 
-## Features
+## Internal changes
 
-## Bug fixes and other updates
+* Integration test script now displays output interactively (#1700)
+* Fixed a few issues with error response documentation in Swagger (#1707)
+* Make mapping between (team) permissions and roles more lenient (#1711)
+* The `DELETE /conversations/:cnv/members/:usr` endpoint rewritten to Servant (#1697)
+* Remove leftover auto-connect internal endpoint and code (#1716)
+* Bump wire-webapp (#1720)
+* Bump team-settings (#1721)
+* Bump account-pages (#1666)
 
-## Documentation
+## Federation changes
 
-## Internal changes
+* Added client certificate support for server to server authentication (#1682)
+* Implemented full server-to-server authentication (#1687)
+* Add an endpoint for removing a qualified user from a local conversation (#1697)
 
 
 # [2021-08-16]
@@ -72,7 +73,6 @@ This is a routine release requiring only the routine upgrade steps.
 * Added a mechanism to derive `AsUnion` instances automatically (#1693)
 * Integration test coverage (#1696, #1704)
 
-
 # [2021-08-02]
 
 ## Release Notes
@@ -124,7 +124,6 @@ Upgrade nginz (#1658)
 * Renamed `DomainHeader` type to `OriginDomainHeader` (#1689)
 * Added golden tests for protobuf serialisation / deserialisation (#1644).
 
-
 # [2021-07-09]
 
 ## Release Notes

Makefile

@@ -334,6 +334,10 @@ chart-%:
 .PHONY: charts-integration
 charts-integration: $(foreach chartName,$(CHARTS_INTEGRATION),chart-$(chartName))
 
+.PHONY: charts-serve
+charts-serve: charts-integration
+	./hack/bin/serve-charts.sh $(CHARTS_INTEGRATION)
+
 # Usecase for this make target:
 # 1. for releases of helm charts
 # 2. for testing helm charts more generally

charts/account-pages/values.yaml

@@ -9,7 +9,7 @@ resources:
     cpu: "1"
 image:
   repository: quay.io/wire/account
-  tag: 2.1.6-7ee369-v0.27.5-production
+  tag: "2.2.1-v0.28.21-0-6bfd7c5"
 service:
   https:
     externalPort: 443

charts/federator/templates/ca.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "federator-ca"
+  labels:
+    wireService: federator
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+{{- if .Values.remoteCAContents }}
+  ca.crt: {{ .Values.remoteCAContents | quote }}
+{{- else }}
+  {}
+{{- end }}

charts/federator/templates/configmap.yaml

@@ -43,8 +43,10 @@ data:
       # Filepath to one or more PEM-encoded server certificates to use as a trust
       # store when making grpc requests to remote backends
       {{- if $.Values.remoteCAContents }}
-      remoteCAStore: "/etc/wire/federator/ca/remote-ca.pem"
+      remoteCAStore: "/etc/wire/federator/ca/ca.crt"
       {{- end }}
+      clientCertificate: "/etc/wire/federator/secrets/tls.crt"
+      clientPrivateKey: "/etc/wire/federator/secrets/tls.key"
       useSystemCAStore: {{ .useSystemCAStore }}
       federationStrategy:
         {{- if .federationStrategy.allowAll }}

charts/federator/templates/deployment.yaml

@@ -25,25 +25,43 @@ spec:
       annotations:
         # An annotation of the configmap checksum ensures changes to the configmap cause a redeployment upon `helm upgrade`
         checksum/configmap: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum }}
-        checksum/configmap-ca: {{ include (print .Template.BasePath "/configmap-ca.yaml") . | sha256sum }}
+        {{- if not .Values.tls.shareFederatorSecret }}
+        checksum/secret: {{ include (print .Template.BasePath "/secret.yaml") . | sha256sum }}
+        {{- end }}
         fluentbit.io/parser: json
     spec:
       volumes:
         - name: "federator-config"
           configMap:
             name: "federator"
-        # federator-ca holds CA certificates to use as a trust store
-        # when making requests to remote backends
-        - name: "federator-ca"
+
+        # federator-secrets contains the client certificate and the
+        # corresponding private key to use when making requests to remote
+        # backends.
+        # NOTE: if tls.useSharedFederatorSecret is set, we use the same secret
+        # as the one for the federator ingress
+        - name: "federator-secrets"
           secret:
-            secretName: "federator-ca"
+            secretName: {{ if .Values.tls.useSharedFederatorSecret -}}
+              "federator-certificate-secret"
+              {{- else if .Values.clientCertificateContents -}}
+              "federator-secret"
+              {{- else }}
+              {{ fail "must set .Values.tls.useSharedFederatorSecret to true or specify .Values.clientCertificateContents" }}
+              {{- end }}
+
+        - name: "federator-ca"
+          configMap:
+            name: "federator-ca"
       containers:
         - name: federator
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ default "" .Values.imagePullPolicy | quote }}
           volumeMounts:
             - name: "federator-config"
               mountPath: "/etc/wire/federator/conf"
+            - name: "federator-secrets"
+              mountPath: "/etc/wire/federator/secrets"
             - name: "federator-ca"
               mountPath: "/etc/wire/federator/ca"
           ports:

charts/federator/templates/secret.yaml

@@ -0,0 +1,19 @@
+{{- if not .Values.tls.useSharedFederatorSecret -}}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "federator-secret"
+  labels:
+    wireService: federator
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+type: kubernetes.io/tls
+data:
+  {{- if .Values.clientPrivateKeyContents }}
+  tls.key: {{ .Values.clientPrivateKeyContents | b64enc | quote }}
+  {{- end -}}
+  {{- if .Values.clientCertificateContents }}
+  tls.crt: {{ .Values.clientCertificateContents | b64enc | quote }}
+  {{- end -}}
+{{- end -}}

charts/federator/templates/tests/configmap.yaml

@@ -19,3 +19,4 @@ data:
     nginxIngress:
       host: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local
       port: 443
+    originDomain: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local

charts/federator/templates/tests/federator-integration.yaml

@@ -13,10 +13,14 @@ spec:
     - name: "federator-config"
       configMap:
         name: "federator"
+    # integration tests need access to the client certificate private key
+    - name: "federator-secrets"
+      secret:
+        secretName: "federator-secret"
     # integration tests need access to the CA
     - name: "federator-ca"
-      secret:
-        secretName: "federator-ca"
+      configMap:
+        name: "federator-ca"
   containers:
   - name: integration
     command: ["federator-integration"]
@@ -26,6 +30,8 @@ spec:
       mountPath: "/etc/wire/integration"
     - name: "federator-config"
       mountPath: "/etc/wire/federator/conf"
+    - name: "federator-secrets"
+      mountPath: "/etc/wire/federator/secrets"
     - name: "federator-ca"
       mountPath: "/etc/wire/federator/ca"
   restartPolicy: Never

charts/federator/values.yaml

@@ -8,6 +8,11 @@ service:
   internalFederatorPort: 8080
   externalFederatorPort: 8081
 
+tls:
+  # if enabled, federator will get its client certificate and private key from
+  # the secret used by the federator ingress
+  useSharedFederatorSecret: false
+
 resources:
   # FUTUREWORK: come up with numbers which didn't appear out of thin air
   requests:
@@ -30,6 +35,9 @@ config:
     #
     # Using custom CA doesn't automatically disable system CA store, it should
     # be disabled explicitly by setting useSystemCAStore to false.
+    #
+    # A client certificate and corresponding private key can be specified
+    # similarly to a custom CA store.
     useSystemCAStore: true
     federationStrategy:
       allowedDomains: []

charts/nginx-ingress-services/templates/ca_federator.yaml

@@ -0,0 +1,19 @@
+{{- /* This is the CA used by the federator ingress to verify client
+certificates. This does not need to be a secret in principle, but the ingress
+controller requires it to be. Also, this could in principle be bundled with the
+corresponding certificate (in secret_federator.yaml), but it is a separate
+secret because cert-manager interferes with the ca.crt field when setting the
+certificate in a secret. */ -}}
+
+{{- if .Values.federator.enabled -}}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: federator-ca-secret
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+data:
+  ca.crt: {{ .Values.secrets.tlsClientCA | b64enc | quote }}
+{{- end -}}

charts/nginx-ingress-services/templates/certificate.yaml

@@ -36,7 +36,4 @@ spec:
     {{- if .Values.accountPages.enabled }}
     - {{ .Values.config.dns.accountPages }}
     {{- end }}
-    {{- if .Values.federator.enabled }}
-    - {{ .Values.config.dns.federator }}
-    {{- end }}
 {{- end -}}

charts/nginx-ingress-services/templates/certificate_federator.yaml

@@ -0,0 +1,34 @@
+{{- if and .Values.federator.enabled (not .Values.tls.enabled) }}
+{{- fail "TLS is required by federator. Either disable federation or enable tls." }}
+{{- end }}
+{{- if and .Values.tls.enabled .Values.tls.useCertManager }}
+apiVersion: cert-manager.io/v1alpha2
+kind: Certificate
+metadata:
+  name: "federator-{{ include "nginx-ingress-services.zone" . | replace "." "-" }}-csr"
+  namespace: {{ .Release.Namespace }}
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+spec:
+  issuerRef:
+    name: letsencrypt-http01
+    kind: Issuer
+  usages:
+    - server auth
+    - client auth
+  duration: 2160h     # 90d, Letsencrypt default; NOTE: changes are ignored by Letsencrypt
+  renewBefore: 360h   # 15d
+  isCA: false
+  keyAlgorithm: ecdsa
+  keySize: 256        # hs-tls only supports p256
+  keyEncoding: pkcs1
+  secretName: federator-certificate-secret
+  # NOTE: disabled due to https://github.com/jetstack/cert-manager/issues/2978
+  # TODO: enable when fixed (probably when cert-manager:v0.16 released)
+  #privateKey:
+  #  rotationPolicy: Always
+  dnsNames:
+    - {{ .Values.config.dns.federator }}
+{{- end -}}

charts/nginx-ingress-services/templates/ingress_federator.yaml

@@ -12,11 +12,15 @@ metadata:
     kubernetes.io/ingress.class: "nginx"
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
     nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    nginx.ingress.kubernetes.io/auth-tls-verify-client: "on"
+    nginx.ingress.kubernetes.io/auth-tls-secret: "{{ .Release.Namespace }}/federator-ca-secret"
+    nginx.ingress.kubernetes.io/configuration-snippet: |
+      grpc_set_header "X-SSL-Certificate" $ssl_client_escaped_cert;
 spec:
   tls:
   - hosts:
       - {{ .Values.config.dns.federator }}
-    secretName: {{ include "nginx-ingress-services.getCertificateSecretName" . | quote }}
+    secretName: "federator-certificate-secret"
   rules:
     - host: {{ .Values.config.dns.federator }}
       http: