Merge pull request #1421 from wireapp/release_2021_03_23

Release 2021-03-23

CHANGELOG.md

@@ -13,6 +13,38 @@
 ## Internal changes
 
 -->
+# [2020-03-23]
+
+## Release Notes
+
+Note that you should never skip a release when upgrading Wire. If you are upgrading to this release, make sure you have deployed the previous releases in order beforehand.
+
+## Features
+
+* [federation] Handle errors which could happen while talking to remote federator (#1408)
+* [federation] Forward grpc traffic to federator via ingress (or nginz for local integration tests) (#1386)
+* [federation] Return UserProfile when getting user by qualified handle (#1397)
+
+## Bug fixes and other updates
+
+* [SCIM] Fix: Invalid requests raise 5xxs (#1392)
+* [SAML] Fix: permissions for IdP CRUD operations. (#1405)
+
+## Documentation
+
+*  Tweak docs about team search visibility configuration. (#1407)
+*  Move docs around. (#1399)
+*  Describe how to look at swagger locally (#1388)
+
+## Internal changes
+
+* Optimize /users/list-clients to only fetch required things from DB (#1398)
+* [SCIM] Remove usage of spar.scim_external_ids table (#1418)
+* Add-license. (#1394)
+* Bump nixpkgs for hls-1.0 (#1412)
+* stack-deps.nix: Use nixpkgs from niv (#1406)
+
+
 # [2020-03-21]
 
 ## Release Notes

Makefile

@@ -8,7 +8,7 @@ DOCKER_TAG            ?= $(USER)
 # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything)
 HELM_SEMVER           ?= 0.0.42
 # The list of helm charts needed for integration tests on kubernetes
-CHARTS_INTEGRATION    := wire-server databases-ephemeral fake-aws
+CHARTS_INTEGRATION    := wire-server databases-ephemeral fake-aws nginx-ingress-controller nginx-ingress-services
 # The list of helm charts to publish on S3
 # FUTUREWORK: after we "inline local subcharts",
 # (e.g. move charts/brig to charts/wire-server/brig)
@@ -273,8 +273,8 @@ kube-restart-%:
 	kubectl delete pod -n $(NAMESPACE) -l wireService=$(*)
 	kubectl delete pod -n $(NAMESPACE)-fed2 -l wireService=$(*)
 
-.PHONY: latest-brig-tag
-latest-brig-tag:
+.PHONY: latest-tag
+latest-tag:
 	./hack/bin/find-latest-docker-tag.sh
 
 .PHONY: release-chart-%

charts/federator/templates/service.yaml

@@ -10,11 +10,11 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-    - name: http
+    - name: internal
       port: {{ .Values.service.internalFederatorPort }}
       targetPort: {{ .Values.service.internalFederatorPort }}
 
-    - name: wire-server-federator
+    - name: federator-ext
       port: {{ .Values.service.externalFederatorPort }}
       targetPort: {{ .Values.service.externalFederatorPort }}
   selector:

charts/nginx-ingress-services/templates/federation-test-helper.yaml

@@ -0,0 +1,19 @@
+# Assumes that the controller is deployed in the same namespace. Only used for
+# enabling discovery by creating SRV records while running integration tests.
+{{- if (and .Values.federator.enabled .Values.federator.integrationTestHelper) }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: federation-test-helper
+  namespace: {{ .Release.namespace }}
+spec:
+  ports:
+  - name: wire-server-federator
+    port: 443
+    protocol: TCP
+    targetPort: https
+  selector:
+    app: nginx-ingress
+    component: controller
+  type: ClusterIP
+{{- end }}

charts/nginx-ingress-services/templates/ingress.yaml

@@ -5,9 +5,8 @@ metadata:
   annotations:
     kubernetes.io/ingress.class: "nginx"
 spec:
-  # This assumes you have created the given cert
+  # This assumes you have created the given cert (see secret.yaml)
   # https://github.com/kubernetes/ingress-nginx/blob/master/docs/examples/PREREQUISITES.md#tls-certificates
-{{- if .Values.tls.enabled }}
   tls:
   - hosts:
       - {{ .Values.config.dns.https }}
@@ -21,7 +20,6 @@ spec:
       - {{ .Values.config.dns.accountPages }}
 {{- end }}
     secretName: {{ include "nginx-ingress-services.getCertificateSecretName" . | quote }}
-{{- end }}
   rules:
     - host: {{ .Values.config.dns.https }}
       http:

charts/nginx-ingress-services/templates/ingress_federator.yaml

@@ -0,0 +1,27 @@
+{{- if .Values.federator.enabled }}
+# We use a separate ingress for federator/grpc since we can't forward
+# both normal http1 traffic and grpc traffic in the same kubernetes ingress it appears.
+# Setting backend-protocol annotation to "GRPC" for everything is likely incorrect.
+# see also example https://github.com/kubernetes/ingress-nginx/blob/master/docs/examples/grpc/ingress.yaml
+# and docs https://kubernetes.github.io/ingress-nginx/examples/grpc/
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: federator-ingress
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+spec:
+  tls:
+  - hosts:
+      - {{ .Values.config.dns.federator }}
+    secretName: {{ include "nginx-ingress-services.getCertificateSecretName" . | quote }}
+  rules:
+    - host: {{ .Values.config.dns.federator }}
+      http:
+        paths:
+          - backend:
+              serviceName: federator
+              servicePort: federator-ext # name must be below 15 chars
+{{- end }}

charts/nginx-ingress-services/templates/secret.yaml

@@ -7,13 +7,13 @@ metadata:
     release: "{{ .Release.Name }}"
     heritage: "{{ .Release.Service }}"
 type: kubernetes.io/tls
-{{ if and .Values.tls.enabled .Values.tls.useCertManager -}}
-{{- /* NOTE: providing `data` (and empty strings) allows to manage this secret resource with Helm if cert-manager is user */ -}}
+{{ if .Values.tls.useCertManager -}}
+{{- /* NOTE: providing `data` (and empty strings) allows to manage this secret resource with Helm if cert-manager is used */ -}}
 data:
   tls.crt: ""
   tls.key: ""
 {{- end -}}
-{{- if and .Values.tls.enabled (not .Values.tls.useCertManager) -}}
+{{- if (not .Values.tls.useCertManager) -}}
 data:
   {{- /* for_helm_linting is necessary only since the 'with' block below does not throw an error upon an empty .Values.secrets */}}
   for_helm_linting: {{ required "No .secrets found in configuration. Did you forget to helm <command> -f path/to/secrets.yaml ?" .Values.secrets | quote | b64enc | quote }}

charts/nginx-ingress-services/values.yaml

@@ -6,7 +6,9 @@ teamSettings:
 # Account pages may be useful to enable password reset or email validation done after the initial registration
 accountPages:
   enabled: false
-
+federator:
+  enabled: false
+  integrationTestHelper: false
 # If you want to use TLS termination on the ingress,
 # then set this variable to true and ensure that there
 # is a valid wildcard TLS certificate
@@ -59,6 +61,8 @@ service:
 #     ssl: nginz-ssl.<domain>
 #     webapp: webapp.<domain>
 #     fakeS3: assets.<domain>
+#     federator: federator.<domain>
+#     ^ federator is ignored unless federator.enabled == true
 #     teamSettings: teams.<domain>
 #     ^ teamSettings is ignored unless teamSettings.enabled == true
 #     accountPages: account.<domain>

deploy/services-demo/conf/nginz/integration-ca-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEApwf/2d2YraQDpCipPVtYR+7BNu47AgkD7kFvGhoxJhDP7CsU
+VdpqU5gsVVo8kvhkh4k1tsJyuWWeKn6piNSXxUCFIc80KkUPgsYf5v+RBXr73Fdg
+ezHQNhNi0dRZCh+YG/hN7pOX46+B0PyKwUEMTeUqizkmFU5tILPMMyDAGx1Bp2LB
+oJi4u+48fzTDMaWSXnCVF04G9+A4LDzw0fPdDMgKLEiXJ8GPoPs0cNs6MJoFDgpe
+gzy1mv7X7otmRVTaafZGd4TTo6lGC2VVSS5tpj4Qfz/PxyCLK7tf5033HNWEJzAw
+6izRXp849VferHuYEbP+2lexNk9tl45BsFhkrwIDAQABAoIBAQCFkzYeSsJginuG
++iVttfEBhYPqo9V4qTEFhjqNS0jmwiclHMZkagkB1P4PO9yZRB9Q7H+SKiqI7STx
+ot19WVYOHqzY/tUewJ/I2xyEJPkawuFLsmyr2IhD1nj+iKy0FdQU+huIoWukX6SX
+Nn7YUWa/nHbLY+Z6v38x2deBQ72dcBtDcOh1vtUR3fVfsiX5uzCcfvNZAw4cCyB2
+j8ySDIiP10Ic81da3FIeCm8g2yp3DrnvTa77xsr0IfSykB3UcSrGqDwZxs9pS82Q
+1fog//4xAfBYC9LEcnQrCvz2kqLSLICtjkgK+dlzgvY3rZMq9c/OY1nR7Wp2BIyp
+kKB5AEnRAoGBANTM3fq4YGzUodf+Xla4MDvQFJsYjQuig/CJboQ7JSFZi2uLnSHX
++7JDiHtQd3uifYMhzSxXXKV82CK7SsJOQlIVoCZ5eTsyYGyAu1fUqfBvfHYN4Gbr
+3QyZJE0Hut2rvn5DaT/dpgh7Uy9QWKhpAsmxzhKa/iADUTiNAO8pxxRFAoGBAMjw
+iZV43XWLvzP90P5jANHuk9tR/B5cM9zK40aWglNsMlK9cUgW3ovohMzTFce/LQWy
+zGZ1WZZcUUcR/pHot3fyjWKeJadZhSZ/7hN/0d/UDuFY5nQ8eGQoy2qrrtY+6MMU
+Eiz09EFnKKA7hUoDnbhOH1hCKsfrOVse55RDkTZjAoGABrzRzm1mCCwXT7prDD3a
+sRoefOajGJo1qTkAuckRnOOz6VzLRdYLzxIaUSU0E0MKzEsWru+5LDgus7LQZCSM
+LwMmRfGUqA4pRWYyCE7gbo9pFmfMEhYnso1qu9Gh1gDpECBcRbxj1GLrOFVH6VUh
+1Hb/ulET+LmCKdM1E110Qy0CgYEAimbDHSUGxHPg2pq0XMMsSWyegq3RjcfMIQPN
+z0zTr0oSz1KUuCaoWo1pCvtJQS+4fvhMOTYS4rHreZw3T6CO3hs+rvJm1QGf6Iit
+HtknYZfaN/TXprAP7Ez87xgZcJAcGmG0syp1Iqc/ID5e7D/ZXpzQkiXg+ZpXAyAi
+OcjgOCkCgYEAmsCsqtPn5vgB+/vr0n28UsFS4Of9whlgEPYndNss3nAmVEohQJRg
+QlBlJd2iDa7R0TrJZCuAwuqK7TxB/RoHL8UkryUt2nag39GYAyE+lfPM558/AWyt
+9yyLQNfiJnqTC2Ne2j7EyicBLha4J9NoBeNE5UqLlzrH4LRJ3fRX9Ps=
+-----END RSA PRIVATE KEY-----

deploy/services-demo/conf/nginz/integration-ca.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDAjCCAeqgAwIBAgIUJ+1IpVKexlyGhgZaOZi2Ka01Q2QwDQYJKoZIhvcNAQEL
+BQAwGTEXMBUGA1UEAxMOY2EuZXhhbXBsZS5jb20wHhcNMjEwMzA5MjA0MDAwWhcN
+MjYwMzA4MjA0MDAwWjAZMRcwFQYDVQQDEw5jYS5leGFtcGxlLmNvbTCCASIwDQYJ
+KoZIhvcNAQEBBQADggEPADCCAQoCggEBAKcH/9ndmK2kA6QoqT1bWEfuwTbuOwIJ
+A+5BbxoaMSYQz+wrFFXaalOYLFVaPJL4ZIeJNbbCcrllnip+qYjUl8VAhSHPNCpF
+D4LGH+b/kQV6+9xXYHsx0DYTYtHUWQofmBv4Te6Tl+OvgdD8isFBDE3lKos5JhVO
+bSCzzDMgwBsdQadiwaCYuLvuPH80wzGlkl5wlRdOBvfgOCw88NHz3QzICixIlyfB
+j6D7NHDbOjCaBQ4KXoM8tZr+1+6LZkVU2mn2RneE06OpRgtlVUkubaY+EH8/z8cg
+iyu7X+dN9xzVhCcwMOos0V6fOPVX3qx7mBGz/tpXsTZPbZeOQbBYZK8CAwEAAaNC
+MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFFLJ
+Dc8PpKsUGFi33sK+r5Xgi97XMA0GCSqGSIb3DQEBCwUAA4IBAQBtiav0b3r4k3CC
+N1DRNNpcvpHR8odPJkxR2r0kCLRqwu2YDxQYxCK5wCPNcpzGDd6nyg/nLWBzl4Vd
+UwBIUPSVavDAeQ8VL0YNCNhQlL4x6z5rT1aSRdp0VlRnnl4zjilWSN/IB5Y61i0Q
+Ww7Sd5T9hZUCOJm5bAeLo4+cxkOwN6jdT2wmLtgFkj4CFVvS/8nL8ZwC1qvt3mLs
+E8q7/KelKgddU5AET8Viim20m6ZxgGNWIX33LBiJ3Rg3QuJMnWyfBI6PCORRrdTb
+0AH/F3iUKQL8Zv6twr++S7VPeDllVgEXkq457ZBi2qJPikL5cUprV/0bWQxgf4+1
+L70TaNMy
+-----END CERTIFICATE-----

deploy/services-demo/conf/nginz/integration-leaf-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA7AcV1TSTJRGMOM+qB2F4wWbsV+M2C/4KWt0LKC6k3EeqYo+R
+n9utlcH42S3UgzjB9zVMXYItwtGSCNVbxQ4s/nCNsyZR2JB689gnR1hetnKCFJTY
+2ETBsqFgoyNHU1ZkoTIq0ZQVeOEkcn2qnM981xuPvptY1ChdFtD52ybJRhAcgrHX
++P3aWns0N7Hn7EgFWinGDCZ27QXA/iiuSteVr/3Fh39RF7+YYix4Y7TRqXQqRhK/
+sp7fZQjLeXB/oECh5tEm/WEFJoX4mDNf96GAVzHynjNysCSZd44iZxbMlgNynYv0
+1Q/LLddI1KBz4MqqtBvHgh61ddtcM6vi15iK8QIDAQABAoIBAGMj9r+/fa8sV8Hl
+OH4xKw/Rs6SXA7S12JOiFXWjYgxE7gTIWVrUY21sNKLE5WbZCGBWOuVsMNyMOtML
+C6kR80RM8rg1eFuHFJ4oRRdqvx3Tq2AJxavR9aqaroTX5LXUrCApemLd2McVisGs
+l2WWodGY+iAOEJnM+o0C6nBrMlAC1fHQGDaIeGCdrRWS1RO47z+wMxnd9n0Lmek1
+YHg+Cylg16ZbZvcl8KfJ3CzD8QGpu4hF1BMzJwzYg2qmLUPrlhB1bJomW8asbZtu
+MyKsr5zl95f7lOtn3DwsxNOYHlQFqYs8u4rogOqb09yIRoYm1n4v6Z1DKdQ1cm/U
+yIPcp4ECgYEA+P7ZPF412TLlHo337ryF10SvuWJJCYZLrVaeiTecgbqbbqtUcR9c
+NnTzd8LTBMAtl6lcyTzrTi5+RGrGiicKunZiUnWl8g67E7EMUsM/tWrZah3yOPDa
+yTNtXtpGkBqa4+sujky3WqCDicBpATeeE9xdIP3KS+OjKUR1gKs1dgUCgYEA8qrZ
+LyWGeI07C0ql3I6iFlhwbSMfQ+8mx8yYj+5rCEiY3Ns/D7WbxvqXOgBVD+QK26a9
+4f20z6HTXBnPAmYoMlG/uh+MJj2VlgsgElRy4fi33ky+F67NWcniaO5gU0iO5P0m
+W6yKBuEg34NAuBZ85Q/HzZoCXGijaugA0H+DyP0CgYEAtSuRWthdq/TZOxsnTCEF
+7XT/uw6lf/WkOLJu1f6NtOLXV3X4Efs0eDcuM3ShX7KJOfG0HoRh3df8bcGXRkxU
+BQpATilmHjLTsec/xTRltZXSLeEuLnopZv4xVTPS5eVF0BJ+JHHzc1CZhPBoNseG
+EINli6y9tewi+tLiAEYStxUCgYBj46Hs/1RgETqpxjuKE4hzDEqTEi8PhsT+36A5
+NxoF2eRanUFTFR2fhY10iah6FxPFINjuysF/6owXUGRfB3Aqbm4ujkfhgd4uWjwE
+b5CupfQB5LMnCoRrHmg9a0ppXee8KNx+RgK3HDqs5tlgLZmRrLJBVtD7+vKx0fhr
+2uGDfQKBgQCVu/KGMBrS1M/05nrg6MS8vrDO0Wwyk3X9/mmGh651omQeW39IvFq+
+5rI0w46f6mslksFYqCJhfxXIKHN0sB3cBw/290j6FY3+TxpD2sJLIOeTllr+ivvA
+4BBs0XcecrPF7Ykw+E9hOHTo+dBRoTz1yUPHJYsTN5vI8los9XIyBg==
+-----END RSA PRIVATE KEY-----

deploy/services-demo/conf/nginz/integration-leaf.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDXDCCAkSgAwIBAgIUcLcIBd4tb4neS1GuVAk3wGKOclQwDQYJKoZIhvcNAQEL
+BQAwGTEXMBUGA1UEAxMOY2EuZXhhbXBsZS5jb20wHhcNMjEwMzA5MjA0MDAwWhcN
+MjIwMzA5MjA0MDAwWjAAMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
+7AcV1TSTJRGMOM+qB2F4wWbsV+M2C/4KWt0LKC6k3EeqYo+Rn9utlcH42S3UgzjB
+9zVMXYItwtGSCNVbxQ4s/nCNsyZR2JB689gnR1hetnKCFJTY2ETBsqFgoyNHU1Zk
+oTIq0ZQVeOEkcn2qnM981xuPvptY1ChdFtD52ybJRhAcgrHX+P3aWns0N7Hn7EgF
+WinGDCZ27QXA/iiuSteVr/3Fh39RF7+YYix4Y7TRqXQqRhK/sp7fZQjLeXB/oECh
+5tEm/WEFJoX4mDNf96GAVzHynjNysCSZd44iZxbMlgNynYv01Q/LLddI1KBz4Mqq
+tBvHgh61ddtcM6vi15iK8QIDAQABo4G0MIGxMA4GA1UdDwEB/wQEAwIFoDAdBgNV
+HSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4E
+FgQU8Xe12Pp5uejfckcNvS4m6s/3aCgwHwYDVR0jBBgwFoAUUskNzw+kqxQYWLfe
+wr6vleCL3tcwMgYDVR0RAQH/BCgwJoIZKi5pbnRlZ3JhdGlvbi5leGFtcGxlLmNv
+bYIJbG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IBAQCcNy2uaVgh2aSxIwkbBJdk
+496vmzpdyvNZP5x3NHNVsUo3Ldb7llt72Rdwe0szGeZ0Df3s5y9F7jEmmM8efINF
+jTvh6R7zWJCQor64JHgeh0qFur8DV6hv7CH3j/RA3Cu+bmwxd6DDlOwroU97NOcf
+fKvRD3F5iH5wcz174NNq+E0lWX5QsscM8K4nQ/JfVbvmVmi5N4MCigTAxzsRF/Dr
+ZylL5ikwP/rcUyVySbm/5JT4bLFnhhLjHORPLDwM3l/AMKEJUzv9kt/HLoZAA9Im
+la+k91bt//8XMhmfP01IRkuMMfN9/nVkfhwmIk5b2MqTiWZzjDVqHulYMq4BNU54
+-----END CERTIFICATE-----

deploy/services-demo/conf/nginz/nginx.conf

@@ -106,23 +106,48 @@ http {
   #
 
   server {
+    # plain TCP/http listening for integration tests only.
     listen 8080;
     listen 8081;
 
+    # for nginx-without-tls, we need to use a separate port for http2 traffic,
+    # as nginx cannot handle unencrypted http1 and http2 trafic on the same
+    # port.
+    # This port is only used for trying out nginx grpc forwarding without TLS locally and should not
+    # be ported to any production nginz config.
+    listen 8090 http2;
+
+    ######## TLS/SSL block start ##############
+    #
+    # Most integration tests simply use the http ports 8080 and 8081
+    # But to also test tls forwarding, this port can be used.
+    # This applies only locally, as for kubernetes (helm chart) based deployments,
+    # TLS is terminated at the ingress level, not at nginz level
+    listen 8443 ssl http2;
+
+    # self-signed certificates generated using wire-server/hack/bin/selfsigned.sh
+    ssl_certificate integration-leaf.pem;
+    ssl_certificate_key integration-leaf-key.pem;
+
+    ######## TLS/SSL block end ##############
+
     zauth_keystore resources/zauth/pubkeys.txt;
     zauth_acl conf/nginz/zauth_acl.txt;
 
     location /status {
+        set $sanitized_request $request;
         zauth off;
         return 200;
     }
 
     location /i/status {
+        set $sanitized_request $request;
         zauth off;
         return 200;
     }
 
     location /vts {
+        set $sanitized_request $request;
         zauth off;
         vhost_traffic_status_display;
         vhost_traffic_status_display_format html;
@@ -132,6 +157,17 @@ http {
     # Service Routing
     #
 
+    # Federator endpoints: expose the federatorExternal port (Inward grpc
+    # service)
+    location /wire.federator.Inward {
+        set $sanitized_request $request;
+        zauth off;
+        grpc_pass grpc://federator_external;
+        # FUTUREWORK(federation): are any other settings
+        # (e.g. timeouts, body size, buffers, headers,...)
+        # useful/recommended/important-for-security?)
+    }
+
     # Brig Endpoints
     #
     ## brig unauthenticated endpoints

deploy/services-demo/conf/nginz/upstreams

@@ -33,3 +33,6 @@ upstream spar {
     keepalive 32;
     server 127.0.0.1:8088 max_fails=3 weight=1;
 }
+upstream federator_external {
+    server 127.0.0.1:8098 max_fails=3 weight=1;
+}

docs/developer/how-to.md

@@ -0,0 +1,19 @@
+# Developer how-to's
+
+The following assume you have a working developer environment with all the dependencies listed in [./dependencies.md](./dependencies.md) available to you.
+
+## How to look at the swagger docs / UI locally
+
+Terminal 1:
+* Set up backing services: `./deploy/dockerephemeral/run.sh`
+
+Terminal 2:
+* Compile all services: `make services`
+* Run services including nginz: `export INTEGRATION_USE_NGINZ=1; ./services/start-services-only.sh`
+
+Open your browser at:
+
+- http://localhost:8080/api/swagger-ui for the swagger 2.0 endpoints (in development as of Feb 2021 - more endpoints will be added here as time goes on)
+- http://localhost:8080/swagger-ui/ for the old swagger 1.2 API (old swagger, endpoints will disappear from here (and become available in the previous link) as time progresses)
+
+Swagger json (for swagger 2.0 endpoints) is available under http://localhost:8080/api/swagger.json