Merge pull request #1764 from wireapp/release-2021-09-14

Release 2021-09-14

.github/pull_request_template.md

@@ -4,8 +4,8 @@
  - [ ] The **PR description** provides context as to why the change should occur and what the code contributes to that effect. This could also be a link to a JIRA ticket or a Github issue, if there is one.
  - [ ] If end-points have been added or changed: the **endpoint / config-flag checklist** (see Wire-employee only backend [wiki page](https://github.com/zinfra/backend-wiki/wiki/Checklists)) has been followed.
  - [ ] If a schema migration has been added, I ran **`make git-add-cassandra-schema`** to update the cassandra schema documentation.
- - [ ] Section *Unreleased* of **CHANGELOG-draft.md** contains the following bits of information:
-   - [ ] A line with the title and number of the PR in one or more suitable sub-sections.
+ - [ ] **changelog.d** contains the following bits of information:
+   - [ ] A file with the changelog entry in one or more suitable sub-sections. The sub-sections are marked by directories inside `changelog.d`.
    - [ ] If /a: measures to be taken by instance operators.
    - [ ] If /a: list of cassandra migrations.
    - [ ] If public end-points have been changed or added: does nginz need upgrade?

CHANGELOG.md

@@ -1,5 +1,44 @@
+<!-- if you're not the release manager, do your edits to changelog under CHANGELOG.d/ -->
+
+# [2021-09-14]
+
+## API changes
+
+* Remove the long-deprecated `message` field in `POST /connections` (#1726)
+* Add `PUT /conversations/:domain/:cnv/name` (#1737)
+* Deprecate `PUT /conversations/:cnv/name` (#1737)
+* Add `GET & PUT /conversations/:domain/:cnv/self` (#1740)
+* Deprecate `GET & PUT /conversations/:cnv/self` (#1740)
+* Remove endpoint `GET /conversations/:domain/:cnv/self` (#1752)
+* The `otr_muted` field in `Member` and `MemberUpdate` has been removed. (#1751)
+* Removed the ability to update one's own role (#1752)
+
+## Features
+
+* Disallow changing phone number to a black listed phone number (#1758)
+* Support using a single IDP with a single EntityID (aka issuer ID) to set up two teams. Required to support multiple teams in environments where the IDP software cannot present anything but one EntityID (E.G.: DualShield). (#1755)
+
+## Documentation
+
+* Added documentation of federation errors (#1674)
+* Better swagger schema for the Range type (#1748)
+* Add better example for Domain in swagger (#1748)
+
+## Internal changes
+
+* Introduce new process for writing changelogs (#1749)
+* Clean up JSON golden tests (Part 4, Part 5) (#1756, #1762)
+* Increased timeout on certificate update tests to 10s (#1750)
+* Fix for flaky test in spar (#1760)
+* Rewrite the `POST /connections` endpoint to Servant (#1726)
+* Various improvements and fixes around SAML/SCIM (#1735)
+
+## Federation changes
+
+* Avoid remote calls to get conversation when it is not found locally (#1749)
+* Federator CA store and client credentials are now automatically reloaded (#1730)
+* Ensure clients only receive messages meant for them in remote convs (#1739)
 
-<!-- if you're not the release manager, do your edits to changelog in CHANGELOG-draft.md -->
 
 # [2021-09-08]
 

changelog.d/0-release-notes/.title

@@ -0,0 +1 @@
+Release notes

changelog.d/1-api-changes/.title

@@ -0,0 +1 @@
+API changes

changelog.d/2-features/.title

@@ -0,0 +1 @@
+Features

changelog.d/2-features/pr-1755

@@ -0,0 +1,3 @@
+Support using a single IDP with a single EntityID (aka issuer ID) to set up two teams.
+Sets up a migration, and makes teamID + EntityID unique, rather than relying on EntityID to be unique.
+Required to support multiple teams in environments where the IDP software cannot present anything but one EntityID (E.G.: DualShield).

changelog.d/3-bug-fixes/.title

@@ -0,0 +1 @@
+Bug fixes and other updates

changelog.d/4-docs/.title

@@ -0,0 +1 @@
+Documentation

changelog.d/5-internal/.title

@@ -0,0 +1 @@
+Internal changes

changelog.d/6-federation/.title

@@ -0,0 +1 @@
+Federation changes

changelog.d/mk-changelog.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+shopt -s nullglob
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+getPRNumber() {
+    git log --reverse --format=%s -- $1 | sed -rn '1 { /\((#.*)\)$/ s|^.*\((#.*)\)$|\1|p; }' | grep "" ||
+      echo "#PR_NOT_FOUND"
+}
+
+for d in "$DIR"/*; do
+    if [[ ! -d "$d" ]]; then continue; fi
+
+    echo -n "## "
+    sed '$ a\' "$d/.title"
+    echo ""
+    for f in "$d"/*; do
+        pr=$(getPRNumber $f)
+        sed -r '
+          # create a bullet point on the first line
+          1 { s/^/\* /; }
+
+          # indent subsequent lines
+          1 !{ s/^/  /; }
+
+          # replace ## with PR number throughout
+          s/##/'"$pr"'/g
+
+          # add PR number at the end (unless already present)
+          $ { /^.*\((#.*)\)$/ ! { s/$/ ('"$pr"')/; } }
+
+          # remove trailing whitespace
+          s/\s+$//
+
+          # make sure there is a trailing newline
+          $ a\' "$f"
+    done
+    echo ""
+done

changelog.d/mk-cleanup.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+shopt -s nullglob
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+rm -f "$DIR"/*/*
+git add "$DIR"

docs/developer/cassandra-interaction.md

@@ -1,5 +1,21 @@
 # Writing code interacting with cassandra
 
+<!-- vim-markdown-toc GFM -->
+
+* [Anti-patterns](#anti-patterns)
+    * [Anti-pattern: Using full table scans in production code](#anti-pattern-using-full-table-scans-in-production-code)
+    * [Anti-pattern: Using IN queries on a field in the partition key](#anti-pattern-using-in-queries-on-a-field-in-the-partition-key)
+    * [Anti-pattern: Designing for a lot of deletes or updates](#anti-pattern-designing-for-a-lot-of-deletes-or-updates)
+* [Understanding more about cassandra](#understanding-more-about-cassandra)
+    * [primary partition clustering keys](#primary-partition-clustering-keys)
+    * [optimizing parallel request performance](#optimizing-parallel-request-performance)
+* [Cassandra schema migrations](#cassandra-schema-migrations)
+    * [Backwards compatible schema changes](#backwards-compatible-schema-changes)
+    * [Backwards incompatible schema changes](#backwards-incompatible-schema-changes)
+        * [What to do about backwards incompatible schema changes](#what-to-do-about-backwards-incompatible-schema-changes)
+
+<!-- vim-markdown-toc -->
+
 ## Anti-patterns
 
 ### Anti-pattern: Using full table scans in production code
@@ -36,3 +52,47 @@ Confused about primary key, partition key, and clustering key? See e.g. [this po
 ### optimizing parallel request performance
 
 See the thoughts in https://github.com/wireapp/wire-server/pull/1345#discussion_r567829234 - measuring overall and per-request performance and trying out different settings here might be worthwhile if increasing read or write performance is critical.
+
+## Cassandra schema migrations
+
+### Backwards compatible schema changes
+
+Most cassandra schema changes are backwards compatible, or *should* be designed to be so. Looking at the changes under `services/{brig,spar,galley,gundeck}/schema` you'll find this to be mostly the case.
+
+The general deployment setup for services interacting with cassandra have the following assumption:
+
+* cassandra schema updates happen *before* new code is deployed.
+  * This is safeguarded by the concourse deployment pipelines
+  * This is also safeguarded by the `wire-server` helm chart, which deploys the `cassandra-migrations` job as part of a helm [pre-install/pre-upgrade hook](https://github.com/wireapp/wire-server/blob/b3b1af6757194aa1dc86a8f387887936f2afd2fb/charts/cassandra-migrations/templates/migrate-schema.yaml#L10-L13): that means the schema changes are applied, and helm waits before launching the new code in the brig/galley/spar/gundeck pods until the changes have completed applying.
+  * This is further safeguarded by the code e.g. in brig refusing to even start the service up if the applied schema migration is not at least at a version the code expects it to. See [versionCheck](https://github.com/wireapp/wire-server/blob/b3b1af6757194aa1dc86a8f387887936f2afd2fb/services/brig/src/Brig/App.hs#L411) and [schemaVersion](https://github.com/wireapp/wire-server/blob/b3b1af6757194aa1dc86a8f387887936f2afd2fb/services/brig/src/Brig/App.hs#L136-L137)
+
+So usually with these safeguards in place, and backwards-compatible changes, we have the following:
+
+* At time t=0, old schema, old code serves traffic; all good.
+* At time t=1, new schema, old code serves traffic: all good since backwards compatible.
+* At time t=2, new schema, old code AND new code serve traffic: all good since backwards compatible.
+* At time t=3, new schema, new code serves traffic: all good since backwards compatible.
+
+If this order (apply schema first; then deploy code) is not safeguarded, then there will be code running in e.g. production which `SELECT my_new_field FROM my_new_table` even though this doesn't yet exist, leading to 500 server errors for as long as the mismatch between applied schema and code version persists.
+
+### Backwards incompatible schema changes
+
+In the case where a schema migration is **not backwards compatible**, such as in the form of `ALTER TABLE my_table DROP my_column`, the reverse problem exists:
+
+During a deployment:
+
+* At time t=0, old schema, old code serves traffic; all good.
+* At time t=1, new schema, old code serves traffic: 500 server errors as the old code is still active, bit columns or tables have been deleted, so old queries of `SELECT x from my_table` cause exceptions / HTTP 5xx results.
+    * In the worst deployment scenario, this could raise an alarm and lead operators or automation to stop the deployment or roll back; but that doesn't solve the issue, 500s will continue being thrown until the new code is deployed.
+* At time t=2, new schema, old code AND new code serve traffic: partial 500s (all traffic still serves by old code)
+* At time t=3, new schema, new code serves traffic: all good again.
+
+#### What to do about backwards incompatible schema changes
+
+Options from most to least desirable:
+
+* Never make backwards-incompatbile changes to the database schema :)
+* Do changes in a two-step process:
+    * First make a change that stops the code from using queries involving `my_column` or `my_table` (assuming you wish to remove those), and deploy this all the way across all of your environments (staging, prod, customers, ...).
+    * After all deployments have gone through (this can take weeks or months), do the schema change removing that column or table from the database. Often there is no urgency with removal of data, so it's fine to do this e.g. 6 months later.
+* Do changes in a one-step process, but accept partial service interruption for several minutes up to a few hours, accept communication overhead across teams to warn them about upcoming interruption or explain previous interruption; accept communication and documentation to warn all operators deploying that or a future version of the code, and accept some amount of frustration that may arise from a lack of said communication and understanding of what is happening.

docs/reference/cassandra-schema.cql

@@ -1513,6 +1513,7 @@ CREATE TABLE spar_test.issuer_idp (
 
 CREATE TABLE spar_test.idp (
     idp uuid PRIMARY KEY,
+    api_version int,
     extra_public_keys list<blob>,
     issuer text,
     old_issuers list<text>,
@@ -1681,6 +1682,27 @@ CREATE TABLE spar_test.team_idp (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
+CREATE TABLE spar_test.issuer_idp_v2 (
+    issuer text,
+    team uuid,
+    idp uuid,
+    PRIMARY KEY (issuer, team)
+) WITH CLUSTERING ORDER BY (team ASC)
+    AND bloom_filter_fp_chance = 0.1
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0.1
+    AND default_time_to_live = 0
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE';
+
 CREATE TABLE spar_test.scim_user_times (
     uid uuid PRIMARY KEY,
     created_at timestamp,

libs/brig-types/src/Brig/Types/Connection.hs

@@ -29,7 +29,6 @@ module Brig.Types.Connection
     UpdateConnectionsInternal (..),
 
     -- * re-exports
-    Message (..),
     Relation (..),
     UserConnection (..),
     ConnectionRequest (..),

libs/galley-types/src/Galley/Types/Conversations/Members.hs

@@ -43,8 +43,6 @@ data RemoteMember = RemoteMember
 data InternalMember id = InternalMember
   { memId :: id,
     memService :: Maybe ServiceRef,
-    -- | DEPRECATED, remove it once enough clients use `memOtrMutedStatus`
-    memOtrMuted :: Bool,
     memOtrMutedStatus :: Maybe MutedStatus,
     memOtrMutedRef :: Maybe Text,
     memOtrArchived :: Bool,

libs/hscim/.gitignore

@@ -25,3 +25,4 @@ cabal.project.local~
 *~
 *.el
 \#*
+.ghci

libs/hscim/src/Web/Scim/Test/Util.hs

@@ -101,6 +101,7 @@ shouldEventuallyRespondWith action matcher =
 
 data AcceptanceConfig tag = AcceptanceConfig
   { scimAppAndConfig :: IO (Application, AcceptanceQueryConfig tag),
+    -- TODO: add a destructor, something like: @destroy :: CustomEnv tag -> IO ()@,
     genUserName :: IO Text,
     -- | some acceptance tests match against a fully rendered
     -- response body, which will not work when running the test

libs/tasty-cannon/package.yaml

@@ -12,6 +12,7 @@ dependencies:
 - aeson
 - async
 - base >=4.6 && <5
+- bilge
 - bytestring
 - bytestring-conversion
 - data-timeout