From 304d870c7a35676ab870db9e2e62f56331ac5808 Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Tue, 29 Nov 2022 08:11:50 -0500
Subject: [PATCH 01/55] Remove unused Synapse config: worker_main_http_uri.
 (#1319)

This was removed in matrix-org/synapse#14400 and is
currently a no-op that adds warnings to the logs.
---
 lib/SyTest/Homeserver/Synapse.pm | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm
index 1b9f86960..49c2d1075 100644
--- a/lib/SyTest/Homeserver/Synapse.pm
+++ b/lib/SyTest/Homeserver/Synapse.pm
@@ -942,7 +942,6 @@ sub _start_synapse
          "worker_log_config"            => $self->configure_logger("frontend_proxy"),
          "worker_replication_host"      => "$bind_host",
          "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
-         "worker_main_http_uri"         => "http://$bind_host:$self->{ports}{synapse_unsecure}",
          "worker_listeners"             => [
             {
                type      => "http",
@@ -975,7 +974,6 @@ sub _start_synapse
          "worker_log_config"            => $self->configure_logger("background_worker"),
          "worker_replication_host"      => "$bind_host",
          "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
-         "worker_main_http_uri"         => "http://$bind_host:$self->{ports}{synapse_unsecure}",
       };
 
       push @worker_configs, $background_worker_config;
@@ -989,7 +987,6 @@ sub _start_synapse
          "worker_log_config"            => $self->configure_logger("event_persister1"),
          "worker_replication_host"      => "$bind_host",
          "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
-         "worker_main_http_uri"         => "http://$bind_host:$self->{ports}{synapse_unsecure}",
          "worker_listeners"             => [
             {
                type      => "http",
@@ -1022,7 +1019,6 @@ sub _start_synapse
          "worker_log_config"            => $self->configure_logger("event_persister2"),
          "worker_replication_host"      => "$bind_host",
          "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
-         "worker_main_http_uri"         => "http://$bind_host:$self->{ports}{synapse_unsecure}",
          "worker_listeners"             => [
             {
                type      => "http",
@@ -1055,7 +1051,6 @@ sub _start_synapse
          "worker_log_config"            => $self->configure_logger("stream_writer"),
          "worker_replication_host"      => "$bind_host",
          "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
-         "worker_main_http_uri"         => "http://$bind_host:$self->{ports}{synapse_unsecure}",
          "worker_listeners"             => [
             {
                type      => "http",

From 33e03f28e4fdbdebe5a9a9282aea39e6ee5ee90f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 30 Nov 2022 13:59:17 +0000
Subject: [PATCH 02/55] Use the `dendrite` branch by default for Dendrite
 (#1237)

* Use the `dendrite` branch by default for Dendrite

That way we can choose when to merge the latest `develop` changes into our branch rather than find CI turns into a sea of red randomly.

* Add comment, remove building sytest-dendrite image

Co-authored-by: kegsay <kegan@matrix.org>
Co-authored-by: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
---
 .github/workflows/docker.yml |  4 ----
 docker/bootstrap.sh          | 14 +++++---------
 docker/dendrite.Dockerfile   |  2 ++
 3 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index a55df93e0..6bc93186c 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -79,10 +79,6 @@ jobs:
             build_args: |
               SYTEST_IMAGE_TAG=bookworm
               PYTHON_VERSION=python3.10
-          - sytest_image_tag: buster
-            dockerfile: dendrite
-            tags: "matrixdotorg/sytest-dendrite:latest"
-            build_args: "SYTEST_IMAGE_TAG=buster"
 
     steps:
       - name: Set up QEMU
diff --git a/docker/bootstrap.sh b/docker/bootstrap.sh
index fc15cf59b..6b32ec3e3 100755
--- a/docker/bootstrap.sh
+++ b/docker/bootstrap.sh
@@ -5,6 +5,7 @@
 set -ex
 
 export SYTEST_TARGET="$1"
+export SYTEST_DEFAULT_BRANCH="${SYTEST_DEFAULT_BRANCH:-develop}"
 shift
 
 if [ -d "/sytest" ]; then
@@ -19,20 +20,15 @@ else
         branch_name="$SYTEST_BRANCH"
     else
         # Otherwise, try and find the branch that the Synapse/Dendrite checkout
-        # is using. Fall back to develop if unknown.
-        branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="develop"
-    fi
-
-    if [ "$SYTEST_TARGET" == "dendrite" ] && [ "$branch_name" == "master" ]; then
-        # Dendrite uses master as its main branch. If the branch is master, we probably want sytest develop
-        branch_name="develop"
+        # is using. Fall back to the default branch if unknown.
+        branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="${SYTEST_DEFAULT_BRANCH}"
     fi
 
     # Try and fetch the branch
     wget -q https://github.com/matrix-org/sytest/archive/$branch_name.tar.gz -O sytest.tar.gz || {
         # Probably a 404, fall back to develop
-        echo "Using develop instead..."
-        wget -q https://github.com/matrix-org/sytest/archive/develop.tar.gz -O sytest.tar.gz
+        echo "Using ${SYTEST_DEFAULT_BRANCH} instead..."
+        wget -q https://github.com/matrix-org/sytest/archive/${SYTEST_DEFAULT_BRANCH}.tar.gz -O sytest.tar.gz
     }
 
     mkdir -p /sytest
diff --git a/docker/dendrite.Dockerfile b/docker/dendrite.Dockerfile
index e6196c53f..c9af13f20 100644
--- a/docker/dendrite.Dockerfile
+++ b/docker/dendrite.Dockerfile
@@ -11,6 +11,8 @@ RUN tar xf go.tar.gz -C /goroot --strip-components=1
 ENV GOROOT=/goroot
 ENV GOPATH=/gopath
 ENV PATH="/goroot/bin:${PATH}"
+# This is used in bootstrap.sh to pull in a dendrite specific Sytest branch
+ENV SYTEST_DEFAULT_BRANCH dendrite
 
 # This is where we expect Dendrite to be binded to from the host
 RUN mkdir -p /src

From 2e67c9b0c1c2f96a6b1f1440c8e35f1ad53ea632 Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Thu, 8 Dec 2022 11:37:06 -0500
Subject: [PATCH 03/55] A missing trailing slash should be a 404. (#1323)

Per MSC3743, note there are related tests that maybe
should also return a 404, but those are unclear.

The trailing slash on the endpoint definitely does not exist,
however.
---
 tests/61push/80torture.pl | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tests/61push/80torture.pl b/tests/61push/80torture.pl
index 0013c895f..99dc49bbc 100644
--- a/tests/61push/80torture.pl
+++ b/tests/61push/80torture.pl
@@ -42,8 +42,19 @@
       };
 };
 
+test "Trying to get push rules with no trailing slash fails with 404",
+  requires => [ local_user_fixture() ],
+
+  check => sub {
+     my ( $user ) = @_;
+
+     do_request_json_for( $user,
+        method  => "GET",
+        uri     => "/v3/pushrules",
+     )->main::expect_http_404;
+  };
+
 my $TO_CHECK_GET_400 = [
-   [ "no trailing slash", "" ],
    [ "scope without trailing slash", "/global" ],
    [ "template without tailing slash", "/global/room" ],
    [ "unknown scope", "/not_a_scope/" ],

From 51b86b04cf216e222961c6f4de29b4f768afd419 Mon Sep 17 00:00:00 2001
From: Brendan Abolivier <babolivier@matrix.org>
Date: Fri, 9 Dec 2022 15:35:32 +0100
Subject: [PATCH 04/55] Install libicu in Docker images (#1324)

---
 docker/base.Dockerfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile
index c8874b97e..bcdd0b815 100644
--- a/docker/base.Dockerfile
+++ b/docker/base.Dockerfile
@@ -24,6 +24,8 @@ RUN apt-get -qq update && apt-get -qq install -y \
     rsync \
     sqlite3 \
     wget \
+    libicu-dev \
+    pkg-config \
     && rm -rf /var/lib/apt/lists/*
 
 # Set up the locales, as the default Debian image only has C, and PostgreSQL needs the correct locales to make a UTF-8 database

From 12b95c6e4f76972bf14dbda10da6627c03e5274a Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 5 Jan 2023 09:18:16 +0100
Subject: [PATCH 05/55] Remove deprecated `home_server` from checks (#1325)

* Remove deprecated home_server from checks

* Validate home_server if present
---
 tests/10apidoc/02login.pl                | 18 +++++++++++-------
 tests/10apidoc/13ui-auth.pl              | 12 ++++++++----
 tests/12login/01threepid-and-password.pl |  8 +++++---
 tests/60app-services/01as-create.pl      |  6 +++---
 4 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/tests/10apidoc/02login.pl b/tests/10apidoc/02login.pl
index 1454f4dcb..eb54815ed 100644
--- a/tests/10apidoc/02login.pl
+++ b/tests/10apidoc/02login.pl
@@ -91,10 +91,12 @@
       )->then( sub {
          my ( $body ) = @_;
 
-         assert_json_keys( $body, qw( access_token home_server ));
+         assert_json_keys( $body, qw( access_token ));
 
-         assert_eq( $body->{home_server}, $http->server_name,
-            'Response home_server' );
+         if (defined $body->{home_server}) {
+            assert_eq( $body->{home_server}, $http->server_name,
+               'Response home_server' );
+         }
 
          Future->done(1);
       });
@@ -161,10 +163,12 @@
       )->then( sub {
          my ( $body ) = @_;
 
-         assert_json_keys( $body, qw( access_token home_server ));
+         assert_json_keys( $body, qw( access_token ));
 
-         assert_eq( $body->{home_server}, $http->server_name,
-            'Response home_server' );
+         if (defined $body->{home_server}) {
+            assert_eq( $body->{home_server}, $http->server_name,
+               'Response home_server' );
+         }
 
          Future->done(1);
       });
@@ -250,7 +254,7 @@ sub matrix_login_again_with_user
    )->then( sub {
       my ( $body ) = @_;
 
-      assert_json_keys( $body, qw( access_token home_server ));
+      assert_json_keys( $body, qw( access_token ));
 
       my $new_user = new_User(
          http          => $user->http,
diff --git a/tests/10apidoc/13ui-auth.pl b/tests/10apidoc/13ui-auth.pl
index bcfe2c93f..8775391e8 100644
--- a/tests/10apidoc/13ui-auth.pl
+++ b/tests/10apidoc/13ui-auth.pl
@@ -185,12 +185,16 @@ sub matrix_login_with_cas
 
       log_if_fail "Response from /login", $body;
 
-      assert_json_keys( $body, qw( access_token home_server user_id device_id ));
+      assert_json_keys( $body, qw( access_token user_id device_id ));
 
-      assert_eq( $body->{home_server}, $http->server_name,
-                 'home_server in /login response' );
+      if (defined $body->{home_server}) {
+         assert_eq( $body->{home_server}, $http->server_name,
+            'home_server in /login response' );
+      }
+
+      
       assert_eq( $body->{user_id}, $user_id,
-                 'user_id in /login response' );
+         'user_id in /login response' );
 
       Future->done(1);
    });
diff --git a/tests/12login/01threepid-and-password.pl b/tests/12login/01threepid-and-password.pl
index f4639c1c2..3ad4691dc 100644
--- a/tests/12login/01threepid-and-password.pl
+++ b/tests/12login/01threepid-and-password.pl
@@ -26,10 +26,12 @@
       })->then( sub {
          my ( $body ) = @_;
 
-         assert_json_keys( $body, qw( access_token home_server ));
+         assert_json_keys( $body, qw( access_token ));
 
-         assert_eq( $body->{home_server}, $http->server_name,
-            'Response home_server' );
+         if (defined $body->{home_server}) {
+            assert_eq( $body->{home_server}, $http->server_name,
+               'Response home_server' );
+         }
 
          Future->done(1);
       });
diff --git a/tests/60app-services/01as-create.pl b/tests/60app-services/01as-create.pl
index 84607df38..08a083b4d 100644
--- a/tests/60app-services/01as-create.pl
+++ b/tests/60app-services/01as-create.pl
@@ -21,7 +21,7 @@
 
          log_if_fail "Body", $body;
 
-         assert_json_keys( $body, qw( user_id home_server access_token device_id ));
+         assert_json_keys( $body, qw( user_id access_token device_id ));
 
          Future->done(1);
       });
@@ -46,7 +46,7 @@
 
          log_if_fail "Body", $body;
 
-         assert_json_keys( $body, qw( user_id home_server access_token device_id ));
+         assert_json_keys( $body, qw( user_id access_token device_id ));
 
          Future->done(1);
       });
@@ -73,7 +73,7 @@
 
          log_if_fail "Body", $body;
 
-         assert_json_keys( $body, qw( user_id home_server ));
+         assert_json_keys( $body, qw( user_id ));
          foreach ( qw( device_id access_token )) {
             exists $body->{$_} and die "Got an unexpected '$_' key";
          }

From 588ff5225783ab7b33ab95d0f00b88f392b0f73d Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Mon, 16 Jan 2023 16:22:14 +0000
Subject: [PATCH 06/55] Bump poetry-version to 1.2.2 (#1326)

Should be backwards-compatible, see
https://github.com/matrix-org/setup-python-poetry/pull/14 for rationale
---
 docker/synapse.Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/synapse.Dockerfile b/docker/synapse.Dockerfile
index c3897c4e6..55549e7a8 100644
--- a/docker/synapse.Dockerfile
+++ b/docker/synapse.Dockerfile
@@ -24,7 +24,7 @@ RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-tool
 # pip version provided by Debian Buster. See
 # https://github.com/pypa/setuptools/issues/3457#issuecomment-1190125849
 RUN ${PYTHON_VERSION} -m pip install -q --upgrade pip
-RUN ${PYTHON_VERSION} -m pip install -q --no-cache-dir poetry==1.2.0
+RUN ${PYTHON_VERSION} -m pip install -q --no-cache-dir poetry==1.2.2
 
 # As part of the Docker build, we attempt to pre-install Synapse's dependencies
 # in the hope that it speeds up the real install of Synapse. To make this work,

From 9caca948ce350efd5eda824347dd5ae9be44c322 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 17 Jan 2023 16:01:12 +0000
Subject: [PATCH 07/55] Bump poetry version to 1.3.2. (#1327)

Part of https://github.com/matrix-org/synapse/issues/14857
---
 docker/synapse.Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/synapse.Dockerfile b/docker/synapse.Dockerfile
index 55549e7a8..0a29bc55c 100644
--- a/docker/synapse.Dockerfile
+++ b/docker/synapse.Dockerfile
@@ -24,7 +24,7 @@ RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-tool
 # pip version provided by Debian Buster. See
 # https://github.com/pypa/setuptools/issues/3457#issuecomment-1190125849
 RUN ${PYTHON_VERSION} -m pip install -q --upgrade pip
-RUN ${PYTHON_VERSION} -m pip install -q --no-cache-dir poetry==1.2.2
+RUN ${PYTHON_VERSION} -m pip install -q --no-cache-dir poetry==1.3.2
 
 # As part of the Docker build, we attempt to pre-install Synapse's dependencies
 # in the hope that it speeds up the real install of Synapse. To make this work,
@@ -77,7 +77,7 @@ RUN /venv/bin/pip install -q --no-cache-dir \
 # value.
 # See https://github.com/matrix-org/synapse/issues/12419
 # TODO: Once poetry 1.2.0 has been released, use the `installer.max-workers`
-#       config option instead.
+#       or `installer.parallel` config option instead.
 # poetry has a bug where this environment variable is not converted to a
 # boolean, so we choose a falsy string value for it. It's fixed in 1.2.0,
 # where we'll be wanting to use `installer.max-workers` anyway.

From 7f2e85b7ca2f57e8d2dd9bf54598dd823db77686 Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Wed, 18 Jan 2023 09:31:21 -0500
Subject: [PATCH 08/55] Add room version 10 to supported room versions. (#1328)

---
 lib/SyTest/Federation/Client.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SyTest/Federation/Client.pm b/lib/SyTest/Federation/Client.pm
index 1a3cec00b..8e3912be3 100644
--- a/lib/SyTest/Federation/Client.pm
+++ b/lib/SyTest/Federation/Client.pm
@@ -16,7 +16,7 @@ use SyTest::Assertions qw( :all );
 use URI::Escape qw( uri_escape );
 
 use constant SUPPORTED_ROOM_VERSIONS => [qw(
-   1 2 3 4 5 6 7 8 9
+   1 2 3 4 5 6 7 8 9 10
 )];
 
 sub configure

From d1c931af1179848526d323f3955a9f21f81d600f Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 20 Jan 2023 12:43:28 +0100
Subject: [PATCH 09/55] Retry getting unread counts (#1330)

---
 tests/61push/03_unread_count.pl | 71 +++++++++++++++++++--------------
 1 file changed, 41 insertions(+), 30 deletions(-)

diff --git a/tests/61push/03_unread_count.pl b/tests/61push/03_unread_count.pl
index d33a70fe5..e3dfe4159 100644
--- a/tests/61push/03_unread_count.pl
+++ b/tests/61push/03_unread_count.pl
@@ -68,38 +68,46 @@ sub user_with_push_rule_fixture
 
             matrix_advance_room_receipt_synced( $user1, $room_id, "m.read" => $event_id );
          })->then( sub {
-            matrix_sync( $user1 );
-         })->then( sub {
-            my ( $body ) = @_;
+            retry_until_success {
+               matrix_sync( $user1 )->then( sub {
+                  my ( $body ) = @_;
 
-            my $room = $body->{rooms}{join}{$room_id};
-            log_if_fail "room in sync response", $room;
+                  my $room = $body->{rooms}{join}{$room_id};
 
-            assert_json_keys( $room, "unread_notifications" );
-            my $unread = $room->{unread_notifications};
-            assert_json_keys( $unread, $counter );
+                  assert_json_keys( $room, "unread_notifications" );
+                  my $unread = $room->{unread_notifications};
+                  assert_json_keys( $unread, $counter );
+                  log_if_fail "room notifications:", $unread;
+                  
+                  $unread->{$counter} == 0
+                     or die "Expected $counter to be 0";
 
-            $unread->{$counter} == 0
-               or die "Expected $counter to be 0";
 
+                  Future->done(1);
+               });
+            }
+         })->then( sub {
             matrix_send_room_text_message_synced( $user2, $room_id,
                body => "Test message 2",
             );
          })->then( sub {
-            matrix_sync( $user1 );
-         })->then( sub {
-            my ( $body ) = @_;
+            retry_until_success {
+               matrix_sync( $user1 )->then( sub {
+                  my ( $body ) = @_;
 
-            my $room = $body->{rooms}{join}{$room_id};
+                  my $room = $body->{rooms}{join}{$room_id};
 
-            assert_json_keys( $room, "unread_notifications" );
-            my $unread = $room->{unread_notifications};
-            assert_json_keys( $unread, $counter );
+                  assert_json_keys( $room, "unread_notifications" );
+                  my $unread = $room->{unread_notifications};
+                  assert_json_keys( $unread, $counter );
 
-            $unread->{$counter} == 1
-               or die "Expected $counter to be 1";
+                  log_if_fail "room notifications:", $unread;
+                  $unread->{$counter} == 1
+                     or die "Expected $counter to be 1";
 
-            Future->done(1);
+                  Future->done(1);
+               });
+            }
          });
       };
 }
@@ -149,19 +157,22 @@ sub user_with_push_rule_fixture
             body => "Test message 2",
          );
       })->then( sub {
-         matrix_sync( $user1 );
-      })->then( sub {
-         my ( $body ) = @_;
+         retry_until_success {
+            matrix_sync( $user1 )->then( sub {
+               my ( $body ) = @_;
 
-         my $room = $body->{rooms}{join}{$room_id};
+               my $room = $body->{rooms}{join}{$room_id};
 
-         assert_json_keys( $room, "unread_notifications" );
-         my $unread = $room->{unread_notifications};
-         assert_json_keys( $unread, "highlight_count" );
+               assert_json_keys( $room, "unread_notifications" );
+               my $unread = $room->{unread_notifications};
+               assert_json_keys( $unread, "highlight_count" );
 
-         $unread->{highlight_count} == 1
-            or die "Expected unread highlight count to be 1";
+               log_if_fail "room notifications:", $unread;
+               $unread->{highlight_count} == 1
+                  or die "Expected unread highlight count to be 1";
 
-         Future->done(1);
+               Future->done(1);
+            });
+         }
       })
    };

From 0fd984a997e6685429510ff69b179a9ed9aa4aed Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Fri, 24 Feb 2023 23:32:32 +0000
Subject: [PATCH 10/55] Build docker images overnight (#1335)

Even if there are no sytest changes, there are often Synapse
dependency changes which we spend time `poetry install`ing.
---
 .github/workflows/docker.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 6bc93186c..948843f69 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -5,6 +5,8 @@ name: Build and deploy docker images
 on:
   push:
     branches: [develop]
+  schedule:
+    - cron: 0 1 * * *  # 1am, daily.
   workflow_dispatch:
 
 permissions:

From 17196040d30fc3f753acd6b8169527759b4da6bb Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 28 Feb 2023 15:13:20 +0100
Subject: [PATCH 11/55] Change binary for Dendrite (#1336)

* Add possibility to run Sytest with coverage enabled
* Change binary name
* More fixes, remove API mode tests
---
 .github/workflows/pipeline.yml    | 7 -------
 lib/SyTest/Homeserver/Dendrite.pm | 8 ++++----
 scripts/dendrite_sytest.sh        | 8 +++++++-
 3 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 9e18f52f8..350c0f7b1 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -129,16 +129,9 @@ jobs:
         include:
           - label: SQLite
 
-          - label: SQLite, full HTTP APIs
-            api: full-http
-
           - label: Postgres
             postgres: postgres
 
-          - label: Postgres, full HTTP APIs
-            postgres: postgres
-            api: full-http
-
     container:
       image: matrixdotorg/sytest-dendrite
       volumes:
diff --git a/lib/SyTest/Homeserver/Dendrite.pm b/lib/SyTest/Homeserver/Dendrite.pm
index 6217b48c2..666d08e22 100644
--- a/lib/SyTest/Homeserver/Dendrite.pm
+++ b/lib/SyTest/Homeserver/Dendrite.pm
@@ -248,17 +248,17 @@ sub _start_monolith
 
    $output->diag( "Starting monolith server" );
    my @command = (
-      $self->{bindir} . '/dendrite-monolith-server',
+      $self->{bindir} . '/dendrite',
       '--config', $self->{paths}{config},
       '--http-bind-address', $self->{bind_host} . ':' . $self->unsecure_port,
       '--https-bind-address', $self->{bind_host} . ':' . $self->secure_port,
-      '--api-bind-address', $self->{bind_host} . ':1' . $self->unsecure_port,
       '--tls-cert', $self->{paths}{tls_cert},
       '--tls-key', $self->{paths}{tls_key},
       '--really-enable-open-registration',
    );
 
-   push(@command, '-api') if $ENV{'API'} == '1';
+   push(@command, '--test.coverprofile=' . $self->{hs_dir} . '/integrationcover.log') if $ENV{'COVER'} == '1';
+
    $output->diag( "Starting Dendrite with: @command" );
 
    return $self->_start_process_and_await_connectable(
@@ -274,7 +274,7 @@ sub _start_monolith
       command => [ @command ],
       connect_host => $self->{bind_host},
       connect_port => $self->secure_port,
-      name => "dendrite-$idx-monolith",
+      name => "dendrite-$idx",
    )->else( sub {
       die "Unable to start dendrite monolith: $_[0]\n";
    })->on_done( sub {
diff --git a/scripts/dendrite_sytest.sh b/scripts/dendrite_sytest.sh
index 6c4bbcca2..16de2159a 100755
--- a/scripts/dendrite_sytest.sh
+++ b/scripts/dendrite_sytest.sh
@@ -34,7 +34,13 @@ export GOBIN=/tmp/bin
 echo >&2 "--- Building dendrite from source"
 cd /src
 mkdir -p $GOBIN
-go install -buildvcs=false -race=${RACE_DETECTION:-0} -tags vw -v ./cmd/dendrite-monolith-server
+
+if [[ -z ${COVER} || ${COVER} -eq 0 ]]; then
+    go install -buildvcs=false -race=${RACE_DETECTION:-0} -tags vw -v ./cmd/dendrite
+else 
+    go test -c -cover -covermode=atomic -race=${RACE_DETECTION:-0} -buildvcs=false -tags vw -o $GOBIN/dendrite -coverpkg "github.com/matrix-org/..." ./cmd/dendrite
+fi
+
 go install -buildvcs=false -race=${RACE_DETECTION:-0} -tags vw -v ./cmd/generate-keys
 go install -buildvcs=false -race=${RACE_DETECTION:-0} -tags vw -v ./cmd/generate-config
 cd -

From 169fb03f9a3ce420b3e50ec3e59a0d572ea91437 Mon Sep 17 00:00:00 2001
From: reivilibre <oliverw@matrix.org>
Date: Tue, 28 Feb 2023 16:37:32 +0000
Subject: [PATCH 12/55] Add /filter to client_reader (#1331)

---
 lib/SyTest/Homeserver/Synapse.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm
index 49c2d1075..588b63ae5 100644
--- a/lib/SyTest/Homeserver/Synapse.pm
+++ b/lib/SyTest/Homeserver/Synapse.pm
@@ -1301,6 +1301,7 @@ sub generate_haproxy_map
 ^/_matrix/client/(api/v1|r0|v3|unstable)/joined_rooms                client_reader
 ^/_matrix/client/(api/v1|r0|v3|unstable/.*)/rooms/.*/aliases         client_reader
 ^/_matrix/client/(api/v1|r0|v3|unstable)/search                      client_reader
+^/_matrix/client/(r0|v3|unstable)/user/.*/filter(/|$)                client_reader
 
 ^/_matrix/client/(api/v1|r0|v3|unstable)/devices$                    stream_writer
 ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/query$                 stream_writer

From 10bbbc057380ac8e6e5aa67bdd08dc238192c298 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 2 Mar 2023 12:26:02 +0000
Subject: [PATCH 13/55] Synapse: fix the build and reduce logging noise (#1334)

* Fix synapse-sytest:testing build

* Reduce `poetry install` verbosity

otherwise this means that `poetry install` tells about every single file
it ignores, which includes loads of guff in `.mypy_cache` and
`.ruff_cache` and others

* Limit number of haproxy connections

I don't know the full context, but
https://github.com/haproxy/haproxy/issues/2043 seems to explain what's
going on. The number `2000` was chosen arbitrarily.

* Hopefully fix sytest docker build on bookworm
---
 .github/workflows/docker.yml     | 6 ++----
 docker/synapse.Dockerfile        | 7 +++++--
 lib/SyTest/Homeserver/Synapse.pm | 2 ++
 scripts/synapse_sytest.sh        | 2 +-
 4 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 948843f69..b2846e63b 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -77,10 +77,8 @@ jobs:
             build_args: "SYTEST_IMAGE_TAG=testing"
           - sytest_image_tag: bookworm
             dockerfile: synapse
-            tags: "matrixdotorg/sytest-synapse:bookworm-python3.10"
-            build_args: |
-              SYTEST_IMAGE_TAG=bookworm
-              PYTHON_VERSION=python3.10
+            tags: "matrixdotorg/sytest-synapse:bookworm"
+            # Note: at the time of writing, bookworm's python3 is python 3.11.
 
     steps:
       - name: Set up QEMU
diff --git a/docker/synapse.Dockerfile b/docker/synapse.Dockerfile
index 0a29bc55c..14187ea6d 100644
--- a/docker/synapse.Dockerfile
+++ b/docker/synapse.Dockerfile
@@ -23,8 +23,11 @@ RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-tool
 # Use the latest version of pip. This pulls in fixes not present in the
 # pip version provided by Debian Buster. See
 # https://github.com/pypa/setuptools/issues/3457#issuecomment-1190125849
-RUN ${PYTHON_VERSION} -m pip install -q --upgrade pip
-RUN ${PYTHON_VERSION} -m pip install -q --no-cache-dir poetry==1.3.2
+# For now, we need to tell Debian we don't care that we're editing the system python
+# installation.
+# Some context in https://github.com/pypa/pip/issues/11381#issuecomment-1399263627
+RUN ${PYTHON_VERSION} -m pip install -q --upgrade pip --break-system-packages
+RUN ${PYTHON_VERSION} -m pip install -q --no-cache-dir poetry==1.3.2 --break-system-packages
 
 # As part of the Docker build, we attempt to pre-install Synapse's dependencies
 # in the hope that it speeds up the real install of Synapse. To make this work,
diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm
index 588b63ae5..9b0065166 100644
--- a/lib/SyTest/Homeserver/Synapse.pm
+++ b/lib/SyTest/Homeserver/Synapse.pm
@@ -1201,6 +1201,8 @@ global
     ssl-default-bind-ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS !RC4"
     ssl-default-bind-options no-sslv3
 
+    maxconn 2000
+
 defaults
     mode http
     log global
diff --git a/scripts/synapse_sytest.sh b/scripts/synapse_sytest.sh
index ceb0fb10c..031b6322e 100755
--- a/scripts/synapse_sytest.sh
+++ b/scripts/synapse_sytest.sh
@@ -161,7 +161,7 @@ else
         fi
         ln -s -T /venv /synapse/.venv # reuse the existing virtual env
         pushd /synapse
-        poetry install -vvv --extras all
+        poetry install -vv --extras all
         popd
     else
         # Install Synapse and dependencies using pip. As of pip 20.1, this will

From 160dde018ee6a7f5b6388026aaa066e38f133830 Mon Sep 17 00:00:00 2001
From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com>
Date: Thu, 2 Mar 2023 18:38:52 +0100
Subject: [PATCH 14/55] Add /knock endpoint to the `event_creator`. (#1332)

---
 lib/SyTest/Homeserver/Synapse.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm
index 9b0065166..b992531a8 100644
--- a/lib/SyTest/Homeserver/Synapse.pm
+++ b/lib/SyTest/Homeserver/Synapse.pm
@@ -1320,6 +1320,7 @@ sub generate_haproxy_map
 ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/send                                 event_creator
 ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$  event_creator
 ^/_matrix/client/(api/v1|r0|v3|unstable)/join/                                         event_creator
+^/_matrix/client/(api/v1|r0|v3|unstable)/knock/                                        event_creator
 ^/_matrix/client/(api/v1|r0|v3|unstable)/profile/                                      event_creator
 ^/_matrix/client/(api/v1|r0|v3|unstable)/createRoom                                    event_creator
 

From 35545b63365b99b16ec440ab508ec9c0118ccb38 Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Wed, 8 Mar 2023 17:04:18 +0000
Subject: [PATCH 15/55] Remove tests that were migrated to Complement (#1338)

These tests were migrated in
https://github.com/matrix-org/complement/pull/302.

Signed-off-by: Sean Quah <seanq@matrix.org>
---
 tests/30rooms/31forget.pl | 210 --------------------------------------
 1 file changed, 210 deletions(-)
 delete mode 100644 tests/30rooms/31forget.pl

diff --git a/tests/30rooms/31forget.pl b/tests/30rooms/31forget.pl
deleted file mode 100644
index 1c97c0c2f..000000000
--- a/tests/30rooms/31forget.pl
+++ /dev/null
@@ -1,210 +0,0 @@
-use List::Util qw( any none );
-
-push our @EXPORT, qw( matrix_forget_room );
-
-sub matrix_forget_room
-{
-   my ( $user, $room_id ) = @_;
-   is_User( $user ) or croak "Expected a User; got $user";
-
-   do_request_json_for( $user,
-      method => "POST",
-      uri    => "/v3/rooms/$room_id/forget",
-
-      content => {},
-   )->then_done(1);
-}
-
-test "Forgotten room messages cannot be paginated",
-   requires => [ local_user_and_room_fixtures(), local_user_fixture() ],
-
-   do => sub {
-      my ( $creator, $room_id, $user ) = @_;
-
-      matrix_join_room_synced( $user, $room_id )
-      ->then( sub {
-         matrix_send_room_text_message_synced( $creator, $room_id, body => "sup" )
-      })->then( sub {
-         matrix_get_room_messages( $user, $room_id, limit => 1 );
-      })->then( sub {
-         my ( $body ) = @_;
-
-         assert_json_keys( $body, qw( chunk ) );
-         log_if_fail "Chunk", $body->{chunk};
-         $body->{chunk}[0]{content}{body} eq "sup" or die "Wrong message";
-
-         matrix_leave_room_synced( $user, $room_id )
-      })->then( sub {
-         matrix_get_room_state( $creator, $room_id,
-            type      => "m.room.member",
-            state_key => $user->user_id
-         )
-      })->then( sub {
-         my ( $content ) = @_;
-
-         assert_eq( $content->{membership}, "leave",
-            "membership state" );
-
-         matrix_forget_room( $user, $room_id )
-      })->then( sub {
-         matrix_get_room_state( $creator, $room_id,
-            type      => "m.room.member",
-            state_key => $user->user_id
-         )
-      })->then( sub {
-         my ( $content ) = @_;
-
-         assert_eq( $content->{membership}, "leave",
-            "membership state" );
-
-         # Need to repeat this, because we may need to give Synapse time to
-         # replicate the "forgot" state from the worker handling
-         # POST /rooms/ROOM/forget (master?) to the worker handling
-         # GET /rooms/ROOM/messages (client_reader).
-         # (AFAICS forgetting a room doesn't have any other visible effects.)
-         retry_until_success {
-            matrix_get_room_messages($user, $room_id, limit => 1)
-               ->main::check_http_code(
-               403 => "ok",
-               200 => "redo",
-            );
-         };
-      });
-   };
-
-test "Forgetting room does not show up in v2 /sync",
-   requires => [ local_user_and_room_fixtures(), local_user_fixture() ],
-
-   do => sub {
-      my ( $creator, $room_id, $user ) = @_;
-
-      matrix_join_room_synced( $user, $room_id )
-      ->then( sub {
-         matrix_send_room_text_message_synced( $creator, $room_id, body => "sup" )
-      })->then( sub {
-         matrix_leave_room_synced( $user, $room_id )
-      })->then( sub {
-         matrix_forget_room( $user, $room_id )
-      })->then( sub {
-         matrix_sync( $user )
-      })->then( sub {
-         my ( $body ) = @_;
-
-         log_if_fail "Sync response", $body;
-
-         die "Did not expect room in archived" if $body->{rooms}->{archived}->{$room_id};
-         die "Did not expect room in joined" if $body->{rooms}->{joined}->{$room_id};
-         die "Did not expect room in invited" if $body->{rooms}->{invited}->{$room_id};
-
-         Future->done( 1 );
-      });
-   };
-
-test "Can forget room you've been kicked from",
-   requires => [ local_user_and_room_fixtures(), local_user_fixture() ],
-
-   do => sub {
-      my ( $creator, $room_id, $user ) = @_;
-
-      matrix_join_room_synced( $user, $room_id )
-      ->then( sub {
-         matrix_send_room_text_message_synced( $creator, $room_id, body => "sup" );
-      })->then( sub {
-         do_request_json_for( $creator,
-            method => "POST",
-            uri    => "/v3/rooms/$room_id/kick",
-
-            content => { user_id => $user->user_id },
-         );
-      })->then( sub {
-         matrix_forget_room( $user, $room_id )
-      })->then( sub {
-         matrix_sync( $user )
-      })->then( sub {
-         my ( $body ) = @_;
-
-         log_if_fail "Sync response", $body;
-
-         die "Did not expect room in archived" if $body->{rooms}->{archived}->{$room_id};
-         die "Did not expect room in joined" if $body->{rooms}->{joined}->{$room_id};
-         die "Did not expect room in invited" if $body->{rooms}->{invited}->{$room_id};
-
-         Future->done( 1 );
-      });
-   };
-
-
-test "Can't forget room you're still in",
-   requires => [ local_user_and_room_fixtures(), local_user_fixture() ],
-
-   do => sub {
-      my ( $creator, $room_id, $user ) = @_;
-
-      matrix_join_room_synced( $user, $room_id )
-      ->then( sub {
-         matrix_send_room_text_message_synced( $creator, $room_id, body => "sup" );
-      })->then( sub {
-         matrix_forget_room( $user, $room_id )
-      })->main::expect_http_4xx;
-   };
-
-test "Can re-join room if re-invited",
-   requires => [ local_user_fixture(), local_user_fixture() ],
-
-   do => sub {
-      my ( $creator, $user ) = @_;
-
-      my ( $room_id );
-
-      matrix_create_room_synced( $creator, invite => [ $user->user_id ] )->then( sub {
-         ( $room_id ) = @_;
-
-         log_if_fail "room_id", $room_id;
-
-         matrix_put_room_state_synced( $creator, $room_id,
-            type      => "m.room.join_rules",
-            state_key => "",
-            content   => {
-               join_rule => "invite",
-            }
-         )
-      })->then( sub {
-         matrix_join_room_synced( $user, $room_id );
-      })->then( sub {
-         matrix_send_room_text_message_synced( $creator, $room_id, body => "before leave" );
-      })->then( sub {
-         matrix_get_room_messages( $user, $room_id, limit => 100 );
-      })->then( sub {
-         matrix_leave_room_synced( $user, $room_id );
-      })->then( sub {
-         matrix_forget_room( $user, $room_id );
-      })->then( sub {
-         matrix_join_room( $user, $room_id )->main::expect_http_403;
-      })->then( sub {
-         matrix_invite_user_to_room_synced( $creator, $user, $room_id );
-      })->then( sub {
-         matrix_join_room_synced( $user, $room_id );
-      })->then( sub {
-         matrix_get_room_messages( $user, $room_id, limit => 100 );
-      })->then( sub {
-         my ( $body ) = @_;
-
-         any { $_->{type} eq "m.room.message" && $_->{content}->{body} eq "before leave" } @{ $body->{chunk} }
-            or die "Should have seen before leave message";
-
-         # have the creator send another message, and ensure the joiner can see it.
-         matrix_send_room_text_message_synced( $creator, $room_id, body => "after rejoin" );
-      })->then( sub {
-         matrix_get_room_messages( $user, $room_id, limit => 1 );
-      })->then( sub {
-         my ( $body ) = @_;
-
-         log_if_fail "body", $body;
-
-         @{ $body->{chunk} } == 1 or die "Expected event";
-         $body->{chunk}[0]->{type} eq "m.room.message" && $body->{chunk}[0]->{content}{body} eq "after rejoin"
-            or die "Got wrong event";
-
-         Future->done( 1 );
-      });
-   };

From 3ec41b72fabb73730582ef8651aeb6f9e551fad2 Mon Sep 17 00:00:00 2001
From: Jason Little <realtyem@gmail.com>
Date: Fri, 17 Mar 2023 08:00:10 -0500
Subject: [PATCH 16/55] Add /register/available endpoint to workers for
 Synapse. (#1340)

---
 lib/SyTest/Homeserver/Synapse.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm
index b992531a8..9869945dd 100644
--- a/lib/SyTest/Homeserver/Synapse.pm
+++ b/lib/SyTest/Homeserver/Synapse.pm
@@ -1297,6 +1297,7 @@ sub generate_haproxy_map
 ^/_matrix/client/versions$                                           client_reader
 ^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$            client_reader
 ^/_matrix/client/(r0|v3|unstable)/register$                          client_reader
+^/_matrix/client/(r0|v3|unstable)/register/available$                client_reader
 ^/_matrix/client/(r0|v3|unstable)/auth/.*/fallback/web$              client_reader
 ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/messages$          client_reader
 ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event              client_reader

From cdac5a1cd09251c27dfb4666ccd6d34a223fe335 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Fri, 17 Mar 2023 13:13:03 +0000
Subject: [PATCH 17/55] Fix Synapse docker builds again. (#1339)

By only using the --break-system-packages on the versions
which support it.
---
 .github/workflows/docker.yml | 14 +++++++-------
 docker/synapse.Dockerfile    |  5 +++--
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index b2846e63b..c4d0c1e1a 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -12,6 +12,10 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build-sytest-images:
     name: "Build sytest:${{ matrix.tag }}"
@@ -25,8 +29,6 @@ jobs:
             tag: buster
           - base_image: debian:testing
             tag: testing
-          - base_image: debian:bookworm
-            tag: bookworm
     steps:
       - name: Set up QEMU
         id: QEMU
@@ -74,11 +76,9 @@ jobs:
           - sytest_image_tag: testing
             dockerfile: synapse
             tags: "matrixdotorg/sytest-synapse:testing"
-            build_args: "SYTEST_IMAGE_TAG=testing"
-          - sytest_image_tag: bookworm
-            dockerfile: synapse
-            tags: "matrixdotorg/sytest-synapse:bookworm"
-            # Note: at the time of writing, bookworm's python3 is python 3.11.
+            build_args: |
+              SYTEST_IMAGE_TAG=testing
+              SYSTEM_PIP_INSTALL_SUFFIX=--break-system-packages
 
     steps:
       - name: Set up QEMU
diff --git a/docker/synapse.Dockerfile b/docker/synapse.Dockerfile
index 14187ea6d..fdb5673c9 100644
--- a/docker/synapse.Dockerfile
+++ b/docker/synapse.Dockerfile
@@ -3,6 +3,7 @@ ARG SYTEST_IMAGE_TAG=buster
 FROM matrixdotorg/sytest:${SYTEST_IMAGE_TAG}
 
 ARG PYTHON_VERSION=python3
+ARG SYSTEM_PIP_INSTALL_SUFFIX=""
 
 ENV DEBIAN_FRONTEND noninteractive
 
@@ -26,8 +27,8 @@ RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-tool
 # For now, we need to tell Debian we don't care that we're editing the system python
 # installation.
 # Some context in https://github.com/pypa/pip/issues/11381#issuecomment-1399263627
-RUN ${PYTHON_VERSION} -m pip install -q --upgrade pip --break-system-packages
-RUN ${PYTHON_VERSION} -m pip install -q --no-cache-dir poetry==1.3.2 --break-system-packages
+RUN ${PYTHON_VERSION} -m pip install -q --upgrade pip ${SYSTEM_PIP_INSTALL_SUFFIX}
+RUN ${PYTHON_VERSION} -m pip install -q --no-cache-dir poetry==1.3.2 ${SYSTEM_PIP_INSTALL_SUFFIX}
 
 # As part of the Docker build, we attempt to pre-install Synapse's dependencies
 # in the hope that it speeds up the real install of Synapse. To make this work,

From 9e53896ff4c8ee214e6b8e0f5df3524681f12b5f Mon Sep 17 00:00:00 2001
From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com>
Date: Mon, 27 Mar 2023 13:37:09 +0200
Subject: [PATCH 18/55] Add `/password_policy` endpoint to `client_reader`
 (#1342)

---
 lib/SyTest/Homeserver/Synapse.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm
index 9869945dd..e0cd48278 100644
--- a/lib/SyTest/Homeserver/Synapse.pm
+++ b/lib/SyTest/Homeserver/Synapse.pm
@@ -1305,6 +1305,7 @@ sub generate_haproxy_map
 ^/_matrix/client/(api/v1|r0|v3|unstable/.*)/rooms/.*/aliases         client_reader
 ^/_matrix/client/(api/v1|r0|v3|unstable)/search                      client_reader
 ^/_matrix/client/(r0|v3|unstable)/user/.*/filter(/|$)                client_reader
+^/_matrix/client/(r0|v3|unstable)/password_policy$                   client_reader
 
 ^/_matrix/client/(api/v1|r0|v3|unstable)/devices$                    stream_writer
 ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/query$                 stream_writer

From deb9b98640548186fa72c2cd7098167470255caf Mon Sep 17 00:00:00 2001
From: "DeepBlueV7.X" <nicolas.werner@hotmail.de>
Date: Tue, 28 Mar 2023 07:14:31 +0000
Subject: [PATCH 19/55] Add proper server names to join in unban tests (#1341)

---
 tests/30rooms/07ban.pl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/30rooms/07ban.pl b/tests/30rooms/07ban.pl
index d5956b08e..44c2c8678 100644
--- a/tests/30rooms/07ban.pl
+++ b/tests/30rooms/07ban.pl
@@ -29,7 +29,7 @@
             })
          }
       })->then( sub {
-         matrix_join_room( $banned_user, $room_id )
+         matrix_join_room( $banned_user, $room_id, ( server_name => $creator->server_name, ) )
             ->main::expect_http_403;  # Must be unbanned first
       })->then( sub {
          do_request_json_for( $creator,
@@ -103,7 +103,7 @@
          };
       })->then( sub {
          # Must be unbanned first
-         matrix_join_room( $banned_user, $room_id )->main::check_http_code(
+         matrix_join_room( $banned_user, $room_id, ( server_name => $creator->server_name, ) )->main::check_http_code(
             403 => "ok",
             200 => "redo",
          );

From 6842592db4975d9fb2044502245e9f3c0d55faa9 Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Tue, 28 Mar 2023 12:03:46 -0400
Subject: [PATCH 20/55] Allow running pipeline on workflow dispatch. (#1345)

---
 .github/workflows/pipeline.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 350c0f7b1..a9b138a66 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -3,6 +3,7 @@ on:
   push:
     branches: ["develop", "release-*"]
   pull_request:
+  workflow_dispatch:
 
 # Only run this action once per pull request/branch; restart if a new commit arrives.
 # C.f. https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency

From 0cbd08892d413b68374ee1bb117cd69ac99de809 Mon Sep 17 00:00:00 2001
From: gnieto <gnieto.talo@gmail.com>
Date: Thu, 30 Mar 2023 11:42:33 +0200
Subject: [PATCH 21/55] Improve room send tests (#1344)

- Add origin_server_ts in make_join proto events as it's a required in
  both the [make_join](https://spec.matrix.org/v1.6/server-server-api/#get_matrixfederationv1make_joinroomiduserid)
  specification and all the PDU definitions in rooms v1-v10.
- Do not require event endpoint in user setup since the tests were
  migrated to [use sync](https://github.com/matrix-org/sytest/commit/ab34a0ed0d122c7f44fc71fd9a799fdc410a2f38)
---
 lib/SyTest/Federation/Room.pm     | 1 +
 tests/50federation/31room-send.pl | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/SyTest/Federation/Room.pm b/lib/SyTest/Federation/Room.pm
index 8b5831897..f13aae5b1 100644
--- a/lib/SyTest/Federation/Room.pm
+++ b/lib/SyTest/Federation/Room.pm
@@ -414,6 +414,7 @@ sub make_join_protoevent
 
       auth_events      => $auth_events,
       content          => { membership => "join" },
+      origin_server_ts => JSON::number( int( time() * 1000 )),
       depth            => JSON::number($self->next_depth),
       prev_events      => $prev_events,
       room_id          => $self->room_id,
diff --git a/tests/50federation/31room-send.pl b/tests/50federation/31room-send.pl
index 4a9323670..4b4437627 100644
--- a/tests/50federation/31room-send.pl
+++ b/tests/50federation/31room-send.pl
@@ -50,7 +50,7 @@
 test "Inbound federation can receive events",
    requires => [ $main::OUTBOUND_CLIENT,
                  local_user_and_room_fixtures(
-                   user_opts => { with_events => 1 },
+                   user_opts => { with_events => 0 },
                  ),
                  federation_user_id_fixture() ],
 
@@ -98,7 +98,7 @@
 test "Inbound federation can receive redacted events",
    requires => [ $main::OUTBOUND_CLIENT,
                  local_user_and_room_fixtures(
-                   user_opts => { with_events => 1 },
+                   user_opts => { with_events => 0 },
                  ),
                  federation_user_id_fixture() ],
 

From 5727f78f77d10b5e3ece5bfc3054eac86fc691d1 Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Thu, 30 Mar 2023 13:36:14 +0100
Subject: [PATCH 22/55] Fix "... non-joined user can call /events on
 world_readable room" flakes (#1346)

The test was flaking because it was sometimes waiting for events to come
down /events after they had already appeared in the stream.

Signed-off-by: Sean Quah <seanq@matrix.org>
---
 tests/30rooms/30history-visibility.pl | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/30rooms/30history-visibility.pl b/tests/30rooms/30history-visibility.pl
index c4e42dc15..24d9bea2e 100644
--- a/tests/30rooms/30history-visibility.pl
+++ b/tests/30rooms/30history-visibility.pl
@@ -147,6 +147,13 @@ sub test_history_visibility
             matrix_set_room_history_visibility_synced( $user, $room_id, "world_readable" );
          })->then( sub {
             matrix_initialsync_room( $nonjoined_user, $room_id )
+         })->then( sub {
+            # Get a token and configure the waits for events below to start from it.
+            matrix_get_events( $nonjoined_user, room_id => $room_id, timeout => 0 )
+            ->then( sub {
+               my ( $body ) = @_;
+               $nonjoined_user->eventstream_token = $body->{end};
+            });
          })->then( sub {
             Future->needs_all(
                matrix_send_room_text_message_synced( $user, $room_id, body => "mice" )

From 29193f12f3f61cf20a11a972a61500b11e82bae6 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 3 Apr 2023 19:20:41 +0200
Subject: [PATCH 23/55] Use `v1` and `v3` endpoints for `/thirdparty` tests
 (#1310)

---
 lib/SyTest/ApplicationService.pm    |  2 +-
 tests/60app-services/03passive.pl   |  4 ++--
 tests/60app-services/05lookup3pe.pl | 10 +++++-----
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/SyTest/ApplicationService.pm b/lib/SyTest/ApplicationService.pm
index dc17142c2..9b5e70c65 100644
--- a/lib/SyTest/ApplicationService.pm
+++ b/lib/SyTest/ApplicationService.pm
@@ -42,7 +42,7 @@ sub new
    my %futures_by_type;
 
    my $f = repeat {
-      $await_http->( qr{^\Q$path\E/transactions/\d+$}, sub { 1 },
+      $await_http->( qr{^\Q$path\E/_matrix/app/v1/transactions/\d+$}, sub { 1 },
          timeout => 0,
       )->then( sub {
          my ( $request ) = @_;
diff --git a/tests/60app-services/03passive.pl b/tests/60app-services/03passive.pl
index 35436fedf..15c3f6752 100644
--- a/tests/60app-services/03passive.pl
+++ b/tests/60app-services/03passive.pl
@@ -13,7 +13,7 @@
       my $localpart = "astest-03passive-1";
       my $user_id = "\@$localpart:$server_name";
 
-      require_stub $appserv->await_http_request( "/users/$user_id", sub { 1 } )
+      require_stub $appserv->await_http_request( "/_matrix/app/v1/users/$user_id", sub { 1 } )
          ->then( sub {
             my ( $request ) = @_;
 
@@ -52,7 +52,7 @@
    do => sub {
       my ( $as_user, $appserv, $local_user, $room_id, $room_alias ) = @_;
 
-      require_stub $appserv->await_http_request( "/rooms/$room_alias", sub { 1 } )
+      require_stub $appserv->await_http_request( "/_matrix/app/v1/rooms/$room_alias", sub { 1 } )
          ->then( sub {
             my ( $request ) = @_;
 
diff --git a/tests/60app-services/05lookup3pe.pl b/tests/60app-services/05lookup3pe.pl
index e29ace7af..79a29a86a 100644
--- a/tests/60app-services/05lookup3pe.pl
+++ b/tests/60app-services/05lookup3pe.pl
@@ -1,6 +1,6 @@
 use List::UtilsBy qw( sort_by );
 
-use constant AS_PREFIX => "/_matrix/app/unstable";
+use constant AS_PREFIX => "/_matrix/app/v1";
 
 sub stub_empty_result
 {
@@ -59,7 +59,7 @@ sub stub_empty_result
 
          do_request_json_for( $user,
             method => "GET",
-            uri    => "/unstable/thirdparty/protocols"
+            uri    => "/v3/thirdparty/protocols"
          )->then( sub {
             my ( $body ) = @_;
 
@@ -107,7 +107,7 @@ sub stub_empty_result
 
       do_request_json_for( $user,
          method => "GET",
-         uri    => "/unstable/thirdparty/protocol/ymca"
+         uri    => "/v3/thirdparty/protocol/ymca"
       )->then( sub {
          my ( $body ) = @_;
 
@@ -172,7 +172,7 @@ sub stub_empty_result
 
          do_request_json_for( $user,
             method => "GET",
-            uri    => "/unstable/thirdparty/user/ymca",
+            uri    => "/v3/thirdparty/user/ymca",
 
             params => {
                field1 => "ONE",
@@ -237,7 +237,7 @@ sub stub_empty_result
 
          do_request_json_for( $user,
             method => "GET",
-            uri    => "/unstable/thirdparty/location/ymca",
+            uri    => "/v3/thirdparty/location/ymca",
 
             params => {
                field3 => "THREE",

From 16e026be3a725f1d4a93e75fe78db8dce74b1299 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 4 Apr 2023 17:18:13 +0100
Subject: [PATCH 24/55] Bump docker actions to latest stable release (#1347)

Part of https://github.com/matrix-org/synapse/issues/14203

I've checked changelogs and AFAICS these are backwards compatible for
our purposes. (There's a semver bump because a newer version of GHA
runners and/or node.js is required. But we don't care about those.)
---
 .github/workflows/docker.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index c4d0c1e1a..1c372541d 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -32,23 +32,23 @@ jobs:
     steps:
       - name: Set up QEMU
         id: QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Inspect builder
         run: docker buildx inspect
 
       - name: Log in to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v4
         with:
           pull: true
           push: true
@@ -83,23 +83,23 @@ jobs:
     steps:
       - name: Set up QEMU
         id: QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Inspect builder
         run: docker buildx inspect
 
       - name: Log in to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v4
         with:
           pull: true
           push: true

From 3b0208a1ced1da95d46d88881ebe90f5696b8e27 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 4 Apr 2023 17:21:05 +0100
Subject: [PATCH 25/55] Fix typo from 1347

---
 .github/workflows/docker.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 1c372541d..a4d3e1dc0 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -42,7 +42,7 @@ jobs:
         run: docker buildx inspect
 
       - name: Log in to DockerHub
-        uses: docker/login-action@v4
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_TOKEN }}
@@ -93,7 +93,7 @@ jobs:
         run: docker buildx inspect
 
       - name: Log in to DockerHub
-        uses: docker/login-action@v4
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_TOKEN }}

From f8c7f2eb11ec61bf8a758582359c5c26af97d2d9 Mon Sep 17 00:00:00 2001
From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com>
Date: Fri, 14 Apr 2023 16:46:13 +0200
Subject: [PATCH 26/55] Add `/directory/room/{roomAlias}` endpoint to
 `client_reader` (#1343)

---
 lib/SyTest/Homeserver/Synapse.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm
index e0cd48278..704e9ff06 100644
--- a/lib/SyTest/Homeserver/Synapse.pm
+++ b/lib/SyTest/Homeserver/Synapse.pm
@@ -1306,6 +1306,7 @@ sub generate_haproxy_map
 ^/_matrix/client/(api/v1|r0|v3|unstable)/search                      client_reader
 ^/_matrix/client/(r0|v3|unstable)/user/.*/filter(/|$)                client_reader
 ^/_matrix/client/(r0|v3|unstable)/password_policy$                   client_reader
+^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$          client_reader
 
 ^/_matrix/client/(api/v1|r0|v3|unstable)/devices$                    stream_writer
 ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/query$                 stream_writer

From 3c26f28b44922f6c1248bc37924604f2a077afd7 Mon Sep 17 00:00:00 2001
From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com>
Date: Fri, 14 Apr 2023 19:33:02 +0200
Subject: [PATCH 27/55] Add `/capabilities` endpoint to `client_reader` (#1349)

---
 lib/SyTest/Homeserver/Synapse.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm
index 704e9ff06..06c2e2662 100644
--- a/lib/SyTest/Homeserver/Synapse.pm
+++ b/lib/SyTest/Homeserver/Synapse.pm
@@ -1307,6 +1307,7 @@ sub generate_haproxy_map
 ^/_matrix/client/(r0|v3|unstable)/user/.*/filter(/|$)                client_reader
 ^/_matrix/client/(r0|v3|unstable)/password_policy$                   client_reader
 ^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$          client_reader
+^/_matrix/client/(r0|v3|unstable)/capabilities$                      client_reader
 
 ^/_matrix/client/(api/v1|r0|v3|unstable)/devices$                    stream_writer
 ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/query$                 stream_writer

From 1e389fd2b2e849e17ea5fba61e7c466c15907109 Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Thu, 8 Jun 2023 21:20:11 +0100
Subject: [PATCH 28/55] Include the `destination` field in `X-Matrix` auth
 headers (#1354)

The `destination` field was added in Matrix 1.3.

Signed-off-by: Sean Quah <seanq@matrix.org>
---
 lib/SyTest/Federation/Client.pm | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/lib/SyTest/Federation/Client.pm b/lib/SyTest/Federation/Client.pm
index 8e3912be3..ab2764502 100644
--- a/lib/SyTest/Federation/Client.pm
+++ b/lib/SyTest/Federation/Client.pm
@@ -92,10 +92,15 @@ sub do_request_json
 
    my $signature = $signing_block{signatures}{$origin}{$key_id};
 
+   # Assume that there is no delegation in place, so the hostname and
+   # destination server name are the same.
+   my $destination = $params{hostname};
+
    my $auth = "X-Matrix " . join_header_words(
-      [ origin => $origin ],
-      [ key    => $key_id ],
-      [ sig    => $signature ],
+      [ origin      => $origin      ],
+      [ key         => $key_id      ],
+      [ sig         => $signature   ],
+      [ destination => $destination ],
    );
 
    # TODO: SYN-437 synapse does not like OWS between auth-param elements

From bb77060e3c58e8984a7ebe01e01fbd0c631319b4 Mon Sep 17 00:00:00 2001
From: Josh Qou <97894002+joshqou@users.noreply.github.com>
Date: Thu, 15 Jun 2023 12:27:57 +0100
Subject: [PATCH 29/55] Fix unsafe assertions for media api tests (#1353)

---
 tests/51media/01unicode.pl | 8 ++++----
 tests/51media/03ascii.pl   | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/51media/01unicode.pl b/tests/51media/01unicode.pl
index 54511f8d4..b450d4be6 100644
--- a/tests/51media/01unicode.pl
+++ b/tests/51media/01unicode.pl
@@ -122,10 +122,10 @@ sub parse_content_disposition_params {
    assert_eq( scalar @parts, 1, "number of content-dispostion header lists" );
    @parts = @{$parts[0]};
 
-   # the first part must be 'inline'
+   # the first part must be 'attachment'
    my $k = shift @parts;
    my $v = shift @parts;
-   assert_eq( $k, "inline", "content-disposition" );
+   assert_eq( $k, "attachment", "content-disposition" );
    die "invalid CD" if defined $v;
 
    my %params;
@@ -233,8 +233,8 @@ sub test_using_client
          my ( $body, $response ) = @_;
 
          my $disposition = $response->header( "Content-Disposition" );
-         uc $disposition eq uc "inline; filename*=utf-8''$alt_filename_encoded" or
-            uc $disposition eq uc "inline; filename=utf-8\"$alt_filename\"" or
+         uc $disposition eq uc "attachment; filename*=utf-8''$alt_filename_encoded" or
+            uc $disposition eq uc "attachment; filename=utf-8\"$alt_filename\"" or
             die "Expected a UTF-8 filename parameter";
 
          Future->done(1);
diff --git a/tests/51media/03ascii.pl b/tests/51media/03ascii.pl
index 684b4f323..6adb05958 100644
--- a/tests/51media/03ascii.pl
+++ b/tests/51media/03ascii.pl
@@ -77,7 +77,7 @@ sub assert_cd_params_match_filename {
          my ( $body, $response ) = @_;
 
          my $disposition = $response->header( "Content-Disposition" );
-         $disposition eq "inline; filename=also_ascii" or
+         $disposition eq "attachment; filename=also_ascii" or
             die "Expected an ASCII filename parameter";
 
          Future->done(1);

From 0fc46a418dc2ae8e37ffa3a3a0246b0cdfcbc0a2 Mon Sep 17 00:00:00 2001
From: reivilibre <oliverw@matrix.org>
Date: Fri, 16 Jun 2023 09:13:45 +0000
Subject: [PATCH 30/55] Change expectations of filenameless
 `Content-Disposition` (#1355)

Follows #1353
---
 tests/51media/02nofilename.pl | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/51media/02nofilename.pl b/tests/51media/02nofilename.pl
index c27bebf36..968a7a6cb 100644
--- a/tests/51media/02nofilename.pl
+++ b/tests/51media/02nofilename.pl
@@ -22,8 +22,9 @@ sub test_using_client
    get_media( $client, $content_id )->then( sub {
       my ( $disposition ) = @_;
 
-      defined $disposition and
-         die "Unexpected Content-Disposition header";
+      # Require `attachment` `Content-Disposition` without a filename portion.
+      # See `get_media`: {} means attachment with no key-value components in the header value.
+      assert_eq( scalar %$disposition, 0, "content-disposition" );
 
       Future->done(1);
    });

From 575569e3802462fd462cb47bc4ca1749fa922bc3 Mon Sep 17 00:00:00 2001
From: Jason Little <realtyem@gmail.com>
Date: Mon, 3 Jul 2023 05:41:12 -0500
Subject: [PATCH 31/55] Transition worker_replication_* settings to
 instance_map->main settings (#1357)

* Transition worker_replication_* settings to instance_map->main settings

* Why is the instance_map shown if there are no workers?
---
 lib/SyTest/Homeserver/Synapse.pm | 70 +++++++++++---------------------
 1 file changed, 24 insertions(+), 46 deletions(-)

diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm
index 06c2e2662..f74d2ce61 100644
--- a/lib/SyTest/Homeserver/Synapse.pm
+++ b/lib/SyTest/Homeserver/Synapse.pm
@@ -308,24 +308,30 @@ sub start
            },
         ) : (),
 
-        instance_map => {
-           "event_persister1" => {
-              host => "$bind_host",
-              port => $self->{ports}{event_persister1},
-           },
-           "event_persister2" => {
-              host => "$bind_host",
-              port => $self->{ports}{event_persister2},
-           },
-           "client_reader" => {
-              host => "$bind_host",
-              port => $self->{ports}{client_reader},
-           },
-           "stream_writer" => {
-              host => "$bind_host",
-              port => $self->{ports}{stream_writer},
-           },
-        },
+        $self->{workers} ? (
+            instance_map => {
+               "main" => {
+                  host => "$bind_host",
+                  port => $self->{ports}{synapse_unsecure},
+               },
+               "event_persister1" => {
+                  host => "$bind_host",
+                  port => $self->{ports}{event_persister1},
+               },
+               "event_persister2" => {
+                  host => "$bind_host",
+                  port => $self->{ports}{event_persister2},
+               },
+               "client_reader" => {
+                  host => "$bind_host",
+                  port => $self->{ports}{client_reader},
+               },
+               "stream_writer" => {
+                  host => "$bind_host",
+                  port => $self->{ports}{stream_writer},
+               },
+            },
+        ) : (),
 
         stream_writers => {
            events => $self->{redis_host} ne '' ? [ "event_persister1", "event_persister2" ] : "master",
@@ -670,8 +676,6 @@ sub _start_synapse
          "worker_name"             => "pusher",
          "worker_pid_file"         => "$hsdir/pusher.pid",
          "worker_log_config"       => $self->configure_logger("pusher"),
-         "worker_replication_host" => "$bind_host",
-         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type      => "http",
@@ -696,8 +700,6 @@ sub _start_synapse
          "worker_name"             => "appservice",
          "worker_pid_file"         => "$hsdir/appservice.pid",
          "worker_log_config"       => $self->configure_logger("appservice"),
-         "worker_replication_host" => "$bind_host",
-         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type => "manhole",
@@ -722,8 +724,6 @@ sub _start_synapse
          "worker_name"             => "federation_sender",
          "worker_pid_file"         => "$hsdir/federation_sender.pid",
          "worker_log_config"       => $self->configure_logger("federation_sender"),
-         "worker_replication_host" => "$bind_host",
-         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type => "manhole",
@@ -748,8 +748,6 @@ sub _start_synapse
          "worker_name"             => "synchrotron",
          "worker_pid_file"         => "$hsdir/synchrotron.pid",
          "worker_log_config"       => $self->configure_logger("synchrotron"),
-         "worker_replication_host" => "$bind_host",
-         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type      => "http",
@@ -780,8 +778,6 @@ sub _start_synapse
          "worker_name"             => "federation_reader",
          "worker_pid_file"         => "$hsdir/federation_reader.pid",
          "worker_log_config"       => $self->configure_logger("federation_reader"),
-         "worker_replication_host" => "$bind_host",
-         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type      => "http",
@@ -812,8 +808,6 @@ sub _start_synapse
          "worker_name"             => "media_repository",
          "worker_pid_file"         => "$hsdir/media_repository.pid",
          "worker_log_config"       => $self->configure_logger("media_repository"),
-         "worker_replication_host" => "$bind_host",
-         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type      => "http",
@@ -844,8 +838,6 @@ sub _start_synapse
          "worker_name"                  => "client_reader",
          "worker_pid_file"              => "$hsdir/client_reader.pid",
          "worker_log_config"            => $self->configure_logger("client_reader"),
-         "worker_replication_host"      => "$bind_host",
-         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"             => [
             {
                type      => "http",
@@ -876,8 +868,6 @@ sub _start_synapse
          "worker_name"             => "user_dir",
          "worker_pid_file"         => "$hsdir/user_dir.pid",
          "worker_log_config"       => $self->configure_logger("user_dir"),
-         "worker_replication_host" => "$bind_host",
-         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"        => [
             {
                type      => "http",
@@ -908,8 +898,6 @@ sub _start_synapse
          "worker_name"                  => "event_creator",
          "worker_pid_file"              => "$hsdir/event_creator.pid",
          "worker_log_config"            => $self->configure_logger("event_creator"),
-         "worker_replication_host"      => "$bind_host",
-         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"             => [
             {
                type      => "http",
@@ -940,8 +928,6 @@ sub _start_synapse
          "worker_name"                  => "frontend_proxy1",
          "worker_pid_file"              => "$hsdir/frontend_proxy.pid",
          "worker_log_config"            => $self->configure_logger("frontend_proxy"),
-         "worker_replication_host"      => "$bind_host",
-         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"             => [
             {
                type      => "http",
@@ -972,8 +958,6 @@ sub _start_synapse
          "worker_name"                  => "background_worker1",
          "worker_pid_file"              => "$hsdir/background_worker.pid",
          "worker_log_config"            => $self->configure_logger("background_worker"),
-         "worker_replication_host"      => "$bind_host",
-         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
       };
 
       push @worker_configs, $background_worker_config;
@@ -985,8 +969,6 @@ sub _start_synapse
          "worker_name"                  => "event_persister1",
          "worker_pid_file"              => "$hsdir/event_persister1.pid",
          "worker_log_config"            => $self->configure_logger("event_persister1"),
-         "worker_replication_host"      => "$bind_host",
-         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"             => [
             {
                type      => "http",
@@ -1017,8 +999,6 @@ sub _start_synapse
          "worker_name"                  => "event_persister2",
          "worker_pid_file"              => "$hsdir/event_persister2.pid",
          "worker_log_config"            => $self->configure_logger("event_persister2"),
-         "worker_replication_host"      => "$bind_host",
-         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"             => [
             {
                type      => "http",
@@ -1049,8 +1029,6 @@ sub _start_synapse
          "worker_name"                  => "stream_writer",
          "worker_pid_file"              => "$hsdir/stream_writer.pid",
          "worker_log_config"            => $self->configure_logger("stream_writer"),
-         "worker_replication_host"      => "$bind_host",
-         "worker_replication_http_port" => $self->{ports}{synapse_unsecure},
          "worker_listeners"             => [
             {
                type      => "http",

From 50544bb329119be48643396e65bac585c554903f Mon Sep 17 00:00:00 2001
From: Shay <hillerys@element.io>
Date: Mon, 10 Jul 2023 09:20:16 -0700
Subject: [PATCH 32/55] drop oldstable buster and sub new oldstable bullseye
 (#1358)

---
 .github/workflows/docker.yml | 10 +++++-----
 docker/README.md             | 14 +++++++-------
 docker/base.Dockerfile       |  2 +-
 docker/dendrite.Dockerfile   |  2 +-
 docker/synapse.Dockerfile    |  6 +-----
 5 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index a4d3e1dc0..119db5381 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -25,8 +25,8 @@ jobs:
         include:
           - base_image: ubuntu:focal
             tag: focal
-          - base_image: debian:buster
-            tag: buster
+          - base_image: debian:bullseye
+            tag: bullseye
           - base_image: debian:testing
             tag: testing
     steps:
@@ -69,10 +69,10 @@ jobs:
             dockerfile: synapse
             tags: "matrixdotorg/sytest-synapse:focal"
             build_args: "SYTEST_IMAGE_TAG=focal"
-          - sytest_image_tag: buster
+          - sytest_image_tag: bullseye
             dockerfile: synapse
-            tags: "matrixdotorg/sytest-synapse:buster"
-            build_args: "SYTEST_IMAGE_TAG=buster"
+            tags: "matrixdotorg/sytest-synapse:bullseye"
+            build_args: "SYTEST_IMAGE_TAG=bullseye"
           - sytest_image_tag: testing
             dockerfile: synapse
             tags: "matrixdotorg/sytest-synapse:testing"
diff --git a/docker/README.md b/docker/README.md
index 51eff5367..cbe3f210c 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -8,9 +8,9 @@ but its dependencies are.
 Included currently is:
 
 - `matrixdotorg/sytest` Base container with SyTest dependencies installed
-  - Tagged by underlying Debian/Ubuntu image: `focal`, `buster` or `testing`
+  - Tagged by underlying Debian/Ubuntu image: `focal`, `bullseye` or `testing`
 - `matrixdotorg/sytest-synapse`: Runs SyTest against Synapse
-  - Tagged by underlying Debian/Ubunutu image: `focal`, `buster` or `testing`
+  - Tagged by underlying Debian/Ubunutu image: `focal`, `bullseye` or `testing`
 
 ## Target-specific details
 
@@ -23,7 +23,7 @@ it is useful to mount a volume there too.
 For example:
 
 ```
-docker run --rm -it -v /path/to/synapse\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:buster
+docker run --rm -it -v /path/to/synapse\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:focal
 ```
 
 The following environment variables can be set with `-e` to control the test run:
@@ -43,7 +43,7 @@ An example of running Synapse in worker mode:
 
 ```
 docker run --rm -it -e POSTGRES=1 -e WORKERS=1 -v /path/to/synapse\:/src:ro \
-    -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:buster
+    -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:focal
 ```
 
 ### Dendrite
@@ -68,7 +68,7 @@ for example:
 docker run --rm -it \
     -e SYTEST_BRANCH="my-sytest-branch"
     -v /path/to/synapse\:/src:ro -v /path/to/where/you/want/logs\:/logs
-    matrixdotorg/sytest-synapse:buster
+    matrixdotorg/sytest-synapse:focal
 ```
 
 If the branch referred to by `SYTEST_BRANCH` does not exist, `develop` is used
@@ -80,7 +80,7 @@ the container:
 
 ```
 docker run --rm -it -v /path/to/synapse\:/src:ro -v /path/to/where/you/want/logs\:/logs \
-    -v /path/to/code/sytest\:/sytest:ro matrixdotorg/sytest-synapse:buster
+    -v /path/to/code/sytest\:/sytest:ro matrixdotorg/sytest-synapse:focal
 ```
 
 ## Running a single test file, and other sytest commandline options
@@ -89,7 +89,7 @@ You can pass arguments to sytest by adding them at the end of the
 docker command. For example:
 
 ```
-docker run --rm -it ... matrixdotorg/sytest-synapse:buster tests/20profile-events.pl
+docker run --rm -it ... matrixdotorg/sytest-synapse:focal tests/20profile-events.pl
 ```
 
 ## Building the containers
diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile
index bcdd0b815..f0f35a197 100644
--- a/docker/base.Dockerfile
+++ b/docker/base.Dockerfile
@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=debian:buster
+ARG BASE_IMAGE=debian:bullseye
 
 FROM ${BASE_IMAGE}
 
diff --git a/docker/dendrite.Dockerfile b/docker/dendrite.Dockerfile
index c9af13f20..4c48d2e0a 100644
--- a/docker/dendrite.Dockerfile
+++ b/docker/dendrite.Dockerfile
@@ -1,4 +1,4 @@
-ARG SYTEST_IMAGE_TAG=buster
+ARG SYTEST_IMAGE_TAG=bullseye
 FROM matrixdotorg/sytest:${SYTEST_IMAGE_TAG}
 
 ARG GO_VERSION=1.19.1
diff --git a/docker/synapse.Dockerfile b/docker/synapse.Dockerfile
index fdb5673c9..cb781338f 100644
--- a/docker/synapse.Dockerfile
+++ b/docker/synapse.Dockerfile
@@ -1,4 +1,4 @@
-ARG SYTEST_IMAGE_TAG=buster
+ARG SYTEST_IMAGE_TAG=bullseye
 
 FROM matrixdotorg/sytest:${SYTEST_IMAGE_TAG}
 
@@ -21,13 +21,9 @@ RUN mkdir /rust /cargo
 
 RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable --profile minimal
 
-# Use the latest version of pip. This pulls in fixes not present in the
-# pip version provided by Debian Buster. See
-# https://github.com/pypa/setuptools/issues/3457#issuecomment-1190125849
 # For now, we need to tell Debian we don't care that we're editing the system python
 # installation.
 # Some context in https://github.com/pypa/pip/issues/11381#issuecomment-1399263627
-RUN ${PYTHON_VERSION} -m pip install -q --upgrade pip ${SYSTEM_PIP_INSTALL_SUFFIX}
 RUN ${PYTHON_VERSION} -m pip install -q --no-cache-dir poetry==1.3.2 ${SYSTEM_PIP_INSTALL_SUFFIX}
 
 # As part of the Docker build, we attempt to pre-install Synapse's dependencies

From 5508cbcfe65654e75ddf92e1c51de86e5f3b8846 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 12 Jul 2023 12:02:29 +0100
Subject: [PATCH 33/55] Test that remote invites are correctly pushed (#1356)

---
 tests/61push/01message-pushed.pl | 100 +++++++++++++++++++++++++++++++
 1 file changed, 100 insertions(+)

diff --git a/tests/61push/01message-pushed.pl b/tests/61push/01message-pushed.pl
index 5151eec79..e51230bfa 100644
--- a/tests/61push/01message-pushed.pl
+++ b/tests/61push/01message-pushed.pl
@@ -630,3 +630,103 @@ sub check_received_push_with_name
          );
       });
    };
+
+test "Invites over federation are correctly pushed",
+   requires => [
+      local_user_fixtures( 2, with_events => 0 ),
+      remote_user_fixture(),
+      $main::TEST_SERVER_INFO
+   ],
+
+   check => sub {
+      my ( $alice, $bob, $charlie, $test_server_info ) = @_;
+      my $room_id;
+
+      setup_push( $alice, $bob, $test_server_info )
+      ->then( sub {
+         matrix_create_room_synced( $charlie );
+      })->then( sub {
+         ( $room_id ) = @_;
+
+         Future->needs_all(
+            await_http_request( $PUSH_LOCATION, sub {
+               my ( $request ) = @_;
+               my $body = $request->body_from_json;
+
+               # Respond to all requests, even if we filiter them out
+               $request->respond_json( {} );
+
+               return unless $body->{notification}{type};
+               return unless $body->{notification}{type} eq "m.room.member";
+               return 1;
+            }),
+            matrix_invite_user_to_room( $charlie, $alice, $room_id )
+         )
+      })->then( sub {
+         my ( $request ) = @_;
+         my $body = $request->body_from_json;
+
+         log_if_fail "Message push request body", $body;
+
+         assert_json_keys( my $notification = $body->{notification}, qw(
+            id room_id type sender content devices counts
+         ));
+
+         assert_eq( $notification->{room_id}, $room_id, "room_id");
+         assert_eq( $notification->{sender}, $charlie->user_id, "sender");
+
+         Future->done(1);
+      })
+   };
+
+
+test "Invites over federation are correctly pushed with name",
+   requires => [
+      local_user_fixtures( 2, with_events => 0 ),
+      remote_user_fixture(),
+      $main::TEST_SERVER_INFO
+   ],
+
+   check => sub {
+      my ( $alice, $bob, $charlie, $test_server_info ) = @_;
+      my $room_id;
+
+      my $name = "Test Name";
+
+      setup_push( $alice, $bob, $test_server_info )
+      ->then( sub {
+         matrix_create_room_synced( $charlie, name => $name );
+      })->then( sub {
+         ( $room_id ) = @_;
+
+         Future->needs_all(
+            await_http_request( $PUSH_LOCATION, sub {
+               my ( $request ) = @_;
+               my $body = $request->body_from_json;
+
+               # Respond to all requests, even if we filiter them out
+               $request->respond_json( {} );
+
+               return unless $body->{notification}{type};
+               return unless $body->{notification}{type} eq "m.room.member";
+               return 1;
+            }),
+            matrix_invite_user_to_room( $charlie, $alice, $room_id )
+         )
+      })->then( sub {
+         my ( $request ) = @_;
+         my $body = $request->body_from_json;
+
+         log_if_fail "Message push request body", $body;
+
+         assert_json_keys( my $notification = $body->{notification}, qw(
+            id room_id type sender content devices counts
+         ));
+
+         assert_eq( $notification->{room_id}, $room_id, "room_id");
+         assert_eq( $notification->{sender}, $charlie->user_id, "sender");
+         assert_eq( $notification->{room_name}, $name, "room_name");
+
+         Future->done(1);
+      })
+   };

From cfab2c7a4a9ca5d27ca3cbab95545cc2118bdf1e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 24 Jul 2023 13:43:31 +0100
Subject: [PATCH 34/55] Allow processing of device lists to be async (#1359)

---
 tests/50federation/40devicelists.pl | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/50federation/40devicelists.pl b/tests/50federation/40devicelists.pl
index 8dde16e38..aafe57e3a 100644
--- a/tests/50federation/40devicelists.pl
+++ b/tests/50federation/40devicelists.pl
@@ -738,6 +738,9 @@
                }
             )
          )
+      })->then( sub {
+         # Wait for the device list update to be handled
+         sync_until_user_in_device_list_id( $user, $creator_id )
       })->then( sub {
          # in stream_id 2, we add keys to the device but deliberately don't
          # tell the remote server about it.
@@ -781,6 +784,9 @@
                }
             )
          );
+      })->then( sub {
+         # Wait for the device list update to be handled
+         sync_until_user_in_device_list_id( $user, $creator_id )
       })->then( sub {
          do_request_json_for( $user,
             method  => "POST",

From 27a3d1fa9d11528d9f239c94ace0c07fb23b040c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 28 Jul 2023 13:49:09 +0100
Subject: [PATCH 35/55] Add test to ensure push/notifications work after room
 purge (#1361)

* Add test to ensure push/notifications work after room purge

* Apply suggestions from code review

Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>

---------

Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>
---
 tests/48admin.pl | 118 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 118 insertions(+)

diff --git a/tests/48admin.pl b/tests/48admin.pl
index 3171340d2..2d38e3d84 100644
--- a/tests/48admin.pl
+++ b/tests/48admin.pl
@@ -498,3 +498,121 @@ sub await_message_in_room
       log_if_fail "Found event $event_id for $user_id";
    })
 }
+
+
+test "After /purge_history users still get pushed for new messages",
+   requires => [ local_admin_fixture(), local_user_and_room_fixtures(), $main::TEST_SERVER_INFO, ],
+   implementation_specific => ['synapse'],
+
+   do => sub {
+      my ( $admin, $user, $room_id, $test_server_info ) = @_;
+
+      my $PUSH_LOCATION = "/_matrix/push/v1/notify";
+
+      my $last_event_id;
+
+      matrix_join_room( $admin, $room_id )
+      ->then( sub {
+         matrix_sync( $user )
+      })->then( sub {
+         matrix_sync( $admin )
+      })->then( sub {
+         matrix_send_room_text_message_synced( $user, $room_id,
+            body => "First message",
+         )
+      })->then( sub {
+         my ( $event_id ) = @_;
+
+         matrix_advance_room_receipt_synced( $admin, $room_id, "m.read" => $event_id )
+      })->then( sub {
+         repeat( sub {
+            my $msgnum = $_[0];
+
+            matrix_send_room_text_message_synced( $user, $room_id,
+               body => "Message $msgnum",
+            )
+         }, foreach => [ 1 .. 10 ])
+      })->then( sub {
+         ( $last_event_id ) = @_;
+
+         await_message_in_room( $user, $room_id, $last_event_id ),
+      })->then( sub {
+         log_if_fail "Got message down sync";
+
+         do_request_json_for( $admin,
+            method   => "POST",
+            full_uri => "/_synapse/admin/v1/purge_history/$room_id/${ \uri_escape( $last_event_id ) }",
+            content  => {}
+         )
+      })->then( sub {
+         my ( $body ) = @_;
+
+         assert_json_keys( $body, "purge_id" );
+         my $purge_id = $body->{purge_id};
+         await_purge_complete( $admin, $purge_id );
+      })->then( sub {
+         my ( $purge_status ) = @_;
+         assert_eq( $purge_status, 'complete' );
+
+         # Test that /sync with an existing token still works.
+         matrix_sync_again( $admin )
+      })->then( sub {
+         # Check that the notification count increases if we see a new message.
+         matrix_send_room_text_message_synced( $user, $room_id,
+            body => "New Message",
+         )
+      })->then( sub {
+         matrix_sync_again( $admin )
+      })->then( sub {
+         my ( $body ) = @_;
+
+         assert_json_keys( $body->{rooms}{join}, $room_id );
+         my $room =  $body->{rooms}{join}{$room_id};
+
+         log_if_fail( "Room", $room );
+
+         # We expect notification count to be 2, as we'll still see two events:
+         # the first being $last_event_id and the second the "new message"
+         assert_eq( $room->{unread_notifications}{"notification_count"}, 2 );
+
+         pass( "Notification count was increased on new message" );
+
+         # Check that we get pushed messages
+         do_request_json_for(
+            $admin,
+            method  => "POST",
+            uri     => "/v3/pushers/set",
+            content => {
+               profile_tag         => "tag",
+               kind                => "http",
+               app_id              => "sytest",
+               app_display_name    => "sytest_display_name",
+               device_display_name => "device_display_name",
+               pushkey             => "a_push_key",
+               lang                => "en",
+               data                => { url => $test_server_info->client_location . $PUSH_LOCATION, },
+            },
+         )
+      })->then( sub {
+         Future->needs_all(
+            await_http_request( $PUSH_LOCATION, sub {
+               my ( $request ) = @_;
+               my $body = $request->body_from_json;
+
+               # Respond to all requests, even if we filiter them out
+               $request->respond_json( {} );
+
+               log_if_fail( "Push", $body );
+
+               return unless $body->{notification}{type};
+               return unless $body->{notification}{type} eq "m.room.message";
+               return unless $body->{notification}{content}{body} eq "Pushed room message";
+               return 1;
+            }),
+
+            matrix_send_room_text_message_synced( $user, $room_id,
+               body => "Pushed room message",
+            )->SyTest::pass_on_done( "Message sent" ),
+         )->SyTest::pass_on_done( "New messages was pushed" )
+      })
+   };

From 961e863ddf5d9ef7f6f163a0f0f014479b44e56f Mon Sep 17 00:00:00 2001
From: Shay <hillerys@element.io>
Date: Thu, 3 Aug 2023 11:43:13 -0700
Subject: [PATCH 36/55] update tests to check for access token in header vs
 query params (#1362)

---
 tests/60app-services/05lookup3pe.pl | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/60app-services/05lookup3pe.pl b/tests/60app-services/05lookup3pe.pl
index 79a29a86a..0a1f4823d 100644
--- a/tests/60app-services/05lookup3pe.pl
+++ b/tests/60app-services/05lookup3pe.pl
@@ -147,13 +147,15 @@ sub stub_empty_result
             my ( $request ) = @_;
             
             my $access_token = $appserv1->info->hs2as_token;
+            my $auth_header = $request->header("Authorization");
+
+            assert_eq($auth_header, "Bearer " . $access_token, 'Access token');
 
             assert_deeply_eq(
                { $request->query_form },
                {
                   field1 => "ONE",
                   field2 => "TWO",
-                  access_token => $access_token,
                },
                'fields in received AS request'
             );
@@ -213,12 +215,14 @@ sub stub_empty_result
          )->then( sub {
             my ( $request ) = @_;
             my $access_token = $appserv1->info->hs2as_token;
+            my $auth_header = $request->header("Authorization");
+
+            assert_eq($auth_header, "Bearer " . $access_token, 'Access token');
 
             assert_deeply_eq(
                { $request->query_form },
                {
                   field3 => "THREE",
-                  access_token => $access_token,
                },
                'fields in received AS request'
             );

From 8d4b37ab58abce5ddf05e3c24a6dd9dee0395916 Mon Sep 17 00:00:00 2001
From: Will Hunt <will@half-shot.uk>
Date: Wed, 9 Aug 2023 15:14:05 +0100
Subject: [PATCH 37/55] Remove media tests supported by complement (#1367)

---
 tests/51media/01unicode.pl    | 242 ----------------------------------
 tests/51media/02nofilename.pl |  47 -------
 tests/51media/03ascii.pl      | 126 ------------------
 tests/51media/10thumbnail.pl  | 107 ---------------
 4 files changed, 522 deletions(-)
 delete mode 100644 tests/51media/01unicode.pl
 delete mode 100644 tests/51media/02nofilename.pl
 delete mode 100644 tests/51media/03ascii.pl
 delete mode 100644 tests/51media/10thumbnail.pl

diff --git a/tests/51media/01unicode.pl b/tests/51media/01unicode.pl
deleted file mode 100644
index b450d4be6..000000000
--- a/tests/51media/01unicode.pl
+++ /dev/null
@@ -1,242 +0,0 @@
-use HTTP::Headers::Util qw( split_header_words );
-use URI::Escape qw( uri_escape );
-use SyTest::TCPProxy;
-use utf8;
-
-my $FILENAME = "\xf0\x9f\x90\x94";
-my $FILENAME_ENCODED = uc uri_escape( $FILENAME );
-
-my $content_id;
-
-my $PROXY_SERVER = fixture(
-   name => 'PROXY_SERVER',
-
-   requires => [ $main::HOMESERVER_INFO[0] ],
-
-   setup => sub {
-      my ( $server_info ) = @_;
-
-      $OUTPUT->diag( "Starting proxy server" );
-
-      my $listener = SyTest::TCPProxy->new(
-         host   => $server_info->federation_host,
-         port   => $server_info->federation_port,
-         output => $OUTPUT,
-      );
-
-      $loop->add( $listener );
-
-      $listener->listen(
-         addr => { family => "inet" },
-      )->on_done( sub {
-         my $sock = $listener->read_handle;
-         $OUTPUT->diag( "Proxy now listening at port " . $sock->sockport );
-         return $listener;
-      });
-   },
-
-   teardown => sub {
-      my ( $listener ) = @_;
-      $listener->close();
-   },
-);
-
-
-=head2 upload_test_content
-
-   my ( $content_id, $content_uri ) = upload_test_content(
-      $user, filename => "filename",
-   ) -> get;
-
-Uploads some test content with the given filename.
-
-Returns the content id of the uploaded content.
-
-=cut
-sub upload_test_content
-{
-   my ( $user, %params ) = @_;
-
-   $user->http->do_request(
-      method       => "POST",
-      full_uri     => "/_matrix/media/v3/upload",
-      content      => "Test media file",
-      content_type => "text/plain",
-
-      params => {
-         access_token => $user->access_token,
-         %params,
-      },
-   )->then( sub {
-      my ( $body ) = @_;
-
-      assert_json_keys( $body, qw( content_uri ));
-
-      my $content_uri = $body->{content_uri};
-
-      my $parsed_uri = URI->new( $body->{content_uri} );
-      my $server = $parsed_uri->authority;
-      my $path = $parsed_uri->path;
-
-      my $content_id = "$server$path";
-
-      Future->done( $content_id, $content_uri );
-   });
-}
-push our @EXPORT, qw( upload_test_content );
-
-
-=head2 get_media
-
-   my ( $content_disposition_params, $content ) = get_media( $http, $content_id ) -> get;
-
-Fetches a piece of media from the server.
-
-=cut
-sub get_media
-{
-   my ( $http, $content_id ) = @_;
-
-   $http->do_request(
-      method   => "GET",
-      full_uri => "/_matrix/media/v3/download/$content_id",
-   )->then( sub {
-      my ( $body, $response ) = @_;
-
-      my $disposition = $response->header( "Content-Disposition" );
-
-      my $cd_params;
-      if ( defined $disposition ) {
-         $cd_params = parse_content_disposition_params( $disposition );
-      }
-      Future->done( $cd_params, $body );
-   });
-}
-push @EXPORT, qw( get_media );
-
-sub parse_content_disposition_params {
-   my ( $disposition ) = @_;
-   my @parts = split_header_words( $disposition );
-
-   # should be only one list of words
-   assert_eq( scalar @parts, 1, "number of content-dispostion header lists" );
-   @parts = @{$parts[0]};
-
-   # the first part must be 'attachment'
-   my $k = shift @parts;
-   my $v = shift @parts;
-   assert_eq( $k, "attachment", "content-disposition" );
-   die "invalid CD" if defined $v;
-
-   my %params;
-   while (@parts) {
-      my $k = shift @parts;
-      my $v = shift @parts;
-      die "multiple $k params" if exists $params{$k};
-      die "unknown param $k" unless ( $k eq 'filename' || $k eq 'filename*' );
-      $params{$k} = $v;
-   }
-   return \%params;
-}
-
-
-test "Can upload with Unicode file name",
-   requires => [ local_user_fixture(),
-                 qw( can_upload_media )],
-
-   proves => [qw( can_upload_media_unicode )],
-
-   do => sub {
-      my ( $user ) = @_;
-
-      upload_test_content( $user, filename=>$FILENAME )->then( sub {
-         ( $content_id ) = @_;
-         Future->done(1)
-      });
-   };
-
-# These next two tests do the same thing with two different HTTP clients, to
-# test locally and via federation
-
-sub test_using_client
-{
-   my ( $client, $content ) = @_;
-
-   if( ! defined( $content )) {
-       $content = $content_id;
-   }
-
-   get_media( $client, $content )->then( sub {
-      my ( $cd_params ) = @_;
-
-      # The Content-Disposition header can take two formats - either:
-      #  * filename=utf-8"filename_here"
-      #  * filename*=utf-8''filename_here
-      # This checks the format of whichever one we were provided with. 
-      if (exists( $cd_params->{filename} )) {
-         assert_eq( $cd_params->{filename}, "utf-8\"$FILENAME\"", "filename" );
-      } else {
-         assert_eq( $cd_params->{'filename*'}, "utf-8''$FILENAME_ENCODED", "filename*" );
-      }
-
-      Future->done(1);
-   });
-}
-
-test "Can download with Unicode file name locally",
-   requires => [ $main::API_CLIENTS[0],
-                 qw( can_upload_media_unicode )],
-
-   check => sub {
-      my ( $http ) = @_;
-      test_using_client( $http );
-   };
-
-test "Can download with Unicode file name over federation",
-   requires => [ $main::API_CLIENTS[1],
-                 qw( can_upload_media_unicode ) ],
-
-   check => sub {
-      my ( $http ) = @_;
-      test_using_client( $http );
-   };
-
-test "Alternative server names do not cause a routing loop",
-   # https://github.com/matrix-org/synapse/issues/1991
-   requires => [ $main::API_CLIENTS[0], $PROXY_SERVER ],
-
-   check => sub {
-      my ( $http, $proxy ) = @_;
-      # we use a proxy server which routes connections straight back to the
-      # homeserver, to mimic the behaviour when the remote server name points
-      # back to the homeserver.
-      my $sock = $proxy->read_handle;
-      my $proxy_address = "localhost:" . $proxy->read_handle->sockport;
-      my $content = "$proxy_address/test_content";
-      test_using_client( $http, $content )->main::expect_http_404;
-   };
-
-test "Can download specifying a different Unicode file name",
-   requires => [ $main::API_CLIENTS[0],
-                 qw( can_upload_media_unicode )],
-
-   check => sub {
-      my ( $http ) = @_;
-
-      my $alt_filename = "☕";
-      my $alt_filename_encoded = "%E2%98%95";
-
-      $http->do_request(
-         method   => "GET",
-         full_uri => "/_matrix/media/v3/download/$content_id/$alt_filename_encoded",
-      )->then( sub {
-         my ( $body, $response ) = @_;
-
-         my $disposition = $response->header( "Content-Disposition" );
-         uc $disposition eq uc "attachment; filename*=utf-8''$alt_filename_encoded" or
-            uc $disposition eq uc "attachment; filename=utf-8\"$alt_filename\"" or
-            die "Expected a UTF-8 filename parameter";
-
-         Future->done(1);
-      });
-   };
diff --git a/tests/51media/02nofilename.pl b/tests/51media/02nofilename.pl
deleted file mode 100644
index 968a7a6cb..000000000
--- a/tests/51media/02nofilename.pl
+++ /dev/null
@@ -1,47 +0,0 @@
-my $content_id;
-
-test "Can upload without a file name",
-   requires => [ local_user_fixture() ],
-
-   do => sub {
-      my ( $user ) = @_;
-
-      upload_test_content( $user, )->then( sub {
-         ( $content_id ) = @_;
-         Future->done(1)
-      });
-   };
-
-# These next two tests do the same thing with two different HTTP clients, to
-# test locally and via federation
-
-sub test_using_client
-{
-   my ( $client ) = @_;
-
-   get_media( $client, $content_id )->then( sub {
-      my ( $disposition ) = @_;
-
-      # Require `attachment` `Content-Disposition` without a filename portion.
-      # See `get_media`: {} means attachment with no key-value components in the header value.
-      assert_eq( scalar %$disposition, 0, "content-disposition" );
-
-      Future->done(1);
-   });
-}
-
-test "Can download without a file name locally",
-   requires => [ $main::API_CLIENTS[0] ],
-
-   check => sub {
-      my ( $http ) = @_;
-      test_using_client( $http );
-   };
-
-test "Can download without a file name over federation",
-   requires => [ $main::API_CLIENTS[1] ],
-
-   check => sub {
-      my ( $http ) = @_;
-      test_using_client( $http );
-   };
diff --git a/tests/51media/03ascii.pl b/tests/51media/03ascii.pl
deleted file mode 100644
index 6adb05958..000000000
--- a/tests/51media/03ascii.pl
+++ /dev/null
@@ -1,126 +0,0 @@
-my $content_id;
-my $content_uri;
-
-test "Can upload with ASCII file name",
-   requires => [ local_user_fixture() ],
-
-   do => sub {
-      my ( $user ) = @_;
-      upload_test_content( $user, filename=>"ascii" )->then( sub {
-         ( $content_id, $content_uri ) = @_;
-         Future->done(1);
-      });
-   };
-
-# we only need one user for these tests
-my $user_fixture = local_user_fixture();
-
-sub assert_cd_params_match_filename {
-   my ( $filename, $cd_params ) = @_;
-
-   # either we need a valid "filename*" param
-   if ( $cd_params->{"filename*"} ) {
-      my $f = $cd_params->{"filename*"};
-      $f =~ s/%(..)/chr hex $1/eg;
-      assert_eq( $f, "utf-8''$filename", "filename*" );
-
-      # there might also be a 'filename', but it doesn't really matter what it
-      # is.
-      return;
-   }
-
-   # or we need a valid filename
-   my $f = $cd_params->{"filename"};
-   assert_eq( $f, $filename, "filename" );
-}
-
-foreach my $filename ( "ascii", "name with spaces", "name;with;semicolons" ) {
-   test "Can download file '$filename'",
-      requires => [
-         $user_fixture, $main::API_CLIENTS[1]
-      ],
-
-      check => sub {
-         my ( $user, $federation_client ) = @_;
-
-         my $content_id;
-
-         # first upload the content with the given filename
-         upload_test_content( $user, filename=>$filename )->then( sub {
-            ( $content_id ) = @_;
-
-            # try and fetch it as a local user
-            get_media( $user->http, $content_id );
-         })->then( sub {
-            assert_cd_params_match_filename( $filename, $_[0] );
-
-            # do the same over federation
-            get_media( $federation_client, $content_id );
-         })->then( sub {
-            assert_cd_params_match_filename( $filename, $_[0] );
-            Future->done(1);
-         });
-      };
-}
-
-
-test "Can download specifying a different ASCII file name",
-   requires => [ $main::API_CLIENTS[0] ],
-
-   check => sub {
-      my ( $http ) = @_;
-
-      $http->do_request(
-         method   => "GET",
-         full_uri => "/_matrix/media/v3/download/$content_id/also_ascii",
-      )->then( sub {
-         my ( $body, $response ) = @_;
-
-         my $disposition = $response->header( "Content-Disposition" );
-         $disposition eq "attachment; filename=also_ascii" or
-            die "Expected an ASCII filename parameter";
-
-         Future->done(1);
-      });
-   };
-
-test "Can send image in room message",
-   requires => [ $main::API_CLIENTS[0], local_user_and_room_fixtures() ],
-
-   check => sub {
-      my ( $http, $user, $room_id ) = @_;
-      matrix_send_room_message( $user, $room_id,
-         content => { msgtype => "m.file", body => "test.txt", url => $content_uri }
-      );
-   };
-
-test "Can fetch images in room",
-   requires => [ $main::API_CLIENTS[0], local_user_and_room_fixtures() ],
-
-   check => sub {
-      my ( $http, $user, $room_id ) = @_;
-      matrix_send_room_message_synced( $user, $room_id,
-         content => { msgtype => "m.text", body => "test" }
-      )->then( sub {
-         matrix_send_room_message_synced( $user, $room_id,
-            content => { msgtype => "m.file", body => "test.txt", url => $content_uri }
-         )
-      })->then( sub {
-         do_request_json_for( $user,
-            method => "GET",
-            uri    => "/v3/rooms/$room_id/messages",
-            params => {
-               filter => '{"contains_url":true}',
-               dir    => 'b',
-            }
-         )
-      })->then( sub {
-         my ( $body ) = @_;
-
-         assert_json_keys( $body, qw( start end chunk ));
-
-         assert_eq( scalar @{ $body->{chunk} }, 1, "Expected 1 message" );
-
-         Future->done( 1 );
-      });
-   };
diff --git a/tests/51media/10thumbnail.pl b/tests/51media/10thumbnail.pl
deleted file mode 100644
index 0951c55b0..000000000
--- a/tests/51media/10thumbnail.pl
+++ /dev/null
@@ -1,107 +0,0 @@
-use File::Basename qw( dirname );
-use File::Slurper qw( read_binary );
-
-my $dir = dirname __FILE__;
-
-test "POSTed media can be thumbnailed",
-   requires => [ $main::API_CLIENTS[0], local_user_fixture(),
-                 qw( can_upload_media can_download_media )],
-
-   do => sub {
-      my ( $http, $user ) = @_;
-
-      upload_test_image(
-         $user,
-      )->then( sub {
-         my ( $content_uri ) = @_;
-         fetch_and_validate_thumbnail( $http, $content_uri );
-      });
-   };
-
-
-test "Remote media can be thumbnailed",
-   requires => [ local_user_fixture(), remote_user_fixture(),
-                 qw( can_upload_media can_download_media )],
-   do => sub {
-      my ( $local_user, $remote_user ) = @_;
-
-      upload_test_image(
-         $local_user,
-      )->then( sub {
-         my ( $content_uri ) = @_;
-         fetch_and_validate_thumbnail( $remote_user->http, $content_uri );
-      });
-   };
-
-
-sub upload_test_image
-{
-   my ( $user ) = @_;
-
-   my $pngdata = read_binary( "$dir/test.png" );
-
-   # Because we're POST'ing non-JSON
-   return $user->http->do_request(
-      method => "POST",
-      full_uri => "/_matrix/media/v3/upload",
-      params => {
-         access_token => $user->access_token,
-      },
-
-      content_type => "image/png",
-      content      => $pngdata,
-   )->then( sub {
-       my ( $body ) = @_;
-       log_if_fail "Upload response", $body;
-       my $content_uri = URI->new( $body->{content_uri} );
-       Future->done( $content_uri );
-    });
-}
-
-sub fetch_and_validate_thumbnail
-{
-   my ( $http, $mxc_uri ) = @_;
-
-   return $http->do_request(
-      method => "GET",
-      full_uri => "/_matrix/media/v3/thumbnail/" .
-         join( "", $mxc_uri->authority, $mxc_uri->path ),
-      params => {
-         width  => 32,
-         height => 32,
-         method => "scale",
-      }
-   )->then( sub {
-       my ( $body, $response ) = @_;
-       for( $response->content_type ) {
-          m{^image/png$} and validate_png( $body ), last;
-
-          # TODO: should probably write a JPEG recogniser too
-
-          die "Unrecognised Content-Type ($_) - unable to detect if this is a valid image";
-       }
-
-       Future->done(1);
-   });
-}
-
-# We won't assert too heavily that it's a valid image as that's hard to do
-# without using a full image parsing library like Imager. Instead we'll just
-# detect file magic of a likely-valid encoding as this should cover most common
-# implementation bugs, such as sending plain-text error messages with image
-# MIME headers, or claiming one MIME type while being another.
-
-sub validate_png
-{
-   my ( $body ) = @_;
-
-   # All PNG images begin with the same 8 byte header
-   $body =~ s/^\x89PNG\x0D\x0A\x1A\x0A// or
-      die "Invalid PNG magic";
-
-   # All PNG images have an IHDR header first. This header is 13 bytes long
-   $body =~ s/^\0\0\0\x0DIHDR// or
-      die "Invalid IHDR chunk";
-
-   return 1;
-}

From 5193e4c10514d3e0b2157dfe43a27eccf5c78de4 Mon Sep 17 00:00:00 2001
From: Cameron Nemo <CameronNemo@users.noreply.github.com>
Date: Tue, 15 Aug 2023 06:09:25 -0700
Subject: [PATCH 38/55] Fix typo and dead link in Dendrite docs (#1363)

---
 README.rst             | 2 +-
 docs/dendrite-setup.md | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 2249c8038..d48a76888 100644
--- a/README.rst
+++ b/README.rst
@@ -6,7 +6,7 @@ primarily at present the Synapse server. It performs "black-box" testing, by
 starting up multiple home server instances and testing the interaction using
 regular HTTP interaction much as any other Matrix client would do. It can
 output test results either to an interactive terminal, or as a TAP-style test
-report, for continuous-integration test harnsses such as Jenkins.
+report, for continuous-integration test harnesses such as Jenkins.
 
 Installing
 ----------
diff --git a/docs/dendrite-setup.md b/docs/dendrite-setup.md
index 14da397f1..b179a3df8 100644
--- a/docs/dendrite-setup.md
+++ b/docs/dendrite-setup.md
@@ -1,3 +1,5 @@
 # How to set up SyTest for Dendrite
 
-See https://github.com/matrix-org/dendrite/blob/master/docs/sytest.md#using-the-sytest-docker-image
+See [dendrite's developer documentation](https://github.com/matrix-org/dendrite/blob/main/docs/development/sytest.md#using-the-sytest-docker-image)
+
+([permalink](https://github.com/matrix-org/dendrite/blob/3f727485d6e21a603e4df1cb31c3795cc1023caa/docs/development/sytest.md#using-the-sytest-docker-image)).

From 845764081c890180e3842f135f87e0aa778557a6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 7 Sep 2023 11:53:16 +0100
Subject: [PATCH 39/55] Route /notifications to client_reader (#1369)

---
 lib/SyTest/Homeserver/Synapse.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm
index f74d2ce61..2c010452f 100644
--- a/lib/SyTest/Homeserver/Synapse.pm
+++ b/lib/SyTest/Homeserver/Synapse.pm
@@ -1286,6 +1286,7 @@ sub generate_haproxy_map
 ^/_matrix/client/(r0|v3|unstable)/password_policy$                   client_reader
 ^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$          client_reader
 ^/_matrix/client/(r0|v3|unstable)/capabilities$                      client_reader
+^/_matrix/client/(r0|v3|unstable)/notifications$                     client_reader
 
 ^/_matrix/client/(api/v1|r0|v3|unstable)/devices$                    stream_writer
 ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/query$                 stream_writer

From bd882ce4baa9d0fad8b0781cbf7d1e7ecbf54493 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 22 Sep 2023 17:10:04 +0200
Subject: [PATCH 40/55] Add roomversion 11 (#1370)

---
 lib/SyTest/Federation/Client.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SyTest/Federation/Client.pm b/lib/SyTest/Federation/Client.pm
index ab2764502..028c5228e 100644
--- a/lib/SyTest/Federation/Client.pm
+++ b/lib/SyTest/Federation/Client.pm
@@ -16,7 +16,7 @@ use SyTest::Assertions qw( :all );
 use URI::Escape qw( uri_escape );
 
 use constant SUPPORTED_ROOM_VERSIONS => [qw(
-   1 2 3 4 5 6 7 8 9 10
+   1 2 3 4 5 6 7 8 9 10 11
 )];
 
 sub configure

From 4914c2f56cd02afe9ed00ab181c8a4b410fb6dd7 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 16 Nov 2023 12:35:58 +0000
Subject: [PATCH 41/55] Fix 'Inbound federation correctly soft fails events'
 (#1371)

---
 tests/50federation/52soft-fail.pl | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/50federation/52soft-fail.pl b/tests/50federation/52soft-fail.pl
index ff6607ebf..45604cf49 100644
--- a/tests/50federation/52soft-fail.pl
+++ b/tests/50federation/52soft-fail.pl
@@ -115,10 +115,16 @@
             event => $denied_event,
             destination => $first_home_server,
          );
+      })->then( sub {
+         # Wait until we receive the power levels
+         retry_until_success {
+            $inbound_server->datastore->get_event( $power_level_event_id );
+            Future->done( 1 )
+         }
       })->then( sub {
          # Now send a non-message (event D)
          my $pl_event = $inbound_server->datastore->get_event( $power_level_event_id );
-         die "did not receive PL event" unless $pl_event;
+
          my $event = $room->create_and_insert_event(
             type => "m.room.other_message_type",
 
@@ -518,6 +524,12 @@
             event => $event_sf2,
             destination => $first_home_server,
          );
+      })->then( sub {
+         # Wait until we receive the power levels
+         retry_until_success {
+            $inbound_server->datastore->get_event( $event_id_pl1 );
+            Future->done( 1 )
+         }
       })->then( sub {
          # send a regular message (event m2), which should be accepted
          my $event_pl1 = $inbound_server->datastore->get_event( $event_id_pl1 );

From 888b56d0d1054e99025e1414f24a09ad7387945a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 24 Nov 2023 09:11:55 +0000
Subject: [PATCH 42/55] Build a sytest-synapse docker image with 3.12 (#1372)

---
 .github/workflows/docker.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 119db5381..68d680d76 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -25,6 +25,8 @@ jobs:
         include:
           - base_image: ubuntu:focal
             tag: focal
+          - base_image: ubuntu:mantic
+            tag: mantic
           - base_image: debian:bullseye
             tag: bullseye
           - base_image: debian:testing
@@ -69,6 +71,12 @@ jobs:
             dockerfile: synapse
             tags: "matrixdotorg/sytest-synapse:focal"
             build_args: "SYTEST_IMAGE_TAG=focal"
+          - sytest_image_tag: mantic
+            dockerfile: synapse
+            tags: "matrixdotorg/sytest-synapse:mantic-3.12"
+            build_args: |
+              SYTEST_IMAGE_TAG=mantic
+              PYTHON_VERSION=python3.12
           - sytest_image_tag: bullseye
             dockerfile: synapse
             tags: "matrixdotorg/sytest-synapse:bullseye"

From e2c4b0ea8136e754ec0423a3ccdbe5650475d204 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 24 Nov 2023 11:30:52 +0000
Subject: [PATCH 43/55] Fix docker image building (#1373)

Broke by #1372
---
 .github/workflows/docker.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 68d680d76..2c99cd83f 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -77,6 +77,7 @@ jobs:
             build_args: |
               SYTEST_IMAGE_TAG=mantic
               PYTHON_VERSION=python3.12
+              SYSTEM_PIP_INSTALL_SUFFIX=--break-system-packages
           - sytest_image_tag: bullseye
             dockerfile: synapse
             tags: "matrixdotorg/sytest-synapse:bullseye"

From bc4aebaf7d52682b47c6cd9fc5dec1b1634c190d Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 27 Mar 2024 15:56:31 +0100
Subject: [PATCH 44/55] Use element-hq instead of matrix-org when pulling the
 corresponding (#1375)

---
 .github/workflows/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index a9b138a66..416e1b326 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -89,7 +89,7 @@ jobs:
             if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
               continue
             fi
-            (wget -O - "https://github.com/matrix-org/synapse/archive/$BRANCH_NAME.tar.gz" \
+            (wget -O - "https://github.com/element-hq/synapse/archive/$BRANCH_NAME.tar.gz" \
               | tar -xz --strip-components=1 -C /src/) \
               && echo "Successfully downloaded and extracted $BRANCH_NAME.tar.gz" \
               && break

From ae8a8cf998f5418ecbf0bb6809a67d72b7acd96d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 28 Mar 2024 15:44:17 +0000
Subject: [PATCH 45/55] Test push rules off main (#1376)

---
 lib/SyTest/Homeserver/Synapse.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm
index 2c010452f..c353fde31 100644
--- a/lib/SyTest/Homeserver/Synapse.pm
+++ b/lib/SyTest/Homeserver/Synapse.pm
@@ -340,6 +340,7 @@ sub start
            account_data => $self->{redis_host} ne '' ? [ "stream_writer" ] : "master",
            receipts     => $self->{redis_host} ne '' ? [ "stream_writer" ] : "master",
            presence     => $self->{redis_host} ne '' ? [ "stream_writer" ] : "master",
+           push         => $self->{redis_host} ne '' ? [ "stream_writer" ] : "master",
            typing       => $self->{redis_host} ne '' ? [ "stream_writer" ] : "master",
         },
 
@@ -1294,6 +1295,7 @@ sub generate_haproxy_map
 ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/claim                  stream_writer
 ^/_matrix/client/(api/v1|r0|v3|unstable)/room_keys                   stream_writer
 ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/                   stream_writer
+^/_matrix/client/(api/v1|r0|v3|unstable)/pushrules/                  stream_writer
 
 ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload  frontend_proxy
 
@@ -1320,8 +1322,6 @@ EOCONFIG
 sub generate_haproxy_get_map
 {
     return <<'EOCONFIG';
-# pushrules should be here, but the tests seem to be racy.
-# ^/_matrix/client/(api/v1|r0|v3|unstable)/pushrules/            client_reader
 ^/_matrix/client/(r0|v3)/user/[^/]*/account_data/                client_reader
 ^/_matrix/client/(r0|v3)/user/[^/]*/rooms/[^/]*/account_data/    client_reader
 EOCONFIG

From 7d3f92d4c2083a3efd16ad5c4046f6c91acfc5e8 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 2 Apr 2024 13:53:18 +0100
Subject: [PATCH 46/55] Fix renamed Synapse stream (#1377)

It got renamed.
---
 lib/SyTest/Homeserver/Synapse.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm
index c353fde31..2b5e41bbf 100644
--- a/lib/SyTest/Homeserver/Synapse.pm
+++ b/lib/SyTest/Homeserver/Synapse.pm
@@ -340,7 +340,7 @@ sub start
            account_data => $self->{redis_host} ne '' ? [ "stream_writer" ] : "master",
            receipts     => $self->{redis_host} ne '' ? [ "stream_writer" ] : "master",
            presence     => $self->{redis_host} ne '' ? [ "stream_writer" ] : "master",
-           push         => $self->{redis_host} ne '' ? [ "stream_writer" ] : "master",
+           push_rules   => $self->{redis_host} ne '' ? [ "stream_writer" ] : "master",
            typing       => $self->{redis_host} ne '' ? [ "stream_writer" ] : "master",
         },
 

From 464864e1cb930c4fd2397f513fba351ec3fd1859 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Wed, 3 Apr 2024 14:32:56 +0100
Subject: [PATCH 47/55] Fix lazy-loading test which was looking for the wrong
 thing (#1374)

As the comment says, this test was looking for the wrong thing, and actually
enforcing buggy behaviour.
---
 tests/31sync/15lazy-members.pl | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/tests/31sync/15lazy-members.pl b/tests/31sync/15lazy-members.pl
index 2970abba8..384686a56 100644
--- a/tests/31sync/15lazy-members.pl
+++ b/tests/31sync/15lazy-members.pl
@@ -494,6 +494,7 @@ sub check_filler_event {
          # Ensure server has propagated those messages to Alice's sync stream
          await_sync_timeline_contains( $alice, $room_id, check => check_filler_event( $bob->user_id, 10 ));
       })->then( sub {
+         # Alice does an initial sync; this stashes the "next_batch" token for use below.
          matrix_sync( $alice, filter => $filter_id );
       })->then( sub {
          my ( $body ) = @_;
@@ -503,16 +504,23 @@ sub check_filler_event {
       })->then( sub {
          matrix_send_filler_messages_synced( $bob, $room_id, 5 );
       })->then( sub {
+         # Make sure alice can see the filler messages. This starts with an initialsync, and doesn't update the "next batch".
          await_sync_timeline_contains( $alice, $room_id, check => check_filler_event( $bob->user_id, 5 ));
       })->then( sub {
+         # Finally, do an incremental sync, which should include all of the changes since the `matrix_sync` above
+         # (ie, before charlie left).
          matrix_sync_again( $alice, filter => $filter_id );
       })->then( sub {
          my ( $body ) = @_;
 
-         # XXX: i'm surprised we have an explicit state entry here at all,
-         # given the state transition is included in the timeline.
-         my $state = $body->{rooms}{join}{$room_id}{state}{events};
-         assert_state_room_members_match( $state, { $charlie->user_id => 'leave' });
+         log_if_fail "Incremental sync", $body;
+
+         # The timeline should include Charlie's leave event (followed by the 5 padding events).
+         my $room = $body->{rooms}{join}{$room_id};
+         my $event = $room->{timeline}{events}[0];
+
+         assert_eq( $event->{type}, "m.room.member" );
+         assert_eq( $event->{state_key}, $charlie->user_id );
 
          Future->done(1);
       });

From 45351a5ec4891b035c5f22d70ef4240bd83171a6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 29 Apr 2024 10:13:14 +0100
Subject: [PATCH 48/55] Download Synapse from correct repo in Dockerfile
 (#1378)

This will fix prebuilding of dependencies.
---
 docker/synapse.Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/synapse.Dockerfile b/docker/synapse.Dockerfile
index cb781338f..f480e80a4 100644
--- a/docker/synapse.Dockerfile
+++ b/docker/synapse.Dockerfile
@@ -51,7 +51,7 @@ RUN ${PYTHON_VERSION} -m pip download --dest /pypi-offline-cache \
 
 # Create the virtual env upfront so we don't need to keep reinstalling
 # dependencies.
-RUN wget -q https://github.com/matrix-org/synapse/archive/develop.tar.gz \
+RUN wget -q https://github.com/element-hq/synapse/archive/develop.tar.gz \
         -O /synapse.tar.gz && \
         mkdir /synapse && \
         tar -C /synapse --strip-components=1 -xf synapse.tar.gz && \

From aafa964a230e8ae9a430b3162104427f8852100d Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Wed, 15 May 2024 14:10:44 +0100
Subject: [PATCH 49/55] Route media-related Synapse Admin API paths to media
 worker (#1381)

---
 lib/SyTest/Homeserver/Synapse.pm | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm
index 2b5e41bbf..95ce93470 100644
--- a/lib/SyTest/Homeserver/Synapse.pm
+++ b/lib/SyTest/Homeserver/Synapse.pm
@@ -1244,7 +1244,13 @@ sub generate_haproxy_map
 ^/_matrix/client/(api/v1|r0|v3)/initialSync$             synchrotron
 ^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$ synchrotron
 
-^/_matrix/media/    media_repository
+^/_matrix/media/                           media_repository
+^/_synapse/admin/v1/purge_media_cache$     media_repository
+^/_synapse/admin/v1/room/.*/media.*$       media_repository
+^/_synapse/admin/v1/user/.*/media.*$       media_repository
+^/_synapse/admin/v1/media/.*$              media_repository
+^/_synapse/admin/v1/quarantine_media/.*$   media_repository
+^/_synapse/admin/v1/users/.*/media$        media_repository
 
 ^/_matrix/federation/v1/event/                        federation_reader
 ^/_matrix/federation/v1/state/                        federation_reader

From 940103768ce1a71a3873268119c747517005f102 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Wed, 15 May 2024 16:43:42 +0100
Subject: [PATCH 50/55] Update Synapse Admin API quarantine path (#1380)

---
 tests/51media/48admin-quarantine.pl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/51media/48admin-quarantine.pl b/tests/51media/48admin-quarantine.pl
index be82e81dd..ff808c8ba 100644
--- a/tests/51media/48admin-quarantine.pl
+++ b/tests/51media/48admin-quarantine.pl
@@ -44,7 +44,7 @@
       })->then( sub {
          do_request_json_for( $admin,
             method   => "POST",
-            full_uri => "/_synapse/admin/v1/quarantine_media/$room_id",
+            full_uri => "/_synapse/admin/v1/room/$room_id/media/quarantine",
             content  => {},
          );
       })->SyTest::pass_on_done( "Quarantine returns success" )

From caccc72f1ae347f093b9160b9995316a72db07a7 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Thu, 16 May 2024 17:34:05 +0100
Subject: [PATCH 51/55] Send `/{send,make}_knock` to Synapse workers (#1379)

---
 lib/SyTest/Homeserver/Synapse.pm | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm
index 95ce93470..0aac10b44 100644
--- a/lib/SyTest/Homeserver/Synapse.pm
+++ b/lib/SyTest/Homeserver/Synapse.pm
@@ -1240,7 +1240,7 @@ sub generate_haproxy_map
    return <<'EOCONFIG';
 
 ^/_matrix/client/(v2_alpha|r0|v3)/sync$                  synchrotron
-^/_matrix/client/(api/v1|v2_alpha|r0)/events$         synchrotron
+^/_matrix/client/(api/v1|v2_alpha|r0)/events$            synchrotron
 ^/_matrix/client/(api/v1|r0|v3)/initialSync$             synchrotron
 ^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$ synchrotron
 
@@ -1263,6 +1263,8 @@ sub generate_haproxy_map
 ^/_matrix/federation/v1/make_leave/                   federation_reader
 ^/_matrix/federation/v1/send_join/                    federation_reader
 ^/_matrix/federation/v1/send_leave/                   federation_reader
+^/_matrix/federation/v1/make_knock/                   federation_reader
+^/_matrix/federation/v1/send_knock/                   federation_reader
 ^/_matrix/federation/v1/invite/                       federation_reader
 ^/_matrix/federation/v1/query_auth/                   federation_reader
 ^/_matrix/federation/v1/event_auth/                   federation_reader

From 9b80feaa947bce9e181b47362879b0e94d3a9f1d Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Fri, 17 May 2024 14:55:24 +0100
Subject: [PATCH 52/55] Add missing v2 variants from Synapse reverse proxy
 config. (#1382)

---
 lib/SyTest/Homeserver/Synapse.pm | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm
index 0aac10b44..2b8623ec9 100644
--- a/lib/SyTest/Homeserver/Synapse.pm
+++ b/lib/SyTest/Homeserver/Synapse.pm
@@ -1261,11 +1261,11 @@ sub generate_haproxy_map
 ^/_matrix/federation/v1/query/                        federation_reader
 ^/_matrix/federation/v1/make_join/                    federation_reader
 ^/_matrix/federation/v1/make_leave/                   federation_reader
-^/_matrix/federation/v1/send_join/                    federation_reader
-^/_matrix/federation/v1/send_leave/                   federation_reader
+^/_matrix/federation/(v1|v2)/send_join/               federation_reader
+^/_matrix/federation/(v1|v2)/send_leave/              federation_reader
 ^/_matrix/federation/v1/make_knock/                   federation_reader
 ^/_matrix/federation/v1/send_knock/                   federation_reader
-^/_matrix/federation/v1/invite/                       federation_reader
+^/_matrix/federation/(v1|v2)/invite/                  federation_reader
 ^/_matrix/federation/v1/query_auth/                   federation_reader
 ^/_matrix/federation/v1/event_auth/                   federation_reader
 ^/_matrix/federation/v1/exchange_third_party_invite/  federation_reader

From 420f5bd5bd33f88477d39b423539d65b0f2a7b2b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 13 Jun 2024 14:23:30 +0100
Subject: [PATCH 53/55] Attempt to fix synapse-sytest docker images (#1384)

BBind to 127.0.0.1 when using docker

I'm not sure why this is a problem, but emprically it is now required
for me to run it locally.

The exact thing that needs to change is that HaProxy server addresses
need to be changed. I assume its something like when HaProxy starts
`localhost` can't be resolved.
---
 lib/SyTest/SSL.pm         | 8 +++++++-
 scripts/synapse_sytest.sh | 4 +++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/lib/SyTest/SSL.pm b/lib/SyTest/SSL.pm
index d5ac88c74..02d99b09e 100644
--- a/lib/SyTest/SSL.pm
+++ b/lib/SyTest/SSL.pm
@@ -62,7 +62,13 @@ sub create_ssl_cert
    # Create extension file
    my $ext_file = "$cert_file.ext";
    open(my $fh, '>', $ext_file) or die "Could not open file '$ext_file': $!";
-   print $fh "subjectAltName=DNS:$server_name\n";
+   if ( $server_name =~ m/^[\d\.:]+$/ ) {
+      # We assume that a server name that is purely numeric (plus ':' and '.')
+      # is an IP.
+      print $fh "subjectAltName=IP:$server_name\n";
+   } else {
+      print $fh "subjectAltName=DNS:$server_name\n";
+   }
    close $fh;
 
    # sign it with the CA
diff --git a/scripts/synapse_sytest.sh b/scripts/synapse_sytest.sh
index 031b6322e..9cfabf924 100755
--- a/scripts/synapse_sytest.sh
+++ b/scripts/synapse_sytest.sh
@@ -192,9 +192,11 @@ echo >&2 "+++ Running tests"
 
 export COVERAGE_PROCESS_START="/src/.coveragerc"
 
+# We set the `--bind-host` as 127.0.0.1 as docker sometimes can't find
+# localhost.
 RUN_TESTS=(
     perl -I "$SYTEST_LIB" /sytest/run-tests.pl --python=/venv/bin/python --synapse-directory=/src -B "/src/$BLACKLIST" --coverage -O tap --all
-    --work-directory="/work"
+    --work-directory="/work" --bind-host 127.0.0.1
 )
 
 if [ -n "$ASYNCIO_REACTOR" ]; then

From 2d5e02f25331f871b7877eb5f8d44bcdcf90065b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Fri, 14 Jun 2024 11:14:47 +0100
Subject: [PATCH 54/55] Update tests to allow x-signing key upload without UIA
 (#1383)

* Update test for MSC3967

* Remove redundant UIA for cross-signing-key upload

MSC3967 says this is no longer needed.
---
 tests/41end-to-end-keys/08-cross-signing.pl | 73 +++++++--------------
 1 file changed, 22 insertions(+), 51 deletions(-)

diff --git a/tests/41end-to-end-keys/08-cross-signing.pl b/tests/41end-to-end-keys/08-cross-signing.pl
index 345e4f301..73a32c53c 100644
--- a/tests/41end-to-end-keys/08-cross-signing.pl
+++ b/tests/41end-to-end-keys/08-cross-signing.pl
@@ -12,14 +12,6 @@
       my $user_id = $user->user_id;
 
       matrix_set_cross_signing_key( $user, {
-          "auth" => {
-              type     => "m.login.password",
-              identifier => {
-                 type => "m.id.user",
-                 user => $user->user_id,
-              },
-              password => $user->password,
-          },
           "master_key" => {
               # private key: 2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0
               "user_id" => $user_id,
@@ -52,23 +44,37 @@
       });
    };
 
-test "Fails to upload self-signing keys with no auth",
+test "Fails to replace cross-signing keys with no auth",
    requires => [ local_user_fixture(), qw( can_upload_self_signing_keys ) ],
 
    do => sub {
       my ( $user ) = @_;
       my $user_id = $user->user_id;
 
+      # Initial upload does not require UIA
       matrix_set_cross_signing_key( $user, {
-          "master_key" => {
-              # private key: 2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0
-              "user_id" => $user_id,
-              "usage" => ["master"],
-              "keys" => {
+         "master_key" => {
+            # private key: HvQBbU+hc2Zr+JP1sE0XwBe1pfZZEYtJNPJLZJtS+F8
+            "user_id" => $user_id,
+            "usage" => ["master"],
+            "keys" => {
+                "ed25519:EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ"
+                   => "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ",
+            },
+         },
+      })->then( sub {
+         # Replacing the key with no auth should be rejected
+         matrix_set_cross_signing_key( $user, {
+            "master_key" => {
+               # private key: 2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0
+               "user_id" => $user_id,
+               "usage" => ["master"],
+               "keys" => {
                   "ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk"
                       => "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk",
-              },
-          },
+               },
+            },
+         });
       })->main::expect_http_401;
    };
 
@@ -80,11 +86,6 @@
       my $user_id = $user->user_id;
 
       matrix_set_cross_signing_key( $user, {
-          "auth" => {
-              "type"     => "m.login.password",
-              "user"     => $user_id,
-              "password" => $user->password,
-          },
           "self_signing_key" => {
               # private key: 2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0
               "user_id" => $user_id,
@@ -134,11 +135,6 @@
             origin => $user_id, key_id => "ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk"
          );
          matrix_set_cross_signing_key( $user1, {
-             "auth" => {
-                 "type"     => "m.login.password",
-                 "user"     => $user_id,
-                 "password" => $user1->password,
-             },
              "master_key" => {
                  # private key: 2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0
                  "user_id" => $user_id,
@@ -285,11 +281,6 @@
             origin => $user_id, key_id => "ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk"
          );
          matrix_set_cross_signing_key( $user1, {
-             "auth" => {
-                 "type"     => "m.login.password",
-                 "user"     => $user_id,
-                 "password" => $user1->password,
-             },
              "master_key" => {
                  # private key: 2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0
                  "user_id" => $user_id,
@@ -329,11 +320,6 @@
          });
       })->then( sub {
          matrix_set_cross_signing_key( $user2, {
-             "auth" => {
-                 "type"     => "m.login.password",
-                 "user"     => $user2_id,
-                 "password" => $user2->password,
-             },
              "master_key" => {
                  # private key: OMkooTr76ega06xNvXIGPbgvvxAOzmQncN8VObS7aBA
                  "user_id" => $user2_id,
@@ -436,11 +422,6 @@
           );
 
       matrix_set_cross_signing_key( $user2, {
-          "auth" => {
-              "type"     => "m.login.password",
-              "user"     => $user2_id,
-              "password" => $user2->password,
-          },
           "master_key" => {
               # private key: 2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0
               "user_id" => $user2_id,
@@ -532,11 +513,6 @@
          sync_until_user_in_device_list( $user1, $user2 );
       })->then( sub {
          matrix_set_cross_signing_key( $user2, {
-             "auth" => {
-                 "type"     => "m.login.password",
-                  "user"     => $user2_id,
-                  "password" => $user2->password,
-             },
              "master_key" => {
                  # private key: 2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0
                  "user_id" => $user2_id,
@@ -621,11 +597,6 @@
 
       matrix_put_e2e_keys( $user2, device_keys => $device)->then( sub {
          matrix_set_cross_signing_key( $user2, {
-             "auth" => {
-                 "type"     => "m.login.password",
-                 "user"     => $user2_id,
-                 "password" => $user2->password,
-             },
              "master_key" => {
                  # private key: 2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0
                  "user_id" => $user2_id,

From e0ef8d85718f10b76a3f425ec42af3c3ed4cf02d Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Mon, 1 Jul 2024 18:17:31 +0100
Subject: [PATCH 55/55] link to complement

---
 README.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.rst b/README.rst
index d48a76888..786d270a1 100644
--- a/README.rst
+++ b/README.rst
@@ -8,6 +8,12 @@ regular HTTP interaction much as any other Matrix client would do. It can
 output test results either to an interactive terminal, or as a TAP-style test
 report, for continuous-integration test harnesses such as Jenkins.
 
+SyTest is written in Perl, and is the original integration testing system for
+Matrix.  New tests tend to be written in Complement_, a Go-based integration
+testing system.
+
+.. _Complement: https://github.com/matrix-org/complement
+
 Installing
 ----------