Merge pull request #3505 from wireapp/release_2023-08-16_09_54

Release 2023-08-16 - (expected chart version 4.37.0)

CHANGELOG.md

@@ -1,3 +1,23 @@
+# [2023-08-16] (Chart Release 4.37.0)
+
+## API changes
+
+
+* Conversation creation endpoints can now return `unreachable_backends` error responses with status code 533 if any of the involved backends are unreachable. The conversation is not created in that case. (#3486)
+
+
+## Bug fixes and other updates
+
+
+* Make sure cassandra updates do not re-introduce removed content. (#3504)
+
+
+## Federation changes
+
+
+* Return `unreachable_backends` error when some backends of newly added users to a conversation are not reachable (#3496)
+
+
 # [2023-08-11] (Chart Release 4.36.0)
 
 ## Release notes

hack/bin/create-user

@@ -64,8 +64,12 @@ def add_team_member(baseurl, team, access_token, basic_auth, i=1):
 
     return member
 
-def create_user(baseurl, basic_auth, create_team, n_members):
-    email = random_email()
+def create_user(baseurl, basic_auth, create_team, n_members, manual_email, has_inbucket):
+    if manual_email is None:
+        email = random_email()
+    else:
+        email = manual_email
+
     password = random_string(20)
 
     body = {
@@ -99,25 +103,37 @@ def create_user(baseurl, basic_auth, create_team, n_members):
         'team': team
     }
 
-    r = requests.post(f'{baseurl}/login', json={'email': email, 'password': password})
-    access_token = r.json()['access_token']
-
     result = {'admin': admin}
 
-    if team is not None:
-        members = []
-        for i in range(n_members):
-            member = add_team_member(baseurl, team, access_token, basic_auth, i)
-            members.append(member)
-        result['members'] = members
+    r = requests.get(f'{baseurl}/i/teams/{team}/features/sndFactorPasswordChallenge', headers=basicauth_headers)
+    d = r.json()
+    second_factor_enabled = d['status'] == 'enabled'
+    # FUTUREWORK: Create team members for 2fa backends. To login 1) send verification code 2) get verification code via internal api 3) use code when logging in as authentication code
+    if second_factor_enabled:
+        if manual_email is None and not has_inbucket:
+            fail("Backend has 2FA enabled. Yout must provide an existing email adress via the -m flag. Also no team members will be created by this script.")
+
+    else:
+        login_request = {'email': email, 'password': password}
+
+        r = requests.post(f'{baseurl}/login', json=login_request)
+
+        access_token = r.json()['access_token']
+
+        if team is not None and not second_factor_enabled:
+            members = []
+            for i in range(n_members):
+                member = add_team_member(baseurl, team, access_token, basic_auth, i)
+                members.append(member)
+            result['members'] = members
 
     return result
 
-def maybe_to_list(x):
-    if x is not None:
-        return [x]
-    else:
-        return []
+def fail(msg):
+    sys.stderr.write(msg)
+    sys.stderr.write('\n')
+    sys.exit(1)
+
 
 def main():
     known_envs = {
@@ -172,6 +188,37 @@ def main():
             'baseurl': 'https://nginz-https.unicorns.dogfood.wire.link',
             'webapp': 'https://webapp.unicorns.dogfood.wire.link/'
         },
+        'bund-next-column-offline-android': {
+            'baseurl': 'https://nginz-https.bund-next-column-offline-android.wire.link',
+            'webapp': 'https://webapp.bund-next-column-offline-android.wire.link/'
+        },
+        'bund-next-column-offline-web': {
+            'baseurl': 'https://nginz-https.bund-next-column-offline-web.wire.link',
+            'webapp': 'https://webapp.bund-next-column-offline-web.wire.link/'
+        },
+        'bund-next-column-offline-ios': {
+            'baseurl': 'https://nginz-https.bund-next-column-offline-ios.wire.link',
+            'webapp': 'https://webapp.bund-next-column-offline-ios.wire.link/'
+        },
+        'bund-next-external': {
+            'baseurl': 'https://nginz-https.bund-next-external.wire.link',
+            'webapp': 'https://webapp.bund-next-external.wire.link/'
+        },
+        'bund-next-column-1': {
+            'baseurl': 'https://nginz-https.bund-next-column-1.wire.link',
+            'webapp': 'https://webapp.bund-next-column-1.wire.link/',
+            'inbucket': 'https://inbucket.bund-next-column-1.wire.link/'
+        },
+        'bund-next-column-2': {
+            'baseurl': 'https://nginz-https.bund-next-column-2.wire.link',
+            'webapp': 'https://webapp.bund-next-column-2.wire.link/',
+            'inbucket': 'https://inbucket.bund-next-column-2.wire.link/'
+        },
+        'bund-next-column-3': {
+            'baseurl': 'https://nginz-https.bund-next-column-3.wire.link',
+            'webapp': 'https://webapp.bund-next-column-3.wire.link/',
+            'inbucket': 'https://inbucket.bund-next-column-3.wire.link/'
+        }
     }
 
     parser = argparse.ArgumentParser(
@@ -180,35 +227,37 @@ def main():
     parser.add_argument('-e', '--env', default='choose_env', help=f'One of: {", ".join(known_envs.keys())}')
     parser.add_argument('-p', '--personal', action='store_true', help="Create a personal user, instead of a team admin.")
     parser.add_argument('-n', '--members', default='1', help="Number of members to add.")
+    parser.add_argument('-m', '--email', default='', help="Email of created user. If omitted a random non-existing @wire.com email will be used.")
     args = parser.parse_args()
 
     if args.env == 'choose_env':
-        print(parser.format_help())
-        sys.exit(1)
+        fail(parser.format_help())
 
     env = known_envs.get(args.env)
     if env is None:
-        print(f'Unknown environment: {args.env}. If missing then add it to the script.')
-        sys.exit(1)
+        fail(f'Unknown environment: {args.env}. If missing then add it to the script.')
 
     basic_auths_json = os.environ.get('CREATE_USER_BASICAUTH')
     if basic_auths_json is None:
-        print(r'Please set CREATE_USER_BASICAUTH to a json object of form {"env_name": {"username": "xx", "password": "xx"}} containing the basicauth credentials for each environment.')
-        sys.exit(1)
+        fail(r'Please set CREATE_USER_BASICAUTH to a json object of form {"env_name": {"username": "xx", "password": "xx"}} containing the basicauth credentials for each environment.')
 
     basic_auths = json.loads(basic_auths_json)
+    if args.env not in basic_auths:
+        fail(f'Environment "{args.env}" is missing in CREATE_USER_BASICAUTH.')
+
     b_user = basic_auths[args.env]['username']
     b_password = basic_auths[args.env]['password']
 
     basic_auth = base64.b64encode(f'{b_user}:{b_password}'.encode('utf8')).decode('utf8')
 
     n_members = int(args.members)
 
-    result = create_user(env['baseurl'], basic_auth, not args.personal, n_members)
+    manual_email = args.email if len(args.email) > 0 else None
+
+    result = create_user(env['baseurl'], basic_auth, not args.personal, n_members, manual_email, 'inbucket' in env)
 
-    links = maybe_to_list(env.get('webapp')) + maybe_to_list(env.get('teams'))
-    if links:
-        result['comment'] = f'These credentials can be used at: {", ".join(links)}'
+    result['env'] = env
+    result['basicauth'] = {'username': b_user, 'password': b_password, 'header': basic_auth}
 
     print(json.dumps(result, indent=4))
 

integration/test/Test/Conversation.hs

@@ -51,8 +51,8 @@ testDynamicBackendsNotFederating = do
         $ bindResponse
           (getFederationStatus uidA [domainB, domainC])
         $ \resp -> do
-          resp.status `shouldMatchInt` 422
-          resp.json %. "label" `shouldMatch` "federation-denied"
+          resp.status `shouldMatchInt` 533
+          resp.json %. "unreachable_backends" `shouldMatchSet` [domainB, domainC]
 
 testDynamicBackendsFullyConnectedWhenAllowDynamic :: HasCallStack => App ()
 testDynamicBackendsFullyConnectedWhenAllowDynamic = do
@@ -123,8 +123,8 @@ testFederationStatus = do
   bindResponse
     (getFederationStatus uid [invalidDomain])
     $ \resp -> do
-      resp.status `shouldMatchInt` 422
-      resp.json %. "label" `shouldMatch` "invalid-domain"
+      resp.status `shouldMatchInt` 533
+      resp.json %. "unreachable_backends" `shouldMatchSet` [invalidDomain]
 
   bindResponse
     (getFederationStatus uid [federatingRemoteDomain])
@@ -327,3 +327,64 @@ testAddMembersNonFullyConnectedProteus = do
     bindResponse (addMembers u1 cid members) $ \resp -> do
       resp.status `shouldMatchInt` 409
       resp.json %. "non_federating_backends" `shouldMatchSet` [domainB, domainC]
+
+testConvWithUnreachableRemoteUsers :: HasCallStack => App ()
+testConvWithUnreachableRemoteUsers = do
+  let overrides =
+        def {dbBrig = setField "optSettings.setFederationStrategy" "allowAll"}
+          <> fullSearchWithAll
+  ([alice, alex, bob, charlie, dylan], domains) <-
+    startDynamicBackends [overrides, overrides] $ \domains -> do
+      own <- make OwnDomain & asString
+      other <- make OtherDomain & asString
+      users <- createAndConnectUsers $ [own, own, other] <> domains
+      pure (users, domains)
+
+  let newConv = defProteus {qualifiedUsers = [alex, bob, charlie, dylan]}
+  bindResponse (postConversation alice newConv) $ \resp -> do
+    resp.status `shouldMatchInt` 533
+    resp.json %. "unreachable_backends" `shouldMatchSet` domains
+
+  convs <- getAllConvs alice >>= asList
+  regConvs <- filterM (\c -> (==) <$> (c %. "type" & asInt) <*> pure 0) convs
+  regConvs `shouldMatch` ([] :: [Value])
+
+testAddReachableWithUnreachableRemoteUsers :: HasCallStack => App ()
+testAddReachableWithUnreachableRemoteUsers = do
+  let overrides =
+        def {dbBrig = setField "optSettings.setFederationStrategy" "allowAll"}
+          <> fullSearchWithAll
+  ([alex, bob], conv) <-
+    startDynamicBackends [overrides, overrides] $ \domains -> do
+      own <- make OwnDomain & asString
+      other <- make OtherDomain & asString
+      [alice, alex, bob, charlie, dylan] <-
+        createAndConnectUsers $ [own, own, other] <> domains
+
+      let newConv = defProteus {qualifiedUsers = [alex, charlie, dylan]}
+      conv <- postConversation alice newConv >>= getJSON 201
+      pure ([alex, bob], conv)
+
+  bobId <- bob %. "qualified_id"
+  bindResponse (addMembers alex conv [bobId]) $ \resp -> do
+    resp.status `shouldMatchInt` 200
+
+testAddUnreachable :: HasCallStack => App ()
+testAddUnreachable = do
+  let overrides =
+        def {dbBrig = setField "optSettings.setFederationStrategy" "allowAll"}
+          <> fullSearchWithAll
+  ([alex, charlie], [charlieDomain, _dylanDomain], conv) <-
+    startDynamicBackends [overrides, overrides] $ \domains -> do
+      own <- make OwnDomain & asString
+      [alice, alex, charlie, dylan] <-
+        createAndConnectUsers $ [own, own] <> domains
+
+      let newConv = defProteus {qualifiedUsers = [alex, dylan]}
+      conv <- postConversation alice newConv >>= getJSON 201
+      pure ([alex, charlie], domains, conv)
+
+  charlieId <- charlie %. "qualified_id"
+  bindResponse (addMembers alex conv [charlieId]) $ \resp -> do
+    resp.status `shouldMatchInt` 533
+    resp.json %. "unreachable_backends" `shouldMatchSet` [charlieDomain]

integration/test/Testlib/App.hs

@@ -8,13 +8,14 @@ import Data.IORef
 import Data.Text qualified as T
 import Data.Yaml qualified as Yaml
 import GHC.Exception
+import GHC.Stack (HasCallStack)
 import System.FilePath
 import Testlib.Env
 import Testlib.JSON
 import Testlib.Types
 import Prelude
 
-failApp :: String -> App a
+failApp :: HasCallStack => String -> App a
 failApp msg = throw (AppFailure msg)
 
 getPrekey :: App Value

integration/test/Testlib/HTTP.hs

@@ -82,18 +82,18 @@ withResponse :: HasCallStack => Response -> (Response -> App a) -> App a
 withResponse r k = onFailureAddResponse r (k r)
 
 -- | Check response status code, then return body.
-getBody :: Int -> Response -> App ByteString
+getBody :: HasCallStack => Int -> Response -> App ByteString
 getBody status resp = withResponse resp $ \r -> do
   r.status `shouldMatch` status
   pure r.body
 
 -- | Check response status code, then return JSON body.
-getJSON :: Int -> Response -> App Aeson.Value
+getJSON :: HasCallStack => Int -> Response -> App Aeson.Value
 getJSON status resp = withResponse resp $ \r -> do
   r.status `shouldMatch` status
   r.json
 
-onFailureAddResponse :: Response -> App a -> App a
+onFailureAddResponse :: HasCallStack => Response -> App a -> App a
 onFailureAddResponse r m = App $ do
   e <- ask
   liftIO $ E.catch (runAppWithEnv e m) $ \(AssertionFailure stack _ msg) -> do

integration/test/Testlib/ModService.hs

@@ -83,8 +83,8 @@ copyDirectoryRecursively from to = do
 -- continuation, the main continuation is run in an environment that
 -- accumulates all the individual environment changes.
 traverseConcurrentlyCodensity ::
-  (a -> Codensity App (Env -> Env)) ->
-  ([a] -> Codensity App (Env -> Env))
+  (HasCallStack => a -> Codensity App (Env -> Env)) ->
+  (HasCallStack => [a] -> Codensity App (Env -> Env))
 traverseConcurrentlyCodensity f args = do
   -- Create variables for synchronisation of the various threads:
   --  * @result@ is used to store the environment change, or possibly an exception
@@ -138,15 +138,19 @@ traverseConcurrentlyCodensity f args = do
     liftIO $ traverse_ wait asyncs
     pure result
 
-startDynamicBackends :: [ServiceOverrides] -> ([String] -> App a) -> App a
-startDynamicBackends beOverrides = runCodensity $ do
-  when (Prelude.length beOverrides > 3) $ lift $ failApp "Too many backends. Currently only 3 are supported."
-  pool <- asks (.resourcePool)
-  resources <- acquireResources (Prelude.length beOverrides) pool
-  void $ traverseConcurrentlyCodensity (\(res, overrides) -> startDynamicBackend res mempty overrides) (zip resources beOverrides)
-  pure $ map (.berDomain) resources
+startDynamicBackends :: HasCallStack => [ServiceOverrides] -> (HasCallStack => [String] -> App a) -> App a
+startDynamicBackends beOverrides k =
+  runCodensity
+    ( do
+        when (Prelude.length beOverrides > 3) $ lift $ failApp "Too many backends. Currently only 3 are supported."
+        pool <- asks (.resourcePool)
+        resources <- acquireResources (Prelude.length beOverrides) pool
+        void $ traverseConcurrentlyCodensity (\(res, overrides) -> startDynamicBackend res mempty overrides) (zip resources beOverrides)
+        pure $ map (.berDomain) resources
+    )
+    k
 
-startDynamicBackend :: BackendResource -> Map.Map Service Word16 -> ServiceOverrides -> Codensity App (Env -> Env)
+startDynamicBackend :: HasCallStack => BackendResource -> Map.Map Service Word16 -> ServiceOverrides -> Codensity App (Env -> Env)
 startDynamicBackend resource staticPorts beOverrides = do
   defDomain <- asks (.domain1)
   let services =

integration/test/Testlib/ResourcePool.hs

@@ -21,6 +21,7 @@ import Data.String
 import Data.Tuple
 import Data.Word
 import GHC.Generics
+import GHC.Stack (HasCallStack)
 import System.IO
 import Prelude
 
@@ -29,7 +30,7 @@ data ResourcePool a = ResourcePool
     resources :: IORef (Set.Set a)
   }
 
-acquireResources :: forall m a. (Ord a, MonadIO m, MonadMask m) => Int -> ResourcePool a -> Codensity m [a]
+acquireResources :: forall m a. (Ord a, MonadIO m, MonadMask m, HasCallStack) => Int -> ResourcePool a -> Codensity m [a]
 acquireResources n pool = Codensity $ \f -> bracket acquire release (f . Set.toList)
   where
     release :: Set.Set a -> m ()

libs/wai-utilities/src/Network/Wai/Utilities/Error.hs

@@ -31,8 +31,6 @@ import Control.Error
 import Data.Aeson hiding (Error)
 import Data.Aeson.Types (Pair)
 import Data.Domain
-import Data.List.NonEmpty (NonEmpty)
-import Data.List.NonEmpty qualified as NE
 import Data.Text.Lazy.Encoding (decodeUtf8)
 import Imports
 import Network.HTTP.Types
@@ -51,24 +49,23 @@ mkError c l m = Error c l m Nothing
 instance Exception Error
 
 data ErrorData = FederationErrorData
-  { federrDomains :: NonEmpty Domain,
+  { federrDomain :: !Domain,
     federrPath :: !Text
   }
   deriving (Eq, Show, Typeable)
 
 instance ToJSON ErrorData where
-  toJSON (FederationErrorData ds p) =
+  toJSON (FederationErrorData d p) =
     object
       [ "type" .= ("federation" :: Text),
-        "domain" .= NE.head ds, -- deprecated in favour for `domains`
-        "domains" .= ds,
+        "domain" .= d,
         "path" .= p
       ]
 
 instance FromJSON ErrorData where
   parseJSON = withObject "ErrorData" $ \o ->
     FederationErrorData
-      <$> o .: "domains"
+      <$> o .: "domain"
       <*> o .: "path"
 
 -- | Assumes UTF-8 encoding.