Merge pull request #1069 from wireapp/release-2020-04-21

Release 2020-04-21

CHANGELOG.md

@@ -1,3 +1,25 @@
+# 2020-04-21
+
+## New Features
+
+* Allow for `report_missing` in `NewOtrMessage`. (#1056, #1062)
+* List team members by UserId (#1048)
+* Support idp update.  (#1065 for issuer, #1026 for everything else)
+* Support synchronous purge-deletion of idps (via query param).  (#1068)
+
+## Bug fixes
+
+* Test that custom backend domains are case-insensitive (#1051)
+* Swagger improvements. (#1059, #1054)
+
+## Internal Changes
+
+* Count team members using es (#1046)
+* Make delete or downgrade team owners scale (#1029)
+* services-demo/demo.sh: mkdir zauth (if not exists) (#1055)
+* Use fork of bloodhound to support ES 5.2 (#1050)
+
+
 # 2020-04-15
 
 ## Upgrade steps (IMPORTANT)

deploy/services-demo/demo.sh

@@ -60,6 +60,7 @@ function check_secrets() {
     fi
     if [[ ! -f ${SCRIPT_DIR}/resources/zauth/privkeys.txt || ! -f ${SCRIPT_DIR}/resources/zauth/pubkeys.txt ]]; then
         echo "Generate private and public keys (used both by brig and nginz)..."
+        mkdir -p ${SCRIPT_DIR}/resources/zauth/
         TMP_KEYS=$(mktemp "/tmp/demo.keys.XXXXXXXXXXX")
         run_zauth -m gen-keypair -i 1 > $TMP_KEYS
         cat $TMP_KEYS | sed -n 's/public: \(.*\)/\1/p' > ${SCRIPT_DIR}/resources/zauth/pubkeys.txt

libs/api-bot/src/Network/Wire/Bot/Crypto.hs

@@ -141,6 +141,7 @@ encryptMessage clt cnv msg =
     <*> pure False -- Transient?
     <*> pure Nothing -- Priority
     <*> pure Nothing -- Extra data distributed to all recipients
+    <*> pure Nothing
 
 decryptMessage :: BotClient -> ConvEvent OtrMessage -> BotSession ByteString
 decryptMessage clt e = do

libs/brig-types/test/unit/Test/Brig/Roundtrip.hs

@@ -19,18 +19,18 @@ module Test.Brig.Roundtrip where
 
 import Data.Aeson (FromJSON, ToJSON, parseJSON, toJSON)
 import Data.Aeson.Types (parseEither)
-import Data.Typeable (typeOf)
 import Imports
 import Test.Tasty (TestTree)
 import Test.Tasty.QuickCheck ((===), Arbitrary, counterexample, testProperty)
+import Type.Reflection (typeRep)
 
 testRoundTrip ::
   forall a.
   (Arbitrary a, Typeable a, ToJSON a, FromJSON a, Eq a, Show a) =>
   TestTree
 testRoundTrip = testProperty msg trip
   where
-    msg = show $ typeOf (undefined :: a)
+    msg = show (typeRep @a)
     trip (v :: a) =
       counterexample (show $ toJSON v) $
         Right v === (parseEither parseJSON . toJSON) v

libs/brig-types/test/unit/Test/Brig/Types/Common.hs

@@ -72,7 +72,6 @@ tests =
       testRoundTrip @SSOTeamConfig,
       testRoundTrip @CustomBackend,
       testRoundTrip @FeatureFlags,
-      testRoundTrip @TruncatedTeamSize,
       testCase "{} is a valid TeamMemberDeleteData" $ do
         assertEqual "{}" (Right $ newTeamMemberDeleteData Nothing) (eitherDecode "{}")
     ]
@@ -97,9 +96,3 @@ instance Arbitrary FeatureFlags where
     FeatureFlags
       <$> Test.Tasty.QuickCheck.elements [minBound ..]
       <*> Test.Tasty.QuickCheck.elements [minBound ..]
-
-instance Arbitrary TruncatedTeamSize where
-  arbitrary =
-    mkTruncatedTeamSize
-      <$> arbitrary
-      <*> arbitrary

libs/federation-util/src/Network/Federation/Util/Internal.hs

@@ -14,7 +14,7 @@
 --
 -- You should have received a copy of the GNU Affero General Public License along
 -- with this program. If not, see <https://www.gnu.org/licenses/>.
---
+
 -- Parts of this code, namely functions srvLookup'' and orderSrvResult,
 -- which were taken from http://hackage.haskell.org/package/pontarius-xmpp
 -- are also licensed under the three-clause BSD license:

libs/galley-types/src/Galley/Types.hs

@@ -239,8 +239,8 @@ data NewConv
         newConvTeam :: !(Maybe ConvTeamInfo),
         newConvMessageTimer :: !(Maybe Milliseconds),
         newConvReceiptMode :: !(Maybe ReceiptMode),
+        -- | Every member except for the creator will have this role
         newConvUsersRole :: !RoleName
-        -- Every member except for the creator will have this role
       }
 
 deriving instance Eq NewConv
@@ -342,7 +342,12 @@ data NewOtrMessage
         newOtrNativePush :: !Bool,
         newOtrTransient :: !Bool,
         newOtrNativePriority :: !(Maybe Priority),
-        newOtrData :: !(Maybe Text)
+        newOtrData :: !(Maybe Text),
+        newOtrReportMissing :: !(Maybe [OpaqueUserId])
+        -- FUTUREWORK: if (and only if) clients can promise this uid list will always exactly
+        -- be the list of uids we could also extract from the messages' recipients field, we
+        -- should do the latter, for two reasons: (1) no need for an artificial limit on the
+        -- body field length, because it'd be just a boolean; (2) less network consumption.
       }
 
 newtype UserClients
@@ -374,7 +379,7 @@ newtype Accept
 
 -- Members ------------------------------------------------------------------
 
--- The semantics of the possible different values is entirely up to clients,
+-- | The semantics of the possible different values is entirely up to clients,
 -- the server will not interpret this value in any way.
 newtype MutedStatus = MutedStatus {fromMutedStatus :: Int32}
   deriving (Eq, Num, Ord, Show, FromJSON, ToJSON, Generic)
@@ -413,7 +418,7 @@ data OtherMember
 instance Ord OtherMember where
   compare a b = compare (omId a) (omId b)
 
--- Inbound self member updates.  This is what galley expects on its endpoint.  See also
+-- | Inbound self member updates.  This is what galley expects on its endpoint.  See also
 -- 'MemberUpdateData' - that event is meant to be sent only to the _self_ user.
 data MemberUpdate
   = MemberUpdate
@@ -434,7 +439,7 @@ deriving instance Eq MemberUpdate
 
 deriving instance Show MemberUpdate
 
--- Inbound other member updates.  This is what galley expects on its endpoint.  See also
+-- | Inbound other member updates.  This is what galley expects on its endpoint.  See also
 -- 'OtherMemberUpdateData' - that event is meant to be sent to all users in a conversation.
 data OtherMemberUpdate
   = OtherMemberUpdate
@@ -448,7 +453,8 @@ deriving instance Show OtherMemberUpdate
 data Invite
   = Invite
       { invUsers :: !(List1 OpaqueUserId),
-        invRoleName :: !RoleName -- This role name is to be applied to all users
+        -- | This role name is to be applied to all users
+        invRoleName :: !RoleName
       }
 
 newInvite :: List1 OpaqueUserId -> Invite
@@ -523,7 +529,7 @@ newtype SimpleMembers
       }
   deriving (Eq, Show, Generic)
 
--- This datatype replaces the old `Members` datatype,
+-- | This datatype replaces the old `Members` datatype,
 -- which has been replaced by `SimpleMembers`. This is
 -- needed due to backwards compatible reasons since old
 -- clients will break if we switch these types. Also, this
@@ -543,14 +549,16 @@ data Connect
       }
   deriving (Eq, Show, Generic)
 
--- Outbound member updates. When a user A acts upon a user B,
+-- | Outbound member updates. When a user A acts upon a user B,
 -- then a user event is generated where B's user ID is set
 -- as misTarget.
 -- Used for events (sent over the websocket, etc.).  See also
 -- 'MemberUpdate' and 'OtherMemberUpdate'.
 data MemberUpdateData
   = MemberUpdateData
-      { misTarget :: !(Maybe UserId), -- Target user of this action
+      { -- | Target user of this action, should not be optional anymore.
+        -- <https://github.com/zinfra/backend-issues/issues/1309>
+        misTarget :: !(Maybe UserId),
         misOtrMuted :: !(Maybe Bool),
         misOtrMutedStatus :: !(Maybe MutedStatus),
         misOtrMutedRef :: !(Maybe Text),
@@ -717,6 +725,7 @@ instance ToJSON NewOtrMessage where
         # "transient" .= newOtrTransient otr
         # "native_priority" .= newOtrNativePriority otr
         # "data" .= newOtrData otr
+        # "report_missing" .= newOtrReportMissing otr
         # []
 
 instance FromJSON NewOtrMessage where
@@ -727,6 +736,7 @@ instance FromJSON NewOtrMessage where
       <*> o .:? "transient" .!= False
       <*> o .:? "native_priority"
       <*> o .:? "data"
+      <*> o .:? "report_missing"
 
 instance FromJSON Accept where
   parseJSON = withObject "accept" $ \o ->
@@ -1081,7 +1091,7 @@ instance ToJSON OtherMemberUpdate where
 instance FromJSON MemberUpdateData where
   parseJSON = withObject "member-update event data" $ \m ->
     MemberUpdateData <$> m .:? "target"
-      -- NOTE: ^-- This is really not a maybe and should
+      -- NOTE: This is really not a maybe and should
       --       be made compulsory 28 days after the next
       --       release to prod to guaratee that no events
       --       out there do not contain id.

libs/galley-types/src/Galley/Types/Conversations/Roles.hs

@@ -120,10 +120,6 @@ instance ToJSON ConversationRolesList where
       [ "conversation_roles" .= r
       ]
 
-instance FromJSON ConversationRolesList where
-  parseJSON = withObject "conversation-roles-list" $ \o ->
-    ConversationRolesList <$> o .: "convesation_roles"
-
 -- RoleNames with `wire_` prefix are reserved
 -- and cannot be created by externals. Therefore, never
 -- expose this constructor outside of this module.

libs/galley-types/src/Galley/Types/Proto.hs

@@ -50,6 +50,7 @@ module Galley.Types.Proto
     newOtrMessageNativePriority,
     newOtrMessageData,
     newOtrMessageTransient,
+    newOtrMessageReportMissing,
     toNewOtrMessage,
   )
 where
@@ -227,7 +228,8 @@ data NewOtrMessage
         _newOtrNativePush :: !(Optional 3 (Value Bool)),
         _newOtrData :: !(Optional 4 (Value ByteString)),
         _newOtrNativePriority :: !(Optional 5 (Enumeration Priority)), -- See note [orphans]
-        _newOtrTransient :: !(Optional 6 (Value Bool))
+        _newOtrTransient :: !(Optional 6 (Value Bool)),
+        _newOtrReportMissing :: !(Repeated 7 (Message UserId))
       }
   deriving (Eq, Show, Generic)
 
@@ -243,7 +245,8 @@ newOtrMessage c us =
       _newOtrNativePush = putField Nothing,
       _newOtrData = putField Nothing,
       _newOtrNativePriority = putField Nothing,
-      _newOtrTransient = putField Nothing
+      _newOtrTransient = putField Nothing,
+      _newOtrReportMissing = putField []
     }
 
 newOtrMessageSender :: Functor f => (ClientId -> f ClientId) -> NewOtrMessage -> f NewOtrMessage
@@ -268,6 +271,9 @@ newOtrMessageData f c = (\x -> c {_newOtrData = x}) <$> field f (_newOtrData c)
 newOtrMessageNativePriority :: Functor f => (Maybe Priority -> f (Maybe Priority)) -> NewOtrMessage -> f NewOtrMessage
 newOtrMessageNativePriority f c = (\x -> c {_newOtrNativePriority = x}) <$> field f (_newOtrNativePriority c)
 
+newOtrMessageReportMissing :: Functor f => ([UserId] -> f [UserId]) -> NewOtrMessage -> f NewOtrMessage
+newOtrMessageReportMissing f c = (\x -> c {_newOtrReportMissing = x}) <$> field f (_newOtrReportMissing c)
+
 toNewOtrMessage :: NewOtrMessage -> Galley.NewOtrMessage
 toNewOtrMessage msg =
   Galley.NewOtrMessage
@@ -276,9 +282,14 @@ toNewOtrMessage msg =
       Galley.newOtrNativePush = view newOtrMessageNativePush msg,
       Galley.newOtrTransient = view newOtrMessageTransient msg,
       Galley.newOtrData = toBase64Text <$> view newOtrMessageData msg,
-      Galley.newOtrNativePriority = toPriority <$> view newOtrMessageNativePriority msg
+      Galley.newOtrNativePriority = toPriority <$> view newOtrMessageNativePriority msg,
+      Galley.newOtrReportMissing = toReportMissing $ view newOtrMessageReportMissing msg
     }
 
+toReportMissing :: [UserId] -> Maybe [Id.OpaqueUserId]
+toReportMissing [] = Nothing
+toReportMissing us = Just $ view userId <$> us
+
 -- Utilities ----------------------------------------------------------------
 
 fromBase64Text :: Text -> ByteString

libs/galley-types/src/Galley/Types/Swagger.hs

@@ -60,6 +60,7 @@ galleyModels =
     otrRecipients,
     otrClientMap,
     userClients,
+    userIdList,
     clientMismatch,
     serviceRef,
     teamInfo,
@@ -267,6 +268,9 @@ newOtrMessage = defineModel "NewOtrMessage" $ do
       "Extra (symmetric) data (i.e. ciphertext) that is replicated \
       \for each recipient."
     optional
+  property "report_missing" (unique $ array bytes') $ do
+    description "List of user IDs"
+    optional
 
 otrRecipients :: Model
 otrRecipients = defineModel "OtrRecipients" $ do
@@ -298,6 +302,12 @@ userClients =
     $ property "" (unique $ array bytes')
     $ description "Map of user IDs to sets of client IDs ({ UserId: [ClientId] })."
 
+userIdList :: Model
+userIdList = defineModel "UserIdList" $ do
+  description "list of user ids"
+  property "user_ids" (unique $ array bytes') $
+    description "the array of team conversations"
+
 members :: Model
 members =
   defineModel "Members"