Merge pull request #1139 from wireapp/r2020-06-19

Release 2020-06-19

CHANGELOG.md

@@ -1,3 +1,25 @@
+# 2020-06-19
+
+## Release Notes
+
+- run galley schema migrations
+- no need to upgrade nginz
+
+## New Features
+
+* Add team level flag for digital signtaures (#1132)
+
+## Bug fixes
+
+* Bump http-client (#1138)
+
+## Internal changes
+
+* Script for finding undead users in elasticsearch (#1137)
+* DB changes for federation (#1070)
+* Refactor team feature tests (#1136)
+
+
 # 2020-06-10
 
 ## Release Notes

docs/reference/cassandra-schema.cql

@@ -103,6 +103,7 @@ CREATE TABLE galley_test.data_migration (
 
 CREATE TABLE galley_test.team_features (
     team_id uuid PRIMARY KEY,
+    digital_signatures int,
     legalhold_status int,
     search_visibility_status int,
     sso_status int,
@@ -175,6 +176,8 @@ CREATE TABLE galley_test.member (
     provider uuid,
     service uuid,
     status int,
+    user_remote_domain text,
+    user_remote_id uuid,
     PRIMARY KEY (conv, user)
 ) WITH CLUSTERING ORDER BY (user ASC)
     AND bloom_filter_fp_chance = 0.1
@@ -262,6 +265,8 @@ CREATE TABLE galley_test.meta (
 CREATE TABLE galley_test.user (
     user uuid,
     conv uuid,
+    conv_remote_domain text,
+    conv_remote_id uuid,
     PRIMARY KEY (user, conv)
 ) WITH CLUSTERING ORDER BY (conv ASC)
     AND bloom_filter_fp_chance = 0.1

docs/reference/elastic-search.md

@@ -112,6 +112,13 @@ REFRESH_INTERVAL=<REFRESH_INTERVAL_FOR_THE_INDEX>
 
 Now you can delete the old index.
 
+**NOTE**: There is a bug hidden when using this way. Sometimes a user won't get
+deleted from the index. Attempts at reproducing this issue in a simpler
+environment have failed. As a workaround, there is a tool in
+[tools/db/find-undead](../../tools/db/find-undead) which can be used to find the
+undead users right after the migration. If they exist, please run refill the ES
+documents from cassandra as described [above](#refill-es-documents-from-cassandra)
+
 ## Recreate an index (Requires downtime)
 
 When analysis settings of an index need to be changed, e.g. for changes

libs/api-bot/src/Network/Wire/Bot/Assert.hs

@@ -20,7 +20,7 @@
 
 module Network.Wire.Bot.Assert where
 
-import Data.Id (ConvId, UserId)
+import Data.Id (ConvId, UserId, makeIdOpaque)
 import qualified Data.Set as Set
 import Imports
 import Network.Wire.Bot.Monad
@@ -39,7 +39,7 @@ assertConvCreated ::
 assertConvCreated c b bs = do
   let everyone = b : bs
   forM_ bs $ \u ->
-    let others = Set.fromList . filter (/= botId u) . map botId $ everyone
+    let others = Set.fromList . map makeIdOpaque . filter (/= botId u) . map botId $ everyone
      in assertEvent u TConvCreate (convCreate (botId u) others)
   where
     convCreate self others = \case
@@ -50,7 +50,7 @@ assertConvCreated c b bs = do
          in cnvId cnv == c
               && convEvtFrom e == botId b
               && cnvType cnv == RegularConv
-              && memId (cmSelf mems) == self
+              && memId (cmSelf mems) == makeIdOpaque self
               && omems == others
       _ -> False
 

libs/galley-types/galley-types.cabal

@@ -4,7 +4,7 @@ cabal-version: 1.12
 --
 -- see: https://github.com/sol/hpack
 --
--- hash: fb74da023d3b1f2a3ad91fca661ee239b7010e42ec219bec603378749aafabea
+-- hash: 73ad5a5126cffda9d353014c94e6f72b68f8dfbe7ecad75a7f03f55f13e06d7b
 
 name:           galley-types
 version:        0.81.0
@@ -22,6 +22,7 @@ library
       Galley.Types
       Galley.Types.Bot
       Galley.Types.Bot.Service
+      Galley.Types.Conversations.Members
       Galley.Types.Conversations.Roles
       Galley.Types.Teams
       Galley.Types.Teams.Intra

libs/galley-types/src/Galley/Types.hs

@@ -24,7 +24,9 @@ module Galley.Types
 
     -- * re-exports
     Conversation (..),
-    Member (..),
+    LocalMember,
+    Member,
+    InternalMember (..),
     ConvMembers (..),
     OtherMember (..),
     Connect (..),
@@ -75,8 +77,9 @@ import Data.Id (ClientId, ConvId, OpaqueUserId, TeamId, UserId)
 import Data.Json.Util ((#))
 import qualified Data.Map.Strict as Map
 import Data.Misc (Milliseconds)
+import Galley.Types.Conversations.Members (InternalMember (..), LocalMember, Member)
 import Imports
-import Wire.API.Conversation
+import Wire.API.Conversation hiding (Member (..))
 import Wire.API.Conversation.Code
 import Wire.API.Conversation.Typing
 import Wire.API.CustomBackend
@@ -85,7 +88,8 @@ import Wire.API.Message
 import Wire.API.User (UserIdList (..))
 import Wire.API.User.Client
 
--- Conversations ------------------------------------------------------------
+--------------------------------------------------------------------------------
+-- ConversationMeta
 
 data ConversationMeta = ConversationMeta
   { cmId :: !ConvId,
@@ -100,56 +104,58 @@ data ConversationMeta = ConversationMeta
   }
   deriving (Eq, Show)
 
---------------------------------------------------------------------------------
+instance ToJSON ConversationMeta where
+  toJSON c =
+    object $
+      "id" .= cmId c
+        # "type" .= cmType c
+        # "creator" .= cmCreator c
+        # "access" .= cmAccess c
+        # "access_role" .= cmAccessRole c
+        # "name" .= cmName c
+        # "team" .= cmTeam c
+        # "message_timer" .= cmMessageTimer c
+        # "receipt_mode" .= cmReceiptMode c
+        # []
 
-foldrOtrRecipients :: (OpaqueUserId -> ClientId -> Text -> a -> a) -> a -> OtrRecipients -> a
-foldrOtrRecipients f a =
-  Map.foldrWithKey go a
-    . userClientMap
-    . otrRecipientsMap
-  where
-    go u cs acc = Map.foldrWithKey (f u) acc cs
+instance FromJSON ConversationMeta where
+  parseJSON = withObject "conversation-meta" $ \o ->
+    ConversationMeta <$> o .: "id"
+      <*> o .: "type"
+      <*> o .: "creator"
+      <*> o .: "access"
+      <*> o .: "access_role"
+      <*> o .: "name"
+      <*> o .:? "team"
+      <*> o .:? "message_timer"
+      <*> o .:? "receipt_mode"
+
+--------------------------------------------------------------------------------
+-- Accept
 
 -- | Request payload for accepting a 1-1 conversation.
 newtype Accept = Accept
   { aUser :: UserId
   }
   deriving (Eq, Show, Generic)
 
--- Instances ----------------------------------------------------------------
-
-instance FromJSON Accept where
-  parseJSON = withObject "accept" $ \o ->
-    Accept <$> o .: "user"
-
 instance ToJSON Accept where
   toJSON a =
     object
       [ "user" .= aUser a
       ]
 
-instance FromJSON ConversationMeta where
-  parseJSON = withObject "conversation-meta" $ \o ->
-    ConversationMeta <$> o .: "id"
-      <*> o .: "type"
-      <*> o .: "creator"
-      <*> o .: "access"
-      <*> o .: "access_role"
-      <*> o .: "name"
-      <*> o .:? "team"
-      <*> o .:? "message_timer"
-      <*> o .:? "receipt_mode"
+instance FromJSON Accept where
+  parseJSON = withObject "accept" $ \o ->
+    Accept <$> o .: "user"
 
-instance ToJSON ConversationMeta where
-  toJSON c =
-    object $
-      "id" .= cmId c
-        # "type" .= cmType c
-        # "creator" .= cmCreator c
-        # "access" .= cmAccess c
-        # "access_role" .= cmAccessRole c
-        # "name" .= cmName c
-        # "team" .= cmTeam c
-        # "message_timer" .= cmMessageTimer c
-        # "receipt_mode" .= cmReceiptMode c
-        # []
+--------------------------------------------------------------------------------
+-- utility functions
+
+foldrOtrRecipients :: (OpaqueUserId -> ClientId -> Text -> a -> a) -> a -> OtrRecipients -> a
+foldrOtrRecipients f a =
+  Map.foldrWithKey go a
+    . userClientMap
+    . otrRecipientsMap
+  where
+    go u cs acc = Map.foldrWithKey (f u) acc cs

libs/galley-types/src/Galley/Types/Conversations/Members.hs

@@ -0,0 +1,52 @@
+{-# LANGUAGE StrictData #-}
+
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2020 Wire Swiss GmbH <[email protected]>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module Galley.Types.Conversations.Members
+  ( LocalMember,
+    Member,
+    InternalMember (..),
+  )
+where
+
+import Data.Id as Id
+import Data.IdMapping (MappedOrLocalId)
+import Imports
+import Wire.API.Conversation.Member (MutedStatus)
+import Wire.API.Conversation.Role (RoleName)
+import Wire.API.Provider.Service (ServiceRef)
+
+type LocalMember = InternalMember Id.UserId
+
+type Member = InternalMember (MappedOrLocalId Id.U)
+
+-- | Internal representation of a conversation member.
+data InternalMember id = Member
+  { memId :: id,
+    memService :: Maybe ServiceRef,
+    -- | DEPRECATED, remove it once enough clients use `memOtrMutedStatus`
+    memOtrMuted :: Bool,
+    memOtrMutedStatus :: Maybe MutedStatus,
+    memOtrMutedRef :: Maybe Text,
+    memOtrArchived :: Bool,
+    memOtrArchivedRef :: Maybe Text,
+    memHidden :: Bool,
+    memHiddenRef :: Maybe Text,
+    memConvRoleName :: RoleName
+  }
+  deriving stock (Functor, Show)

libs/galley-types/src/Galley/Types/Teams.hs

@@ -336,6 +336,7 @@ roleHiddenPermissions role = HiddenPermissions p p
           ViewTeamFeature TeamFeatureSSO,
           ViewTeamFeature TeamFeatureSearchVisibility,
           ViewTeamFeature TeamFeatureValidateSAMLEmails,
+          ViewTeamFeature TeamFeatureDigitalSignatures,
           ViewLegalHoldUserSettings,
           ViewTeamSearchVisibility
         ]

libs/types-common/src/Data/Domain.hs

@@ -52,7 +52,7 @@ import Util.Attoparsec (takeUpToWhile)
 --
 -- The domain will be normalized to lowercase when parsed.
 newtype Domain = Domain {_domainText :: Text}
-  deriving (Eq, Generic, Show)
+  deriving stock (Eq, Ord, Generic, Show)
 
 domainText :: Domain -> Text
 domainText = _domainText

libs/types-common/src/Data/Id.hs

@@ -70,13 +70,18 @@ data Mapped a
 
 data Opaque a
 
+data Remote a
+
 type AssetId = Id A
 
 type InvitationId = Id I
 
 -- | A local conversation ID
 type ConvId = Id C
 
+-- | A UUID local to another backend, only meaningful together with its domain.
+type RemoteConvId = Id (Remote C)
+
 -- | A UUID local to this backend, for which we know a mapping to a
 -- remote qualified conversation ID exists.
 -- These IDs should never leak to other backends or their clients.
@@ -91,6 +96,9 @@ type OpaqueConvId = Id (Opaque C)
 -- | A local user ID
 type UserId = Id U
 
+-- | A UUID local to another backend, only meaningful together with its domain.
+type RemoteUserId = Id (Remote U)
+
 -- | A UUID local to this backend, for which we know a mapping to a
 -- remote qualified user ID exists.
 -- These IDs should never leak to other backends or their clients.

libs/types-common/src/Data/IdMapping.hs

@@ -27,7 +27,7 @@ import Test.QuickCheck (Arbitrary (arbitrary), oneof)
 data MappedOrLocalId a
   = Mapped (IdMapping a)
   | Local (Id a)
-  deriving (Show)
+  deriving stock (Eq, Ord, Show)
 
 opaqueIdFromMappedOrLocal :: MappedOrLocalId a -> Id (Opaque a)
 opaqueIdFromMappedOrLocal = \case
@@ -41,9 +41,9 @@ partitionMappedOrLocalIds = foldMap $ \case
 
 data IdMapping a = IdMapping
   { idMappingLocal :: Id (Mapped a),
-    idMappingGlobal :: Qualified (Id a)
+    idMappingGlobal :: Qualified (Id (Remote a))
   }
-  deriving (Show)
+  deriving stock (Eq, Ord, Show)
 
 ----------------------------------------------------------------------
 -- ARBITRARY

libs/types-common/src/Data/Qualified.hs

@@ -92,7 +92,7 @@ data Qualified a = Qualified
   { _qLocalPart :: a,
     _qDomain :: Domain
   }
-  deriving (Eq, Show, Generic)
+  deriving stock (Eq, Ord, Show, Generic)
 
 renderQualified :: (a -> Text) -> Qualified a -> Text
 renderQualified renderLocal (Qualified localPart domain) =

libs/wire-api/src/Wire/API/Conversation/Member.hs

@@ -83,7 +83,7 @@ instance FromJSON ConvMembers where
 -- Members
 
 data Member = Member
-  { memId :: UserId,
+  { memId :: OpaqueUserId,
     memService :: Maybe ServiceRef,
     -- | DEPRECATED, remove it once enough clients use `memOtrMutedStatus`
     memOtrMuted :: Bool,
@@ -165,7 +165,7 @@ newtype MutedStatus = MutedStatus {fromMutedStatus :: Int32}
   deriving newtype (Num, FromJSON, ToJSON, Arbitrary)
 
 data OtherMember = OtherMember
-  { omId :: UserId,
+  { omId :: OpaqueUserId,
     omService :: Maybe ServiceRef,
     omConvRoleName :: RoleName
   }