Merge pull request #868 from wireapp/release_2019_09_30

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,27 @@
+---
+name: Bug report
+about: Create a bug report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Run  '....'
+3. '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Additional context**
+Add any other context about the problem here. Example: how did you compile or install wire-server? Which configuration are you using? Which version (if using docker images) or git branch are you using?

.github/ISSUE_TEMPLATE/other.md

@@ -0,0 +1,10 @@
+---
+name: Other
+about: Other
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,12 @@
+---
+name: Question
+about: Question about wire-server
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+* [ ] I have seen https://docs.wire.com/ and https://github.com/wireapp/wire-server-deploy - the documentation there does not answer my question.
+
+**My question:**

CHANGELOG.md

@@ -1,3 +1,23 @@
+# 2019-09-30 #868
+
+## Relevant for self-hosters
+- More information is logged about user actions (#856)
+
+## Relevant for client developers
+- Make team member property size configurable (#867)
+
+## Bug fixes
+- Fix bugs related to metrics (#853, #866)
+- Sneak up on flaky test. (#863)
+
+## Internal Changes
+- Derive Generic everywhere (#864)
+- Add issue templates (#862)
+- Cleanup stern (#845)
+- Log warnings only when users are suspended (#854)
+- Documentation update for restund and smoketester (#855)
+
+
 # 2019-09-16 #858
 
 ## Relevant for self-hosters

deploy/services-demo/README.md

@@ -75,16 +75,24 @@ If you wish to send verification SMS/calls (to support registration using phone
 
 Note: This demo setup comes bundled with a postfix email sending docker image; however due to the minimal setup, emails will likely land in the Spam/Junk folder of the target email address, if you configure a common email provider. To get the smoketester to check the Spam folder as well, use e.g. (in the case of gmail) `--mailbox-folder INBOX --mailbox-folder '[Gmail]/Spam'`.
 
-Example:
+Configure an email inbox for the smoketester:
 
 ```
-# from the wire-server directory, after having compiled everything with 'make install'
+# from the root of wire-server directory
+cp tools/api-simulations/mailboxes.example.json mailboxes.json
+```
+
+Now adjust `mailboxes.json` and use credentials for an email account you own.
+
+Next, from the wire-server directory, after having compiled everything with 'make install':
+
+```bash
 ./dist/api-smoketest \
     --api-host=127.0.0.1 \
     --api-port=8080 \
     --api-websocket-host=127.0.0.1 \
     --api-websocket-port=8081 \
-    --mailbox-config=<path_to_mailboxes_file> \
+    --mailbox-config=mailboxes.json \
     [email protected] \
     --mailbox-folder INBOX \
     --mailbox-folder '[Gmail]/Spam' \

libs/bilge/src/Bilge/Request.hs

@@ -28,7 +28,7 @@ module Bilge.Request
     , showRequest
     , noRedirect
     , timeout
-    , expect2xx, expect3xx, expect4xx
+    , expect2xx, expect3xx, expect4xx, expectStatus
     , checkStatus
     , cookie
     , cookieRaw

libs/brig-types/src/Brig/Types/Client.hs

@@ -79,22 +79,22 @@ data Client = Client
     , clientCookie   :: !(Maybe CookieLabel)
     , clientLocation :: !(Maybe Location)
     , clientModel    :: !(Maybe Text)
-    } deriving (Eq, Show)
+    } deriving (Eq, Show, Generic)
 
 data PubClient = PubClient
     { pubClientId     :: !ClientId
     , pubClientClass  :: !(Maybe ClientClass)
-    } deriving (Eq, Show)
+    } deriving (Eq, Show, Generic)
 
 newtype RmClient = RmClient
     { rmPassword :: Maybe PlainTextPassword
-    }
+    } deriving (Generic)
 
 data UpdateClient = UpdateClient
     { updateClientPrekeys :: ![Prekey]
     , updateClientLastKey :: !(Maybe LastPrekey)
     , updateClientLabel   :: !(Maybe Text)
-    }
+    } deriving (Generic)
 
 -- * JSON instances:
 

libs/brig-types/src/Brig/Types/Connection.hs

@@ -79,6 +79,7 @@ data UserConnectionList = UserConnectionList
 -- 'Accepted' state.
 data UserIds = UserIds
     { cUsers :: [UserId] }
+    deriving (Eq, Show, Generic)
 
 -- | Data that is passed to the @\/i\/users\/connections-status@ endpoint.
 data ConnectionsStatusRequest = ConnectionsStatusRequest

libs/brig-types/src/Brig/Types/Intra.hs

@@ -42,6 +42,7 @@ instance ToJSON AccountStatus where
 
 newtype AccountStatusUpdate = AccountStatusUpdate
     { suStatus :: AccountStatus }
+    deriving (Generic)
 
 instance FromJSON AccountStatusUpdate where
     parseJSON = withObject "account-status-update" $ \o ->

libs/galley-types/src/Galley/Types.hs

@@ -281,7 +281,7 @@ data NewOtrMessage = NewOtrMessage
 
 newtype UserClients = UserClients
     { userClients :: Map UserId (Set ClientId)
-    } deriving (Eq, Show, Semigroup, Monoid)
+    } deriving (Eq, Show, Semigroup, Monoid, Generic)
 
 filterClients :: (Set ClientId -> Bool) -> UserClients -> UserClients
 filterClients p (UserClients c) = UserClients $ Map.filter p c
@@ -293,19 +293,19 @@ data ClientMismatch = ClientMismatch
     -- | Clients that the message /should not/ have been encrypted for, but was.
     , redundantClients :: !UserClients
     , deletedClients   :: !UserClients
-    } deriving (Eq, Show)
+    } deriving (Eq, Show, Generic)
 
 -- | Request payload for accepting a 1-1 conversation.
 newtype Accept = Accept
     { aUser :: UserId
-    } deriving (Eq, Show)
+    } deriving (Eq, Show, Generic)
 
 -- Members ------------------------------------------------------------------
 
 -- The semantics of the possible different values is entirely up to clients,
 -- the server will not interpret this value in any way.
 newtype MutedStatus = MutedStatus { fromMutedStatus :: Int32 }
-    deriving (Eq, Num, Ord, Show, FromJSON, ToJSON)
+    deriving (Eq, Num, Ord, Show, FromJSON, ToJSON, Generic)
 
 data Member = Member
     { memId             :: !UserId
@@ -317,12 +317,12 @@ data Member = Member
     , memOtrArchivedRef :: !(Maybe Text)
     , memHidden         :: !Bool
     , memHiddenRef      :: !(Maybe Text)
-    } deriving (Eq, Show)
+    } deriving (Eq, Show, Generic)
 
 data OtherMember = OtherMember
     { omId      :: !UserId
     , omService :: !(Maybe ServiceRef)
-    } deriving (Eq, Show)
+    } deriving (Eq, Show, Generic)
 
 instance Ord OtherMember where
     compare a b = compare (omId a) (omId b)
@@ -357,7 +357,7 @@ data Event = Event
     , evtFrom :: !UserId
     , evtTime :: !UTCTime
     , evtData :: !(Maybe EventData)
-    } deriving Eq
+    } deriving (Eq, Generic)
 
 data EventType
     = MemberJoin
@@ -374,7 +374,7 @@ data EventType
     | ConvReceiptModeUpdate
     | OtrMessageAdd
     | Typing
-    deriving (Eq, Show)
+    deriving (Eq, Show, Generic)
 
 data EventData
     = EdMembers             !Members
@@ -388,25 +388,25 @@ data EventData
     | EdConversation        !Conversation
     | EdTyping              !TypingData
     | EdOtrMessage          !OtrMessage
-    deriving (Eq, Show)
+    deriving (Eq, Show, Generic)
 
 data OtrMessage = OtrMessage
     { otrSender     :: !ClientId
     , otrRecipient  :: !ClientId
     , otrCiphertext :: !Text
     , otrData       :: !(Maybe Text)
-    } deriving (Eq, Show)
+    } deriving (Eq, Show, Generic)
 
 newtype Members = Members
     { mUsers :: [UserId]
-    } deriving (Eq, Show)
+    } deriving (Eq, Show, Generic)
 
 data Connect = Connect
     { cRecipient :: !UserId
     , cMessage   :: !(Maybe Text)
     , cName      :: !(Maybe Text)
     , cEmail     :: !(Maybe Text)
-    } deriving (Eq, Show)
+    } deriving (Eq, Show, Generic)
 
 -- Outbound member updates.  Used for events (sent over the websocket, etc.).  See also
 -- 'MemberUpdate'.
@@ -418,22 +418,22 @@ data MemberUpdateData = MemberUpdateData
     , misOtrArchivedRef :: !(Maybe Text)
     , misHidden         :: !(Maybe Bool)
     , misHiddenRef      :: !(Maybe Text)
-    } deriving (Eq, Show)
+    } deriving (Eq, Show, Generic)
 
 newtype TypingData = TypingData
     { tdStatus :: TypingStatus
-    } deriving (Eq, Show)
+    } deriving (Eq, Show, Generic)
 
 data TypingStatus
     = StartedTyping
     | StoppedTyping
-    deriving (Eq, Ord, Show)
+    deriving (Eq, Ord, Show, Generic)
 
 data ConversationCode = ConversationCode
     { conversationKey   :: !Code.Key
     , conversationCode  :: !Code.Value
     , conversationUri   :: !(Maybe HttpsUrl)
-    } deriving (Eq, Show)
+    } deriving (Eq, Show, Generic)
 
 mkConversationCode :: Code.Key -> Code.Value -> HttpsUrl -> ConversationCode
 mkConversationCode k v (HttpsUrl prefix) = ConversationCode

libs/galley-types/src/Galley/Types/Teams/Intra.hs

@@ -18,7 +18,7 @@ data TeamStatus
     | Deleted
     | Suspended
     | PendingActive
-    deriving (Eq, Show)
+    deriving (Eq, Show, Generic)
 
 instance ToJSON TeamStatus where
     toJSON Active        = String "active"
@@ -40,6 +40,7 @@ data TeamData = TeamData
     , tdStatus     :: !TeamStatus
     , tdStatusTime :: !(Maybe UTCTime) -- This needs to be a Maybe due to backwards compatibility
     }
+    deriving (Eq, Show, Generic)
 
 instance ToJSON TeamData where
     toJSON (TeamData t s st) = object
@@ -59,6 +60,7 @@ data TeamStatusUpdate = TeamStatusUpdate
     , tuCurrency :: !(Maybe Currency.Alpha)
     -- TODO: Remove Currency selection once billing supports currency changes after team creation
     }
+    deriving (Eq, Show, Generic)
 
 instance FromJSON TeamStatusUpdate where
     parseJSON = withObject "team-status-update" $ \o ->
@@ -72,5 +74,6 @@ instance ToJSON TeamStatusUpdate where
 
 newtype TeamName = TeamName
     { tnName :: Text }
+    deriving (Eq, Show, Generic)
 
 deriveJSON toJSONFieldName ''TeamName

libs/metrics-wai/src/Data/Metrics/Servant.hs

@@ -17,6 +17,26 @@ import Data.Tree
 import GHC.TypeLits
 import Servant.API
 
+import qualified Data.Metrics.Types as Metrics
+import qualified Network.Wai as Wai
+import qualified Network.Wai.Middleware.Prometheus as Promth
+
+
+-- | This does not catch errors, so it must be called outside of 'WU.catchErrors'.
+servantPrometheusMiddleware :: forall proxy api. (RoutesToPaths api) => proxy api -> Wai.Middleware
+servantPrometheusMiddleware _ = Promth.prometheus conf . Promth.instrumentHandlerValue promthNormalize
+  where
+    conf = Promth.def
+      { Promth.prometheusEndPoint = ["i", "metrics"]
+      , Promth.prometheusInstrumentApp = False
+      }
+
+    promthNormalize :: Wai.Request -> Text
+    promthNormalize req = pathInfo
+      where
+        mPathInfo  = Metrics.treeLookup (routesToPaths @api) $ cs <$> Wai.pathInfo req
+        pathInfo   = cs $ fromMaybe "N/A" mPathInfo
+
 
 routesToPaths :: forall routes. RoutesToPaths routes => Paths
 routesToPaths = Paths (meltTree (getRoutes @routes))

libs/types-common/src/Data/Id.hs

@@ -57,13 +57,13 @@ type ScimTokenId  = Id STo
 
 -- Id -------------------------------------------------------------------------
 
-data NoId = NoId deriving (Eq, Show)
+data NoId = NoId deriving (Eq, Show, Generic)
 
 instance NFData NoId where rnf a = seq a ()
 
 newtype Id a = Id
     { toUUID :: UUID
-    } deriving (Eq, Ord, NFData, Hashable)
+    } deriving (Eq, Ord, NFData, Hashable, Generic)
 
 -- REFACTOR: non-derived, custom show instances break pretty-show and violate the law
 -- that @show . read == id@.  can we derive Show here?
@@ -149,6 +149,7 @@ newtype ConnId = ConnId
                , ToByteString
                , Hashable
                , NFData
+               , Generic
                )
 
 instance ToJSON ConnId where
@@ -164,7 +165,7 @@ instance FromJSON ConnId where
 -- lives as long as the device is registered.  See also: 'ConnId'.
 newtype ClientId = ClientId
     { client :: Text
-    } deriving (Eq, Ord, Show, ToByteString, Hashable, NFData, ToJSON, ToJSONKey)
+    } deriving (Eq, Ord, Show, ToByteString, Hashable, NFData, ToJSON, ToJSONKey, Generic)
 
 newClientId :: Word64 -> ClientId
 newClientId = ClientId . toStrict . toLazyText . hexadecimal
@@ -215,6 +216,7 @@ newtype BotId = BotId
              , NFData
              , FromJSON
              , ToJSON
+             , Generic
              )
 
 instance Show BotId where
@@ -243,6 +245,7 @@ newtype RequestId = RequestId
                , ToByteString
                , Hashable
                , NFData
+               , Generic
                )
 
 -- | Returns "N/A"