Merge pull request #858 from wireapp/release_2019_09_16

Release 2019 09 16

.gitignore

@@ -63,5 +63,14 @@ swagger-ui
 deploy/services-demo/resources/templates/*
 deploy/services-demo/conf/nginz/zwagger-ui/*
 
+deploy/docker-ephemeral/build/airdock_base-all/
+deploy/docker-ephemeral/build/airdock_base/
+deploy/docker-ephemeral/build/airdock_fakesqs-all/
+deploy/docker-ephemeral/build/airdock_fakesqs/
+deploy/docker-ephemeral/build/airdock_rvm-all/
+deploy/docker-ephemeral/build/airdock_rvm/
+deploy/docker-ephemeral/build/dynamodb_local/
+deploy/docker-ephemeral/build/smtp/
+
 # Ignore cabal files; use package.yaml instead
 *.cabal

CHANGELOG.md

@@ -1,4 +1,28 @@
-# 2019-09-03
+# 2019-09-16 #858
+
+## Relevant for self-hosters
+
+- Documentation changes for Twilio configurations and TURN setup. (#775)
+
+## Relevant for client developers
+
+- Better events for deletion of team conversations (also send `conversation.delete` to team members) (#849)
+- Add a new type of authorization tokens for legalhold (for details on legalhold, see https://github.com/wireapp/wire-server/blob/develop/docs/reference/team/legalhold.md) (#761)
+
+## Bug fixes
+
+- Fix swagger docs. (#852)
+- Fix intra call in stern (aka customer support, aka backoffice) (#844)
+
+## Internal Changes
+
+- Change feature flags from boolean to custom enum types. (#850)
+- Fix flaky integration test. (#848)
+- Cleanup: incoherent functions for response body parsing. (#847)
+- add route for consistency (#851)
+
+
+# 2019-09-03 #843
 
 ## Relevant for self-hosters
 

deploy/services-demo/conf/brig.demo.yaml

@@ -84,6 +84,8 @@ zauth:
     sessionTokenTimeout: 604800  # 7 days
     accessTokenTimeout: 900      # 15 minutes
     providerTokenTimeout: 604800 # 7 days
+    legalHoldUserTokenTimeout: 4838400    # 56 days
+    legalHoldSessionTokenTimeout: 604800  # 7 days
 
 turn:
   serversV2: resources/turn/servers-v2.txt

deploy/services-demo/conf/nginz/nginx-docker.conf

@@ -114,6 +114,11 @@ http {
         return 200;
     }
 
+    location /i/status {
+        zauth off;
+        return 200;
+    }
+
     location /vts {
         zauth off;
         vhost_traffic_status_display;

deploy/services-demo/conf/nginz/nginx.conf

@@ -113,6 +113,11 @@ http {
         return 200;
     }
 
+    location /i/status {
+        zauth off;
+        return 200;
+    }
+
     location /vts {
         zauth off;
         vhost_traffic_status_display;

deploy/services-demo/conf/nginz/zauth_acl.txt

@@ -4,10 +4,14 @@ a (blacklist (path "/provider")
              (path "/bot/**")
              (path "/i/**"))
 
-u (whitelist (path "/access"))
-
 b (whitelist (path "/bot")
              (path "/bot/**"))
 
 p (whitelist (path "/provider")
              (path "/provider/**"))
+
+# LegalHold Access Tokens
+la (whitelist (path "/notifications")
+              (path "/assets/v3/**")
+              (path "/users")
+              (path "/users/**"))

docs/reference/config-options.md

@@ -0,0 +1,50 @@
+# Config Options {#RefConfigOptions}
+
+Fragment.
+
+This page is about the yaml files that determine the configuration of
+the Wire backend services.
+
+
+## Feature flags
+
+Feature flags can be used to turn features on or off, or determine the
+behavior of the features.  Example:
+
+```
+# [galley.yaml]
+settings:
+  featureFlags:
+    sso: disabled-by-default
+    legalhold: disabled-by-default
+```
+
+The `featureFlags` field in the galley settings is mandatory, and all
+features must be listed.  Each feature defines its own set of allowed
+flag values.  (The reason for that is that as we will see, the
+semantics is slightly different (or more specific) than boolean.)
+
+### SSO
+
+This sets the default setting for all teams, and can be overridden by
+customer support / backoffice.  [Allowed
+values](https://github.com/wireapp/wire-server/blob/46713382a1a6544de3936eb03e987b9f76df3faa/libs/galley-types/src/Galley/Types/Teams.hs#L327-L329):
+`disabled-by-default`, `enabled-by-default`.
+
+IMPORTANT: if you change this from 'enabled-by-default' to
+'disabled-by-default' in production, you need to run [this migration
+script](https://github.com/wireapp/wire-server/tree/master/tools/db/migrate-sso-feature-flag)
+to fix all teams that have registered an idp.  (if you don't, the idp
+will keep working, but the admin won't be able to register new idps.)
+
+### LegalHold
+
+Optionally block customer support / backoffice from enabling legal
+hold for individual teams.  [Allowed
+values](https://github.com/wireapp/wire-server/blob/46713382a1a6544de3936eb03e987b9f76df3faa/libs/galley-types/src/Galley/Types/Teams.hs#L332-L334):
+'disabled-permanently', 'disabled-by-default'.
+
+IMPORTANT: If you switch this back to `disabled-permanently` from
+`disabled-by-default`, LegalHold devices may still be active in teams
+that have created them while it was allowed.  This may change in the
+future.

libs/api-client/src/Network/Wire/Client/API/Auth.hs

@@ -15,6 +15,7 @@ import Bilge
 import Brig.Types.User.Auth as Auth hiding (Cookie, user)
 import Control.Monad.Catch (MonadMask)
 import Data.List.NonEmpty
+import Data.Text (pack)
 import Data.Time (getCurrentTime)
 import Network.HTTP.Client (generateCookie)
 import Network.HTTP.Types.Method
@@ -95,7 +96,7 @@ tokenResponse rq rs ck
   where
     mkAuth = do
         cok <- mkCookie $ parseSetCookie <$> getHeader "Set-Cookie" rs
-        tok <- fromBody rs
+        tok <- responseJsonThrow (ParseError . pack) rs
         return . Just $ Auth cok tok
 
     mkCookie Nothing    = maybe (unexpected rs "missing set-cookie") return ck

libs/api-client/src/Network/Wire/Client/API/Conversation.hs

@@ -13,16 +13,19 @@ module Network.Wire.Client.API.Conversation
 
 import Imports
 import Bilge
+import Control.Monad.Catch (MonadThrow)
 import Data.ByteString.Conversion
 import Data.Id
 import Data.List.NonEmpty hiding (cons, toList)
 import Data.List1
+import Data.Text (pack)
 import Galley.Types as M hiding (Event, EventType)
 import Network.HTTP.Types.Method
 import Network.HTTP.Types.Status hiding (statusCode)
 import Network.Wire.Client.HTTP
 import Network.Wire.Client.Session
 import Network.Wire.Client.API.Push (ConvEvent)
+import Network.Wire.Client.Monad (ClientException(ParseError))
 
 postOtrMessage :: MonadSession m => ConvId -> NewOtrMessage -> m ClientMismatch
 postOtrMessage cnv msg = sessionRequest req rsc readBody
@@ -40,11 +43,11 @@ postOtrMessage cnv msg = sessionRequest req rsc readBody
 -- If some users can not be added to the conversation, 'UnexpectedResponse'
 -- will be thrown. It's not possible that some users will be added and
 -- others will not.
-addMembers :: MonadSession m => ConvId -> List1 UserId -> m (Maybe (ConvEvent Members))
+addMembers :: (MonadSession m, MonadThrow m) => ConvId -> List1 UserId -> m (Maybe (ConvEvent Members))
 addMembers cnv mems = do
     rs <- sessionRequest req rsc consumeBody
     case statusCode rs of
-        200 -> Just <$> fromBody rs
+        200 -> Just <$> responseJsonThrow (ParseError . pack) rs
         204 -> return Nothing
         _   -> unexpected rs "addMembers: status code"
   where
@@ -57,11 +60,11 @@ addMembers cnv mems = do
 
 -- | Remove a user and (in case of success) return the event corresponding
 -- to the user removal.
-removeMember :: MonadSession m => ConvId -> UserId -> m (Maybe (ConvEvent Members))
+removeMember :: (MonadSession m, MonadThrow m) => ConvId -> UserId -> m (Maybe (ConvEvent Members))
 removeMember cnv mem = do
     rs <- sessionRequest req rsc consumeBody
     case statusCode rs of
-        200 -> Just <$> fromBody rs
+        200 -> Just <$> responseJsonThrow (ParseError . pack) rs
         204 -> return Nothing
         _   -> unexpected rs "removeMember: status code"
   where
@@ -81,11 +84,11 @@ memberUpdate cnv updt = sessionRequest req rsc (const $ return ())
         $ empty
     rsc = status200 :| []
 
-getConv :: MonadSession m => ConvId -> m (Maybe Conversation)
+getConv :: (MonadSession m, MonadThrow m) => ConvId -> m (Maybe Conversation)
 getConv cnv = do
     rs <- sessionRequest req rsc consumeBody
     case statusCode rs of
-        200 -> fromBody rs
+        200 -> responseJsonThrow (ParseError . pack) rs
         404 -> return Nothing
         _   -> unexpected rs "getConv: status code"
   where

libs/api-client/src/Network/Wire/Client/API/Push.hs

@@ -29,11 +29,13 @@ import Brig.Types
 import Control.Concurrent (myThreadId)
 import Control.Concurrent.Async
 import Control.Exception (bracket, finally, onException)
+import Control.Monad.Catch (MonadThrow)
 import Data.Aeson hiding (Error)
 import Data.Aeson.Types (Parser)
 import Data.Default.Class
 import Data.Id
 import Data.List.NonEmpty
+import Data.Text (pack)
 import Data.Time.Clock
 import Data.UUID (UUID, fromString)
 import Galley.Types hiding (Event, EventType)
@@ -103,14 +105,14 @@ awaitNotifications f = do
     readChunk  c = (\x -> if C.null x then Nothing else Just x) <$> connectionGetChunk c
     writeChunk c = maybe (return ()) (connectionPut c . L.toStrict)
 
-fetchNotifications :: MonadSession m
+fetchNotifications :: (MonadSession m, MonadThrow m)
                    => Maybe ByteString
                    -> m (Bool, [Notification])
 fetchNotifications snc = do
     rs <- sessionRequest req rsc consumeBody
     case statusCode rs of
-        200 -> (True,)  <$> fromBody rs
-        404 -> (False,) <$> fromBody rs
+        200 -> (True,)  <$> responseJsonThrow (ParseError . pack) rs
+        404 -> (False,) <$> responseJsonThrow (ParseError . pack) rs
         _   -> unexpected rs "fetch: status code"
   where
     req = method GET
@@ -120,11 +122,11 @@ fetchNotifications snc = do
         $ empty
     rsc = status200 :| [status404]
 
-lastNotification :: MonadSession m => m (Maybe Notification)
+lastNotification :: (MonadSession m, MonadThrow m) => m (Maybe Notification)
 lastNotification = do
     rs <- sessionRequest req rsc consumeBody
     case statusCode rs of
-        200 -> Just <$> fromBody rs
+        200 -> Just <$> responseJsonThrow (ParseError . pack) rs
         404 -> return Nothing
         _   -> unexpected rs "last: status code"
   where

libs/api-client/src/Network/Wire/Client/API/User.hs

@@ -18,6 +18,7 @@ import Control.Monad.Catch (MonadMask)
 import Data.ByteString.Conversion
 import Data.Id
 import Data.List.NonEmpty
+import Data.Text (pack)
 import Network.HTTP.Types.Method
 import Network.HTTP.Types.Status hiding (statusCode)
 import Network.Wire.Client.HTTP
@@ -95,7 +96,7 @@ getConnection :: (MonadSession m, MonadUnliftIO m, MonadMask m) => UserId -> m (
 getConnection u = do
     rs <- sessionRequest req rsc consumeBody
     case statusCode rs of
-        200 -> fromBody rs
+        200 -> responseJsonThrow (ParseError . pack) rs
         404 -> return Nothing
         _   -> unexpected rs "getConnection: status code"
   where

libs/api-client/src/Network/Wire/Client/HTTP.hs

@@ -4,7 +4,6 @@
 module Network.Wire.Client.HTTP
     ( clientRequest
     , readBody
-    , fromBody
     , unexpected
     , mkErrorResponse
     ) where
@@ -77,16 +76,8 @@ clientRequest rq expected f = do
 -------------------------------------------------------------------------------
 -- Utilities
 
-readBody :: FromJSON a => Response BodyReader -> IO a
-readBody = consumeBody >=> fromBody
-
-fromBody :: (MonadIO m, FromJSON a) => Response (Maybe Lazy.ByteString) -> m a
-fromBody = either (liftIO . throwIO . ParseError . ("fromBody: "<>)) return . parse
-  where
-    parse = maybe (Left "missing response body")
-                  (fmapL pack . eitherDecode)
-                  .
-                  responseBody
+readBody :: (Typeable a, FromJSON a) => Response BodyReader -> IO a
+readBody = consumeBody >=> responseJsonThrow (ParseError . pack)
 
 unexpected :: MonadIO m => Response a -> Text -> m b
 unexpected r = liftIO . throwIO . UnexpectedResponse (responseStatus r) (responseHeaders r)