Merge pull request #843 from wireapp/release_2019_09_03

CHANGELOG.md

@@ -1,16 +1,42 @@
+# 2019-09-03
+
+## Relevant for self-hosters
+
+- Option for limiting login retries (#830)
+- Option for suspending inactive users (#831)
+- Add json logging (#828) (#836)
+- Feature Flags in galley options. (#825)
+
+## Relevant for client developers
+
+- Specialize the error cases on conversation lookup. (#841)
+
+## Bug fixes
+
+- Fix is-team-owner logic (don't require email in all cases) (#833)
+- Typos in swagger (#826)
+
+## Internal changes
+
+- Fix flaky integration test. (#834)
+- Remove `exposed-modules` sections from all package.yaml files. (#832)
+- Remove Debug.Trace from Imports. (#838)
+- Cleanup integration tests (#839)
+
+
 # 2019-08-08 #822
 
 ## Features
 
 - legalhold (#802), but block feature activation (#823)
 - a few shell scripts for self-hosters (#805, #801)
-- release nginz_disco (#759)
+- Release nginz_disco (#759)
 
-## Client-facing internal changes
+## Public API changes
 
+- SSO is disabled by default now; but enabled for all teams that already have an IdP.
 - feature flags (starting with legalhold, sso) (#813, #818)
-  - SSO is disabled by default now; but enabled for all teams that already have an IdP.
-  - new public end-points:
+  - new public end-points (#813, #818):
     - get "/teams/:tid/features/legalhold"
     - get "/teams/:tid/features/sso"
   - new internal end-points:

libs/api-bot/package.yaml

@@ -1,4 +1,4 @@
-defaults: 
+defaults:
   local: ../../package-defaults.yaml
 name: api-bot
 version: '0.4.2'
@@ -58,15 +58,4 @@ dependencies:
 - vector >=0.10
 library:
   source-dirs: src
-  exposed-modules:
-  - Network.Wire.Bot
-  - Network.Wire.Bot.Assert
-  - Network.Wire.Bot.Clients
-  - Network.Wire.Bot.Crypto
-  - Network.Wire.Bot.Email
-  - Network.Wire.Bot.Metrics
-  - Network.Wire.Bot.Monad
-  - Network.Wire.Bot.Report
-  - Network.Wire.Bot.Report.Text
-  - Network.Wire.Bot.Settings
 stability: experimental

libs/api-client/package.yaml

@@ -1,4 +1,4 @@
-defaults: 
+defaults:
   local: ../../package-defaults.yaml
 name: api-client
 version: '0.4.2'
@@ -47,16 +47,4 @@ dependencies:
 - websockets >=0.9
 library:
   source-dirs: src
-  exposed-modules:
-  - Network.Wire.Client
-  - Network.Wire.Client.API.Auth
-  - Network.Wire.Client.API.Asset
-  - Network.Wire.Client.API.Conversation
-  - Network.Wire.Client.API.Push
-  - Network.Wire.Client.API.Search
-  - Network.Wire.Client.API.User
-  - Network.Wire.Client.API.Client
-  - Network.Wire.Client.HTTP
-  - Network.Wire.Client.Monad
-  - Network.Wire.Client.Session
 stability: experimental

libs/bilge/package.yaml

@@ -40,12 +40,4 @@ dependencies:
 - wai-extra
 library:
   source-dirs: src
-  exposed-modules:
-  - Bilge
-  - Bilge.Assert
-  - Bilge.IO
-  - Bilge.Request
-  - Bilge.Response
-  - Bilge.Retry
-  - Bilge.RPC
 stability: experimental

libs/bilge/src/Bilge/IO.hs

@@ -48,7 +48,7 @@ module Bilge.IO
     , HttpException (..)
     ) where
 
-import Imports hiding (head, trace)
+import Imports hiding (head)
 import Control.Monad.Base
 import Control.Monad.Catch
 import Control.Monad.Trans.Control

libs/brig-types/test/unit/Test/Brig/Types/Common.hs

@@ -16,7 +16,6 @@ import Brig.Types.Test.Arbitrary ()
 import Control.Lens
 import Data.Aeson
 import Data.Aeson.Types
-import Data.Proxy
 import Data.Typeable (typeOf)
 import Galley.Types.Teams
 import Galley.Types.Teams.SSO
@@ -29,40 +28,41 @@ import Test.Tasty.QuickCheck
 
 tests :: TestTree
 tests = testGroup "Common (types vs. aeson)"
-    [ run @Handle Proxy
-    , run @Name Proxy
-    , run @ColourId Proxy
-    , run @Email Proxy
-    , run @Phone Proxy
-    , run @UserIdentity Proxy
-    , run @UserSSOId Proxy
-    , run @AssetSize Proxy
-    , run @Asset Proxy
-    , run @ExcludedPrefix Proxy
-    , run @ManagedBy Proxy
-    , run @TeamMemberDeleteData Proxy
-    , run @LegalHoldStatus Proxy
-    , run @LegalHoldTeamConfig Proxy
-    , run @NewLegalHoldService Proxy
-    , run @LegalHoldService Proxy
-    , run @ViewLegalHoldService Proxy
-    , run @NewLegalHoldClient Proxy
-    , run @RequestNewLegalHoldClient Proxy
-    , run @UserLegalHoldStatusResponse Proxy
-    , run @LegalHoldServiceConfirm Proxy
-    , run @LegalHoldClientRequest Proxy
-    , run @RemoveLegalHoldSettingsRequest Proxy
-    , run @DisableLegalHoldForUserRequest Proxy
-    , run @ApproveLegalHoldForUserRequest Proxy
-    , run @SSOStatus Proxy
-    , run @SSOTeamConfig Proxy
+    [ run @Handle
+    , run @Name
+    , run @ColourId
+    , run @Email
+    , run @Phone
+    , run @UserIdentity
+    , run @UserSSOId
+    , run @AssetSize
+    , run @Asset
+    , run @ExcludedPrefix
+    , run @ManagedBy
+    , run @TeamMemberDeleteData
+    , run @LegalHoldStatus
+    , run @LegalHoldTeamConfig
+    , run @NewLegalHoldService
+    , run @LegalHoldService
+    , run @ViewLegalHoldService
+    , run @NewLegalHoldClient
+    , run @RequestNewLegalHoldClient
+    , run @UserLegalHoldStatusResponse
+    , run @LegalHoldServiceConfirm
+    , run @LegalHoldClientRequest
+    , run @RemoveLegalHoldSettingsRequest
+    , run @DisableLegalHoldForUserRequest
+    , run @ApproveLegalHoldForUserRequest
+    , run @SSOStatus
+    , run @SSOTeamConfig
+    , run @FeatureFlags
     , testCase "{} is a valid TeamMemberDeleteData" $ do
         assertEqual "{}" (Right $ newTeamMemberDeleteData Nothing) (eitherDecode "{}")
     ]
   where
     run :: forall a. (Arbitrary a, Typeable a, ToJSON a, FromJSON a, Eq a, Show a)
-         => Proxy a -> TestTree
-    run Proxy = testProperty msg trip
+         => TestTree
+    run = testProperty msg trip
       where
         msg = show $ typeOf (undefined :: a)
         trip (v :: a) = counterexample (show $ toJSON v)
@@ -83,3 +83,10 @@ instance Arbitrary SSOStatus where
 
 instance Arbitrary SSOTeamConfig where
   arbitrary = SSOTeamConfig <$> arbitrary
+
+instance Arbitrary FeatureFlags where
+  arbitrary = FeatureFlags <$> arbitrary
+  shrink (FeatureFlags ls) = FeatureFlags <$> shrink ls
+
+instance Arbitrary FeatureFlag where
+  arbitrary = Test.Tasty.QuickCheck.elements [minBound..]

libs/cargohold-types/package.yaml

@@ -1,4 +1,4 @@
-defaults: 
+defaults:
   local: ../../package-defaults.yaml
 name: cargohold-types
 version: '1.5.0'
@@ -31,7 +31,3 @@ dependencies:
 - uuid >=1.2
 library:
   source-dirs: src
-  exposed-modules:
-  - CargoHold.Types
-  - CargoHold.Types.V3
-  - CargoHold.Types.V3.Resumable

libs/cassandra-util/package.yaml

@@ -35,10 +35,3 @@ dependencies:
 - retry
 library:
   source-dirs: src
-  exposed-modules:
-  - Cassandra
-  - Cassandra.CQL
-  - Cassandra.Exec
-  - Cassandra.Schema
-  - Cassandra.Settings
-  - Cassandra.Util

libs/extended/package.yaml

@@ -17,6 +17,7 @@ dependencies:
 - base
 - bytestring
 - extra
+- aeson
 - imports
 - optparse-applicative
 - tinylog
@@ -34,6 +35,7 @@ dependencies:
 - servant-swagger
 - string-conversions
 - transformers
+- text
 - wai
 library:
   source-dirs: src

libs/extended/src/System/Logger/Extended.hs

@@ -1,29 +1,90 @@
+{-# LANGUAGE DeriveAnyClass #-}
 {-# LANGUAGE GeneralizedNewtypeDeriving #-}
-{-# LANGUAGE DerivingStrategies         #-}
+{-# OPTIONS_GHC -Wno-orphans #-}
 -- | Tinylog convenience things.
 module System.Logger.Extended
     ( module Log
+    , LogFormat(..)
     , mkLogger
     , mkLogger'
     , LoggerT(..)
     , runWithLogger
+    , netStringsToLogFormat
     ) where
 
-import Imports
 import Control.Monad.Catch
+import Data.Aeson
+import Data.Aeson.Encoding (pair, list, text)
 import Database.CQL.IO
+import GHC.Generics
+import Imports
 import System.Logger as Log
-
-import qualified Data.ByteString.Lazy.Char8 as L
+import Data.String.Conversions (cs)
 import qualified Data.ByteString.Lazy.Builder as B
+import qualified Data.ByteString.Lazy.Char8 as L
 import qualified System.Logger.Class as LC
 
-mkLogger :: Log.Level -> Bool -> IO Log.Logger
-mkLogger lvl netstr = Log.new
+
+deriving instance Generic LC.Level
+instance FromJSON LC.Level
+instance ToJSON LC.Level
+
+-- | The log formats supported
+data LogFormat = JSON | Plain | Netstring 
+  deriving stock (Eq, Show, Generic)
+  deriving anyclass (ToJSON, FromJSON)
+
+-- | We use this as an intermediate structure to ease the implementation of the
+-- ToJSON instance but we could just inline everything. I think this has
+-- negligible impact and makes the code a bit more readable. Let me know
+data Element' = Element' Series [Builder]
+
+elementToEncoding :: Element' -> Encoding
+elementToEncoding (Element' fields msgs) = pairs $ fields <> msgsToSeries msgs
+    where
+      msgsToSeries :: [Builder] -> Series
+      msgsToSeries  = pair "msgs" . list (text . cs . eval)
+
+collect :: [Element] -> Element'
+collect = foldr go (Element' mempty [])
+  where
+    go :: Element -> Element' -> Element'
+    go (Bytes b) (Element' f m) =
+      Element' f (b : m)
+    go (Field k v) (Element' f m) = 
+      Element' (f <> pair (cs . eval $ k) (text . cs . eval $ v)) m
+
+jsonRenderer :: Renderer
+jsonRenderer _sep _dateFormat _logLevel = fromEncoding  . elementToEncoding . collect
+
+-- | Here for backwards-compatibility reasons
+netStringsToLogFormat :: Bool -> LogFormat
+netStringsToLogFormat True = Netstring
+netStringsToLogFormat False = Plain
+
+-- | Creates a logger given a log format Also takes an useNetstrings argument
+-- which is there because we cannot immediatelly deprecate the old interface.
+-- Old configs only provide the useNetstrings argument and not the logFormat
+-- argument, and in that case implement the old behaviour of either enabling
+-- plain text logging or netstring logging.  If both arguments are set,
+-- logFormat takes presedence over useNetstrings
+--
+-- FUTUREWORK: Once we get rid of the useNetstrings in our config files, we can
+-- remove this function and rename 'mkLoggerNew' to 'mkLogger'
+mkLogger :: Log.Level -> Maybe (Last Bool) -> Maybe (Last LogFormat) -> IO  Log.Logger
+mkLogger lvl useNetstrings logFormat = do
+  mkLoggerNew lvl $
+    case (fmap netStringsToLogFormat <$> useNetstrings) <> logFormat of
+      Just x -> getLast x
+      Nothing -> Plain
+
+-- | Version of mkLogger that doesn't support the deprecated useNetstrings option
+mkLoggerNew :: Log.Level -> LogFormat -> IO Log.Logger
+mkLoggerNew lvl logFormat = Log.new
     . Log.setReadEnvironment False
     . Log.setOutput Log.StdOut
     . Log.setFormat Nothing
-    $ simpleSettings (Just lvl) (Just netstr)
+    $ simpleSettings lvl logFormat
 
 -- | Variant of Log.defSettings:
 --
@@ -33,15 +94,16 @@ mkLogger lvl netstr = Log.new
 --
 --   * use 'canonicalizeWhitespace'.
 --
-simpleSettings :: Maybe Level -> Maybe Bool -> Log.Settings
-simpleSettings lvl netstr
-  = maybe id setLogLevel lvl
-  . setRenderer (canonicalizeWhitespace rndr)
+simpleSettings :: Log.Level -> LogFormat -> Log.Settings
+simpleSettings lvl logFormat
+  = Log.setLogLevel lvl
+  . Log.setRenderer (canonicalizeWhitespace rndr)
   $ Log.defSettings
   where
-    rndr = case netstr of
-      Just True  -> \_ _ _ -> renderNetstr
-      _          -> \s _ _ -> renderDefault s
+    rndr = case logFormat of
+      Netstring  -> \_separator _dateFormat _level -> Log.renderNetstr
+      Plain -> \ separator _dateFormat _level -> Log.renderDefault separator
+      JSON -> jsonRenderer
 
 -- | Replace all whitespace characters in the output of a renderer by @' '@.
 -- Log output must be ASCII encoding.
@@ -50,7 +112,7 @@ simpleSettings lvl netstr
 -- places and situations in your code and your dependencies that inject newlines
 -- into your log messages, you can choose to call 'canonicalizeWhitespace' on
 -- your renderer.)
-canonicalizeWhitespace :: Renderer -> Renderer
+canonicalizeWhitespace :: Log.Renderer -> Log.Renderer
 canonicalizeWhitespace rndrRaw delim df lvl
   = B.lazyByteString . nl2sp . B.toLazyByteString . rndrRaw delim df lvl
   where