From 262b7c2c5dae467759f8e693b8caac27437471c6 Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Mon, 13 Dec 2021 11:57:38 +0100 Subject: [PATCH 01/74] Do not purge bucket in fake-s3 (#1981) * Do not purge bucket in fake-s3 With `purge: true` the reaper script will delete the bucket's content when it runs, causing integration tests to fail occasionally. This fixes a very frequent flake in CI integration tests (e.g. https://bridge-ie-concourse.zinfra.io/builds/21651258). --- changelog.d/5-internal/fix-cargohold-flake | 1 + charts/fake-aws-s3/values.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/5-internal/fix-cargohold-flake diff --git a/changelog.d/5-internal/fix-cargohold-flake b/changelog.d/5-internal/fix-cargohold-flake new file mode 100644 index 00000000000..3281ab93589 --- /dev/null +++ b/changelog.d/5-internal/fix-cargohold-flake @@ -0,0 +1 @@ +Set `purge: false` in fake-s3 chart diff --git a/charts/fake-aws-s3/values.yaml b/charts/fake-aws-s3/values.yaml index 3c7ffd4774d..a736eb82cb0 100644 --- a/charts/fake-aws-s3/values.yaml +++ b/charts/fake-aws-s3/values.yaml @@ -21,7 +21,7 @@ minio: memory: 200Mi buckets: - name: dummy-bucket - purge: true + purge: false policy: none - name: assets purge: false From fb912aeec18e2890e3d2c4a04f9d5c634cabfe85 Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Mon, 13 Dec 2021 18:03:19 +0100 Subject: [PATCH 02/74] Fix `make c test=1` (#1987) --- changelog.d/5-internal/cabal-make-c-test-all | 1 + hack/bin/cabal-run-tests.sh | 22 ++++++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) create mode 100644 changelog.d/5-internal/cabal-make-c-test-all diff --git a/changelog.d/5-internal/cabal-make-c-test-all b/changelog.d/5-internal/cabal-make-c-test-all new file mode 100644 index 00000000000..305c0c8d6ce --- /dev/null +++ b/changelog.d/5-internal/cabal-make-c-test-all @@ -0,0 +1 @@ +Fix test runner for global cabal make target diff --git a/hack/bin/cabal-run-tests.sh b/hack/bin/cabal-run-tests.sh index a9b17b4c462..46a9099ecad 100755 --- a/hack/bin/cabal-run-tests.sh +++ b/hack/bin/cabal-run-tests.sh @@ -4,15 +4,19 @@ set -euo pipefail DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" TOP_LEVEL="$(cd "$DIR/../.." && pwd)" -pkgName=${1:-Please specify package name} +package=${1:-all} -# This is required because some tests (e.g. golden tests) depend on the path -# where they are run from. -pkgDir=$(find "$TOP_LEVEL" -name "$pkgName.cabal" | grep -v dist-newstyle | head -1 | xargs -n 1 dirname) -cd "$pkgDir" - -test_suites=$(cabal-plan list-bins "$pkgName"':test:*' | awk '{print $2}') - -for test_suite in $test_suites; do +if [[ "$package" == all ]]; then + pattern='*.cabal' +else + pattern="$package.cabal" +fi +for cabal in $(find "$TOP_LEVEL" -name "$pattern" | grep -v dist-newstyle); do + # This is required because some tests (e.g. golden tests) must be run from + # the package root. + cd "$(dirname $cabal)" + package="$(basename ${cabal%.*})" + for test_suite in $(cabal-plan list-bins "$package:test:*" | awk '{print $2}'); do $test_suite "${@:2}" + done done From 4c04c3f0486e8a3d20edf3764e93fa6055cb66fd Mon Sep 17 00:00:00 2001 From: Julia Longtin Date: Tue, 14 Dec 2021 13:27:50 +0100 Subject: [PATCH 03/74] Tag/add integration tests for certification (#1869) * add first mapping * further discovered tests. * ormolu * changelog entry * fixup; add two tests * Add another test * fixup * comments * Add annotation for access token tests * Add subsystem tags in ZAuth.hs * Add subsystem tags * add testmapping to test from documentation * Update services/brig/test/integration/API/User/Auth.hs * Update services/brig/test/integration/API/User/Auth.hs * Update services/brig/test/integration/API/User/Auth.hs * Update services/brig/test/integration/API/User/Client.hs * Update services/brig/test/integration/API/User/Client.hs * Update services/brig/test/integration/API/User/Handles.hs * Update services/brig/test/integration/API/User/Auth.hs * Update services/brig/test/integration/API/User/Client.hs * Update services/brig/test/integration/API/User/Client.hs * Update services/spar/test-integration/Test/Spar/Scim/AuthSpec.hs * Update services/spar/test-integration/Test/Spar/Scim/UserSpec.hs * hi ci Co-authored-by: jschaul Co-authored-by: VictorWissink Co-authored-by: fisx Co-authored-by: Leif Battermann --- changelog.d/4-docs/pr-1869 | 1 + libs/zauth/test/ZAuth.hs | 4 ++ .../brig/test/integration/API/User/Account.hs | 65 ++++++++++++++++++- .../brig/test/integration/API/User/Auth.hs | 12 ++++ .../brig/test/integration/API/User/Client.hs | 8 +++ .../brig/test/integration/API/User/Handles.hs | 4 ++ services/brig/test/integration/Util.hs | 9 +++ .../Test/Spar/Scim/AuthSpec.hs | 3 +- .../Test/Spar/Scim/UserSpec.hs | 5 +- 9 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 changelog.d/4-docs/pr-1869 diff --git a/changelog.d/4-docs/pr-1869 b/changelog.d/4-docs/pr-1869 new file mode 100644 index 00000000000..5b3e780a9e3 --- /dev/null +++ b/changelog.d/4-docs/pr-1869 @@ -0,0 +1 @@ +Annotate a first batch of integration and unit tests to map them to externally-facing documentation diff --git a/libs/zauth/test/ZAuth.hs b/libs/zauth/test/ZAuth.hs index 43e27cac49f..67729073b45 100644 --- a/libs/zauth/test/ZAuth.hs +++ b/libs/zauth/test/ZAuth.hs @@ -93,6 +93,10 @@ testNotExpired p = do x <- liftIO $ runValidate p $ check t liftIO $ assertBool "testNotExpired: validation failed" (isRight x) +-- The testExpired test conforms to the following testing standards: +-- @SF.Channel @TSFI.RESTfulAPI @S2 @S3 +-- +-- Using an expired access token should fail testExpired :: V.Env -> Create () testExpired p = do u <- liftIO nextRandom diff --git a/services/brig/test/integration/API/User/Account.hs b/services/brig/test/integration/API/User/Account.hs index f723e318aa0..f09af13c4a9 100644 --- a/services/brig/test/integration/API/User/Account.hs +++ b/services/brig/test/integration/API/User/Account.hs @@ -90,9 +90,12 @@ tests _ at opts p b c ch g aws = testGroup "account" [ test' aws p "post /register - 201 (with preverified)" $ testCreateUserWithPreverified opts b aws, + test' aws p "post /register - 400 (with preverified)" $ testCreateUserWithInvalidVerificationCode b, test' aws p "post /register - 201" $ testCreateUser b g, test' aws p "post /register - 201 + no email" $ testCreateUserNoEmailNoPassword b, test' aws p "post /register - 201 anonymous" $ testCreateUserAnon b g, + test' aws p "post /register - 400 empty name" $ testCreateUserEmptyName b, + test' aws p "post /register - 400 name too long" $ testCreateUserLongName b, test' aws p "post /register - 201 anonymous expiry" $ testCreateUserAnonExpiry b, test' aws p "post /register - 201 pending" $ testCreateUserPending opts b, test' aws p "post /register - 201 existing activation" $ testCreateAccountPendingActivationKey opts b, @@ -148,6 +151,32 @@ tests _ at opts p b c ch g aws = ] ] +-- The testCreateUserWithInvalidVerificationCode test conforms to the following testing standards: +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 +-- +-- Registering with an invalid verification code and valid account details should fail. +testCreateUserWithInvalidVerificationCode :: Brig -> Http () +testCreateUserWithInvalidVerificationCode brig = do + -- Attempt to register (pre verified) user with phone + p <- randomPhone + code <- randomActivationCode -- incorrect but syntactically valid activation code + let Object regPhone = + object + [ "name" .= Name "Alice", + "phone" .= fromPhone p, + "phone_code" .= code + ] + postUserRegister' regPhone brig !!! const 404 === statusCode + -- Attempt to register (pre verified) user with email + e <- randomEmail + let Object regEmail = + object + [ "name" .= Name "Alice", + "email" .= fromEmail e, + "email_code" .= code + ] + postUserRegister' regEmail brig !!! const 404 === statusCode + testUpdateUserEmailByTeamOwner :: Brig -> Http () testUpdateUserEmailByTeamOwner brig = do (_, teamOwner, emailOwner : otherTeamMember : _) <- createPopulatedBindingTeamWithNamesAndHandles brig 2 @@ -263,6 +292,33 @@ testCreateUser brig galley = do b <- responseBody r b ^? key "conversations" . nth 0 . key "type" >>= maybeFromJSON +-- The testCreateUserEmptyName test conforms to the following testing standards: +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 +-- +-- An empty name is not allowed on registration +testCreateUserEmptyName :: Brig -> Http () +testCreateUserEmptyName brig = do + let p = + RequestBodyLBS . encode $ + object + ["name" .= ("" :: Text)] + post (brig . path "/register" . contentJson . body p) + !!! const 400 === statusCode + +-- The testCreateUserLongName test conforms to the following testing standards: +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 +-- +-- a name with > 128 characters is not allowed. +testCreateUserLongName :: Brig -> Http () +testCreateUserLongName brig = do + let nameTooLong = cs $ concat $ replicate 129 "a" + let p = + RequestBodyLBS . encode $ + object + ["name" .= (nameTooLong :: Text)] + post (brig . path "/register" . contentJson . body p) + !!! const 400 === statusCode + testCreateUserAnon :: Brig -> Galley -> Http () testCreateUserAnon brig galley = do let p = @@ -348,7 +404,10 @@ testCreateUserNoEmailNoPassword brig = do getPhoneLoginCode brig p initiateEmailUpdateLogin brig e (SmsLogin p code Nothing) uid !!! (const 202 === statusCode) --- | email address must not be taken on @/register@. +-- The testCreateUserConflict test conforms to the following testing standards: +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 +-- +-- email address must not be taken on @/register@. testCreateUserConflict :: Opt.Opts -> Brig -> Http () testCreateUserConflict (Opt.setRestrictUserCreation . Opt.optSettings -> Just True) _ = pure () testCreateUserConflict _ brig = do @@ -378,6 +437,10 @@ testCreateUserConflict _ brig = do const 409 === statusCode const (Just "key-exists") === fmap Error.label . responseJsonMaybe +-- The testCreateUserInvalidEmailOrPhone test conforms to the following testing standards: +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 +-- +-- Test to make sure a new user cannot be created with an invalid email address or invalid phone number. testCreateUserInvalidEmailOrPhone :: Opt.Opts -> Brig -> Http () testCreateUserInvalidEmailOrPhone (Opt.setRestrictUserCreation . Opt.optSettings -> Just True) _ = pure () testCreateUserInvalidEmailOrPhone _ brig = do diff --git a/services/brig/test/integration/API/User/Auth.hs b/services/brig/test/integration/API/User/Auth.hs index 37e803e54bd..c13b3b77315 100644 --- a/services/brig/test/integration/API/User/Auth.hs +++ b/services/brig/test/integration/API/User/Auth.hs @@ -361,6 +361,10 @@ testSendLoginCode brig = do let _timeout = fromLoginCodeTimeout <$> responseJsonMaybe rsp2 liftIO $ assertEqual "timeout" (Just (Code.Timeout 600)) _timeout +-- The testLoginFailure test conforms to the following testing standards: +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 +-- +-- Test that trying to log in with a wrong password or non-existent email fails. testLoginFailure :: Brig -> Http () testLoginFailure brig = do Just email <- userEmail <$> randomUser brig @@ -395,6 +399,8 @@ testThrottleLogins conf b = do threadDelay (1000000 * (n + 1)) login b (defEmailLogin e) SessionCookie !!! const 200 === statusCode +-- The testLimitRetries test conforms to the following testing standards: +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 testLimitRetries :: HasCallStack => Opts.Opts -> Brig -> Http () testLimitRetries conf brig = do let Just opts = Opts.setLimitFailedLogins . Opts.optSettings $ conf @@ -566,6 +572,10 @@ testNoUserSsoLogin brig = do ------------------------------------------------------------------------------- -- Token Refresh +-- The testInvalidCookie test conforms to the following testing standards: +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 +-- +-- Test that invalid and expired tokens do not work. testInvalidCookie :: forall u. ZAuth.UserTokenLike u => ZAuth.Env -> Brig -> Http () testInvalidCookie z b = do -- Syntactically invalid @@ -898,6 +908,8 @@ testRemoveCookiesByLabelAndId b = do let lbl = cookieLabel c4 listCookies b (userId u) >>= liftIO . ([lbl] @=?) . map cookieLabel +-- The testTooManyCookies test conforms to the following testing standards: +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 testTooManyCookies :: Opts.Opts -> Brig -> Http () testTooManyCookies config b = do u <- randomUser b diff --git a/services/brig/test/integration/API/User/Client.hs b/services/brig/test/integration/API/User/Client.hs index f086ebbdafd..d538c3208b9 100644 --- a/services/brig/test/integration/API/User/Client.hs +++ b/services/brig/test/integration/API/User/Client.hs @@ -416,6 +416,8 @@ testMultiUserGetPrekeysQualified brig opts = do const 200 === statusCode const (Right $ expectedUserClientMap) === responseJsonEither +-- The testTooManyClients test conforms to the following testing standards: +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 testTooManyClients :: Opt.Opts -> Brig -> Http () testTooManyClients opts brig = do uid <- userId <$> randomUser brig @@ -436,6 +438,8 @@ testTooManyClients opts brig = do const (Just "too-many-clients") === fmap Error.label . responseJsonMaybe const (Just "application/json;charset=utf-8") === getHeader "Content-Type" +-- The testRemoveClient test conforms to the following testing standards: +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 testRemoveClient :: Bool -> Brig -> Cannon -> Http () testRemoveClient hasPwd brig cannon = do u <- randomUser' hasPwd brig @@ -475,6 +479,8 @@ testRemoveClient hasPwd brig cannon = do newClientCookie = Just defCookieLabel } +-- The testRemoveClientShortPwd test conforms to the following testing standards: +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 testRemoveClientShortPwd :: Brig -> Http () testRemoveClientShortPwd brig = do u <- randomUser brig @@ -661,6 +667,8 @@ testMissingClient brig = do const ["text/plain;charset=utf-8"] === map snd . filter ((== "Content-Type") . fst) . responseHeaders +-- The testAddMultipleTemporary test conforms to the following testing standards: +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 -- Legacy (galley) testAddMultipleTemporary :: Brig -> Galley -> Http () testAddMultipleTemporary brig galley = do diff --git a/services/brig/test/integration/API/User/Handles.hs b/services/brig/test/integration/API/User/Handles.hs index 356e70e5bd0..a1e4cba18d4 100644 --- a/services/brig/test/integration/API/User/Handles.hs +++ b/services/brig/test/integration/API/User/Handles.hs @@ -66,6 +66,10 @@ tests _cl _at conf p b c g = test p "GET /users/by-handle// : no federation" $ testGetUserByQualifiedHandleNoFederation conf b ] +-- The next line contains a mapping from the testHandleUpdate test to the following test standards: +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 +-- +-- Test changes to the user's handle. testHandleUpdate :: Brig -> Cannon -> Http () testHandleUpdate brig cannon = do user <- randomUser brig diff --git a/services/brig/test/integration/Util.hs b/services/brig/test/integration/Util.hs index 2b9879c7d91..79eb10e70de 100644 --- a/services/brig/test/integration/Util.hs +++ b/services/brig/test/integration/Util.hs @@ -60,6 +60,7 @@ import Data.Misc (PlainTextPassword (..)) import Data.Proxy import Data.Qualified import Data.Range +import qualified Data.Text as T import qualified Data.Text as Text import qualified Data.Text.Ascii as Ascii import Data.Text.Encoding (encodeUtf8) @@ -76,6 +77,7 @@ import qualified Network.Wai as Wai import qualified Network.Wai.Handler.Warp as Warp import Network.Wai.Test (Session) import qualified Network.Wai.Test as WaiTest +import OpenSSL.BN (randIntegerZeroToNMinusOne) import Servant.Client.Generic (AsClientT) import System.Random (randomIO, randomRIO) import qualified System.Timeout as System @@ -83,6 +85,7 @@ import Test.Tasty (TestName, TestTree) import Test.Tasty.Cannon import qualified Test.Tasty.Cannon as WS import Test.Tasty.HUnit +import Text.Printf (printf) import qualified UnliftIO.Async as Async import Util.AWS import Util.Options (Endpoint (Endpoint)) @@ -740,6 +743,12 @@ randomPhone = liftIO $ do let phone = parsePhone . Text.pack $ "+0" ++ concat nrs return $ fromMaybe (error "Invalid random phone#") phone +randomActivationCode :: (HasCallStack, MonadIO m) => m ActivationCode +randomActivationCode = + liftIO $ + ActivationCode . Ascii.unsafeFromText . T.pack . printf "%06d" + <$> randIntegerZeroToNMinusOne 1000000 + updatePhone :: HasCallStack => Brig -> UserId -> Phone -> Http () updatePhone brig uid phn = do -- update phone diff --git a/services/spar/test-integration/Test/Spar/Scim/AuthSpec.hs b/services/spar/test-integration/Test/Spar/Scim/AuthSpec.hs index 20c3bdc071c..6ac1b2bbf30 100644 --- a/services/spar/test-integration/Test/Spar/Scim/AuthSpec.hs +++ b/services/spar/test-integration/Test/Spar/Scim/AuthSpec.hs @@ -147,7 +147,8 @@ testNumIdPs = do createToken_ owner (CreateScimToken "drei" (Just defPassword)) (env ^. teSpar) !!! checkErr 400 (Just "more-than-one-idp") --- | Test that a token can only be created as a team owner +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 +-- Test that a token can only be created as a team owner testCreateTokenAuthorizesOnlyAdmins :: TestSpar () testCreateTokenAuthorizesOnlyAdmins = do env <- ask diff --git a/services/spar/test-integration/Test/Spar/Scim/UserSpec.hs b/services/spar/test-integration/Test/Spar/Scim/UserSpec.hs index a73b15ed147..e04cf943236 100644 --- a/services/spar/test-integration/Test/Spar/Scim/UserSpec.hs +++ b/services/spar/test-integration/Test/Spar/Scim/UserSpec.hs @@ -466,7 +466,10 @@ testExternalIdIsRequired = do createUser_ (Just tok) user' (env ^. teSpar) !!! const 400 === statusCode --- | Test that user creation fails if handle is invalid +-- The next line contains a mapping from this test to the following test standards: +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 +-- +-- Test that user creation fails if handle is invalid testCreateRejectsInvalidHandle :: TestSpar () testCreateRejectsInvalidHandle = do env <- ask From 5ace5956b1482a428b3e7833191f15612156176d Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Tue, 14 Dec 2021 13:28:28 +0100 Subject: [PATCH 04/74] Tag/add integration tests (Missing Clients) (BSI Docs) (#1985) * added stand alone test for checking that sending messages with missing clients should fail * naming Co-authored-by: fisx * naming Co-authored-by: fisx * description and more tags * changelog * wip * test for point 1. Co-authored-by: fisx --- changelog.d/5-internal/sqservices-1118 | 1 + services/galley/test/integration/API.hs | 66 ++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 changelog.d/5-internal/sqservices-1118 diff --git a/changelog.d/5-internal/sqservices-1118 b/changelog.d/5-internal/sqservices-1118 new file mode 100644 index 00000000000..0d240283035 --- /dev/null +++ b/changelog.d/5-internal/sqservices-1118 @@ -0,0 +1 @@ +Tag integration tests for certification. diff --git a/services/galley/test/integration/API.hs b/services/galley/test/integration/API.hs index d394a30c025..801d020109d 100644 --- a/services/galley/test/integration/API.hs +++ b/services/galley/test/integration/API.hs @@ -220,7 +220,9 @@ tests s = test s "cannot join private conversation" postJoinConvFail, test s "remove user with only local convs" removeUserNoFederation, test s "remove user with local and remote convs" removeUser, - test s "iUpsertOne2OneConversation" testAllOne2OneConversationRequests + test s "iUpsertOne2OneConversation" testAllOne2OneConversationRequests, + test s "post message - reject if missing client" postMessageRejectIfMissingClients, + test s "post message - client that is not in group doesn't receive message" postMessageClientNotInGroupDoesNotReceiveMsg ] emptyFederatedBrig :: F.BrigApi (AsServerT Handler) @@ -366,6 +368,7 @@ postConvWithRemoteUsersOk = do -- | This test verifies whether a message actually gets sent all the way to -- cannon. +-- @SF.Separation @TSFI.RESTfulAPI @S2 postCryptoMessage1 :: TestM () postCryptoMessage1 = do localDomain <- viewFederationDomain @@ -450,6 +453,7 @@ postCryptoMessage1 = do assertNoMsg wsB2 (wsAssertOtr qconv qalice ac bc cipher) -- | This test verifies basic mismatch behaviour of the the JSON endpoint. +-- @SF.Separation @TSFI.RESTfulAPI @S2 postCryptoMessage2 :: TestM () postCryptoMessage2 = do b <- view tsBrig @@ -475,6 +479,7 @@ postCryptoMessage2 = do Map.keys <$> Map.lookup eve (userClientMap (getUserClientPrekeyMap p)) @=? Just [ec] -- | This test verifies basic mismatch behaviour of the protobuf endpoint. +-- @SF.Separation @TSFI.RESTfulAPI @S2 postCryptoMessage3 :: TestM () postCryptoMessage3 = do b <- view tsBrig @@ -501,7 +506,7 @@ postCryptoMessage3 = do Map.keys (userClientMap (getUserClientPrekeyMap p)) @=? [eve] Map.keys <$> Map.lookup eve (userClientMap (getUserClientPrekeyMap p)) @=? Just [ec] --- | This test verfies behaviour when an unknown client posts the message. Only +-- | This test verifies behaviour when an unknown client posts the message. Only -- tests the Protobuf endpoint. postCryptoMessage4 :: TestM () postCryptoMessage4 = do @@ -516,8 +521,63 @@ postCryptoMessage4 = do postProtoOtrMessage alice (ClientId "172618352518396") conv m !!! const 403 === statusCode +-- | This test verifies the following scenario. +-- A client sends a message to all clients of a group and one more who is not part of the group. +-- The server must not send this message to client ids not part of the group. +-- @SF.Separation @TSFI.RESTfulAPI @S2 +postMessageClientNotInGroupDoesNotReceiveMsg :: TestM () +postMessageClientNotInGroupDoesNotReceiveMsg = do + localDomain <- viewFederationDomain + cannon <- view tsCannon + (alice, ac) <- randomUserWithClient (someLastPrekeys !! 0) + (bob, bc) <- randomUserWithClient (someLastPrekeys !! 1) + (eve, ec) <- randomUserWithClient (someLastPrekeys !! 2) + (chad, cc) <- randomUserWithClient (someLastPrekeys !! 3) + connectUsers alice (list1 bob [eve, chad]) + conversationWithAllButChad <- decodeConvId <$> postConv alice [bob, eve] (Just "gossip") [] Nothing Nothing + let qalice = Qualified alice localDomain + qconv = Qualified conversationWithAllButChad localDomain + WS.bracketR3 cannon bob eve chad $ \(wsBob, wsEve, wsChad) -> do + let msgToAllIncludingChad = [(bob, bc, toBase64Text "ciphertext2"), (eve, ec, toBase64Text "ciphertext2"), (chad, cc, toBase64Text "ciphertext2")] + postOtrMessage id alice ac conversationWithAllButChad msgToAllIncludingChad !!! const 201 === statusCode + let checkBobGetsMsg = void . liftIO $ WS.assertMatch (5 # Second) wsBob (wsAssertOtr qconv qalice ac bc (toBase64Text "ciphertext2")) + let checkEveGetsMsg = void . liftIO $ WS.assertMatch (5 # Second) wsEve (wsAssertOtr qconv qalice ac ec (toBase64Text "ciphertext2")) + let checkChadDoesNotGetMsg = assertNoMsg wsChad (wsAssertOtr qconv qalice ac ac (toBase64Text "ciphertext2")) + checkBobGetsMsg + checkEveGetsMsg + checkChadDoesNotGetMsg + +-- | This test verifies that when a client sends a message not to all clients of a group then the server should reject the message and sent a notification to the sender (412 Missing clients). +-- The test is somewhat redundant because this is already tested as part of other tests already. This is a stand alone test that solely tests the behavior described above. +-- @SF.Separation @TSFI.RESTfulAPI @S2 +postMessageRejectIfMissingClients :: TestM () +postMessageRejectIfMissingClients = do + (sender, senderClient) : allReceivers <- randomUserWithClient `traverse` someLastPrekeys + let (receiver1, receiverClient1) : otherReceivers = allReceivers + connectUsers sender (list1 receiver1 (fst <$> otherReceivers)) + conv <- decodeConvId <$> postConv sender (receiver1 : (fst <$> otherReceivers)) (Just "gossip") [] Nothing Nothing + let msgToAllClients = mkMsg "hello!" <$> allReceivers + let msgMissingClients = mkMsg "hello!" <$> drop 1 allReceivers + + let checkSendToAllClientShouldBeSuccessful = + postOtrMessage id sender senderClient conv msgToAllClients !!! do + const 201 === statusCode + assertMismatch [] [] [] + + let checkSendWitMissingClientsShouldFail = + postOtrMessage id sender senderClient conv msgMissingClients !!! do + const 412 === statusCode + assertMismatch [(receiver1, Set.singleton receiverClient1)] [] [] + + checkSendToAllClientShouldBeSuccessful + checkSendWitMissingClientsShouldFail + where + mkMsg :: ByteString -> (UserId, ClientId) -> (UserId, ClientId, Text) + mkMsg text (userId, clientId) = (userId, clientId, toBase64Text text) + -- | This test verifies behaviour under various values of ignore_missing and -- report_missing. Only tests the JSON endpoint. +-- @SF.Separation @TSFI.RESTfulAPI @S2 postCryptoMessage5 :: TestM () postCryptoMessage5 = do (alice, ac) <- randomUserWithClient (someLastPrekeys !! 0) @@ -709,6 +769,7 @@ postMessageQualifiedLocalOwningBackendSuccess = do -- | Sets up a conversation on Backend A known as "owning backend". One of the -- users from Backend A will send the message but have a missing client. It is -- expected that the message will not be sent. +-- @SF.Separation @TSFI.RESTfulAPI @S2 postMessageQualifiedLocalOwningBackendMissingClients :: TestM () postMessageQualifiedLocalOwningBackendMissingClients = do -- Cannon for local users @@ -868,6 +929,7 @@ postMessageQualifiedLocalOwningBackendRedundantAndDeletedClients = do -- users from Backend A will send the message but have a missing client. It is -- expected that the message will be sent except when it is specifically -- requested to report on missing clients of a user. +-- @SF.Separation @TSFI.RESTfulAPI @S2 postMessageQualifiedLocalOwningBackendIgnoreMissingClients :: TestM () postMessageQualifiedLocalOwningBackendIgnoreMissingClients = do -- WS receive timeout From 9d1ae79ad1904a2f5f82af652a3a346d4139c315 Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Tue, 14 Dec 2021 15:08:48 +0100 Subject: [PATCH 05/74] Streaming federator (#1966) * Add body argument to Service effect Also remove use of polysemy-mock from ExternalServer tests. * Remove dependency on polysemy-mocks * Federator: initial impl of service streaming * Remove unused function interpretService * Forward Content-Type header of service calls Co-authored-by: Sven Tennie * Use Codensity IO as a base monad for Service This implements the bracketing pattern needed for closing a streaming response at the level of the base monad, instead of introducing CPS for the main polysemy actions. * Formatting fixes * Add basic response streaming primitive for HTTP2 Also use the same return type for the response as the one in Servant.Client, namely `ResponseF`. * RunStreamingClient instance for FederatorClient * Create test for withHTTP2Request * Fix CPS to Codensity in federator client * Test federator client streaming * Enable streaming in outward service Co-authored-by: Sven Tennie --- cabal.project | 2 +- cabal.project.freeze | 1 - changelog.d/6-federation/federator-streaming | 1 + libs/wire-api-federation/package.yaml | 1 + .../src/Wire/API/Federation/Client.hs | 204 ++++++++++------ .../wire-api-federation.cabal | 4 +- services/federator/federator.cabal | 15 +- services/federator/package.yaml | 4 +- services/federator/src/Federator/App.hs | 2 - services/federator/src/Federator/Env.hs | 4 +- .../federator/src/Federator/ExternalServer.hs | 20 +- .../federator/src/Federator/InternalServer.hs | 5 +- services/federator/src/Federator/Remote.hs | 28 ++- services/federator/src/Federator/Response.hs | 32 ++- services/federator/src/Federator/Run.hs | 10 +- services/federator/src/Federator/Service.hs | 87 +++++-- .../integration/Test/Federator/IngressSpec.hs | 16 +- .../test/unit/Test/Federator/Client.hs | 77 +++++- .../unit/Test/Federator/ExternalServer.hs | 230 +++++++++--------- .../unit/Test/Federator/InternalServer.hs | 21 +- .../test/unit/Test/Federator/Remote.hs | 3 + stack.yaml | 6 +- stack.yaml.lock | 13 +- 23 files changed, 509 insertions(+), 277 deletions(-) create mode 100644 changelog.d/6-federation/federator-streaming diff --git a/cabal.project b/cabal.project index 35f93fb1000..94b8595b0a0 100644 --- a/cabal.project +++ b/cabal.project @@ -135,7 +135,7 @@ source-repository-package source-repository-package type: git location: https://github.com/wireapp/http2 - tag: 1ee1ce432d923839dab6782410e91dc17df2a880 + tag: aa3501ad58e1abbd196781fac25a84f41ec2a787 source-repository-package type: git diff --git a/cabal.project.freeze b/cabal.project.freeze index 22013305b8f..6e81f6e4022 100644 --- a/cabal.project.freeze +++ b/cabal.project.freeze @@ -1691,7 +1691,6 @@ constraints: any.AC-Angle ==1.0, any.polyparse ==1.13, any.polysemy ==1.7.0.0, any.polysemy-check ==0.8.1.0, - any.polysemy-mocks ==0.2.0.0, any.polysemy-plugin ==0.4.2.0, any.pooled-io ==0.0.2.2, any.port-utils ==0.2.1.0, diff --git a/changelog.d/6-federation/federator-streaming b/changelog.d/6-federation/federator-streaming new file mode 100644 index 00000000000..7901572b90b --- /dev/null +++ b/changelog.d/6-federation/federator-streaming @@ -0,0 +1 @@ +Make federator capable of streaming responses diff --git a/libs/wire-api-federation/package.yaml b/libs/wire-api-federation/package.yaml index 6b4ff64da17..8fb6f5d3003 100644 --- a/libs/wire-api-federation/package.yaml +++ b/libs/wire-api-federation/package.yaml @@ -25,6 +25,7 @@ dependencies: - http-types - http2 - imports +- kan-extensions - lifted-base - metrics-wai - mtl diff --git a/libs/wire-api-federation/src/Wire/API/Federation/Client.hs b/libs/wire-api-federation/src/Wire/API/Federation/Client.hs index 4215def0283..4dae263caf3 100644 --- a/libs/wire-api-federation/src/Wire/API/Federation/Client.hs +++ b/libs/wire-api-federation/src/Wire/API/Federation/Client.hs @@ -22,13 +22,17 @@ module Wire.API.Federation.Client ( FederatorClientEnv (..), FederatorClient, runFederatorClient, + runFederatorClientToCodensity, performHTTP2Request, + withHTTP2Request, + streamingResponseStrictBody, headersFromTable, ) where import qualified Control.Exception as E import Control.Monad.Catch +import Control.Monad.Codensity import Control.Monad.Except import qualified Data.Aeson as Aeson import qualified Data.ByteString as BS @@ -52,6 +56,7 @@ import Network.TLS as TLS import qualified Network.Wai.Utilities.Error as Wai import Servant.Client import Servant.Client.Core +import Servant.Types.SourceT import qualified System.TimeManager import Util.Options (Endpoint (..)) import Wire.API.Federation.Component @@ -65,7 +70,7 @@ data FederatorClientEnv = FederatorClientEnv } newtype FederatorClient (c :: Component) a = FederatorClient - {unFederatorClient :: ReaderT FederatorClientEnv (ExceptT FederatorClientError IO) a} + {unFederatorClient :: ReaderT FederatorClientEnv (ExceptT FederatorClientError (Codensity IO)) a} deriving newtype ( Functor, Applicative, @@ -75,6 +80,9 @@ newtype FederatorClient (c :: Component) a = FederatorClient MonadIO ) +liftCodensity :: Codensity IO a -> FederatorClient c a +liftCodensity = FederatorClient . lift . lift + headersFromTable :: HTTP2.HeaderTable -> [HTTP.Header] headersFromTable (headerList, _) = flip map headerList $ \(token, headerValue) -> (HTTP2.tokenKey token, headerValue) @@ -90,93 +98,136 @@ performHTTP2Request :: HTTP2.Request -> ByteString -> Int -> - IO (Either FederatorClientHTTP2Error (HTTP.Status, [HTTP.Header], Builder)) -performHTTP2Request mtlsConfig req hostname port = do - let drainResponse resp = go mempty - where - go acc = do - chunk <- HTTP2.getResponseBodyChunk resp - if BS.null chunk - then pure acc - else go (acc <> byteString chunk) + IO (Either FederatorClientHTTP2Error (ResponseF Builder)) +performHTTP2Request mtlsConfig req hostname port = try $ do + withHTTP2Request mtlsConfig req hostname port $ \resp -> do + b <- + fmap (either (const mempty) id) + . runExceptT + . runSourceT + . responseBody + $ resp + pure $ resp $> foldMap byteString b + +withHTTP2Request :: + Maybe TLS.ClientParams -> + HTTP2.Request -> + ByteString -> + Int -> + (StreamingResponse -> IO a) -> + IO a +withHTTP2Request mtlsConfig req hostname port k = do let clientConfig = HTTP2.ClientConfig "https" hostname {- cacheLimit: -} 20 - flip - E.catches - [ -- catch FederatorClientHTTP2Error (e.g. connection and TLS errors) - E.Handler (pure . Left), - -- catch HTTP2 exceptions - E.Handler (pure . Left . FederatorClientHTTP2Exception) - ] - $ bracket (connectSocket hostname port) NS.close $ \sock -> do - let withHTTP2Config k = case mtlsConfig of - Nothing -> bracket (HTTP2.allocSimpleConfig sock 4096) HTTP2.freeSimpleConfig k + E.handle (E.throw . FederatorClientHTTP2Exception) $ + bracket (connectSocket hostname port) NS.close $ \sock -> do + let withHTTP2Config k' = case mtlsConfig of + Nothing -> bracket (HTTP2.allocSimpleConfig sock 4096) HTTP2.freeSimpleConfig k' -- FUTUREWORK(federation): Use openssl Just tlsConfig -> do ctx <- E.handle (E.throw . FederatorClientTLSException) $ do ctx <- TLS.contextNew sock tlsConfig TLS.handshake ctx pure ctx - bracket (allocTLSConfig ctx 4096) freeTLSConfig k - withHTTP2Config $ \conf -> - HTTP2.run clientConfig conf $ \sendRequest -> do + bracket (allocTLSConfig ctx 4096) freeTLSConfig k' + withHTTP2Config $ \conf -> do + HTTP2.run clientConfig conf $ \sendRequest -> sendRequest req $ \resp -> do - result <- drainResponse resp let headers = headersFromTable (HTTP2.responseHeaders resp) - pure $ case HTTP2.responseStatus resp of - Nothing -> Left FederatorClientNoStatusCode - Just status -> Right (status, headers, result) + result = fromAction BS.null (HTTP2.getResponseBodyChunk resp) + case HTTP2.responseStatus resp of + Nothing -> E.throw FederatorClientNoStatusCode + Just status -> + k + Response + { responseStatusCode = status, + responseHeaders = Seq.fromList headers, + responseHttpVersion = HTTP.http20, + responseBody = result + } instance KnownComponent c => RunClient (FederatorClient c) where runRequestAcceptStatus expectedStatuses req = do - env <- ask - let baseUrlPath = - HTTP.encodePathSegments - [ "rpc", - domainText (ceTargetDomain env), - componentName (componentVal @c) - ] - let path = baseUrlPath <> requestPath req - body <- case requestBody req of - Just (RequestBodyLBS lbs, _) -> pure lbs - Just (RequestBodyBS bs, _) -> pure (LBS.fromStrict bs) - Just (RequestBodySource _, _) -> - throwError FederatorClientStreamingNotSupported - Nothing -> pure mempty - let req' = - HTTP2.requestBuilder - (requestMethod req) - (LBS.toStrict (toLazyByteString path)) - (toList (requestHeaders req) <> [(originDomainHeaderName, toByteString' (ceOriginDomain env))]) - (lazyByteString body) - let Endpoint (Text.encodeUtf8 -> hostname) (fromIntegral -> port) = ceFederator env - eresp <- liftIO $ performHTTP2Request Nothing req' hostname port - case eresp of - Left err -> throwError (FederatorClientHTTP2Error err) - Right (status, headers, result) - | maybe (HTTP.statusIsSuccessful status) (elem status) expectedStatuses -> - pure $ - Response - { responseStatusCode = status, - responseHeaders = Seq.fromList headers, - responseHttpVersion = HTTP.http20, - responseBody = toLazyByteString result - } - | otherwise -> - throwError $ - FederatorClientError - ( mkFailureResponse - status - (ceTargetDomain env) - (toLazyByteString (requestPath req)) - (toLazyByteString result) - ) + let successfulStatus status = + maybe + (HTTP.statusIsSuccessful status) + (elem status) + expectedStatuses + withHTTP2StreamingRequest successfulStatus req $ \resp -> do + bdy <- + fmap (either (const mempty) (toLazyByteString . foldMap byteString)) + . runExceptT + . runSourceT + . responseBody + $ resp + pure $ resp $> bdy throwClientError = throwError . FederatorClientServantError +instance KnownComponent c => RunStreamingClient (FederatorClient c) where + withStreamingRequest = withHTTP2StreamingRequest HTTP.statusIsSuccessful + +streamingResponseStrictBody :: StreamingResponse -> IO Builder +streamingResponseStrictBody resp = + fmap (either stringUtf8 (foldMap byteString)) + . runExceptT + . runSourceT + . responseBody + $ resp + +withHTTP2StreamingRequest :: + forall c a. + KnownComponent c => + (HTTP.Status -> Bool) -> + Request -> + (StreamingResponse -> IO a) -> + FederatorClient c a +withHTTP2StreamingRequest successfulStatus req handleResponse = do + env <- ask + let baseUrlPath = + HTTP.encodePathSegments + [ "rpc", + domainText (ceTargetDomain env), + componentName (componentVal @c) + ] + let path = baseUrlPath <> requestPath req + body <- case requestBody req of + Just (RequestBodyLBS lbs, _) -> pure lbs + Just (RequestBodyBS bs, _) -> pure (LBS.fromStrict bs) + Just (RequestBodySource _, _) -> + throwError FederatorClientStreamingNotSupported + Nothing -> pure mempty + let req' = + HTTP2.requestBuilder + (requestMethod req) + (LBS.toStrict (toLazyByteString path)) + (toList (requestHeaders req) <> [(originDomainHeaderName, toByteString' (ceOriginDomain env))]) + (lazyByteString body) + let Endpoint (Text.encodeUtf8 -> hostname) (fromIntegral -> port) = ceFederator env + resp <- + (either throwError pure =<<) . liftCodensity $ + Codensity $ \k -> + E.catch + (withHTTP2Request Nothing req' hostname port (k . Right)) + (k . Left . FederatorClientHTTP2Error) + + if successfulStatus (responseStatusCode resp) + then liftIO $ handleResponse resp + else do + -- in case of an error status code, read the whole body to construct the error + bdy <- liftIO $ streamingResponseStrictBody resp + throwError $ + FederatorClientError + ( mkFailureResponse + (responseStatusCode resp) + (ceTargetDomain env) + (toLazyByteString (requestPath req)) + (toLazyByteString bdy) + ) + mkFailureResponse :: HTTP.Status -> Domain -> LByteString -> LByteString -> Wai.Error mkFailureResponse status domain path body -- If the outward federator fails with 403, that means that there was an @@ -211,12 +262,25 @@ mkFailureResponse status domain path body "unknown-federation-error" (LText.decodeUtf8With Text.lenientDecode body) +-- | Run federator client synchronously. runFederatorClient :: KnownComponent c => FederatorClientEnv -> FederatorClient c a -> IO (Either FederatorClientError a) -runFederatorClient env action = runExceptT (runReaderT (unFederatorClient action) env) +runFederatorClient env = + lowerCodensity + . runFederatorClientToCodensity env + +runFederatorClientToCodensity :: + KnownComponent c => + FederatorClientEnv -> + FederatorClient c a -> + Codensity IO (Either FederatorClientError a) +runFederatorClientToCodensity env = + runExceptT + . flip runReaderT env + . unFederatorClient freeTLSConfig :: HTTP2.Config -> IO () freeTLSConfig cfg = free (HTTP2.confWriteBuffer cfg) diff --git a/libs/wire-api-federation/wire-api-federation.cabal b/libs/wire-api-federation/wire-api-federation.cabal index a1592e461ea..5d665a791a6 100644 --- a/libs/wire-api-federation/wire-api-federation.cabal +++ b/libs/wire-api-federation/wire-api-federation.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: 621c254076cf520b525269ca4fc550df57f410aea52a288f6cb68bd2d6f1ada3 +-- hash: f62744549c34b54e9900b31c33a4ebad5c51ac13ec44010ad2da6c6ca67fdc29 name: wire-api-federation version: 0.1.0 @@ -51,6 +51,7 @@ library , http-types , http2 , imports + , kan-extensions , lifted-base , metrics-wai , mtl @@ -110,6 +111,7 @@ test-suite spec , http-types , http2 , imports + , kan-extensions , lifted-base , metrics-wai , mtl diff --git a/services/federator/federator.cabal b/services/federator/federator.cabal index fca3d7731cf..7bfdce95e1c 100644 --- a/services/federator/federator.cabal +++ b/services/federator/federator.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: 2131bf1a367dd734cbccd900c4724c5b48e3f494501b2221e3f32ecaf0a12ec4 +-- hash: 371580e0c3adbdb4994c74a23ada11c91023bc654f2f0fe304ead7e88a3b8ba6 name: federator version: 1.0.0 @@ -83,6 +83,7 @@ library , http-types , http2 , imports + , kan-extensions , lens , metrics-core , metrics-wai @@ -94,7 +95,7 @@ library , polysemy-wire-zoo , retry , servant - , servant-server + , servant-client-core , streaming-commons , string-conversions , text @@ -149,6 +150,7 @@ executable federator , http-types , http2 , imports + , kan-extensions , lens , metrics-core , metrics-wai @@ -160,7 +162,7 @@ executable federator , polysemy-wire-zoo , retry , servant - , servant-server + , servant-client-core , streaming-commons , string-conversions , text @@ -224,6 +226,7 @@ executable federator-integration , http-types , http2 , imports + , kan-extensions , lens , metrics-core , metrics-wai @@ -237,7 +240,7 @@ executable federator-integration , random , retry , servant - , servant-server + , servant-client-core , streaming-commons , string-conversions , tasty @@ -308,6 +311,7 @@ test-suite federator-tests , http2 , imports , interpolate + , kan-extensions , lens , metrics-core , metrics-wai @@ -316,12 +320,11 @@ test-suite federator-tests , network-uri , pem , polysemy - , polysemy-mocks , polysemy-wire-zoo , retry , servant , servant-client - , servant-server + , servant-client-core , streaming-commons , string-conversions , tasty diff --git a/services/federator/package.yaml b/services/federator/package.yaml index bb4c0b9655a..0c9abdd8509 100644 --- a/services/federator/package.yaml +++ b/services/federator/package.yaml @@ -31,6 +31,7 @@ dependencies: - http-client-openssl - http-types - http2 +- kan-extensions - imports - lens - metrics-core @@ -43,7 +44,7 @@ dependencies: - polysemy-wire-zoo - retry - servant -- servant-server +- servant-client-core - streaming-commons - string-conversions - text @@ -111,7 +112,6 @@ tests: - directory - federator - interpolate - - polysemy-mocks - QuickCheck - servant-client - streaming-commons diff --git a/services/federator/src/Federator/App.hs b/services/federator/src/Federator/App.hs index e1601ed8900..2a33cd98600 100644 --- a/services/federator/src/Federator/App.hs +++ b/services/federator/src/Federator/App.hs @@ -34,8 +34,6 @@ import Federator.Env (Env, applog, httpManager, requestId) import Imports import Polysemy import Polysemy.Input -import Servant.API.Generic () -import Servant.Server () import System.Logger.Class as LC import qualified System.Logger.Extended as Log diff --git a/services/federator/src/Federator/Env.hs b/services/federator/src/Federator/Env.hs index 4594b4fd852..ced29deb2bd 100644 --- a/services/federator/src/Federator/Env.hs +++ b/services/federator/src/Federator/Env.hs @@ -21,7 +21,6 @@ module Federator.Env where import Bilge (RequestId) -import qualified Bilge as RPC import Control.Lens (makeLenses) import Data.Metrics (Metrics) import Data.X509.CertificateStore @@ -31,6 +30,7 @@ import Network.DNS.Resolver (Resolver) import qualified Network.HTTP.Client as HTTP import qualified Network.TLS as TLS import qualified System.Logger.Class as LC +import Util.Options import Wire.API.Federation.Component data TLSSettings = TLSSettings @@ -44,7 +44,7 @@ data Env = Env _requestId :: RequestId, _dnsResolver :: Resolver, _runSettings :: RunSettings, - _service :: Component -> RPC.Request, + _service :: Component -> Endpoint, _httpManager :: HTTP.Manager, _tls :: IORef TLSSettings } diff --git a/services/federator/src/Federator/ExternalServer.hs b/services/federator/src/Federator/ExternalServer.hs index 59546eb2165..b81c7b18691 100644 --- a/services/federator/src/Federator/ExternalServer.hs +++ b/services/federator/src/Federator/ExternalServer.hs @@ -18,8 +18,9 @@ module Federator.ExternalServer (callInward, serveInward, parseRequestData, RequestData (..)) where import qualified Data.ByteString as BS -import Data.ByteString.Builder (toLazyByteString) +import Data.ByteString.Builder import qualified Data.ByteString.Lazy as LBS +import qualified Data.Sequence as Seq import qualified Data.Text as Text import Federator.Discovery import Federator.Env @@ -36,6 +37,7 @@ import Polysemy.Error import Polysemy.Input import Polysemy.TinyLog (TinyLog) import qualified Polysemy.TinyLog as Log +import Servant.Client.Core import qualified System.Logger.Message as Log import Wire.API.Federation.Component import Wire.API.Federation.Domain @@ -43,7 +45,7 @@ import Wire.API.Federation.Domain -- FUTUREWORK(federation): Versioning of the federation API. callInward :: Members - '[ Service, + '[ ServiceStreaming, Embed IO, TinyLog, DiscoverFederator, @@ -67,12 +69,18 @@ callInward wreq = do let path = LBS.toStrict (toLazyByteString (HTTP.encodePathSegments ["federation", rdRPC req])) - (status, body) <- serviceCall (rdComponent req) path (rdBody req) validatedDomain + resp <- serviceCall (rdComponent req) path (rdBody req) validatedDomain Log.debug $ Log.msg ("Inward Request response" :: ByteString) - . Log.field "status" (show status) - - pure $ Wai.responseLBS status defaultHeaders (fromMaybe mempty body) + . Log.field "status" (show (responseStatusCode resp)) + pure $ + streamingResponseToWai + resp + { responseHeaders = + Seq.filter + (\(name, _) -> name == "Content-Type") + (responseHeaders resp) + } data RequestData = RequestData { rdComponent :: Component, diff --git a/services/federator/src/Federator/InternalServer.hs b/services/federator/src/Federator/InternalServer.hs index a3fadf6f2ec..5493448f32b 100644 --- a/services/federator/src/Federator/InternalServer.hs +++ b/services/federator/src/Federator/InternalServer.hs @@ -65,6 +65,7 @@ import qualified Polysemy.Input as Polysemy import qualified Polysemy.Resource as Polysemy import Polysemy.TinyLog (TinyLog) import qualified Polysemy.TinyLog as Log +import Servant.Client.Core import qualified System.TimeManager as T import qualified System.X509 as TLS import Wire.API.Federation.Component @@ -118,14 +119,14 @@ callOutward req = do rd <- parseRequestData req domain <- parseDomainText (rdTargetDomain rd) ensureCanFederateWith domain - (status, result) <- + resp <- discoverAndCall domain (rdComponent rd) (rdRPC rd) (rdHeaders rd) (fromLazyByteString (rdBody rd)) - pure $ Wai.responseBuilder status defaultHeaders result + pure $ streamingResponseToWai resp serveOutward :: Env -> Int -> IO () serveOutward = serve callOutward diff --git a/services/federator/src/Federator/Remote.hs b/services/federator/src/Federator/Remote.hs index bbf5146da53..a206fcea360 100644 --- a/services/federator/src/Federator/Remote.hs +++ b/services/federator/src/Federator/Remote.hs @@ -26,7 +26,9 @@ module Federator.Remote ) where +import qualified Control.Exception as E import Control.Lens ((^.)) +import Control.Monad.Codensity import Data.Binary.Builder import Data.ByteString.Conversion (toByteString') import qualified Data.ByteString.Lazy as LBS @@ -49,6 +51,7 @@ import qualified Network.TLS.Extra.Cipher as TLS import Polysemy import Polysemy.Error import Polysemy.Input +import Servant.Client.Core import Wire.API.Federation.Client import Wire.API.Federation.Component import Wire.API.Federation.Error @@ -93,13 +96,13 @@ data Remote m a where Text -> [HTTP.Header] -> Builder -> - Remote m (HTTP.Status, Builder) + Remote m StreamingResponse makeSem ''Remote interpretRemote :: Members - '[ Embed IO, + '[ Embed (Codensity IO), DiscoverFederator, Error DiscoveryFailure, Error RemoteError, @@ -117,12 +120,21 @@ interpretRemote = interpret $ \case HTTP.encodePathSegments ["federation", componentName component, rpc] req' = HTTP2.requestBuilder HTTP.methodPost path headers body tlsConfig = mkTLSConfig settings hostname port - (status, _, result) <- - mapError (RemoteError target) . (fromEither =<<) . embed $ - performHTTP2Request (Just tlsConfig) req' hostname (fromIntegral port) - unless (HTTP.statusIsSuccessful status) $ - throw $ RemoteErrorResponse target status (toLazyByteString result) - pure (status, result) + + resp <- mapError (RemoteError target) . (fromEither @FederatorClientHTTP2Error =<<) . embed $ + Codensity $ \k -> + E.catch + (withHTTP2Request (Just tlsConfig) req' hostname (fromIntegral port) (k . Right)) + (k . Left) + + unless (HTTP.statusIsSuccessful (responseStatusCode resp)) $ do + bdy <- embed @(Codensity IO) . liftIO $ streamingResponseStrictBody resp + throw $ + RemoteErrorResponse + target + (responseStatusCode resp) + (toLazyByteString bdy) + pure resp mkTLSConfig :: TLSSettings -> ByteString -> Word16 -> TLS.ClientParams mkTLSConfig settings hostname port = diff --git a/services/federator/src/Federator/Response.hs b/services/federator/src/Federator/Response.hs index 7edab1111b2..76e2649a85f 100644 --- a/services/federator/src/Federator/Response.hs +++ b/services/federator/src/Federator/Response.hs @@ -20,10 +20,13 @@ module Federator.Response serve, runWaiError, runWaiErrors, + streamingResponseToWai, ) where import Control.Lens +import Control.Monad.Codensity +import Data.ByteString.Builder import Federator.Discovery import Federator.Env import Federator.Error @@ -39,10 +42,13 @@ import qualified Network.Wai.Handler.Warp as Warp import qualified Network.Wai.Utilities.Error as Wai import qualified Network.Wai.Utilities.Server as Wai import Polysemy +import Polysemy.Embed import Polysemy.Error import Polysemy.Input import Polysemy.Internal import Polysemy.TinyLog +import Servant.Client.Core +import Servant.Types.SourceT import Wire.Network.DNS.Effect defaultHeaders :: [HTTP.Header] @@ -96,14 +102,13 @@ serve action env port = where app :: Wai.Application app req respond = - runFederator env (action req) - >>= respond + runCodensity (runFederator env (action req)) respond type AllEffects = '[ Remote, DiscoverFederator, DNSLookup, -- needed by DiscoverFederator - Service, + ServiceStreaming, Input RunSettings, Input TLSSettings, -- needed by Remote Input Env, -- needed by Service @@ -112,14 +117,16 @@ type AllEffects = Error ServerError, Error DiscoveryFailure, TinyLog, - Embed IO + Embed IO, + Embed (Codensity IO) ] -- | Run Sem action containing HTTP handlers. All errors have to been handled -- already by this point. -runFederator :: Env -> Sem AllEffects Wai.Response -> IO Wai.Response +runFederator :: Env -> Sem AllEffects Wai.Response -> Codensity IO Wai.Response runFederator env = - runM @IO + runM + . runEmbedded @IO @(Codensity IO) liftIO . runTinyLog (view applog env) -- FUTUREWORK: add request id . runWaiErrors @'[ ValidationError, @@ -130,7 +137,18 @@ runFederator env = . runInputConst env . runInputSem (embed @IO (readIORef (view tls env))) . runInputConst (view runSettings env) - . interpretService + . interpretServiceHTTP . runDNSLookupWithResolver (view dnsResolver env) . runFederatorDiscovery . interpretRemote + +streamingResponseToWai :: StreamingResponse -> Wai.Response +streamingResponseToWai resp = + let headers = toList (responseHeaders resp) + status = responseStatusCode resp + streamingBody output flush = + foreach + (const (pure ())) + (\chunk -> output (byteString chunk) *> flush) + (responseBody resp) + in Wai.responseStream status headers streamingBody diff --git a/services/federator/src/Federator/Run.hs b/services/federator/src/Federator/Run.hs index 3e9312a4604..a41b259451b 100644 --- a/services/federator/src/Federator/Run.hs +++ b/services/federator/src/Federator/Run.hs @@ -34,13 +34,11 @@ module Federator.Run ) where -import qualified Bilge as RPC import Control.Concurrent.Async import Control.Exception (bracket) import Control.Lens ((^.)) import Data.Default (def) import qualified Data.Metrics.Middleware as Metrics -import Data.Text.Encoding (encodeUtf8) import Federator.Env import Federator.ExternalServer (serveInward) import Federator.InternalServer (serveOutward) @@ -95,14 +93,12 @@ newEnv o _dnsResolver = do _applog <- LogExt.mkLogger (Opt.logLevel o) (Opt.logNetStrings o) (Opt.logFormat o) let _requestId = def let _runSettings = Opt.optSettings o - let _service Brig = mkEndpoint (Opt.brig o) - _service Galley = mkEndpoint (Opt.galley o) - _service Cargohold = mkEndpoint (Opt.cargohold o) + let _service Brig = Opt.brig o + _service Galley = Opt.galley o + _service Cargohold = Opt.cargohold o _httpManager <- initHttpManager _tls <- mkTLSSettingsOrThrow _runSettings >>= newIORef return Env {..} - where - mkEndpoint s = RPC.host (encodeUtf8 (s ^. epHost)) . RPC.port (s ^. epPort) $ RPC.empty closeEnv :: Env -> IO () closeEnv e = do diff --git a/services/federator/src/Federator/Service.hs b/services/federator/src/Federator/Service.hs index 99b024fcc31..9514f9ecf24 100644 --- a/services/federator/src/Federator/Service.hs +++ b/services/federator/src/Federator/Service.hs @@ -15,35 +15,57 @@ -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . -module Federator.Service where +module Federator.Service + ( Service (..), + ServiceStreaming, + interpretServiceHTTP, + serviceCall, + ) +where -- FUTUREWORK(federation): Once we authenticate the call, we should send authentication data -- to brig so brig can do some authorization as required. import qualified Bilge as RPC -import Bilge.RPC (rpc') +import Control.Exception import Control.Lens (view) +import Control.Monad.Codensity +import qualified Data.ByteString as BS import Data.Domain +import qualified Data.Sequence as Seq import Data.String.Conversions (cs) -import qualified Data.Text.Lazy as LText -import Federator.App +import qualified Data.Text.Encoding as Text import Federator.Env import Imports +import Network.HTTP.Client import qualified Network.HTTP.Types as HTTP import Polysemy import Polysemy.Input +import Polysemy.TinyLog +import qualified Servant.Client.Core as Servant +import Servant.Types.SourceT +import Util.Options import Wire.API.Federation.Component import Wire.API.Federation.Domain (originDomainHeaderName) -newtype ServiceError = ServiceErrorInvalidStatus HTTP.Status - deriving (Eq, Show) +type ServiceStreaming = Service (SourceT IO ByteString) -data Service m a where - -- | Returns status and body, 'HTTP.Response' is not nice to work with in tests - ServiceCall :: Component -> ByteString -> LByteString -> Domain -> Service m (HTTP.Status, Maybe LByteString) +data Service body m a where + -- | Returns status, headers and body, 'HTTP.Response' is not nice to work with in tests + ServiceCall :: Component -> ByteString -> LByteString -> Domain -> Service body m (Servant.ResponseF body) makeSem ''Service +bodyReaderToStreamT :: Monad m => m ByteString -> SourceT m ByteString +bodyReaderToStreamT action = fromStepT go + where + go = Effect $ do + chunk <- action + pure $ + if BS.null chunk + then Stop + else Yield chunk go + -- FUTUREWORK(federation): Do we want to use servant client here? May make -- everything typed and safe -- @@ -52,19 +74,36 @@ makeSem ''Service -- -- FUTUREWORK: unify this interpretation with similar ones in Galley -- --- FUTUREWORK: does it make sense to use a lower level abstraction instead of bilge here? -interpretService :: - Members '[Embed IO, Input Env] r => - Sem (Service ': r) a -> +interpretServiceHTTP :: + Members '[Embed (Codensity IO), Input Env, TinyLog] r => + Sem (ServiceStreaming ': r) a -> Sem r a -interpretService = interpret $ \case - ServiceCall component path body domain -> embedApp @IO $ do - serviceReq <- view service <$> ask - res <- - rpc' (LText.pack (show component)) (serviceReq component) $ - RPC.method HTTP.POST - . RPC.path path - . RPC.body (RPC.RequestBodyLBS body) - . RPC.contentJson - . RPC.header originDomainHeaderName (cs (domainText domain)) - pure (RPC.responseStatus res, RPC.responseBody res) +interpretServiceHTTP = interpret $ \case + ServiceCall component rpcPath body domain -> do + Endpoint serviceHost servicePort <- inputs (view service) <*> pure component + manager <- inputs (view httpManager) + reqId <- inputs (view requestId) + let req = + defaultRequest + { method = HTTP.methodPost, + host = Text.encodeUtf8 serviceHost, + port = fromIntegral servicePort, + requestBody = RequestBodyLBS body, + path = rpcPath, + requestHeaders = + [ ("Content-Type", "application/json"), + (originDomainHeaderName, cs (domainText domain)), + (RPC.requestIdName, RPC.unRequestId reqId) + ] + } + + embed $ + Codensity $ \k -> + bracket (responseOpen req manager) responseClose $ \resp -> + k $ + Servant.Response + { Servant.responseStatusCode = responseStatus resp, + Servant.responseHeaders = Seq.fromList (responseHeaders resp), + Servant.responseHttpVersion = HTTP.http11, + Servant.responseBody = bodyReaderToStreamT (responseBody resp) + } diff --git a/services/federator/test/integration/Test/Federator/IngressSpec.hs b/services/federator/test/integration/Test/Federator/IngressSpec.hs index 04d80bb147d..08d989b647a 100644 --- a/services/federator/test/integration/Test/Federator/IngressSpec.hs +++ b/services/federator/test/integration/Test/Federator/IngressSpec.hs @@ -18,6 +18,7 @@ module Test.Federator.IngressSpec where import Control.Lens (view) +import Control.Monad.Codensity import qualified Data.Aeson as Aeson import Data.Binary.Builder import Data.Domain @@ -32,11 +33,14 @@ import Federator.Remote import Imports import qualified Network.HTTP.Types as HTTP import Polysemy +import Polysemy.Embed import Polysemy.Error import Polysemy.Input +import Servant.Client.Core import Test.Federator.Util import Test.Hspec import Util.Options (Endpoint (Endpoint)) +import Wire.API.Federation.Client import Wire.API.Federation.Component import Wire.API.Federation.Domain import Wire.API.User @@ -53,14 +57,15 @@ spec env = do _ <- putHandle brig (userId user) hdl let expectedProfile = (publicProfile user UserLegalHoldNoConsent) {profileHandle = Just (Handle hdl)} - (status, resp) <- + resp <- runTestSem . assertNoError @RemoteError $ inwardBrigCallViaIngress "get-user-by-handle" $ (Aeson.fromEncoding (Aeson.toEncoding hdl)) - let actualProfile = Aeson.decode (toLazyByteString resp) liftIO $ do - status `shouldBe` HTTP.status200 + bdy <- streamingResponseStrictBody resp + let actualProfile = Aeson.decode (toLazyByteString bdy) + responseStatusCode resp `shouldBe` HTTP.status200 actualProfile `shouldBe` (Just expectedProfile) it "should not be accessible without a client certificate" $ @@ -102,7 +107,7 @@ inwardBrigCallViaIngress :: Members [Input TestEnv, Embed IO, Error RemoteError] r => Text -> Builder -> - Sem r (HTTP.Status, Builder) + Sem r StreamingResponse inwardBrigCallViaIngress path payload = do tlsSettings <- inputs (view teTLSSettings) inwardBrigCallViaIngressWithSettings tlsSettings path payload @@ -112,7 +117,7 @@ inwardBrigCallViaIngressWithSettings :: TLSSettings -> Text -> Builder -> - Sem r (HTTP.Status, Builder) + Sem r StreamingResponse inwardBrigCallViaIngressWithSettings tlsSettings requestPath payload = do Endpoint ingressHost ingressPort <- cfgNginxIngress . view teTstOpts <$> input @@ -122,5 +127,6 @@ inwardBrigCallViaIngressWithSettings tlsSettings requestPath payload = runInputConst tlsSettings . assertNoError @DiscoveryFailure . discoverConst target + . runEmbedded @(Codensity IO) @IO lowerCodensity . interpretRemote $ discoverAndCall (Domain "example.com") Brig requestPath headers payload diff --git a/services/federator/test/unit/Test/Federator/Client.hs b/services/federator/test/unit/Test/Federator/Client.hs index a0d172050f9..854ac570874 100644 --- a/services/federator/test/unit/Test/Federator/Client.hs +++ b/services/federator/test/unit/Test/Federator/Client.hs @@ -18,14 +18,26 @@ module Test.Federator.Client (tests) where import Control.Exception hiding (handle) +import Control.Monad.Codensity +import Control.Monad.Except import qualified Data.Aeson as Aeson import Data.Bifunctor (first) +import qualified Data.ByteString as BS +import Data.ByteString.Builder (Builder, byteString, toLazyByteString) +import qualified Data.ByteString.Lazy as LBS import Data.Domain +import Data.Proxy +import qualified Data.Text.Encoding as Text import Federator.MockServer import Imports import Network.HTTP.Types as HTTP import qualified Network.HTTP2.Client as HTTP2 -import Network.Wai.Utilities.Error as Wai +import qualified Network.Wai as Wai +import qualified Network.Wai.Utilities.Error as Wai +import Servant.API +import Servant.Client +import Servant.Client.Core +import Servant.Types.SourceT import Test.QuickCheck (arbitrary, generate) import Test.Tasty import Test.Tasty.HUnit @@ -53,6 +65,7 @@ tests = [ testGroup "Servant" [ testCase "testClientSuccess" testClientSuccess, + testCase "testClientStreaming" testClientStreaming, testCase "testClientFailure" testClientFailure, testCase "testFederatorFailure" testFederatorFailure, testCase "testClientException" testClientExceptions, @@ -60,7 +73,8 @@ tests = ], testGroup "HTTP2 client" - [ testCase "testResponseHeaders" testResponseHeaders + [ testCase "testResponseHeaders" testResponseHeaders, + testCase "testStreaming" testStreaming ] ] @@ -106,6 +120,25 @@ testClientSuccess = do ] first (const ()) actualResponse @?= Right (Just expectedResponse) +type StreamingAPI = StreamGet NewlineFraming PlainText (SourceIO Text) + +testClientStreaming :: IO () +testClientStreaming = withInfiniteMockServer $ \port -> do + let env = + FederatorClientEnv + { ceOriginDomain = originDomain, + ceTargetDomain = targetDomain, + ceFederator = Endpoint "127.0.0.1" (fromIntegral port) + } + let c = clientIn (Proxy @StreamingAPI) (Proxy @(FederatorClient 'Brig)) + runCodensity (runFederatorClientToCodensity env c) $ \eout -> + case eout of + Left err -> assertFailure $ "Unexpected error: " <> displayException err + Right out -> do + let expected = mconcat (replicate 500 "Hello") + actual <- takeSourceT (fromIntegral (LBS.length expected)) (fmap Text.encodeUtf8 out) + actual @?= expected + testClientFailure :: IO () testClientFailure = do handle <- generate arbitrary @@ -181,6 +214,40 @@ testResponseHeaders = do Left err -> assertFailure $ "Unexpected error while connecting to mock federator: " <> show err - Right (status, headers, _) -> do - status @?= HTTP.status200 - lookup "X-Foo" headers @?= Just "bar" + Right resp -> do + responseStatusCode resp @?= HTTP.status200 + lookup "X-Foo" (toList (responseHeaders resp)) @?= Just "bar" + +testStreaming :: IO () +testStreaming = withInfiniteMockServer $ \port -> do + let req = HTTP2.requestBuilder HTTP.methodPost "test" [] mempty + withHTTP2Request Nothing req "127.0.0.1" port $ \resp -> do + let expected = mconcat (replicate 512 "Hello\n") + actual <- takeSourceT (fromIntegral (LBS.length expected)) (responseBody resp) + actual @?= expected + +withInfiniteMockServer :: (Int -> IO a) -> IO a +withInfiniteMockServer k = bracket (startMockServer Nothing app) fst (k . snd) + where + app _ respond = respond $ + Wai.responseStream HTTP.ok200 mempty $ \write flush -> + let go n = do + when (n == 0) $ flush + write (byteString "Hello\n") *> go (if n == 0 then 100 else n - 1) + in go (1000 :: Int) + +-- SourceT utilities + +takeStepT :: Builder -> Int -> StepT IO ByteString -> IO LByteString +takeStepT acc _ Stop = pure (toLazyByteString acc) +takeStepT acc _ (Error _) = pure (toLazyByteString acc) +takeStepT acc s (Skip next) = takeStepT acc s next +takeStepT acc s (Yield chunk next) + | BS.length chunk >= s = + pure $ toLazyByteString (acc <> byteString (BS.take s chunk)) + | otherwise = do + takeStepT (acc <> byteString chunk) (s - BS.length chunk) next +takeStepT acc s (Effect m) = m >>= takeStepT acc s + +takeSourceT :: Int -> SourceT IO ByteString -> IO LByteString +takeSourceT s m = unSourceT m (takeStepT mempty s) diff --git a/services/federator/test/unit/Test/Federator/ExternalServer.hs b/services/federator/test/unit/Test/Federator/ExternalServer.hs index a1c749cec4a..a70934f8dac 100644 --- a/services/federator/test/unit/Test/Federator/ExternalServer.hs +++ b/services/federator/test/unit/Test/Federator/ExternalServer.hs @@ -26,7 +26,7 @@ import qualified Data.Text.Encoding as Text import Federator.Discovery import Federator.Error.ServerError (ServerError (..)) import Federator.ExternalServer -import Federator.Service (Service) +import Federator.Service (Service (..), ServiceStreaming) import Federator.Validation import Imports import qualified Network.HTTP.Types as HTTP @@ -35,18 +35,17 @@ import qualified Network.Wai.Utilities.Server as Wai import Polysemy import Polysemy.Error import Polysemy.Input -import qualified Polysemy.TinyLog as TinyLog +import Polysemy.Output +import Polysemy.TinyLog +import qualified Servant.Client.Core as Servant +import Servant.Types.SourceT import Test.Federator.Options (noClientCertSettings) import Test.Federator.Util import Test.Federator.Validation (mockDiscoveryTrivial) -import Test.Polysemy.Mock (Mock (mock), evalMock) -import Test.Polysemy.Mock.TH (genMock) import Test.Tasty import Test.Tasty.HUnit import Wire.API.Federation.Component -genMock ''Service - tests :: TestTree tests = testGroup @@ -72,59 +71,79 @@ exampleRequest certFile path = do trBody = "\"foo\"" } +data Call = Call + { cComponent :: Component, + cPath :: ByteString, + cBody :: LByteString, + cDomain :: Domain + } + deriving (Eq, Show) + +mockService :: + Members [Output Call, Embed IO] r => + HTTP.Status -> + Sem (ServiceStreaming ': r) a -> + Sem r a +mockService status = interpret $ \case + ServiceCall comp path body domain -> do + output (Call comp path body domain) + pure + Servant.Response + { Servant.responseStatusCode = status, + Servant.responseHeaders = mempty, + Servant.responseHttpVersion = HTTP.http11, + Servant.responseBody = source ["\"bar\""] + } + requestBrigSuccess :: TestTree requestBrigSuccess = - testCase "should translate response from brig to 'InwardResponseBody' when response has status 200" $ do + testCase "should forward response from brig when status is 200" $ do request <- exampleRequest "test/resources/unit/localhost.example.com.pem" "/federation/brig/get-user-by-handle" - runM . evalMock @Service @IO $ do - mockServiceCallReturns @IO (\_ _ _ _ -> pure (HTTP.ok200, Just "\"bar\"")) - - res <- - mock @Service @IO - . assertNoError @ValidationError - . assertNoError @DiscoveryFailure - . assertNoError @ServerError - . TinyLog.discardLogs - . mockDiscoveryTrivial - . runInputConst noClientCertSettings - $ callInward request - actualCalls <- mockServiceCallCalls @IO - let expectedCall = (Brig, "/federation/get-user-by-handle", "\"foo\"", aValidDomain) - embed $ assertEqual "one call to brig should be made" [expectedCall] actualCalls - embed $ Wai.responseStatus res @?= HTTP.status200 - body <- embed $ Wai.lazyResponseBody res - embed $ body @?= "\"bar\"" + (actualCalls, res) <- + runM + . runOutputList + . mockService HTTP.ok200 + . assertNoError @ValidationError + . assertNoError @DiscoveryFailure + . assertNoError @ServerError + . discardLogs + . mockDiscoveryTrivial + . runInputConst noClientCertSettings + $ callInward request + let expectedCall = Call Brig "/federation/get-user-by-handle" "\"foo\"" aValidDomain + assertEqual "one call to brig should be made" [expectedCall] actualCalls + Wai.responseStatus res @?= HTTP.status200 + body <- Wai.lazyResponseBody res + body @?= "\"bar\"" requestBrigFailure :: TestTree requestBrigFailure = - testCase "should translate response from brig to 'InwardResponseError' when response has status 404" $ do + testCase "should preserve the status code returned by the service" $ do request <- exampleRequest "test/resources/unit/localhost.example.com.pem" "/federation/brig/get-user-by-handle" - runM . evalMock @Service @IO $ do - let brigResponseBody = "response body" - mockServiceCallReturns @IO (\_ _ _ _ -> pure (HTTP.notFound404, Just brigResponseBody)) - res <- - mock @Service @IO - . assertNoError @ValidationError - . assertNoError @DiscoveryFailure - . assertNoError @ServerError - . TinyLog.discardLogs - . mockDiscoveryTrivial - . runInputConst noClientCertSettings - $ callInward request - - actualCalls <- mockServiceCallCalls @IO - let expectedCall = (Brig, "/federation/get-user-by-handle", "\"foo\"", aValidDomain) - embed $ assertEqual "one call to brig should be made" [expectedCall] actualCalls - embed $ Wai.responseStatus res @?= HTTP.notFound404 - body <- embed $ Wai.lazyResponseBody res - embed $ body @?= brigResponseBody + (actualCalls, res) <- + runM + . runOutputList + . mockService HTTP.notFound404 + . assertNoError @ValidationError + . assertNoError @DiscoveryFailure + . assertNoError @ServerError + . discardLogs + . mockDiscoveryTrivial + . runInputConst noClientCertSettings + $ callInward request + + let expectedCall = Call Brig "/federation/get-user-by-handle" "\"foo\"" aValidDomain + assertEqual "one call to brig should be made" [expectedCall] actualCalls + Wai.responseStatus res @?= HTTP.notFound404 + body <- Wai.lazyResponseBody res + body @?= "\"bar\"" requestGalleySuccess :: TestTree requestGalleySuccess = @@ -134,20 +153,18 @@ requestGalleySuccess = "test/resources/unit/localhost.example.com.pem" "/federation/galley/get-conversations" - runM . evalMock @Service @IO $ do - mockServiceCallReturns @IO (\_ _ _ _ -> pure (HTTP.ok200, Just "\"bar\"")) - - res <- - mock @Service @IO + runM $ do + (actualCalls, res) <- + runOutputList + . mockService HTTP.ok200 . assertNoError @ValidationError . assertNoError @DiscoveryFailure . assertNoError @ServerError - . TinyLog.discardLogs + . discardLogs . mockDiscoveryTrivial . runInputConst noClientCertSettings $ callInward request - actualCalls <- mockServiceCallCalls @IO - let expectedCall = (Galley, "/federation/get-conversations", "\"foo\"", aValidDomain) + let expectedCall = Call Galley "/federation/get-conversations" "\"foo\"" aValidDomain embed $ assertEqual "one call to galley should be made" [expectedCall] actualCalls embed $ Wai.responseStatus res @?= HTTP.status200 body <- embed $ Wai.lazyResponseBody res @@ -164,20 +181,18 @@ requestNoDomain = trPath = "/federation/brig/get-users" } - runM . evalMock @Service @IO $ do - mockServiceCallReturns @IO (\_ _ _ _ -> pure (HTTP.ok200, Just "\"bar\"")) - - res <- - runError - . mock @Service @IO + runM $ do + (actualCalls, res) <- + runOutputList @Call + . mockService HTTP.ok200 + . runError . assertNoError @ValidationError . assertNoError @DiscoveryFailure - . TinyLog.discardLogs + . discardLogs . mockDiscoveryTrivial . runInputConst noClientCertSettings $ callInward request - actualCalls <- mockServiceCallCalls @IO embed $ assertEqual "no calls to services should be made" [] actualCalls embed $ void res @?= Left NoOriginDomain @@ -191,22 +206,20 @@ requestNoCertificate = trPath = "/federation/brig/get-users" } - runM . evalMock @Service @IO $ do - mockServiceCallReturns @IO (\_ _ _ _ -> pure (HTTP.ok200, Just "\"bar\"")) - - res <- - runError - . mock @Service @IO - . assertNoError @ServerError - . assertNoError @DiscoveryFailure - . TinyLog.discardLogs - . mockDiscoveryTrivial - . runInputConst noClientCertSettings - $ callInward request - - actualCalls <- mockServiceCallCalls @IO - embed $ assertEqual "no calls to services should be made" [] actualCalls - embed $ void res @?= Left NoClientCertificate + (actualCalls, res) <- + runM + . runOutputList @Call + . mockService HTTP.ok200 + . runError + . assertNoError @ServerError + . assertNoError @DiscoveryFailure + . discardLogs + . mockDiscoveryTrivial + . runInputConst noClientCertSettings + $ callInward request + + assertEqual "no calls to services should be made" [] actualCalls + void res @?= Left NoClientCertificate testInvalidPaths :: TestTree testInvalidPaths = do @@ -244,23 +257,20 @@ testInvalidPaths = do "test/resources/unit/localhost.example.com.pem" invalidPath - runM . evalMock @Service @IO $ do - mockServiceCallReturns @IO (\_ _ _ _ -> pure (HTTP.ok200, Just "\"bar\"")) - - res <- - runError @ServerError - . mock @Service @IO - . assertNoError @ValidationError - . assertNoError @DiscoveryFailure - . TinyLog.discardLogs - . mockDiscoveryTrivial - . runInputConst noClientCertSettings - $ callInward request - - embed $ assertEqual ("Expected request with path \"" <> cs invalidPath <> "\" to fail") (Left InvalidRoute) (void res) + (actualCalls, res) <- + runM + . runOutputList @Call + . mockService HTTP.ok200 + . runError @ServerError + . assertNoError @ValidationError + . assertNoError @DiscoveryFailure + . discardLogs + . mockDiscoveryTrivial + . runInputConst noClientCertSettings + $ callInward request - actualCalls <- mockServiceCallCalls @IO - embed $ assertEqual "no calls to any service should be made" [] actualCalls + assertEqual ("Expected request with path \"" <> cs invalidPath <> "\" to fail") (Left InvalidRoute) (void res) + assertEqual "no calls to any service should be made" [] actualCalls testInvalidComponent :: TestTree testInvalidComponent = @@ -270,22 +280,20 @@ testInvalidComponent = "test/resources/unit/localhost.example.com.pem" "/federation/mast/get-users" - runM . evalMock @Service @IO $ do - mockServiceCallReturns @IO (\_ _ _ _ -> pure (HTTP.ok200, Just "\"bar\"")) - - res <- - runError @ServerError - . mock @Service @IO - . assertNoError @ValidationError - . assertNoError @DiscoveryFailure - . TinyLog.discardLogs - . mockDiscoveryTrivial - . runInputConst noClientCertSettings - $ callInward request - - embed $ void res @?= Left (UnknownComponent "mast") - actualCalls <- mockServiceCallCalls @IO - embed $ assertEqual "no calls to any service should be made" [] actualCalls + (actualCalls, res) <- + runM + . runOutputList @Call + . mockService HTTP.ok200 + . runError @ServerError + . assertNoError @ValidationError + . assertNoError @DiscoveryFailure + . discardLogs + . mockDiscoveryTrivial + . runInputConst noClientCertSettings + $ callInward request + + void res @?= Left (UnknownComponent "mast") + assertEqual "no calls to any service should be made" [] actualCalls testMethod :: TestTree testMethod = @@ -304,10 +312,10 @@ testMethod = res <- runM . runError @ServerError - . interpret @Service (\_ -> embed $ assertFailure "unexpected call to service") + . interpret @ServiceStreaming (\_ -> embed $ assertFailure "unexpected call to service") . assertNoError @ValidationError . assertNoError @DiscoveryFailure - . TinyLog.discardLogs + . discardLogs . mockDiscoveryTrivial . runInputConst noClientCertSettings $ callInward request diff --git a/services/federator/test/unit/Test/Federator/InternalServer.hs b/services/federator/test/unit/Test/Federator/InternalServer.hs index 3def6a2c2a9..db3393c5290 100644 --- a/services/federator/test/unit/Test/Federator/InternalServer.hs +++ b/services/federator/test/unit/Test/Federator/InternalServer.hs @@ -19,7 +19,7 @@ module Test.Federator.InternalServer (tests) where -import Data.Binary.Builder +import Data.ByteString.Builder import Data.ByteString.Conversion import Data.Default import Data.Domain @@ -36,6 +36,8 @@ import Polysemy import Polysemy.Error import Polysemy.Input import Polysemy.TinyLog +import Servant.Client.Core +import Servant.Types.SourceT import Test.Federator.Options (noClientCertSettings) import Test.Federator.Util import Test.Tasty @@ -78,7 +80,13 @@ federatedRequestSuccess = rpc @?= "get-user-by-handle" headers @?= requestHeaders toLazyByteString body @?= "\"foo\"" - pure (HTTP.status200, fromLazyByteString "\"bar\"") + pure + Response + { responseStatusCode = HTTP.ok200, + responseHeaders = mempty, + responseHttpVersion = HTTP.http20, + responseBody = source ["\"bar\""] + } res <- runM . interpretCall @@ -107,7 +115,14 @@ federatedRequestFailureAllowList = let checkRequest :: Sem (Remote ': r) a -> Sem r a checkRequest = interpret $ \case - DiscoverAndCall {} -> pure (HTTP.status200, fromLazyByteString "\"bar\"") + DiscoverAndCall {} -> + pure + Response + { responseStatusCode = HTTP.ok200, + responseHeaders = mempty, + responseHttpVersion = HTTP.http20, + responseBody = source ["\"bar\""] + } eith <- runM diff --git a/services/federator/test/unit/Test/Federator/Remote.hs b/services/federator/test/unit/Test/Federator/Remote.hs index 66689a26291..ce13842fb9c 100644 --- a/services/federator/test/unit/Test/Federator/Remote.hs +++ b/services/federator/test/unit/Test/Federator/Remote.hs @@ -18,6 +18,7 @@ module Test.Federator.Remote where import Control.Exception (bracket) +import Control.Monad.Codensity import Data.Domain import Federator.Discovery import Federator.Env (TLSSettings) @@ -31,6 +32,7 @@ import Network.Wai import qualified Network.Wai.Handler.Warp as Warp import qualified Network.Wai.Handler.WarpTLS as Warp import Polysemy +import Polysemy.Embed import Polysemy.Error import Polysemy.Input import Test.Federator.Options (defRunSettings) @@ -83,6 +85,7 @@ mkTestCall tlsSettings port = . runInputConst tlsSettings . discoverLocalhost port . assertNoError @DiscoveryFailure + . runEmbedded @(Codensity IO) @IO lowerCodensity . interpretRemote $ discoverAndCall (Domain "localhost") Brig "test" [] mempty diff --git a/stack.yaml b/stack.yaml index 12671354d5b..6081f77af12 100644 --- a/stack.yaml +++ b/stack.yaml @@ -161,7 +161,6 @@ extra-deps: - markov-chain-usage-model-0.0.0 - wai-predicates-1.0.0 - redis-io-1.1.0 -- polysemy-mocks-0.2.0.0 - warp-3.3.17 # Not latest as last one breaks wai-routing @@ -208,10 +207,9 @@ extra-deps: - git: https://github.com/dpwright/HaskellNet-SSL commit: ca84ef29a93eaef7673fa58056cdd8dae1568d2d # master (Sep 14, 2020) -# Fix for connection preface race condition -# https://github.com/kazu-yamamoto/http2/pull/33 +# Fix for server sending too many empty data frames - git: https://github.com/wireapp/http2 - commit: 1ee1ce432d923839dab6782410e91dc17df2a880 # preface-race branch + commit: aa3501ad58e1abbd196781fac25a84f41ec2a787 # avoid-empty-data branch # Fix in PR: https://github.com/bos/snappy/pull/7 - git: https://github.com/wireapp/snappy diff --git a/stack.yaml.lock b/stack.yaml.lock index 6da7fa2060c..c9b4d4ccfc9 100644 --- a/stack.yaml.lock +++ b/stack.yaml.lock @@ -447,13 +447,6 @@ packages: sha256: db61f70aa7387090c26ccca0545ffdeea0adfcf93b76d5eaf6a954c0e5a34064 original: hackage: redis-io-1.1.0 -- completed: - hackage: polysemy-mocks-0.2.0.0@sha256:ed7b4aa8ee29995d0b840ac0c131a141636ca46493b82706b7e5ec5e33b9ffa7,1441 - pantry-tree: - size: 695 - sha256: 8218e3dde278ca1f01d19009fad40f603e1b17993a519d15fe6319e3a827cc01 - original: - hackage: polysemy-mocks-0.2.0.0 - completed: hackage: warp-3.3.17@sha256:3a3ea203141d00d2244b511ee99174b8ed58fc862552755d470a25a44ce5275b,10910 pantry-tree: @@ -646,11 +639,11 @@ packages: git: https://github.com/wireapp/http2 pantry-tree: size: 52771 - sha256: dc6d3868a049d2ed38ef16ca6dd6aeb6b8e8a1e730c664ecdd243ffdb45ee750 - commit: 1ee1ce432d923839dab6782410e91dc17df2a880 + sha256: 71040c20c8e6a766b6b309c03dbc970062b15e450a63e05f8d095a87cdb5082f + commit: aa3501ad58e1abbd196781fac25a84f41ec2a787 original: git: https://github.com/wireapp/http2 - commit: 1ee1ce432d923839dab6782410e91dc17df2a880 + commit: aa3501ad58e1abbd196781fac25a84f41ec2a787 - completed: name: snappy version: 0.2.0.2 From 1e5cb90b24a61e39cc39696b9dda9336c5554b99 Mon Sep 17 00:00:00 2001 From: Stefan Matting Date: Tue, 14 Dec 2021 17:30:44 +0100 Subject: [PATCH 06/74] SFT chart: Add multiSFT option, remove additionalArgs option (#1992) --- changelog.d/5-internal/sftd-multi-sft | 1 + charts/sftd/templates/statefulset.yaml | 57 +++++++++++++++++++++++++- charts/sftd/values.yaml | 10 +++-- 3 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 changelog.d/5-internal/sftd-multi-sft diff --git a/changelog.d/5-internal/sftd-multi-sft b/changelog.d/5-internal/sftd-multi-sft new file mode 100644 index 00000000000..a0324fe7496 --- /dev/null +++ b/changelog.d/5-internal/sftd-multi-sft @@ -0,0 +1 @@ +sftd chart: Add multiSFT option, remove additionalArgs option diff --git a/charts/sftd/templates/statefulset.yaml b/charts/sftd/templates/statefulset.yaml index 2c5d52fce40..8345b6bc1c1 100644 --- a/charts/sftd/templates/statefulset.yaml +++ b/charts/sftd/templates/statefulset.yaml @@ -30,10 +30,13 @@ spec: {{- toYaml .Values.podSecurityContext | nindent 8 }} terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet serviceAccountName: {{ include "sftd.fullname" . }} volumes: - name: external-ip emptyDir: {} + - name: multi-sft-config + emptyDir: {} initContainers: - name: get-external-ip image: bitnami/kubectl:1.19.7 @@ -54,6 +57,47 @@ spec: addr=$(kubectl get node $HOSTNAME -ojsonpath='{.metadata.annotations.wire\.com/external-ip}') fi echo -n "$addr" | tee /dev/stderr > /external-ip/ip + - name: get-multi-sft-config + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + + volumeMounts: + - name: multi-sft-config + mountPath: /multi-sft-config + + command: + - /bin/sh + - -c + - | + set -e + + {{- if .Values.multiSFT.enabled }} + + response=$(curl "{{ .Values.multiSFT.turnDiscoveryURL }}") + if [ -z "$response" ]; then + echo "No response from restund server." + exit 1 + fi + + echo "$response" | jq -r '.username' > /multi-sft-config/username + if [ ! -s /multi-sft-config/username ]; then + echo "Response does not contain a username" + exit 1 + fi + + echo "$response" | jq -r '.password' > /multi-sft-config/password + if [ ! -s /multi-sft-config/password ]; then + echo "Response does not contain a password" + exit 1 + fi + + echo "$response" | jq -r '.uris[0]' > /multi-sft-config/turn_server + if [ ! -s /multi-sft-config/turn_server ]; then + echo "Response does not contain a turn server" + exit 1 + fi + + {{- end }} + containers: - name: {{ .Chart.Name }} securityContext: @@ -72,6 +116,8 @@ spec: volumeMounts: - name: external-ip mountPath: /external-ip + - name: multi-sft-config + mountPath: /multi-sft-config command: - /bin/sh - -c @@ -83,11 +129,20 @@ spec: else ACCESS_ARGS="-A ${EXTERNAL_IP}" fi + + {{- if .Values.multiSFT.enabled }} + MULTI_SFT_ARGS="-t \"$(cat /multi-sft-config/turn_server)\" \ + -x \"$(cat /multi-sft-config/username)\" \ + -c \"$(cat /multi-sft-config/password)\"" + {{- else }} + MULTI_SFT_ARGS="" + {{- end }} + exec sftd \ -I "${POD_IP}" \ -M "${POD_IP}" \ ${ACCESS_ARGS} \ - {{ .Values.additionalArgs }} \ + ${MULTI_SFT_ARGS} \ {{ if .Values.turnDiscoveryEnabled }}-T{{ end }} \ -u "https://{{ required "must specify host" .Values.host }}/sfts/${POD_NAME}" ports: diff --git a/charts/sftd/values.yaml b/charts/sftd/values.yaml index a34ae175892..5e90388b66d 100644 --- a/charts/sftd/values.yaml +++ b/charts/sftd/values.yaml @@ -86,6 +86,10 @@ joinCall: # DOCS: https://docs.wire.com/understand/sft.html#prerequisites turnDiscoveryEnabled: false -# Additional arguments to be passed to `sftd` -# Note: this might be removed in the future. -additionalArgs: "" +# Allow establishing calls involving remote SFT servers (e.g. for Federation) +# Requires appVersion 3.0.9 or later +multiSFT: + enabled: False + # Required. URL that provides TURN connection configuration. These configured + # TURN servers will be used to connect to remote SFT servers. + turnDiscoveryURL: "" From 7329d41726cee1b91ec1023ea98b972c6ca4c66b Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 14 Dec 2021 18:37:27 +0100 Subject: [PATCH 07/74] Add a paragraph about using VSCode in a direnv env --- docs/developer/editor-setup.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/developer/editor-setup.md b/docs/developer/editor-setup.md index 249529677d9..4e030f88b08 100644 --- a/docs/developer/editor-setup.md +++ b/docs/developer/editor-setup.md @@ -110,4 +110,8 @@ Setup steps: - Install the plugins `Haskell` (Haskell Language Server support), `Haskell Syntax` and `Nix Environment Selector` - Generate the `hie.yaml` file: `make hie.yaml` - Select the nix environment from `shell.nix` with the command `Nix-Env: Select environment`. -- Reload the window as proposed by the `Nix Environment Selector` plugin \ No newline at end of file +- Reload the window as proposed by the `Nix Environment Selector` plugin + +An alternative way to make these dependencies accessible to VSCode is to start it in the `direnv` environment. +I.e. from a shell that's current working directory is in the project. The drawbacks of this approach are +that it only works locally (not on a remote connection) and one VSCode process needs to be started per project. \ No newline at end of file From 70316ac5b3f3535a68c81773feaf0fb98308961b Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Wed, 15 Dec 2021 13:29:30 +0100 Subject: [PATCH 08/74] Split cannon benchmarks and tests (#1986) * Split cannon benchmarks and tests --- .../5-internal/split-cannon-benchmarks | 1 + services/cannon/{test => bench}/Bench.hs | 0 services/cannon/bench/Main.hs | 24 ++++++++++++++ services/cannon/cannon.cabal | 32 +++++++++++++++++-- services/cannon/package.yaml | 22 +++++++++++++ services/cannon/test/Main.hs | 4 +-- 6 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 changelog.d/5-internal/split-cannon-benchmarks rename services/cannon/{test => bench}/Bench.hs (100%) create mode 100644 services/cannon/bench/Main.hs diff --git a/changelog.d/5-internal/split-cannon-benchmarks b/changelog.d/5-internal/split-cannon-benchmarks new file mode 100644 index 00000000000..2122a84d6b9 --- /dev/null +++ b/changelog.d/5-internal/split-cannon-benchmarks @@ -0,0 +1 @@ +Split cannon benchmarks and tests diff --git a/services/cannon/test/Bench.hs b/services/cannon/bench/Bench.hs similarity index 100% rename from services/cannon/test/Bench.hs rename to services/cannon/bench/Bench.hs diff --git a/services/cannon/bench/Main.hs b/services/cannon/bench/Main.hs new file mode 100644 index 00000000000..d019b98000f --- /dev/null +++ b/services/cannon/bench/Main.hs @@ -0,0 +1,24 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2021 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Main where + +import Bench +import Imports + +main :: IO () +main = benchmark diff --git a/services/cannon/cannon.cabal b/services/cannon/cannon.cabal index 56f3ef1f40f..e91a0617087 100644 --- a/services/cannon/cannon.cabal +++ b/services/cannon/cannon.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: 6c53d0a25079c3947f669ee6cf4a32b3e9a9472db9050997e478fc8fdb7b3858 +-- hash: 87d683d193f2ad916c72f4c84d0816911f7d894cfbffed36f361436005571339 name: cannon version: 0.31.0 @@ -100,7 +100,6 @@ test-suite cannon-tests type: exitcode-stdio-1.0 main-is: Main.hs other-modules: - Bench Test.Cannon.Dict Paths_cannon hs-source-dirs: @@ -125,3 +124,32 @@ test-suite cannon-tests , uuid , wai-utilities default-language: Haskell2010 + +benchmark cannon-bench + type: exitcode-stdio-1.0 + main-is: Main.hs + other-modules: + Bench + Paths_cannon + hs-source-dirs: + bench + default-extensions: AllowAmbiguousTypes BangPatterns ConstraintKinds DataKinds DefaultSignatures DerivingStrategies DerivingVia DeriveFunctor DeriveGeneric DeriveLift DeriveTraversable EmptyCase FlexibleContexts FlexibleInstances FunctionalDependencies GADTs InstanceSigs KindSignatures LambdaCase MultiParamTypeClasses MultiWayIf NamedFieldPuns NoImplicitPrelude OverloadedStrings PackageImports PatternSynonyms PolyKinds QuasiQuotes RankNTypes ScopedTypeVariables StandaloneDeriving TemplateHaskell TupleSections TypeApplications TypeFamilies TypeFamilyDependencies TypeOperators UndecidableInstances ViewPatterns + ghc-options: -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates -Wpartial-fields -fwarn-tabs -optP-Wno-nonportable-include-path -threaded -with-rtsopts=-N + build-depends: + QuickCheck >=2.7 + , async + , base + , bytestring + , cannon + , criterion >=1.0 + , extended + , imports + , metrics-wai + , random >=1.0 + , tasty >=0.8 + , tasty-hunit >=0.8 + , tasty-quickcheck >=0.8 + , types-common + , uuid + , wai-utilities + default-language: Haskell2010 diff --git a/services/cannon/package.yaml b/services/cannon/package.yaml index 91148258420..c71d26a72b1 100644 --- a/services/cannon/package.yaml +++ b/services/cannon/package.yaml @@ -88,6 +88,28 @@ tests: - types-common - uuid - wai-utilities +benchmarks: + cannon-bench: + main: Main.hs + source-dirs: bench + ghc-options: + - -threaded + - -with-rtsopts=-N + dependencies: + - async + - base + - bytestring + - cannon + - criterion >=1.0 + - metrics-wai + - QuickCheck >=2.7 + - random >=1.0 + - tasty >=0.8 + - tasty-hunit >=0.8 + - tasty-quickcheck >=0.8 + - types-common + - uuid + - wai-utilities flags: static: description: Enable static linking diff --git a/services/cannon/test/Main.hs b/services/cannon/test/Main.hs index eb09db312c9..75517337702 100644 --- a/services/cannon/test/Main.hs +++ b/services/cannon/test/Main.hs @@ -17,7 +17,6 @@ module Main where -import qualified Bench as B import qualified Cannon.API import Data.Metrics.Test (pathsConsistencyCheck) import Data.Metrics.WaiRoute (treeToPaths) @@ -28,8 +27,7 @@ import Test.Tasty import Test.Tasty.HUnit main :: IO () -main = do - B.benchmark +main = defaultMain $ testGroup "Tests" From 215d34d9cd3174df5571684fc72438561b05de1d Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Wed, 15 Dec 2021 15:36:23 +0100 Subject: [PATCH 09/74] Move Integration Test BSI Tags to Top (#1996) * Moved integration test BSI tags to top * More descriptive test names --- services/galley/test/integration/API.hs | 58 ++++++++++++------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/services/galley/test/integration/API.hs b/services/galley/test/integration/API.hs index 801d020109d..0186f3448d8 100644 --- a/services/galley/test/integration/API.hs +++ b/services/galley/test/integration/API.hs @@ -199,11 +199,11 @@ tests s = test s "conversation receipt mode update with remote members" putReceiptModeWithRemotesOk, test s "send typing indicators" postTypingIndicators, test s "leave connect conversation" leaveConnectConversation, - test s "post conversations/:cnv/otr/message: message delivery and missing clients" postCryptoMessage1, - test s "post conversations/:cnv/otr/message: mismatch and prekey fetching" postCryptoMessage2, - test s "post conversations/:cnv/otr/message: mismatch with protobuf" postCryptoMessage3, - test s "post conversations/:cnv/otr/message: unknown sender client" postCryptoMessage4, - test s "post conversations/:cnv/otr/message: ignore_missing and report_missing" postCryptoMessage5, + test s "post conversations/:cnv/otr/message: message delivery and missing clients" postCryptoMessageVerifyMsgSentAndRejectIfMissingClient, + test s "post conversations/:cnv/otr/message: mismatch and prekey fetching" postCryptoMessageVerifyRejectMissingClientAndRepondMissingPrekeysJson, + test s "post conversations/:cnv/otr/message: mismatch with protobuf" postCryptoMessageVerifyRejectMissingClientAndRepondMissingPrekeysProto, + test s "post conversations/:cnv/otr/message: unknown sender client" postCryptoMessageNotAuthorizeUnknownClient, + test s "post conversations/:cnv/otr/message: ignore_missing and report_missing" postCryptoMessageVerifyCorrectResponseIfIgnoreAndReportMissingQueryParam, test s "post message qualified - local owning backend - success" postMessageQualifiedLocalOwningBackendSuccess, test s "post message qualified - local owning backend - missing clients" postMessageQualifiedLocalOwningBackendMissingClients, test s "post message qualified - local owning backend - redundant and deleted clients" postMessageQualifiedLocalOwningBackendRedundantAndDeletedClients, @@ -366,11 +366,11 @@ postConvWithRemoteUsersOk = do EdConversation c' -> assertConvEquals cnv c' _ -> assertFailure "Unexpected event data" --- | This test verifies whether a message actually gets sent all the way to --- cannon. -- @SF.Separation @TSFI.RESTfulAPI @S2 -postCryptoMessage1 :: TestM () -postCryptoMessage1 = do +-- This test verifies whether a message actually gets sent all the way to +-- cannon. +postCryptoMessageVerifyMsgSentAndRejectIfMissingClient :: TestM () +postCryptoMessageVerifyMsgSentAndRejectIfMissingClient = do localDomain <- viewFederationDomain c <- view tsCannon (alice, ac) <- randomUserWithClient (someLastPrekeys !! 0) @@ -452,10 +452,10 @@ postCryptoMessage1 = do liftIO $ assertBool "unexpected equal clients" (bc /= bc2) assertNoMsg wsB2 (wsAssertOtr qconv qalice ac bc cipher) --- | This test verifies basic mismatch behaviour of the the JSON endpoint. -- @SF.Separation @TSFI.RESTfulAPI @S2 -postCryptoMessage2 :: TestM () -postCryptoMessage2 = do +-- This test verifies basic mismatch behavior of the the JSON endpoint. +postCryptoMessageVerifyRejectMissingClientAndRepondMissingPrekeysJson :: TestM () +postCryptoMessageVerifyRejectMissingClientAndRepondMissingPrekeysJson = do b <- view tsBrig (alice, ac) <- randomUserWithClient (someLastPrekeys !! 0) (bob, bc) <- randomUserWithClient (someLastPrekeys !! 1) @@ -478,10 +478,10 @@ postCryptoMessage2 = do Map.keys (userClientMap (getUserClientPrekeyMap p)) @=? [eve] Map.keys <$> Map.lookup eve (userClientMap (getUserClientPrekeyMap p)) @=? Just [ec] --- | This test verifies basic mismatch behaviour of the protobuf endpoint. -- @SF.Separation @TSFI.RESTfulAPI @S2 -postCryptoMessage3 :: TestM () -postCryptoMessage3 = do +-- This test verifies basic mismatch behaviour of the protobuf endpoint. +postCryptoMessageVerifyRejectMissingClientAndRepondMissingPrekeysProto :: TestM () +postCryptoMessageVerifyRejectMissingClientAndRepondMissingPrekeysProto = do b <- view tsBrig (alice, ac) <- randomUserWithClient (someLastPrekeys !! 0) (bob, bc) <- randomUserWithClient (someLastPrekeys !! 1) @@ -508,8 +508,8 @@ postCryptoMessage3 = do -- | This test verifies behaviour when an unknown client posts the message. Only -- tests the Protobuf endpoint. -postCryptoMessage4 :: TestM () -postCryptoMessage4 = do +postCryptoMessageNotAuthorizeUnknownClient :: TestM () +postCryptoMessageNotAuthorizeUnknownClient = do alice <- randomUser bob <- randomUser bc <- randomClient bob (someLastPrekeys !! 0) @@ -521,10 +521,10 @@ postCryptoMessage4 = do postProtoOtrMessage alice (ClientId "172618352518396") conv m !!! const 403 === statusCode --- | This test verifies the following scenario. +-- @SF.Separation @TSFI.RESTfulAPI @S2 +-- This test verifies the following scenario. -- A client sends a message to all clients of a group and one more who is not part of the group. -- The server must not send this message to client ids not part of the group. --- @SF.Separation @TSFI.RESTfulAPI @S2 postMessageClientNotInGroupDoesNotReceiveMsg :: TestM () postMessageClientNotInGroupDoesNotReceiveMsg = do localDomain <- viewFederationDomain @@ -547,9 +547,9 @@ postMessageClientNotInGroupDoesNotReceiveMsg = do checkEveGetsMsg checkChadDoesNotGetMsg --- | This test verifies that when a client sends a message not to all clients of a group then the server should reject the message and sent a notification to the sender (412 Missing clients). --- The test is somewhat redundant because this is already tested as part of other tests already. This is a stand alone test that solely tests the behavior described above. -- @SF.Separation @TSFI.RESTfulAPI @S2 +-- This test verifies that when a client sends a message not to all clients of a group then the server should reject the message and sent a notification to the sender (412 Missing clients). +-- The test is somewhat redundant because this is already tested as part of other tests already. This is a stand alone test that solely tests the behavior described above. postMessageRejectIfMissingClients :: TestM () postMessageRejectIfMissingClients = do (sender, senderClient) : allReceivers <- randomUserWithClient `traverse` someLastPrekeys @@ -575,11 +575,11 @@ postMessageRejectIfMissingClients = do mkMsg :: ByteString -> (UserId, ClientId) -> (UserId, ClientId, Text) mkMsg text (userId, clientId) = (userId, clientId, toBase64Text text) --- | This test verifies behaviour under various values of ignore_missing and --- report_missing. Only tests the JSON endpoint. -- @SF.Separation @TSFI.RESTfulAPI @S2 -postCryptoMessage5 :: TestM () -postCryptoMessage5 = do +-- This test verifies behaviour under various values of ignore_missing and +-- report_missing. Only tests the JSON endpoint. +postCryptoMessageVerifyCorrectResponseIfIgnoreAndReportMissingQueryParam :: TestM () +postCryptoMessageVerifyCorrectResponseIfIgnoreAndReportMissingQueryParam = do (alice, ac) <- randomUserWithClient (someLastPrekeys !! 0) (bob, bc) <- randomUserWithClient (someLastPrekeys !! 1) (chad, cc) <- randomUserWithClient (someLastPrekeys !! 2) @@ -766,10 +766,10 @@ postMessageQualifiedLocalOwningBackendSuccess = do WS.assertMatch_ t wsAlex2 (wsAssertOtr' encodedData convId alice aliceClient alexClient2 encodedTextForAlex2) WS.assertMatch_ t wsAmy (wsAssertOtr' encodedData convId alice aliceClient amyClient encodedTextForAmy) --- | Sets up a conversation on Backend A known as "owning backend". One of the +-- @SF.Separation @TSFI.RESTfulAPI @S2 +-- Sets up a conversation on Backend A known as "owning backend". One of the -- users from Backend A will send the message but have a missing client. It is -- expected that the message will not be sent. --- @SF.Separation @TSFI.RESTfulAPI @S2 postMessageQualifiedLocalOwningBackendMissingClients :: TestM () postMessageQualifiedLocalOwningBackendMissingClients = do -- Cannon for local users @@ -925,11 +925,11 @@ postMessageQualifiedLocalOwningBackendRedundantAndDeletedClients = do -- Wait less for no message WS.assertNoEvent (1 # Second) [wsNonMember] --- | Sets up a conversation on Backend A known as "owning backend". One of the +-- @SF.Separation @TSFI.RESTfulAPI @S2 +-- Sets up a conversation on Backend A known as "owning backend". One of the -- users from Backend A will send the message but have a missing client. It is -- expected that the message will be sent except when it is specifically -- requested to report on missing clients of a user. --- @SF.Separation @TSFI.RESTfulAPI @S2 postMessageQualifiedLocalOwningBackendIgnoreMissingClients :: TestM () postMessageQualifiedLocalOwningBackendIgnoreMissingClients = do -- WS receive timeout From f15892e44b2a0e06dbad9e585abb59b72a6ffc57 Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Wed, 15 Dec 2021 15:42:01 +0100 Subject: [PATCH 10/74] Remove resumable upload API (#1998) --- .../1-api-changes/remove-resumable-uploads | 1 + libs/cargohold-types/cargohold-types.cabal | 3 +- .../src/CargoHold/Types/V3/Resumable.hs | 36 -- libs/wire-api/src/Wire/API/Asset.hs | 1 - .../src/Wire/API/Asset/V3/Resumable.hs | 141 ------ .../golden/Test/Wire/API/Golden/Generated.hs | 29 -- .../API/Golden/Generated/ChunkSize_user.hs | 24 - .../Wire/API/Golden/Generated/Offset_user.hs | 22 - .../Golden/Generated/ResumableAsset_user.hs | 46 -- .../Generated/ResumableSettings_user.hs | 54 --- .../API/Golden/Generated/TotalSize_user.hs | 24 - .../golden/Test/Wire/API/Golden/Generator.hs | 6 - .../golden/testObject_ChunkSize_user_1.json | 1 - .../test/golden/testObject_Offset_user_1.json | 1 - .../testObject_ResumableAsset_user_1.json | 9 - .../testObject_ResumableSettings_user_1.json | 5 - .../testObject_ResumableSettings_user_17.json | 5 - .../testObject_ResumableSettings_user_2.json | 5 - .../testObject_ResumableSettings_user_4.json | 5 - .../testObject_ResumableSettings_user_5.json | 5 - .../golden/testObject_TotalSize_user_1.json | 1 - .../unit/Test/Wire/API/Roundtrip/Aeson.hs | 6 - .../Test/Wire/API/Roundtrip/ByteString.hs | 4 - libs/wire-api/wire-api.cabal | 8 +- services/cargohold/cargohold.cabal | 4 +- services/cargohold/src/CargoHold/API/Error.hs | 27 -- .../cargohold/src/CargoHold/API/Public.hs | 64 +-- .../src/CargoHold/API/V3/Resumable.hs | 149 ------ services/cargohold/src/CargoHold/S3.hs | 459 ------------------ services/cargohold/src/CargoHold/TUS.hs | 89 ---- services/cargohold/test/integration/API/V3.hs | 171 +------ 31 files changed, 10 insertions(+), 1395 deletions(-) create mode 100644 changelog.d/1-api-changes/remove-resumable-uploads delete mode 100644 libs/cargohold-types/src/CargoHold/Types/V3/Resumable.hs delete mode 100644 libs/wire-api/src/Wire/API/Asset/V3/Resumable.hs delete mode 100644 libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/ChunkSize_user.hs delete mode 100644 libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/Offset_user.hs delete mode 100644 libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/ResumableAsset_user.hs delete mode 100644 libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/ResumableSettings_user.hs delete mode 100644 libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/TotalSize_user.hs delete mode 100644 libs/wire-api/test/golden/testObject_ChunkSize_user_1.json delete mode 100644 libs/wire-api/test/golden/testObject_Offset_user_1.json delete mode 100644 libs/wire-api/test/golden/testObject_ResumableAsset_user_1.json delete mode 100644 libs/wire-api/test/golden/testObject_ResumableSettings_user_1.json delete mode 100644 libs/wire-api/test/golden/testObject_ResumableSettings_user_17.json delete mode 100644 libs/wire-api/test/golden/testObject_ResumableSettings_user_2.json delete mode 100644 libs/wire-api/test/golden/testObject_ResumableSettings_user_4.json delete mode 100644 libs/wire-api/test/golden/testObject_ResumableSettings_user_5.json delete mode 100644 libs/wire-api/test/golden/testObject_TotalSize_user_1.json delete mode 100644 services/cargohold/src/CargoHold/API/V3/Resumable.hs delete mode 100644 services/cargohold/src/CargoHold/TUS.hs diff --git a/changelog.d/1-api-changes/remove-resumable-uploads b/changelog.d/1-api-changes/remove-resumable-uploads new file mode 100644 index 00000000000..a73bc51b5bc --- /dev/null +++ b/changelog.d/1-api-changes/remove-resumable-uploads @@ -0,0 +1 @@ +Remove resumable upload API diff --git a/libs/cargohold-types/cargohold-types.cabal b/libs/cargohold-types/cargohold-types.cabal index 6626e3dd6f5..05263ea04d9 100644 --- a/libs/cargohold-types/cargohold-types.cabal +++ b/libs/cargohold-types/cargohold-types.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: 6e7a4ce0ec22392573f2b2764073d89b1096322eff38e0af765761e6bb2257a2 +-- hash: 33860dd6b7263a9496b0ffc60303472a71fc9da43168c9cd017f6c2eb626faa3 name: cargohold-types version: 1.5.0 @@ -21,7 +21,6 @@ library exposed-modules: CargoHold.Types CargoHold.Types.V3 - CargoHold.Types.V3.Resumable other-modules: Paths_cargohold_types hs-source-dirs: diff --git a/libs/cargohold-types/src/CargoHold/Types/V3/Resumable.hs b/libs/cargohold-types/src/CargoHold/Types/V3/Resumable.hs deleted file mode 100644 index f84ac4bdf08..00000000000 --- a/libs/cargohold-types/src/CargoHold/Types/V3/Resumable.hs +++ /dev/null @@ -1,36 +0,0 @@ --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2020 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module CargoHold.Types.V3.Resumable - ( -- * re-exports - ResumableSettings, - mkResumableSettings, - setResumableType, - setResumablePublic, - setResumableRetention, - ResumableAsset, - TotalSize (..), - ChunkSize (..), - Offset (..), - mkResumableAsset, - resumableAsset, - resumableExpires, - resumableChunkSize, - ) -where - -import Wire.API.Asset.V3.Resumable diff --git a/libs/wire-api/src/Wire/API/Asset.hs b/libs/wire-api/src/Wire/API/Asset.hs index 544dca0df52..94522eba44e 100644 --- a/libs/wire-api/src/Wire/API/Asset.hs +++ b/libs/wire-api/src/Wire/API/Asset.hs @@ -21,4 +21,3 @@ module Wire.API.Asset where import Wire.API.Asset.V3 as V3 -import Wire.API.Asset.V3.Resumable as V3 diff --git a/libs/wire-api/src/Wire/API/Asset/V3/Resumable.hs b/libs/wire-api/src/Wire/API/Asset/V3/Resumable.hs deleted file mode 100644 index 60e23ed9a32..00000000000 --- a/libs/wire-api/src/Wire/API/Asset/V3/Resumable.hs +++ /dev/null @@ -1,141 +0,0 @@ -{-# LANGUAGE GeneralizedNewtypeDeriving #-} -{-# LANGUAGE StrictData #-} -{-# LANGUAGE TemplateHaskell #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2020 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Wire.API.Asset.V3.Resumable - ( -- * ResumableSettings - ResumableSettings, - mkResumableSettings, - setResumableType, - setResumablePublic, - setResumableRetention, - - -- * ResumableAsset - ResumableAsset, - mkResumableAsset, - TotalSize (..), - ChunkSize (..), - Offset (..), - resumableAsset, - resumableExpires, - resumableChunkSize, - ) -where - -import qualified Codec.MIME.Parse as MIME -import qualified Codec.MIME.Type as MIME -import Control.Lens (makeLenses) -import Data.Aeson -import Data.Aeson.Types -import Data.ByteString.Conversion -import Data.Json.Util (UTCTimeMillis (fromUTCTimeMillis), toUTCTimeMillis, (#)) -import Data.Time.Clock -import Imports -import Wire.API.Arbitrary (Arbitrary (..), GenericUniform (..)) -import Wire.API.Asset.V3 - --------------------------------------------------------------------------------- --- ResumableSettings - --- | Settings for initiating a resumable upload. -data ResumableSettings = ResumableSettings - { _setResumableRetention :: AssetRetention, - _setResumablePublic :: Bool, - _setResumableType :: MIME.Type - } - deriving stock (Eq, Show, Generic) - deriving (Arbitrary) via (GenericUniform ResumableSettings) - -makeLenses ''ResumableSettings - -mkResumableSettings :: AssetRetention -> Bool -> MIME.Type -> ResumableSettings -mkResumableSettings = ResumableSettings - -instance ToJSON ResumableSettings where - toJSON (ResumableSettings ret pub typ) = - object $ - "retention" .= ret - # "type" .= MIME.showType typ - # "public" .= pub - # [] - -instance FromJSON ResumableSettings where - parseJSON = withObject "ResumableSettings" $ \o -> - ResumableSettings - <$> o .:? "retention" .!= AssetPersistent - <*> o .:? "public" .!= False - <*> (parseMime =<< o .: "type") - -parseMime :: Text -> Parser MIME.Type -parseMime v = - maybe - (fail "Invalid MIME type") - return - (MIME.parseMIMEType v) - --------------------------------------------------------------------------------- --- ResumableAsset - -newtype TotalSize = TotalSize - {totalSizeBytes :: Word} - deriving stock (Eq, Ord, Show) - deriving newtype (Enum, Num, Real, Integral, FromJSON, ToJSON, FromByteString, ToByteString, Arbitrary) - -newtype ChunkSize = ChunkSize - {chunkSizeBytes :: Word} - deriving stock (Eq, Ord, Show) - deriving newtype (Enum, Num, Real, Integral, FromJSON, ToJSON, FromByteString, ToByteString, Arbitrary) - -newtype Offset = Offset - {offsetBytes :: Word} - deriving stock (Eq, Ord, Show) - deriving newtype (Enum, Num, Real, Integral, FromJSON, ToJSON, FromByteString, ToByteString, Arbitrary) - -data ResumableAsset = ResumableAsset - { _resumableAsset :: Asset, - _resumableExpires :: UTCTime, - _resumableChunkSize :: ChunkSize - } - deriving stock (Eq, Show, Generic) - -instance Arbitrary ResumableAsset where - arbitrary = ResumableAsset <$> arbitrary <*> (milli <$> arbitrary) <*> arbitrary - where - milli = fromUTCTimeMillis . toUTCTimeMillis - -makeLenses ''ResumableAsset - -mkResumableAsset :: Asset -> UTCTime -> ChunkSize -> ResumableAsset -mkResumableAsset = ResumableAsset - -instance ToJSON ResumableAsset where - toJSON r = - object $ - "asset" .= _resumableAsset r - # "expires" .= toUTCTimeMillis (_resumableExpires r) - # "chunk_size" .= _resumableChunkSize r - # [] - -instance FromJSON ResumableAsset where - parseJSON = withObject "ResumableAsset" $ \o -> - ResumableAsset - <$> o .: "asset" - <*> o .: "expires" - <*> o .: "chunk_size" diff --git a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated.hs b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated.hs index 7eacbb74d24..f4a5dff0652 100644 --- a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated.hs +++ b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated.hs @@ -42,7 +42,6 @@ import qualified Test.Wire.API.Golden.Generated.BindingNewTeam_team import qualified Test.Wire.API.Golden.Generated.BotConvView_provider import qualified Test.Wire.API.Golden.Generated.BotUserView_provider import qualified Test.Wire.API.Golden.Generated.CheckHandles_user -import qualified Test.Wire.API.Golden.Generated.ChunkSize_user import qualified Test.Wire.API.Golden.Generated.ClientClass_user import qualified Test.Wire.API.Golden.Generated.ClientMismatch_user import qualified Test.Wire.API.Golden.Generated.ClientPrekey_user @@ -127,7 +126,6 @@ import qualified Test.Wire.API.Golden.Generated.NewService_provider import qualified Test.Wire.API.Golden.Generated.NewTeamMember_team import qualified Test.Wire.API.Golden.Generated.NewUserPublic_user import qualified Test.Wire.API.Golden.Generated.NewUser_user -import qualified Test.Wire.API.Golden.Generated.Offset_user import qualified Test.Wire.API.Golden.Generated.OtherMemberUpdate_user import qualified Test.Wire.API.Golden.Generated.OtherMember_user import qualified Test.Wire.API.Golden.Generated.OtrMessage_user @@ -166,8 +164,6 @@ import qualified Test.Wire.API.Golden.Generated.RemoveBotResponse_user import qualified Test.Wire.API.Golden.Generated.RemoveCookies_user import qualified Test.Wire.API.Golden.Generated.RemoveLegalHoldSettingsRequest_team import qualified Test.Wire.API.Golden.Generated.RequestNewLegalHoldClient_team -import qualified Test.Wire.API.Golden.Generated.ResumableAsset_user -import qualified Test.Wire.API.Golden.Generated.ResumableSettings_user import qualified Test.Wire.API.Golden.Generated.RichField_user import qualified Test.Wire.API.Golden.Generated.RichInfoAssocList_user import qualified Test.Wire.API.Golden.Generated.RichInfoMapAndList_user @@ -212,7 +208,6 @@ import qualified Test.Wire.API.Golden.Generated.TeamUpdateData_team import qualified Test.Wire.API.Golden.Generated.Team_team import qualified Test.Wire.API.Golden.Generated.TokenType_user import qualified Test.Wire.API.Golden.Generated.Token_user -import qualified Test.Wire.API.Golden.Generated.TotalSize_user import qualified Test.Wire.API.Golden.Generated.Transport_user import qualified Test.Wire.API.Golden.Generated.TurnHost_user import qualified Test.Wire.API.Golden.Generated.TurnURI_user @@ -293,30 +288,6 @@ tests = (Test.Wire.API.Golden.Generated.AssetKey_user.testObject_AssetKey_user_4, "testObject_AssetKey_user_4.json"), (Test.Wire.API.Golden.Generated.AssetKey_user.testObject_AssetKey_user_5, "testObject_AssetKey_user_5.json") ], - testGroup "Golden: ResumableSettings_user" $ - testObjects - [ (Test.Wire.API.Golden.Generated.ResumableSettings_user.testObject_ResumableSettings_user_1, "testObject_ResumableSettings_user_1.json"), - (Test.Wire.API.Golden.Generated.ResumableSettings_user.testObject_ResumableSettings_user_2, "testObject_ResumableSettings_user_2.json"), - (Test.Wire.API.Golden.Generated.ResumableSettings_user.testObject_ResumableSettings_user_4, "testObject_ResumableSettings_user_4.json"), - (Test.Wire.API.Golden.Generated.ResumableSettings_user.testObject_ResumableSettings_user_5, "testObject_ResumableSettings_user_5.json"), - (Test.Wire.API.Golden.Generated.ResumableSettings_user.testObject_ResumableSettings_user_17, "testObject_ResumableSettings_user_17.json") - ], - testGroup "Golden: TotalSize_user" $ - testObjects - [ (Test.Wire.API.Golden.Generated.TotalSize_user.testObject_TotalSize_user_1, "testObject_TotalSize_user_1.json") - ], - testGroup "Golden: ChunkSize_user" $ - testObjects - [ (Test.Wire.API.Golden.Generated.ChunkSize_user.testObject_ChunkSize_user_1, "testObject_ChunkSize_user_1.json") - ], - testGroup "Golden: Offset_user" $ - testObjects - [ (Test.Wire.API.Golden.Generated.Offset_user.testObject_Offset_user_1, "testObject_Offset_user_1.json") - ], - testGroup "Golden: ResumableAsset_user" $ - testObjects - [ (Test.Wire.API.Golden.Generated.ResumableAsset_user.testObject_ResumableAsset_user_1, "testObject_ResumableAsset_user_1.json") - ], testGroup "Golden: TurnHost_user" $ testObjects [ (Test.Wire.API.Golden.Generated.TurnHost_user.testObject_TurnHost_user_1, "testObject_TurnHost_user_1.json"), diff --git a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/ChunkSize_user.hs b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/ChunkSize_user.hs deleted file mode 100644 index c12c6163d29..00000000000 --- a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/ChunkSize_user.hs +++ /dev/null @@ -1,24 +0,0 @@ -{-# LANGUAGE OverloadedLists #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2021 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . -module Test.Wire.API.Golden.Generated.ChunkSize_user where - -import Wire.API.Asset (ChunkSize (..)) - -testObject_ChunkSize_user_1 :: ChunkSize -testObject_ChunkSize_user_1 = ChunkSize {chunkSizeBytes = 17} diff --git a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/Offset_user.hs b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/Offset_user.hs deleted file mode 100644 index b6aa293045a..00000000000 --- a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/Offset_user.hs +++ /dev/null @@ -1,22 +0,0 @@ --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2021 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . -module Test.Wire.API.Golden.Generated.Offset_user where - -import Wire.API.Asset (Offset (..)) - -testObject_Offset_user_1 :: Offset -testObject_Offset_user_1 = Offset {offsetBytes = 1} diff --git a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/ResumableAsset_user.hs b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/ResumableAsset_user.hs deleted file mode 100644 index 98798a95470..00000000000 --- a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/ResumableAsset_user.hs +++ /dev/null @@ -1,46 +0,0 @@ --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2021 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . -module Test.Wire.API.Golden.Generated.ResumableAsset_user where - -import Control.Lens ((.~), (?~)) -import Data.Id (Id (Id)) -import Data.Text.Ascii (AsciiChars (validate)) -import qualified Data.UUID as UUID (fromString) -import Imports (Functor (fmap), Maybe (Just), fromJust, fromRight, read, undefined, (&)) -import Wire.API.Asset - ( AssetKey (AssetKeyV3), - AssetRetention - ( AssetExpiring - ), - AssetToken (AssetToken, assetTokenAscii), - ChunkSize (ChunkSize, chunkSizeBytes), - ResumableAsset, - assetExpires, - assetToken, - mkAsset, - mkResumableAsset, - ) - -testObject_ResumableAsset_user_1 :: ResumableAsset -testObject_ResumableAsset_user_1 = - mkResumableAsset - ( mkAsset (AssetKeyV3 (Id (fromJust (UUID.fromString "00000010-0000-0008-0000-004300000006"))) AssetExpiring) - & assetExpires .~ fmap read (Just "1864-04-13 11:37:47.393 UTC") - & assetToken ?~ (AssetToken {assetTokenAscii = fromRight undefined (validate "5A==")}) - ) - (read "1864-04-09 06:01:25.576 UTC") - (ChunkSize {chunkSizeBytes = 17}) diff --git a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/ResumableSettings_user.hs b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/ResumableSettings_user.hs deleted file mode 100644 index 3c82024710f..00000000000 --- a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/ResumableSettings_user.hs +++ /dev/null @@ -1,54 +0,0 @@ -{-# LANGUAGE OverloadedLists #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2021 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . -module Test.Wire.API.Golden.Generated.ResumableSettings_user where - -import Codec.MIME.Type (Type (..)) -import qualified Codec.MIME.Type as MIME (MIMEType (Image)) -import Imports (Bool (False, True)) -import Wire.API.Asset - ( AssetRetention - ( AssetEternal, - AssetEternalInfrequentAccess, - AssetExpiring, - AssetPersistent, - AssetVolatile - ), - ResumableSettings, - mkResumableSettings, - ) - -testObject_ResumableSettings_user_1 :: ResumableSettings -testObject_ResumableSettings_user_1 = - mkResumableSettings AssetExpiring False (Type {mimeType = MIME.Image "png", mimeParams = []}) - -testObject_ResumableSettings_user_2 :: ResumableSettings -testObject_ResumableSettings_user_2 = - mkResumableSettings AssetEternal True (Type {mimeType = MIME.Image "png", mimeParams = []}) - -testObject_ResumableSettings_user_4 :: ResumableSettings -testObject_ResumableSettings_user_4 = - mkResumableSettings AssetEternalInfrequentAccess True (Type {mimeType = MIME.Image "png", mimeParams = []}) - -testObject_ResumableSettings_user_5 :: ResumableSettings -testObject_ResumableSettings_user_5 = - mkResumableSettings AssetPersistent False (Type {mimeType = MIME.Image "png", mimeParams = []}) - -testObject_ResumableSettings_user_17 :: ResumableSettings -testObject_ResumableSettings_user_17 = - mkResumableSettings AssetVolatile True (Type {mimeType = MIME.Image "png", mimeParams = []}) diff --git a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/TotalSize_user.hs b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/TotalSize_user.hs deleted file mode 100644 index d60539a5593..00000000000 --- a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/TotalSize_user.hs +++ /dev/null @@ -1,24 +0,0 @@ -{-# LANGUAGE OverloadedLists #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2021 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . -module Test.Wire.API.Golden.Generated.TotalSize_user where - -import Wire.API.Asset (TotalSize (..)) - -testObject_TotalSize_user_1 :: TotalSize -testObject_TotalSize_user_1 = TotalSize {totalSizeBytes = 9} diff --git a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generator.hs b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generator.hs index e4daf3da24b..db70e032203 100644 --- a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generator.hs +++ b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generator.hs @@ -23,7 +23,6 @@ import System.IO (Handle, hPutStr, hPutStrLn, openFile) import Test.Tasty.QuickCheck (Arbitrary (..), generate) import Type.Reflection (typeRep) import qualified Wire.API.Asset as Asset -import qualified Wire.API.Asset.V3.Resumable as Asset.Resumable import qualified Wire.API.Call.Config as Call.Config import qualified Wire.API.Connection as Connection import qualified Wire.API.Conversation as Conversation @@ -136,11 +135,6 @@ generateTestModule = do generateBindingModule @Asset.AssetRetention "user" ref generateBindingModule @Asset.AssetSettings "user" ref generateBindingModule @Asset.AssetKey "user" ref - generateBindingModule @Asset.Resumable.ResumableSettings "user" ref - generateBindingModule @Asset.Resumable.TotalSize "user" ref - generateBindingModule @Asset.Resumable.ChunkSize "user" ref - generateBindingModule @Asset.Resumable.Offset "user" ref - generateBindingModule @Asset.Resumable.ResumableAsset "user" ref generateBindingModule @Call.Config.TurnHost "user" ref generateBindingModule @Call.Config.Scheme "user" ref generateBindingModule @Call.Config.Transport "user" ref diff --git a/libs/wire-api/test/golden/testObject_ChunkSize_user_1.json b/libs/wire-api/test/golden/testObject_ChunkSize_user_1.json deleted file mode 100644 index 98d9bcb75a6..00000000000 --- a/libs/wire-api/test/golden/testObject_ChunkSize_user_1.json +++ /dev/null @@ -1 +0,0 @@ -17 diff --git a/libs/wire-api/test/golden/testObject_Offset_user_1.json b/libs/wire-api/test/golden/testObject_Offset_user_1.json deleted file mode 100644 index d00491fd7e5..00000000000 --- a/libs/wire-api/test/golden/testObject_Offset_user_1.json +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/libs/wire-api/test/golden/testObject_ResumableAsset_user_1.json b/libs/wire-api/test/golden/testObject_ResumableAsset_user_1.json deleted file mode 100644 index b31ce6ad354..00000000000 --- a/libs/wire-api/test/golden/testObject_ResumableAsset_user_1.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "asset": { - "expires": "1864-04-13T11:37:47.393Z", - "key": "3-5-00000010-0000-0008-0000-004300000006", - "token": "5A==" - }, - "chunk_size": 17, - "expires": "1864-04-09T06:01:25.576Z" -} diff --git a/libs/wire-api/test/golden/testObject_ResumableSettings_user_1.json b/libs/wire-api/test/golden/testObject_ResumableSettings_user_1.json deleted file mode 100644 index 6fe2dd17d1f..00000000000 --- a/libs/wire-api/test/golden/testObject_ResumableSettings_user_1.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "public": false, - "retention": "expiring", - "type": "image/png" -} diff --git a/libs/wire-api/test/golden/testObject_ResumableSettings_user_17.json b/libs/wire-api/test/golden/testObject_ResumableSettings_user_17.json deleted file mode 100644 index 462ed70e3eb..00000000000 --- a/libs/wire-api/test/golden/testObject_ResumableSettings_user_17.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "public": true, - "retention": "volatile", - "type": "image/png" -} diff --git a/libs/wire-api/test/golden/testObject_ResumableSettings_user_2.json b/libs/wire-api/test/golden/testObject_ResumableSettings_user_2.json deleted file mode 100644 index 8223d85e2dd..00000000000 --- a/libs/wire-api/test/golden/testObject_ResumableSettings_user_2.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "public": true, - "retention": "eternal", - "type": "image/png" -} diff --git a/libs/wire-api/test/golden/testObject_ResumableSettings_user_4.json b/libs/wire-api/test/golden/testObject_ResumableSettings_user_4.json deleted file mode 100644 index 7b059d3d67e..00000000000 --- a/libs/wire-api/test/golden/testObject_ResumableSettings_user_4.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "public": true, - "retention": "eternal-infrequent_access", - "type": "image/png" -} diff --git a/libs/wire-api/test/golden/testObject_ResumableSettings_user_5.json b/libs/wire-api/test/golden/testObject_ResumableSettings_user_5.json deleted file mode 100644 index 13648db881b..00000000000 --- a/libs/wire-api/test/golden/testObject_ResumableSettings_user_5.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "public": false, - "retention": "persistent", - "type": "image/png" -} diff --git a/libs/wire-api/test/golden/testObject_TotalSize_user_1.json b/libs/wire-api/test/golden/testObject_TotalSize_user_1.json deleted file mode 100644 index ec635144f60..00000000000 --- a/libs/wire-api/test/golden/testObject_TotalSize_user_1.json +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/libs/wire-api/test/unit/Test/Wire/API/Roundtrip/Aeson.hs b/libs/wire-api/test/unit/Test/Wire/API/Roundtrip/Aeson.hs index 2b10b608d54..a3d5e85f7e8 100644 --- a/libs/wire-api/test/unit/Test/Wire/API/Roundtrip/Aeson.hs +++ b/libs/wire-api/test/unit/Test/Wire/API/Roundtrip/Aeson.hs @@ -26,7 +26,6 @@ import qualified Test.Tasty as T import Test.Tasty.QuickCheck (Arbitrary, counterexample, testProperty, (.&&.), (===)) import Type.Reflection (typeRep) import qualified Wire.API.Asset as Asset -import qualified Wire.API.Asset.V3.Resumable as Asset.Resumable import qualified Wire.API.Call.Config as Call.Config import qualified Wire.API.Connection as Connection import qualified Wire.API.Conversation as Conversation @@ -81,11 +80,6 @@ tests = testRoundTrip @Asset.AssetSettings, testRoundTrip @Asset.AssetKey, testRoundTrip @Asset.Asset, - testRoundTrip @Asset.Resumable.ResumableSettings, - testRoundTrip @Asset.Resumable.TotalSize, - testRoundTrip @Asset.Resumable.ChunkSize, - testRoundTrip @Asset.Resumable.Offset, - testRoundTrip @Asset.Resumable.ResumableAsset, testRoundTrip @Call.Config.TurnHost, testRoundTrip @Call.Config.Scheme, testRoundTrip @Call.Config.Transport, diff --git a/libs/wire-api/test/unit/Test/Wire/API/Roundtrip/ByteString.hs b/libs/wire-api/test/unit/Test/Wire/API/Roundtrip/ByteString.hs index 58a2e1c27b5..05cb4d975d0 100644 --- a/libs/wire-api/test/unit/Test/Wire/API/Roundtrip/ByteString.hs +++ b/libs/wire-api/test/unit/Test/Wire/API/Roundtrip/ByteString.hs @@ -24,7 +24,6 @@ import Test.Tasty.QuickCheck (Arbitrary, counterexample, testProperty, (===)) import Type.Reflection (typeRep) import qualified Wire.API.Arbitrary as Arbitrary () import qualified Wire.API.Asset.V3 as Asset.V3 -import qualified Wire.API.Asset.V3.Resumable as Asset.V3.Resumable import qualified Wire.API.Call.Config as Call.Config import qualified Wire.API.Conversation.Code as Conversation.Code import qualified Wire.API.Conversation.Role as Conversation.Role @@ -50,9 +49,6 @@ tests = [ testRoundTrip @Asset.V3.AssetKey, testRoundTrip @Asset.V3.AssetRetention, testRoundTrip @Asset.V3.AssetToken, - testRoundTrip @Asset.V3.Resumable.ChunkSize, - testRoundTrip @Asset.V3.Resumable.Offset, - testRoundTrip @Asset.V3.Resumable.TotalSize, testRoundTrip @Call.Config.Scheme, testRoundTrip @Call.Config.Transport, testRoundTrip @Call.Config.TurnHost, diff --git a/libs/wire-api/wire-api.cabal b/libs/wire-api/wire-api.cabal index ccb35be4219..6d643e1c4d8 100644 --- a/libs/wire-api/wire-api.cabal +++ b/libs/wire-api/wire-api.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: fff87c53181cb0f2cd21a3da254e2a9a5dbae2ee00ea67ac124be1593a640e5e +-- hash: bcda0d57293162140cc50d149ad068df7975f9526cf1621c7aa4f753b45e36a6 name: wire-api version: 0.1.0 @@ -22,7 +22,6 @@ library Wire.API.Arbitrary Wire.API.Asset Wire.API.Asset.V3 - Wire.API.Asset.V3.Resumable Wire.API.Call.Config Wire.API.Connection Wire.API.Conversation @@ -204,7 +203,6 @@ test-suite wire-api-golden-tests Test.Wire.API.Golden.Generated.BotConvView_provider Test.Wire.API.Golden.Generated.BotUserView_provider Test.Wire.API.Golden.Generated.CheckHandles_user - Test.Wire.API.Golden.Generated.ChunkSize_user Test.Wire.API.Golden.Generated.Client_user Test.Wire.API.Golden.Generated.ClientClass_user Test.Wire.API.Golden.Generated.ClientMismatch_user @@ -289,7 +287,6 @@ test-suite wire-api-golden-tests Test.Wire.API.Golden.Generated.NewTeamMember_team Test.Wire.API.Golden.Generated.NewUser_user Test.Wire.API.Golden.Generated.NewUserPublic_user - Test.Wire.API.Golden.Generated.Offset_user Test.Wire.API.Golden.Generated.OtherMember_user Test.Wire.API.Golden.Generated.OtherMemberUpdate_user Test.Wire.API.Golden.Generated.OtrMessage_user @@ -327,8 +324,6 @@ test-suite wire-api-golden-tests Test.Wire.API.Golden.Generated.RemoveCookies_user Test.Wire.API.Golden.Generated.RemoveLegalHoldSettingsRequest_team Test.Wire.API.Golden.Generated.RequestNewLegalHoldClient_team - Test.Wire.API.Golden.Generated.ResumableAsset_user - Test.Wire.API.Golden.Generated.ResumableSettings_user Test.Wire.API.Golden.Generated.RichField_user Test.Wire.API.Golden.Generated.RichInfo_user Test.Wire.API.Golden.Generated.RichInfoAssocList_user @@ -375,7 +370,6 @@ test-suite wire-api-golden-tests Test.Wire.API.Golden.Generated.TeamUpdateData_team Test.Wire.API.Golden.Generated.Token_user Test.Wire.API.Golden.Generated.TokenType_user - Test.Wire.API.Golden.Generated.TotalSize_user Test.Wire.API.Golden.Generated.Transport_user Test.Wire.API.Golden.Generated.TurnHost_user Test.Wire.API.Golden.Generated.TurnURI_user diff --git a/services/cargohold/cargohold.cabal b/services/cargohold/cargohold.cabal index dd689436f55..94c6b8635f6 100644 --- a/services/cargohold/cargohold.cabal +++ b/services/cargohold/cargohold.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: 43240dbac626b3b23a6c7631367cc8c708e417b55fa7b007b4a58885878f8911 +-- hash: e1d35804687900338efb4ea42019d902f778d6b03ea61e3e8a1d3e0fcba0115e name: cargohold version: 1.5.0 @@ -31,7 +31,6 @@ library CargoHold.API.Legacy CargoHold.API.Public CargoHold.API.V3 - CargoHold.API.V3.Resumable CargoHold.App CargoHold.AWS CargoHold.CloudFront @@ -39,7 +38,6 @@ library CargoHold.Options CargoHold.Run CargoHold.S3 - CargoHold.TUS CargoHold.Util Main other-modules: diff --git a/services/cargohold/src/CargoHold/API/Error.hs b/services/cargohold/src/CargoHold/API/Error.hs index d8be728e837..bffd8ba433c 100644 --- a/services/cargohold/src/CargoHold/API/Error.hs +++ b/services/cargohold/src/CargoHold/API/Error.hs @@ -17,9 +17,6 @@ module CargoHold.API.Error where -import CargoHold.Types.V3.Resumable (Offset, TotalSize) -import Data.Text.Lazy.Builder -import Data.Text.Lazy.Builder.Int import Imports import Network.HTTP.Types.Status import Network.Wai.Utilities.Error @@ -48,18 +45,6 @@ requestTimeout = \but none was sent over an extended period of time. Idle connections \ \will be closed." -invalidOffset :: Offset -> Offset -> Error -invalidOffset expected given = - mkError status409 "invalid-offset" $ - toLazyText $ - "Invalid offset: " - <> "expected: " - <> decimal expected - <> ", " - <> "given: " - <> decimal given - <> "." - uploadTooSmall :: Error uploadTooSmall = mkError @@ -76,18 +61,6 @@ uploadTooLarge = "The current chunk size + offset \ \is larger than the full upload size." -uploadIncomplete :: TotalSize -> TotalSize -> Error -uploadIncomplete expected actual = - mkError status403 "client-error" $ - toLazyText $ - "The upload is incomplete: " - <> "expected size: " - <> decimal expected - <> ", " - <> "current size: " - <> decimal actual - <> "." - clientError :: LText -> Error clientError = mkError status400 "client-error" diff --git a/services/cargohold/src/CargoHold/API/Public.hs b/services/cargohold/src/CargoHold/API/Public.hs index 0d141ad808c..b2b4660e5be 100644 --- a/services/cargohold/src/CargoHold/API/Public.hs +++ b/services/cargohold/src/CargoHold/API/Public.hs @@ -24,13 +24,10 @@ where import qualified CargoHold.API.Error as Error import qualified CargoHold.API.Legacy as LegacyAPI import qualified CargoHold.API.V3 as V3 -import qualified CargoHold.API.V3.Resumable as Resumable import CargoHold.App -import CargoHold.Options -import qualified CargoHold.TUS as TUS import qualified CargoHold.Types.V3 as V3 (Principal (..)) import Control.Error -import Control.Lens (view, (^.)) +import Control.Lens ((^.)) import Data.ByteString.Conversion import Data.Id import Data.Predicate @@ -49,6 +46,9 @@ import Network.Wai.Utilities.ZAuth import URI.ByteString import qualified Wire.API.Asset as Public +-- FUTUREWORK: restore (and servantify) resumable upload functionality, removed +-- in https://github.com/wireapp/wire-server/pull/1998 + sitemap :: Routes Doc.ApiBuilder Handler () sitemap = do --------------------------------------------------------------------------- @@ -67,35 +67,6 @@ sitemap = do Doc.errorResponse Error.invalidLength Doc.response 201 "Asset posted" Doc.end - --- Resumable (multi-step) Upload - - -- TODO: swagger doc - options "/assets/v3/resumable" (continue resumableOptionsV3) $ - header "Z-User" - - -- TODO (Compliance): Require and check Tus-Resumable header - -- against supported version(s). - post "/assets/v3/resumable" (continue createResumableV3) $ - header "Z-User" - .&. header "Upload-Length" - .&. jsonRequest @Public.ResumableSettings - - -- TODO (Compliance): Require and check Tus-Resumable header - -- against supported version(s). - head "/assets/v3/resumable/:key" (continue statusResumableV3) $ - header "Z-User" - .&. capture "key" - - -- TODO (Compliance): Require and check Tus-Resumable header - -- against supported version(s). - patch "/assets/v3/resumable/:key" (continue uploadResumableV3) $ - header "Z-User" - .&. header "Upload-Offset" - .&. header "Content-Length" - .&. contentType "application" "offset+octet-stream" - .&. capture "key" - .&. request - --- Download get "/assets/v3/:key" (continue downloadAssetV3) $ @@ -245,33 +216,6 @@ deleteTokenV3 (usr ::: key) = do V3.deleteToken (V3.UserPrincipal usr) key return empty -resumableOptionsV3 :: UserId -> Handler Response -resumableOptionsV3 _ = do - maxTotal <- view (settings . setMaxTotalBytes) - return $ TUS.optionsResponse (fromIntegral maxTotal) empty - -createResumableV3 :: UserId ::: Public.TotalSize ::: JsonRequest Public.ResumableSettings -> Handler Response -createResumableV3 (u ::: size ::: req) = do - sets <- parseBody req !>> Error.clientError - res <- Resumable.create (V3.UserPrincipal u) sets size - let key = res ^. Public.resumableAsset . Public.assetKey - let expiry = res ^. Public.resumableExpires - let loc = "/assets/v3/resumable/" <> toByteString' key - return . TUS.createdResponse loc expiry $ json (res :: Public.ResumableAsset) - -statusResumableV3 :: UserId ::: Public.AssetKey -> Handler Response -statusResumableV3 (u ::: a) = do - stat <- Resumable.status (V3.UserPrincipal u) a - return $ case stat of - Nothing -> setStatus status404 empty - Just st -> TUS.headResponse st empty - --- Request = raw bytestring -uploadResumableV3 :: UserId ::: Public.Offset ::: Word ::: Media "application" "offset+octet-stream" ::: Public.AssetKey ::: Request -> Handler Response -uploadResumableV3 (usr ::: offset ::: size ::: _ ::: aid ::: req) = do - (offset', expiry) <- Resumable.upload (V3.UserPrincipal usr) aid offset size (sourceRequestBody req) - return $ TUS.patchResponse offset' expiry empty - -------------------------------------------------------------------------------- -- Provider API Handlers diff --git a/services/cargohold/src/CargoHold/API/V3/Resumable.hs b/services/cargohold/src/CargoHold/API/V3/Resumable.hs deleted file mode 100644 index 878842375b4..00000000000 --- a/services/cargohold/src/CargoHold/API/V3/Resumable.hs +++ /dev/null @@ -1,149 +0,0 @@ --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2020 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module CargoHold.API.V3.Resumable - ( create, - status, - upload, - ) -where - -import qualified CargoHold.API.Error as Error -import CargoHold.API.V3 (randToken) -import CargoHold.App -import CargoHold.Options -import qualified CargoHold.S3 as S3 -import CargoHold.Types.V3 as V3 -import CargoHold.Types.V3.Resumable as V3 -import Control.Error (throwE) -import Control.Lens (set, view) -import Data.ByteString.Conversion -import Data.Coerce -import Data.Conduit -import Data.Id -import Data.Time.Clock -import Data.UUID.V4 (nextRandom) -import Imports -import System.Logger.Class (field, msg, val, (~~)) -import qualified System.Logger.Class as Log - -create :: V3.Principal -> V3.ResumableSettings -> V3.TotalSize -> Handler V3.ResumableAsset -create own sets size = do - let cl = fromIntegral size - when (cl <= 0) $ - throwE Error.invalidLength - maxTotalBytes <- view (settings . setMaxTotalBytes) - when (cl > maxTotalBytes) $ - throwE Error.assetTooLarge - aid <- liftIO $ Id <$> nextRandom - tok <- - if view setResumablePublic sets - then return Nothing - else Just <$> randToken - let ret = view setResumableRetention sets - let typ = view setResumableType sets - let key = V3.AssetKeyV3 aid ret - astExpire <- case V3.assetRetentionSeconds ret of - Just n -> Just . addUTCTime n <$> liftIO getCurrentTime - Nothing -> return Nothing - Log.debug $ - field "asset" (toByteString aid) - ~~ field "asset.size" (toByteString size) - ~~ msg (val "Initialising resumable upload") - r <- S3.createResumable key own typ size tok - let chunkSize = S3.resumableChunkSize r - let uplExpire = S3.resumableExpires r - let ast = - V3.mkAsset key - & set V3.assetExpires astExpire - & set V3.assetToken tok - return $! mkResumableAsset ast uplExpire chunkSize - -status :: V3.Principal -> AssetKey -> Handler (Maybe (V3.Offset, V3.TotalSize)) -status own key = do - Log.debug $ - field "asset" (toByteString key) - ~~ msg (val "Getting status of resumable upload") - r <- getResumable key - return $ - if own /= S3.resumableOwner r - then Nothing - else - let total = S3.resumableTotalSize r - offset = S3.resumableOffset r - in Just (offset, total) - -upload :: V3.Principal -> AssetKey -> Offset -> Word -> ConduitM () ByteString IO () -> Handler (Offset, UTCTime) -upload own key off len src = do - r <- getResumable key - let offset = S3.resumableOffset r - validate r offset - if off == Offset (totalSize r) - then complete r - else resume r offset - where - complete r = do - fin <- S3.getMetadataV3 key - unless (isJust fin) $ - S3.completeResumable r - return (off, S3.resumableExpires r) - resume r offset = do - Log.debug $ - field "asset" (toByteString key) - ~~ field "asset.offset" (toByteString offset) - ~~ msg (val "Resuming upload") - (r', offset') <- consume r offset len (sealConduitT src) - when (offset' == Offset (totalSize r')) $ - -- TODO: Completion might take a while, such that we may need to - -- keep the client connection alive by sending whitespace after the - -- response status line and headers but before the final response body, - -- just like S3 does when completing multipart uploads. - S3.completeResumable r' - return (offset', S3.resumableExpires r') - consume r offset 0 _ = return (r, offset) - consume r offset remaining rsrc = do - let totalBytes = V3.totalSizeBytes (S3.resumableTotalSize r) - let numBytes = min (chunkSize r) remaining - if numBytes < chunkSize r && coerce offset + remaining < totalBytes - then -- Remaining input that is not a full chunk size and does - -- not constitute the last chunk is ignored, i.e. all chunks - -- except the last must have the same size (the chunk size). - return (r, offset) - else do - (r', rsrc') <- S3.uploadChunk r offset rsrc - let offset' = offset + Offset numBytes - let remaining' = remaining - numBytes - consume r' offset' remaining' rsrc' - validate r o - | invalidOwner r = throwE Error.assetNotFound - | invalidOffset o = throwE (Error.invalidOffset o off) - | tooSmall r o = throwE Error.uploadTooSmall - | tooLarge r = throwE Error.uploadTooLarge - | otherwise = return () - invalidOwner r = own /= S3.resumableOwner r - invalidOffset o = o /= off - tooSmall r o = len < chunkSize r && missingBytes r o > chunkSize r - tooLarge r = proposedBytes > S3.resumableTotalSize r - chunkSize = chunkSizeBytes . S3.resumableChunkSize - totalSize = totalSizeBytes . S3.resumableTotalSize - missingBytes r o = totalSize r - V3.offsetBytes o - proposedBytes = V3.TotalSize (V3.offsetBytes off + len) - -getResumable :: AssetKey -> Handler S3.S3Resumable -getResumable key = do - rs <- S3.getResumable key - maybe (throwE Error.assetNotFound) return rs diff --git a/services/cargohold/src/CargoHold/S3.hs b/services/cargohold/src/CargoHold/S3.hs index c71220e9beb..7ad9c43a038 100644 --- a/services/cargohold/src/CargoHold/S3.hs +++ b/services/cargohold/src/CargoHold/S3.hs @@ -28,19 +28,6 @@ module CargoHold.S3 deleteV3, mkKey, signedURL, - - -- * Resumable Uploads - S3Resumable, - resumableOwner, - resumableTotalSize, - resumableExpires, - resumableChunkSize, - resumableOffset, - createResumable, - getResumable, - completeResumable, - S3Chunk, - uploadChunk, -- Legacy plainKey, otrKey, @@ -54,23 +41,18 @@ import qualified CargoHold.AWS as AWS import CargoHold.App hiding (Env, Handler) import CargoHold.Options import qualified CargoHold.Types.V3 as V3 -import qualified CargoHold.Types.V3.Resumable as V3 import qualified Codec.MIME.Parse as MIME import qualified Codec.MIME.Type as MIME import Conduit import Control.Error (ExceptT, throwE) import Control.Lens hiding (parts, (.=), (:<), (:>)) import Data.ByteString.Builder (toLazyByteString) -import qualified Data.ByteString.Char8 as C8 import Data.ByteString.Conversion import qualified Data.ByteString.Lazy as LBS import qualified Data.CaseInsensitive as CI import qualified Data.Conduit.Binary as Conduit import qualified Data.HashMap.Lazy as HML import Data.Id -import qualified Data.List.NonEmpty as NE -import Data.Sequence (Seq, ViewL (..), ViewR (..)) -import qualified Data.Sequence as Seq import qualified Data.Text as Text import qualified Data.Text.Ascii as Ascii import Data.Text.Encoding (decodeLatin1, encodeUtf8) @@ -82,7 +64,6 @@ import Network.AWS hiding (Error) import Network.AWS.Data.Body import Network.AWS.S3 import Network.Wai.Utilities.Error (Error (..)) -import Safe (readMay) import qualified System.Logger.Class as Log import System.Logger.Message (msg, val, (.=), (~~)) import URI.ByteString @@ -237,408 +218,6 @@ metaHeaders tok prc = Just (setAmzMetaPrincipal prc) ] -------------------------------------------------------------------------------- --- Resumable Uploads - -newtype S3ResumableKey = S3ResumableKey {s3ResumableKey :: Text} - deriving (Eq, Show, ToByteString) - -newtype S3ChunkKey = S3ChunkKey {s3ChunkKey :: Text} - deriving (Eq, Show, ToByteString) - -newtype S3ChunkNr = S3ChunkNr Word - deriving (Eq, Ord, Show, ToByteString, FromByteString, Num, Integral, Enum, Real) - -newtype S3ETag = S3ETag {s3ETag :: Text} - deriving (Eq, Show, ToByteString, FromByteString) - -data S3Resumable = S3Resumable - { -- | The resumable asset key. - resumableKey :: S3ResumableKey, - -- | The final asset key. - resumableAsset :: V3.AssetKey, - -- | The creator (i.e. owner). - resumableOwner :: V3.Principal, - -- | Size of each chunk. - resumableChunkSize :: V3.ChunkSize, - -- | Size of the final asset. - resumableTotalSize :: V3.TotalSize, - -- | MIME type of the final asset. - resumableType :: MIME.Type, - -- | Token of the final asset. - resumableToken :: Maybe V3.AssetToken, - -- | Expiry of the resumable upload. - resumableExpires :: UTCTime, - -- | S3 multipart upload ID, if any. - resumableUploadId :: Maybe Text, - resumableChunks :: Seq S3Chunk - } - deriving (Show) - -data S3Chunk = S3Chunk - { -- | Sequence nr. - chunkNr :: S3ChunkNr, - -- | Offset of the first byte. - chunkOffset :: V3.Offset, - -- | (Actual) Size of the chunk. - chunkSize :: Word, - -- | S3 ETag. - chunkETag :: S3ETag - } - deriving (Show) - -mkChunkNr :: S3Resumable -> V3.Offset -> S3ChunkNr -mkChunkNr r o = S3ChunkNr ((offBytes `quot` chunkBytes) + 1) - where - offBytes = V3.offsetBytes o - chunkBytes = V3.chunkSizeBytes (resumableChunkSize r) - -mkOffset :: S3Resumable -> S3ChunkNr -> V3.Offset -mkOffset r n = V3.Offset ((fromIntegral n - 1) * chunkBytes) - where - chunkBytes = V3.chunkSizeBytes (resumableChunkSize r) - -resumableOffset :: S3Resumable -> V3.Offset -resumableOffset r = case Seq.viewr (resumableChunks r) of - Seq.EmptyR -> V3.Offset 0 - _ :> c -> chunkOffset c + V3.Offset (chunkSize c) - --- | Given a total size for an upload, calculates the desired --- size of individual chunks. Semantically, the calculation grows --- the number of chunks and the chunk size in an alternating fashion --- until the number of chunks multiplied by the chunk size is equal --- or greater than the given total size: --- --- [0. If the total size is less than 'minSmallSize', then 'minSmallSize' --- is the chunk size and we are done.] --- 1. Starting with a chunk size of 'minSmallSize', the number --- of chunks is increased up to 'maxSmallChunks'. --- 2. Staying at 'maxSmallChunks', the chunk size is increased --- up to 'maxSmallSize'. --- 3. Starting with a chunk size of 'minBigSize' and 1 chunk, the number --- of chunks is increased up to 'maxTotalChunks'. --- 4. Staying at 'maxTotalChunks', the chunk size is increased --- until the total size is accommodated. -calculateChunkSize :: V3.TotalSize -> V3.ChunkSize -calculateChunkSize (fromIntegral -> total) = - let smallChunks = max 1 (min maxSmallChunks (total `quot` minSmallSize)) - bigChunks = max 1 (min maxTotalChunks (total `quot` minBigSize)) - smallSize = total `quot` smallChunks - bigSize = total `quot` bigChunks - in V3.ChunkSize $ - if - | smallChunks < maxSmallChunks -> minSmallSize - | smallSize <= maxSmallSize -> smallSize - | bigChunks < maxTotalChunks -> minBigSize - | otherwise -> bigSize - --- | The maximum number of small chunks, sized ['minSmallChunk', 'maxSmallChunk'] --- that we are willing to assemble on our side, to compensate for the 5MiB lower --- bound on S3 multipart uploads. -maxSmallChunks :: Word -maxSmallChunks = 25 - --- | The maximum number of chunks we are willing to process in total for a --- single upload, regardless of where the final assembly is performed. -maxTotalChunks :: Word -maxTotalChunks = 1000 - --- | Lower bound (inclusive) for small chunks. -minSmallSize :: Word -minSmallSize = 100 * 1024 -- 100 KiB - --- | Upper bound (inclusive) for small chunks. -maxSmallSize :: Word -maxSmallSize = 1 * 1024 * 1024 -- 1 MiB - --- | Lower bound (inclusive) for large chunks, i.e. the lower bound for S3 --- multipart upload uploads. -minBigSize :: Word -minBigSize = 5 * 1024 * 1024 -- 5 MiB - -getResumable :: V3.AssetKey -> ExceptT Error App (Maybe S3Resumable) -getResumable k = do - Log.debug $ - "remote" .= val "S3" - ~~ "asset" .= toByteString k - ~~ "asset.key" .= toByteString rk - ~~ "asset.key.meta" .= toByteString mk - ~~ msg (val "Getting resumable asset metadata") - maybe (return Nothing) handle =<< execCatch req - where - rk = mkResumableKey k - mk = mkResumableKeyMeta k - req b = headObject (BucketName b) (ObjectKey $ s3ResumableKey mk) - handle r = do - let ct = fromMaybe octets (MIME.parseMIMEType =<< view horsContentType r) - let meta = HML.toList $ view horsMetadata r - case parse ct meta of - Nothing -> return Nothing - Just r' -> fmap (\cs -> r' {resumableChunks = cs}) <$> listChunks r' - parse ct h = - S3Resumable rk k - <$> getAmzMetaPrincipal h - <*> getAmzMetaChunkSize h - <*> getAmzMetaTotalSize h - <*> pure ct - <*> Just (getAmzMetaToken h) - <*> getAmzMetaUploadExpires h - <*> Just (getAmzMetaUploadId h) - <*> pure Seq.empty - -createResumable :: - V3.AssetKey -> - V3.Principal -> - MIME.Type -> - V3.TotalSize -> - Maybe V3.AssetToken -> - ExceptT Error App S3Resumable -createResumable k p _ size tok = do - let typ = octets -- see note: overrideMimeTypeAsOctetStream - let csize = calculateChunkSize size - ex <- addUTCTime V3.assetVolatileSeconds <$> liftIO getCurrentTime - let key = mkResumableKey k - mk = mkResumableKeyMeta k - let res = S3Resumable key k p csize size typ tok ex Nothing Seq.empty - up <- initMultipart res - let ct = resumableType res - void . exec $ first (s3ResumableKey mk) ct (resumableMeta csize ex up) - return res {resumableUploadId = up} - where - initMultipart r - | canUseMultipart r = do - let cmu b = - createMultipartUpload (BucketName b) (ObjectKey $ s3Key (mkKey k)) - & cmuContentType ?~ MIME.showType (resumableType r) - & cmuMetadata .~ metaHeaders (resumableToken r) p - imur <- exec cmu - return $! view cmursUploadId imur - | otherwise = return Nothing - first key ct meta b = - putObject (BucketName b) (ObjectKey key) (toBody (mempty :: ByteString)) - & poContentType ?~ MIME.showType ct - & poMetadata .~ HML.fromList meta - -- Determine whether a given 'S3Resumable' is eligible for the - -- S3 multipart upload API. That is the case if the chunk size - -- is >= 5 MiB or if there is only 1 chunk (<= 'minSmallSize'). - canUseMultipart r = chunkBytes >= minBigSize || totalBytes <= minSmallSize - where - chunkBytes = V3.chunkSizeBytes (resumableChunkSize r) - totalBytes = V3.totalSizeBytes (resumableTotalSize r) - resumableMeta csize expires upl = - setAmzMetaPrincipal p : - setAmzMetaTotalSize size : - setAmzMetaChunkSize csize : - setAmzMetaUploadExpires expires : - catMaybes - [ setAmzMetaToken <$> tok, - setAmzMetaUploadId <$> upl - ] - -uploadChunk :: - S3Resumable -> - V3.Offset -> - Conduit.SealedConduitT () ByteString IO () -> - ExceptT Error App (S3Resumable, Conduit.SealedConduitT () ByteString IO ()) -uploadChunk r offset rsrc = do - let chunkSize = fromIntegral (resumableChunkSize r) - (rest, chunk) <- liftIO $ rsrc $$++ Conduit.take chunkSize - let size = fromIntegral (LBS.length chunk) - Log.debug $ - "remote" .= val "S3" - ~~ "asset" .= toByteString (resumableAsset r) - ~~ "asset.owner" .= toByteString (resumableOwner r) - ~~ "asset.key" .= toByteString (resumableKey r) - ~~ "asset.chunk" .= toByteString nr - ~~ "asset.offset" .= toByteString offset - ~~ "asset.size" .= toByteString size - ~~ msg (val "Uploading chunk") - c <- case resumableUploadId r of - Nothing -> putChunk chunk size - Just up -> putPart up chunk size - let r' = r {resumableChunks = resumableChunks r Seq.|> c} - return (r', rest) - where - nr = mkChunkNr r offset - ct = MIME.showType octets -- see note overrideMimeTypeAsOctetStream - putChunk chunk size = do - let S3ChunkKey k = mkChunkKey (resumableKey r) nr - let req b = - putObject (BucketName b) (ObjectKey k) (toBody chunk) - & poContentType ?~ ct - void $ exec req - return $! S3Chunk nr offset size (S3ETag "") - putPart up chunk size = do - let S3AssetKey k = mkKey (resumableAsset r) - let req b = - uploadPart (BucketName b) (ObjectKey k) (fromIntegral nr) up (toBody chunk) - tg <- view uprsETag <$> exec req - etag <- case tg of - Just (ETag t) -> return $ S3ETag (Text.decodeLatin1 t) - Nothing -> throwE serverError - return $! S3Chunk nr offset size etag - --- | Complete a resumable upload, assembling all chunks into a final asset. -completeResumable :: S3Resumable -> ExceptT Error App () -completeResumable r = do - Log.debug $ - "remote" .= val "S3" - ~~ "asset" .= toByteString ast - ~~ "asset.owner" .= toByteString own - ~~ "asset.key" .= toByteString (resumableKey r) - ~~ msg (val "Completing resumable upload") - let chunks = resumableChunks r - verifyChunks chunks - case resumableUploadId r of - Nothing -> assembleLocal chunks - Just up -> assembleRemote up (NE.nonEmpty $ toList chunks) - Log.debug $ - "remote" .= val "S3" - ~~ "asset" .= toByteString ast - ~~ "asset.owner" .= toByteString own - ~~ "asset.key" .= toByteString (resumableKey r) - ~~ msg (val "Resumable upload completed") - where - (own, ast) = (resumableOwner r, resumableAsset r) - -- Local assembly for small chunk sizes (< 5 MiB): Download and re-upload - -- the chunks in a streaming fashion one-by-one to create the final object. - assembleLocal :: Seq S3Chunk -> ExceptT Error App () - assembleLocal chunks = do - e <- view aws - let totalSize = fromIntegral (resumableTotalSize r) - let chunkSize = calcChunkSize chunks - let reqBdy = Chunked $ ChunkedBody chunkSize totalSize (chunkSource e chunks) - let putRq b = - putObject (BucketName b) (ObjectKey (s3Key (mkKey ast))) reqBdy - & poContentType ?~ MIME.showType octets -- see note overrideMimeTypeAsOctetStream - & poMetadata .~ metaHeaders (resumableToken r) own - void $ exec putRq - - -- For symmetry with the behavior of the S3 multipart API, where the - -- resumable upload and all parts are removed upon completion, we do - -- the same here. - let rk = resumableKey r - let keys = - s3ResumableKey rk : - map (s3ChunkKey . mkChunkKey rk . chunkNr) (toList chunks) - let del = - delete' & dObjects .~ map (objectIdentifier . ObjectKey) keys - & dQuiet ?~ True - let delRq b = deleteObjects (BucketName b) del - void $ exec delRq - - -- All chunks except for the last should be of the same size so it makes - -- sense to use that as our default - calcChunkSize cs = case Seq.viewl cs of - EmptyL -> defaultChunkSize - c :< _ -> ChunkSize $ fromIntegral (chunkSize c) - -- Remote assembly for large(r) chunk sizes (>= 5 MiB) via the - -- S3 multipart upload API. - assembleRemote _ Nothing = throwE serverError - assembleRemote up (Just chunks) = do - let key = s3Key (mkKey ast) - let parts = fmap mkPart chunks - let completeRq b = - completeMultipartUpload (BucketName b) (ObjectKey key) up - & cMultipartUpload ?~ (completedMultipartUpload & cmuParts ?~ parts) - void $ exec completeRq - let S3ResumableKey rkey = resumableKey r - let delRq b = deleteObject (BucketName b) (ObjectKey rkey) - void $ exec delRq - mkPart c = completedPart (fromIntegral (chunkNr c)) (ETag . Text.encodeUtf8 $ s3ETag (chunkETag c)) - -- Verify that the chunks constitute the full asset, i.e. that the - -- upload is complete. - verifyChunks cs = do - let !total = V3.TotalSize $ foldl' (\t v -> t + chunkSize v) 0 cs - unless (total == resumableTotalSize r) $ - throwE $ - uploadIncomplete (resumableTotalSize r) total - -- Construct a 'Source' by downloading the chunks. - -- chunkSource :: AWS.Env - -- -> Seq S3Chunk - -- -> Source (ResourceT IO) ByteString - chunkSource env cs = case Seq.viewl cs of - EmptyL -> mempty - c :< cc -> do - let S3ChunkKey ck = mkChunkKey (resumableKey r) (chunkNr c) - let b = view AWS.s3Bucket env - let req = getObject (BucketName b) (ObjectKey ck) - v <- - lift $ - AWS.execute env $ - AWS.send req - >>= flip sinkBody Conduit.sinkLbs . view gorsBody - Conduit.yield (LBS.toStrict v) >> chunkSource env cc - -listChunks :: S3Resumable -> ExceptT Error App (Maybe (Seq S3Chunk)) -listChunks r = do - let ast = resumableAsset r - let S3ResumableKey key = resumableKey r - Log.debug $ - "remote" .= val "S3" - ~~ "asset" .= toByteString ast - ~~ "asset.resumable" .= key - ~~ msg (val "Listing chunks") - fmap Seq.fromList <$> case resumableUploadId r of - Nothing -> listBucket key - Just up -> listMultiParts up - where - listBucket k = do - let req b = - listObjects (BucketName b) - & loPrefix ?~ (k <> "/") - & loMaxKeys ?~ fromIntegral maxTotalChunks - maybe (return Nothing) parseObjects =<< execCatch req - parseObjects = - return . Just . mapMaybe chunkFromObject - . view lorsContents - listMultiParts up = do - let req b = - listParts - (BucketName b) - (ObjectKey $ s3Key (mkKey (resumableAsset r))) - up - maybe (return Nothing) parseParts =<< execCatch req - parseParts = - return . Just . mapMaybe chunkFromPart - . view lprsParts - chunkFromObject :: Object -> Maybe S3Chunk - chunkFromObject o = do - let (ObjectKey okey) = view oKey o - nr <- parseNr okey - let etag = - let (ETag t) = (view oETag o) - in S3ETag (Text.decodeLatin1 t) - let size = fromIntegral (view oSize o) - let off = mkOffset r nr - Just $! S3Chunk nr off size etag - chunkFromPart :: Part -> Maybe S3Chunk - chunkFromPart p = case (view pPartNumber p, view pETag p, view pSize p) of - (Just x, Just (ETag y), Just z) -> - let nr = S3ChunkNr (fromIntegral x) - off = mkOffset r nr - size = (fromIntegral z) - etag = S3ETag (Text.decodeLatin1 y) - in Just $! S3Chunk nr off size etag - _ -> Nothing - parseNr = fmap S3ChunkNr . readMay . Text.unpack . snd . Text.breakOnEnd "/" - -mkResumableKey :: V3.AssetKey -> S3ResumableKey -mkResumableKey (V3.AssetKeyV3 aid _) = - S3ResumableKey $ "v3/resumable/" <> UUID.toText (toUUID aid) - -mkResumableKeyMeta :: V3.AssetKey -> S3ResumableKey -mkResumableKeyMeta (V3.AssetKeyV3 aid _) = - S3ResumableKey $ "v3/resumable/" <> UUID.toText (toUUID aid) <> "/meta" - -mkChunkKey :: S3ResumableKey -> S3ChunkNr -> S3ChunkKey -mkChunkKey (S3ResumableKey k) (S3ChunkNr n) = - S3ChunkKey $ k <> "/" <> nr - where - -- Chunk numbers must be between 1 and 10000, as per the S3 - -- multipart upload API, hence the max. left padding of 5 digits. - nr = Text.justifyRight 5 '0' (Text.pack (show n)) - ------------------------------------------------------------------------------- -- S3 Metadata Headers @@ -651,21 +230,9 @@ hAmzMetaBot = "bot" hAmzMetaProvider :: Text hAmzMetaProvider = "provider" -hAmzMetaSize :: Text -hAmzMetaSize = "total-size" - hAmzMetaToken :: Text hAmzMetaToken = "token" -hAmzMetaChunkSize :: Text -hAmzMetaChunkSize = "chunk-size" - -hAmzMetaUploadExpires :: Text -hAmzMetaUploadExpires = "upload-expires" - -hAmzMetaUploadId :: Text -hAmzMetaUploadId = "upload-id" - ------------------------------------------------------------------------------- -- S3 Metadata Setters @@ -681,18 +248,6 @@ setAmzMetaProvider p = (hAmzMetaProvider, UUID.toText (toUUID p)) setAmzMetaToken :: V3.AssetToken -> (Text, Text) setAmzMetaToken t = (hAmzMetaToken, Ascii.toText (V3.assetTokenAscii t)) -setAmzMetaTotalSize :: V3.TotalSize -> (Text, Text) -setAmzMetaTotalSize s = (hAmzMetaSize, Text.decodeLatin1 (toByteString' s)) - -setAmzMetaChunkSize :: V3.ChunkSize -> (Text, Text) -setAmzMetaChunkSize s = (hAmzMetaChunkSize, Text.decodeLatin1 (toByteString' s)) - -setAmzMetaUploadExpires :: UTCTime -> (Text, Text) -setAmzMetaUploadExpires t = (hAmzMetaUploadExpires, Text.pack (show t)) - -setAmzMetaUploadId :: Text -> (Text, Text) -setAmzMetaUploadId i = (hAmzMetaUploadId, i) - setAmzMetaPrincipal :: V3.Principal -> (Text, Text) setAmzMetaPrincipal (V3.UserPrincipal u) = setAmzMetaUser u setAmzMetaPrincipal (V3.BotPrincipal b) = setAmzMetaBot b @@ -724,20 +279,6 @@ getAmzMetaToken h = V3.AssetToken . Ascii.unsafeFromText <$> lookupCI hAmzMetaToken h -getAmzMetaUploadExpires :: [(Text, Text)] -> Maybe UTCTime -getAmzMetaUploadExpires h = - readMay . C8.unpack . encodeUtf8 - =<< lookupCI hAmzMetaUploadExpires h - -getAmzMetaTotalSize :: [(Text, Text)] -> Maybe V3.TotalSize -getAmzMetaTotalSize = parseAmzMeta hAmzMetaSize - -getAmzMetaChunkSize :: [(Text, Text)] -> Maybe V3.ChunkSize -getAmzMetaChunkSize = parseAmzMeta hAmzMetaChunkSize - -getAmzMetaUploadId :: [(Text, Text)] -> Maybe Text -getAmzMetaUploadId = lookupCI hAmzMetaUploadId - parseAmzMeta :: FromByteString a => Text -> [(Text, Text)] -> Maybe a parseAmzMeta k h = lookupCI k h >>= fromByteString . encodeUtf8 diff --git a/services/cargohold/src/CargoHold/TUS.hs b/services/cargohold/src/CargoHold/TUS.hs deleted file mode 100644 index 542833ca0c7..00000000000 --- a/services/cargohold/src/CargoHold/TUS.hs +++ /dev/null @@ -1,89 +0,0 @@ --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2020 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module CargoHold.TUS - ( createdResponse, - headResponse, - patchResponse, - optionsResponse, - ) -where - -import CargoHold.Types.V3.Resumable (Offset, TotalSize) -import qualified Data.ByteString.Char8 as C8 -import Data.ByteString.Conversion -import Data.Time.Clock -import Data.Time.Format -import Data.Time.LocalTime -import Imports -import Network.HTTP.Types.Status -import Network.Wai -import Network.Wai.Utilities hiding (message) - -createdResponse :: ByteString -> UTCTime -> Response -> Response -createdResponse loc expiry = - setStatus status201 - . addHeader "Location" loc - . uploadExpires expiry - --- cf. http://tus.io/protocols/resumable-upload.html#head -headResponse :: (Offset, TotalSize) -> Response -> Response -headResponse (offset, total) = - setStatus status200 - . addHeader "Cache-Control" "no-store" - . hResumable - . hOffset offset - . hLength total - --- cf. http://tus.io/protocols/resumable-upload.html#patch -patchResponse :: Offset -> UTCTime -> Response -> Response -patchResponse offset expiry = - setStatus status204 - . hOffset offset - . hResumable - . uploadExpires expiry - --- cf. http://tus.io/protocols/resumable-upload.html#options -optionsResponse :: Word -> Response -> Response -optionsResponse maxSize = - setStatus status204 - . addHeader "Tus-Extension" "creation,expiration" - . addHeader "Tus-Max-Size" (toByteString' maxSize) - . hVersion - . hResumable - --- Internal -------------------------------------------------------------------- - --- cf. http://tus.io/protocols/resumable-upload.html#expiration -uploadExpires :: UTCTime -> Response -> Response -uploadExpires = addHeader "Upload-Expires" . C8.pack . time - where - -- Must be according to RFC 7231 - time = formatTime defaultTimeLocale "%a, %d %B %Y %H:%M:%S %Z" . utcToZonedTime gmt - gmt = TimeZone 0 False "GMT" - -hVersion :: Response -> Response -hVersion = addHeader "Tus-Version" "1.0.0" - -hResumable :: Response -> Response -hResumable = addHeader "Tus-Resumable" "1.0.0" - -hOffset :: Offset -> Response -> Response -hOffset = addHeader "Upload-Offset" . toByteString' - -hLength :: TotalSize -> Response -> Response -hLength = addHeader "Upload-Length" . toByteString' diff --git a/services/cargohold/test/integration/API/V3.hs b/services/cargohold/test/integration/API/V3.hs index bde128d4f42..a6a34f6ef58 100644 --- a/services/cargohold/test/integration/API/V3.hs +++ b/services/cargohold/test/integration/API/V3.hs @@ -17,16 +17,14 @@ -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . -module API.V3 where +module API.V3 (tests) where import Bilge hiding (body) import Bilge.Assert import qualified CargoHold.Types.V3 as V3 -import qualified CargoHold.Types.V3.Resumable as V3 import qualified Codec.MIME.Parse as MIME import qualified Codec.MIME.Type as MIME import Control.Lens hiding (sets) -import Data.Aeson hiding (json) import Data.ByteString.Builder import qualified Data.ByteString.Char8 as C8 import Data.ByteString.Conversion @@ -41,7 +39,7 @@ import Imports hiding (head) import Network.HTTP.Client (parseUrlThrow) import Network.HTTP.Types.Header import Network.HTTP.Types.Method -import Network.HTTP.Types.Status (status200, status204) +import Network.HTTP.Types.Status (status200) import Network.Wai.Utilities (Error (label)) import Test.Tasty import Test.Tasty.HUnit @@ -57,17 +55,6 @@ tests s = test s "tokens" testSimpleTokens, test s "s3-upstream-closed" testSimpleS3ClosedConnectionReuse, test s "client-compatibility" testUploadCompatibility - ], - testGroup - "RealAWS" - [ testGroup - "resumable" - [ test s "small" testResumableSmall, - test s "large" testResumableBig, - test s "last-small" testResumableLastSmall, - test s "stepwise-small" testResumableStepSmall, - test s "stepwise-big" testResumableStepBig - ] ] ] @@ -241,68 +228,6 @@ testUploadCompatibility c = do \--FrontierIyj6RcVrqMcxNtMEWPsNpuPm325QsvWQ--\r\n\ \\r\n" --------------------------------------------------------------------------------- --- Resumable (multi-step) uploads - -testResumableSmall :: TestSignature () -testResumableSmall c = assertRandomResumable c totalSize chunkSize UploadFull - where - totalSize = 100 -- 100 B - chunkSize = 100 * 1024 -- 100 KiB - -testResumableBig :: TestSignature () -testResumableBig c = assertRandomResumable c totalSize chunkSize UploadFull - where - totalSize = 25 * 1024 * 1024 -- 25 MiB - chunkSize = 1 * 1024 * 1024 -- 1 MiB - -testResumableLastSmall :: TestSignature () -testResumableLastSmall c = assertRandomResumable c totalSize chunkSize UploadFull - where - totalSize = 250 * 1024 + 12345 -- 250 KiB + 12345 B - chunkSize = 100 * 1024 -- 100 KiB - -testResumableStepSmall :: TestSignature () -testResumableStepSmall c = assertRandomResumable c totalSize chunkSize UploadStepwise - where - totalSize = 500 * 1024 + 12345 -- 500 KiB + 12345 B - chunkSize = 100 * 1024 -- 100 KiB - --- This should use the S3 multipart upload behind the scenes. -testResumableStepBig :: TestSignature () -testResumableStepBig c = assertRandomResumable c totalSize chunkSize UploadStepwise - where - totalSize = 26 * 1024 * 1024 -- 26 MiB - chunkSize = 5 * 1024 * 1024 -- 5 MiB - --- Assertions ----------------------------------------------------------------- - -data UploadType = UploadFull | UploadStepwise - -assertRandomResumable :: - HasCallStack => CargoHold -> V3.TotalSize -> V3.ChunkSize -> UploadType -> Http () -assertRandomResumable c totalSize chunkSize typ = do - (uid, dat, ast) <- randomResumable c totalSize - let key = ast ^. V3.resumableAsset . V3.assetKey - liftIO $ assertEqual "chunksize" chunkSize (ast ^. V3.resumableChunkSize) - case typ of - UploadStepwise -> uploadStepwise c uid key chunkSize dat - UploadFull -> void $ uploadResumable c uid key 0 dat - r <- downloadAsset c uid key Nothing - liftIO $ do - assertEqual "status" status200 (responseStatus r) - assertEqual "content-type should always be application/octet-stream" (Just applicationOctetStream) (getContentType r) - assertEqual "user mismatch" uid (decodeHeader "x-amz-meta-user" r) - assertEqual "data mismatch" (Just $ Lazy.fromStrict dat) (responseBody r) - -randomResumable :: CargoHold -> V3.TotalSize -> Http (UserId, ByteString, V3.ResumableAsset) -randomResumable c size = do - uid <- liftIO $ Id <$> nextRandom - let sets = V3.mkResumableSettings V3.AssetPersistent True textPlain - let dat = C8.replicate (fromIntegral size) 'a' - ast <- createResumable c uid sets size - return (uid, dat, ast) - -- API Calls ------------------------------------------------------------------ uploadSimple :: @@ -329,103 +254,17 @@ uploadRaw c usr bs = . content "multipart/mixed" . lbytes bs -createResumable :: - HasCallStack => - CargoHold -> - UserId -> - V3.ResumableSettings -> - V3.TotalSize -> - Http V3.ResumableAsset -createResumable c u sets size = do - rsp <- - post - ( c - . path "/assets/v3/resumable" - . zUser u - . header "Content-Type" "application/json" - . header "Upload-Length" (toByteString' size) - . lbytes (encode sets) - ) - toByteString' (ast ^. V3.resumableAsset . V3.assetKey) - liftIO $ assertEqual "Location" loc' loc - return ast - -getResumableStatus :: HasCallStack => CargoHold -> UserId -> V3.AssetKey -> Http V3.Offset -getResumableStatus c u k = do - r <- - head - ( c - . paths ["assets", "v3", "resumable", toByteString' k] - . zUser u - ) - UserId -> V3.AssetKey -> V3.Offset -> ByteString -> Http V3.Offset -uploadResumable c u k off bs = do - r <- - patch - ( c - . paths ["assets", "v3", "resumable", toByteString' k] - . header "Upload-Offset" (toByteString' off) - . header "Content-Type" applicationOffset - . zUser u - . bytes bs - ) - liftIO $ assertEqual "status" status204 (responseStatus r) - return $ getOffset r - -uploadStepwise :: CargoHold -> UserId -> V3.AssetKey -> V3.ChunkSize -> ByteString -> Http () -uploadStepwise c u k s d = next 0 d - where - totalSize = fromIntegral (C8.length d) - chunkSize = fromIntegral s - next pos dat = do - off <- uploadResumable c u k pos (C8.take chunkSize dat) - unless (V3.offsetBytes off == totalSize) $ do - off' <- getResumableStatus c u k - liftIO $ assertEqual "offset" off off' - next off (C8.drop (fromIntegral (off - pos)) dat) - -getAsset :: CargoHold -> UserId -> V3.AssetKey -> Maybe V3.AssetToken -> Http (Response (Maybe Lazy.ByteString)) -getAsset c u k t = - get $ - c - . paths ["assets", "v3", toByteString' k] - . zUser u - . maybe id (header "Asset-Token" . toByteString') t - . noRedirect - -downloadAsset :: HasCallStack => CargoHold -> UserId -> V3.AssetKey -> Maybe V3.AssetToken -> Http (Response (Maybe Lazy.ByteString)) -downloadAsset c u k t = do - r <- - getAsset c u k t UserId -> V3.AssetKey -> Http (Response (Maybe Lazy.ByteString)) deleteAsset c u k = delete $ c . zUser u . paths ["assets", "v3", toByteString' k] -- Utilities ------------------------------------------------------------------ -type ContentType = ByteString - decodeHeader :: FromByteString a => HeaderName -> Response b -> a decodeHeader h = fromMaybe (error $ "decodeHeader: missing or invalid header: " ++ show h) . fromByteString . getHeader' h -getOffset :: Response b -> V3.Offset -getOffset = decodeHeader "Upload-Offset" - getContentType :: Response a -> Maybe MIME.Type getContentType = MIME.parseContentType . decodeLatin1 . getHeader' "Content-Type" @@ -435,12 +274,6 @@ applicationText = MIME.Type (MIME.Application "text") [] applicationOctetStream :: MIME.Type applicationOctetStream = MIME.Type (MIME.Application "octet-stream") [] -textPlain :: MIME.Type -textPlain = MIME.Type (MIME.Text "plain") [] - -applicationOffset :: ContentType -applicationOffset = "application/offset+octet-stream" - zUser :: UserId -> Request -> Request zUser = header "Z-User" . UUID.toASCIIBytes . toUUID From 3c376f3d10544044d35b10ae03a3d4dd00f0c52a Mon Sep 17 00:00:00 2001 From: Stefan Matting Date: Thu, 16 Dec 2021 01:58:32 +0100 Subject: [PATCH 11/74] Fix bug in SFT chart: multi sft args are quoted (#1999) * Fix bug in SFT chart: multi sft args are quoted * Add changelog entry --- changelog.d/5-internal/sftd-multi-sft-fixup | 1 + charts/sftd/templates/statefulset.yaml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelog.d/5-internal/sftd-multi-sft-fixup diff --git a/changelog.d/5-internal/sftd-multi-sft-fixup b/changelog.d/5-internal/sftd-multi-sft-fixup new file mode 100644 index 00000000000..78b95439e69 --- /dev/null +++ b/changelog.d/5-internal/sftd-multi-sft-fixup @@ -0,0 +1 @@ +sftd chart: Fix quoted args for multiSFT option diff --git a/charts/sftd/templates/statefulset.yaml b/charts/sftd/templates/statefulset.yaml index 8345b6bc1c1..3027ccf601d 100644 --- a/charts/sftd/templates/statefulset.yaml +++ b/charts/sftd/templates/statefulset.yaml @@ -131,9 +131,9 @@ spec: fi {{- if .Values.multiSFT.enabled }} - MULTI_SFT_ARGS="-t \"$(cat /multi-sft-config/turn_server)\" \ - -x \"$(cat /multi-sft-config/username)\" \ - -c \"$(cat /multi-sft-config/password)\"" + MULTI_SFT_ARGS="-t $(cat /multi-sft-config/turn_server) \ + -x $(cat /multi-sft-config/username) \ + -c $(cat /multi-sft-config/password)" {{- else }} MULTI_SFT_ARGS="" {{- end }} From 10592ddf9d66b585b1cd5c79831fa50b61c01e11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Dimja=C5=A1evi=C4=87?= Date: Thu, 16 Dec 2021 11:41:40 +0100 Subject: [PATCH 12/74] Improve Test Documentation for BSI (SQSERVICES-1127) (#1991) * Improve documentation for tests per SQSERVICES-1127 Co-authored-by: fisx * Update the changelog * Add a test: attempt to delete client via an incorrect password * Fixup Co-authored-by: fisx --- changelog.d/4-docs/sqservices-1127 | 1 + .../brig/test/integration/API/User/Auth.hs | 16 +++++- .../brig/test/integration/API/User/Client.hs | 51 +++++++++++++++++++ .../brig/test/integration/API/User/Handles.hs | 7 ++- 4 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 changelog.d/4-docs/sqservices-1127 diff --git a/changelog.d/4-docs/sqservices-1127 b/changelog.d/4-docs/sqservices-1127 new file mode 100644 index 00000000000..bb5629509ae --- /dev/null +++ b/changelog.d/4-docs/sqservices-1127 @@ -0,0 +1 @@ +Add the description to several test cases diff --git a/services/brig/test/integration/API/User/Auth.hs b/services/brig/test/integration/API/User/Auth.hs index c13b3b77315..22ed0ea735e 100644 --- a/services/brig/test/integration/API/User/Auth.hs +++ b/services/brig/test/integration/API/User/Auth.hs @@ -400,7 +400,15 @@ testThrottleLogins conf b = do login b (defEmailLogin e) SessionCookie !!! const 200 === statusCode -- The testLimitRetries test conforms to the following testing standards: --- @SF.Provisioning @TSFI.RESTfulAPI @S2 +-- @SF.Channel @TSFI.RESTfulAPI @S2 +-- +-- The following test tests the login retries. It checks that a user can make +-- only a prespecified number of attempts to log in with an invalid password, +-- after which the user is unable to try again for a configured amount of time. +-- After the configured amount of time has passed, the test asserts the user can +-- successfully log in again. Furthermore, the test asserts that another +-- unrelated user can successfully log-in in parallel to the failed attempts of +-- the aforementioned user. testLimitRetries :: HasCallStack => Opts.Opts -> Brig -> Http () testLimitRetries conf brig = do let Just opts = Opts.setLimitFailedLogins . Opts.optSettings $ conf @@ -910,6 +918,12 @@ testRemoveCookiesByLabelAndId b = do -- The testTooManyCookies test conforms to the following testing standards: -- @SF.Provisioning @TSFI.RESTfulAPI @S2 +-- +-- The test asserts that there is an upper limit for the number of user cookies +-- per cookie type. It does that by concurrently attempting to create more +-- persistent and session cookies than the configured maximum. +-- Creation of new cookies beyond the limit causes deletion of the +-- oldest cookies. testTooManyCookies :: Opts.Opts -> Brig -> Http () testTooManyCookies config b = do u <- randomUser b diff --git a/services/brig/test/integration/API/User/Client.hs b/services/brig/test/integration/API/User/Client.hs index d538c3208b9..27a20c4c3e2 100644 --- a/services/brig/test/integration/API/User/Client.hs +++ b/services/brig/test/integration/API/User/Client.hs @@ -89,6 +89,7 @@ tests _cl _at opts p b c g = test p "delete /clients/:client - 200 (pwd)" $ testRemoveClient True b c, test p "delete /clients/:client - 200 (no pwd)" $ testRemoveClient False b c, test p "delete /clients/:client - 400 (short pwd)" $ testRemoveClientShortPwd b, + test p "delete /clients/:client - 403 (incorrect pwd)" $ testRemoveClientIncorrectPwd b, test p "put /clients/:client - 200" $ testUpdateClient opts b, test p "get /clients/:client - 404" $ testMissingClient b, test p "post /clients - 200 multiple temporary" $ testAddMultipleTemporary b g, @@ -418,6 +419,11 @@ testMultiUserGetPrekeysQualified brig opts = do -- The testTooManyClients test conforms to the following testing standards: -- @SF.Provisioning @TSFI.RESTfulAPI @S2 +-- +-- The test validates the upper bound on the number of permanent clients per +-- user. It does so by trying to create one permanent client more than allowed. +-- The expected outcome is that all the clients up to the limit are successfully +-- created, but the one over the limit is not (error `404 too-many-clients`). testTooManyClients :: Opt.Opts -> Brig -> Http () testTooManyClients opts brig = do uid <- userId <$> randomUser brig @@ -440,6 +446,10 @@ testTooManyClients opts brig = do -- The testRemoveClient test conforms to the following testing standards: -- @SF.Provisioning @TSFI.RESTfulAPI @S2 +-- +-- This test validates creating and deleting a client. A client is created and +-- consequently deleted. Deleting a second time yields response 404 not found. +-- Prekeys and cookies are not there anymore once the client is deleted. testRemoveClient :: Bool -> Brig -> Cannon -> Http () testRemoveClient hasPwd brig cannon = do u <- randomUser' hasPwd brig @@ -481,6 +491,10 @@ testRemoveClient hasPwd brig cannon = do -- The testRemoveClientShortPwd test conforms to the following testing standards: -- @SF.Provisioning @TSFI.RESTfulAPI @S2 +-- +-- The test checks if a client can be deleted by providing a too short password. +-- This is done by using a single-character password, whereas the minimum is 6 +-- characters. The client deletion attempt fails as expected. testRemoveClientShortPwd :: Brig -> Http () testRemoveClientShortPwd brig = do u <- randomUser brig @@ -507,6 +521,38 @@ testRemoveClientShortPwd brig = do newClientCookie = Just defCookieLabel } +-- The testRemoveClientIncorrectPwd test conforms to the following testing standards: +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 +-- +-- The test checks if a client can be deleted by providing a syntax-valid, but +-- incorrect password. The client deletion attempt fails with a 403 error +-- response. +testRemoveClientIncorrectPwd :: Brig -> Http () +testRemoveClientIncorrectPwd brig = do + u <- randomUser brig + let uid = userId u + let Just email = userEmail u + -- Permanent client with attached cookie + login brig (defEmailLogin email) PersistentCookie + !!! const 200 === statusCode + numCookies <- countCookies brig uid defCookieLabel + liftIO $ Just 1 @=? numCookies + c <- responseJsonError =<< addClient brig uid (client PermanentClientType (someLastPrekeys !! 10)) + resp <- + deleteClient brig uid (clientId c) (Just "abcdef") + Brig -> Http () testUpdateClient opts brig = do uid <- userId <$> randomUser brig @@ -670,6 +716,11 @@ testMissingClient brig = do -- The testAddMultipleTemporary test conforms to the following testing standards: -- @SF.Provisioning @TSFI.RESTfulAPI @S2 -- Legacy (galley) +-- +-- Add temporary client, check that all services (both galley and +-- brig) have registered it. Add second temporary client, check +-- again. (NB: temp clients replace each other, there can always be +-- at most one per account.) testAddMultipleTemporary :: Brig -> Galley -> Http () testAddMultipleTemporary brig galley = do uid <- userId <$> randomUser brig diff --git a/services/brig/test/integration/API/User/Handles.hs b/services/brig/test/integration/API/User/Handles.hs index a1e4cba18d4..a3391991284 100644 --- a/services/brig/test/integration/API/User/Handles.hs +++ b/services/brig/test/integration/API/User/Handles.hs @@ -69,7 +69,12 @@ tests _cl _at conf p b c g = -- The next line contains a mapping from the testHandleUpdate test to the following test standards: -- @SF.Provisioning @TSFI.RESTfulAPI @S2 -- --- Test changes to the user's handle. +-- The test validates various updates to the user's handle. First, it attempts +-- to set invalid handles. This fails. Then it successfully sets a valid handle. +-- The user can retry setting the valid handle. The next scenario is for another +-- user to attempt to reuse an already used handle, which fails. Finally, +-- several scenarios of searching users by handle are explored, where users +-- appear by handle. A user can also free a handle and then reclaim it again. testHandleUpdate :: Brig -> Cannon -> Http () testHandleUpdate brig cannon = do user <- randomUser brig From 0f967e0579b3653b19fb9dd3033e5589a3870c4b Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Thu, 16 Dec 2021 12:57:39 +0100 Subject: [PATCH 13/74] SQSERVICES-1128 Tag Integration Tests (PROVISIONING/CHANNEL) (#1997) * added integration test and tag for BSI * clean up * More spar tests. Co-authored-by: fisx --- .../test-integration/Test/Spar/APISpec.hs | 156 ++++++++++++------ .../Test/Spar/Scim/AuthSpec.hs | 23 ++- .../Test/Spar/Scim/UserSpec.hs | 2 +- 3 files changed, 119 insertions(+), 62 deletions(-) diff --git a/services/spar/test-integration/Test/Spar/APISpec.hs b/services/spar/test-integration/Test/Spar/APISpec.hs index 980a761aa04..272674dac7a 100644 --- a/services/spar/test-integration/Test/Spar/APISpec.hs +++ b/services/spar/test-integration/Test/Spar/APISpec.hs @@ -67,10 +67,12 @@ import SAML2.WebSSO (-/), ) import qualified SAML2.WebSSO as SAML +import SAML2.WebSSO.API.Example (SimpleSP) import SAML2.WebSSO.Test.Lenses import SAML2.WebSSO.Test.MockResponse import SAML2.WebSSO.Test.Util import qualified Spar.Intra.BrigApp as Intra +import qualified Spar.Sem.AReqIDStore as AReqIDStore import qualified Spar.Sem.BrigAccess as BrigAccess import qualified Spar.Sem.IdP as IdPEffect import Text.XML.DSig (SignPrivCreds, mkSignCredsWithCert) @@ -198,6 +200,8 @@ specInitiateLogin = do specFinalizeLogin :: SpecWith TestEnv specFinalizeLogin = do describe "POST /sso/finalize-login" $ do + -- @SF.CHANNEL@TSFI.RESTfulAPI @S2 @S3 + -- Receiving an invalid SAML token from client should not give the user a valid access token context "access denied" $ do it "responds with a very peculiar 'forbidden' HTTP response" $ do (_, tid, idp, (_, privcreds)) <- registerTestIdPWithMeta @@ -206,12 +210,6 @@ specFinalizeLogin = do authnresp <- runSimpleSP $ mkAuthnResponse privcreds idp spmeta authnreq False sparresp <- submitAuthnResponse tid authnresp liftIO $ do - -- import Text.XML - -- putStrLn $ unlines - -- [ cs . renderLBS def { rsPretty = True } . fromSignedAuthnResponse $ authnresp - -- , show sparresp - -- , maybe "Nothing" cs (responseBody sparresp) - -- ] statusCode sparresp `shouldBe` 200 let bdy = maybe "" (cs @LBS @String) (responseBody sparresp) bdy `shouldContain` "" @@ -223,9 +221,7 @@ specFinalizeLogin = do bdy `shouldContain` "\"label\":\"forbidden\"" bdy `shouldContain` "}, receiverOrigin)" hasPersistentCookieHeader sparresp `shouldBe` Left "no set-cookie header" - context "user has been deleted" $ do - it "responds with 'forbidden'" $ do - pendingWith "or do we want to un-delete the user? or create a new one?" + context "access granted" $ do let loginSuccess :: HasCallStack => ResponseLBS -> TestSpar () loginSuccess sparresp = liftIO $ do @@ -299,6 +295,8 @@ specFinalizeLogin = do authnresp <- runSimpleSP $ mkAuthnResponse privcreds idp3 spmeta authnreq True loginSuccess =<< submitAuthnResponse tid3 authnresp + -- @SF.CHANNEL@TSFI.RESTfulAPI @S2 @S3 + -- Receiving an invalid SAML token from client should not give the user a valid access token context "idp sends user to two teams with same issuer, nameid" $ do it "fails" $ do skipIdPAPIVersions @@ -358,7 +356,7 @@ specFinalizeLogin = do ) liftIO $ threadDelay 100000 -- make sure deletion is done. if we don't want to take -- the time, we should find another way to robustly - -- confirm that deletion has compelted in the background. + -- confirm that deletion has completed in the background. -- second login do @@ -372,8 +370,8 @@ specFinalizeLogin = do context "known user A, but client device (probably a browser?) is already authenticated as another (probably non-sso) user B" $ do it "logs out user B, logs in user A" $ do + -- TODO(arianvp): Ask Matthias what this even means pending - -- TODO(arianvp): Ask Matthias what this even means context "more than one dsig cert" $ do it "accepts the first of two certs for signatures" $ do @@ -381,47 +379,101 @@ specFinalizeLogin = do it "accepts the second of two certs for signatures" $ do pending - context "unknown IdP Issuer" $ do - it "rejects" $ do - (_, teamid, idp, (_, privcreds)) <- registerTestIdPWithMeta - authnreq <- negotiateAuthnRequest idp - spmeta <- getTestSPMetadata teamid - authnresp <- - runSimpleSP $ - mkAuthnResponse - privcreds - (idp & idpMetadata . edIssuer .~ Issuer [uri|http://unknown-issuer/|]) - spmeta - authnreq - True - sparresp <- submitAuthnResponse teamid authnresp - let shouldContainInBase64 :: String -> String -> Expectation - shouldContainInBase64 hay needle = cs hay'' `shouldContain` needle - where - Right (Just hay'') = decodeBase64 <$> validateBase64 hay' - hay' = cs $ f hay - where - -- exercise to the reader: do this more idiomatically! - f (splitAt 5 -> ("
", s)) = g s
-                    f (_ : s) = f s
-                    f "" = ""
-                    g (splitAt 6 -> ("
", _)) = "" - g (c : s) = c : g s - g "" = "" - liftIO $ do - statusCode sparresp `shouldBe` 404 - -- body should contain the error label in the title, the verbatim haskell error, and the request: - (cs . fromJust . responseBody $ sparresp) `shouldContain` "wire:sso:error:not-found" - (cs . fromJust . responseBody $ sparresp) `shouldContainInBase64` "CustomError (SparIdPNotFound" - (cs . fromJust . responseBody $ sparresp) `shouldContainInBase64` "Input {iName = \"SAMLResponse\"" - -- TODO(arianvp): Ask Matthias what this even means - context "AuthnResponse does not match any request" $ do - it "rejects" $ do - pending - -- TODO(arianvp): Ask Matthias what this even means - context "AuthnResponse contains assertions that have been offered before" $ do - it "rejects" $ do - pending + context "bad AuthnResponse" $ do + let check :: + (IdP -> TestSpar SAML.AuthnRequest) -> + (SignPrivCreds -> IdP -> SAML.SPMetadata -> SAML.AuthnRequest -> SimpleSP SignedAuthnResponse) -> + (TeamId -> SignedAuthnResponse -> TestSpar (Response (Maybe LByteString))) -> + (ResponseLBS -> IO ()) -> + TestSpar () + check mkareq mkaresp submitaresp checkresp = do + (_, teamid, idp, (_, privcreds)) <- registerTestIdPWithMeta + authnreq <- mkareq idp + spmeta <- getTestSPMetadata teamid + authnresp <- + runSimpleSP $ + mkaresp + privcreds + idp + spmeta + authnreq + sparresp <- submitaresp teamid authnresp + liftIO $ checkresp sparresp + + shouldContainInBase64 :: String -> String -> Expectation + shouldContainInBase64 hay needle = cs hay'' `shouldContain` needle + where + Right (Just hay'') = decodeBase64 <$> validateBase64 hay' + hay' = cs $ f hay + where + -- exercise to the reader: do this more idiomatically! + f (splitAt 5 -> ("
", s)) = g s
+                  f (_ : s) = f s
+                  f "" = ""
+                  g (splitAt 6 -> ("
", _)) = "" + g (c : s) = c : g s + g "" = "" + + -- @SF.CHANNEL@TSFI.RESTfulAPI @S2 @S3 + it "rejects saml responses with invalid issuer entity id" $ do + let mkareq = negotiateAuthnRequest + mkaresp privcreds idp spmeta authnreq = + mkAuthnResponse + privcreds + (idp & idpMetadata . edIssuer .~ Issuer [uri|http://unknown-issuer/|]) + spmeta + authnreq + True + submitaresp = submitAuthnResponse + checkresp sparresp = do + statusCode sparresp `shouldBe` 404 + -- body should contain the error label in the title, the verbatim haskell error, and the request: + (cs . fromJust . responseBody $ sparresp) `shouldContain` "wire:sso:error:not-found" + (cs . fromJust . responseBody $ sparresp) `shouldContainInBase64` "CustomError (SparIdPNotFound" + (cs . fromJust . responseBody $ sparresp) `shouldContainInBase64` "Input {iName = \"SAMLResponse\"" + check mkareq mkaresp submitaresp checkresp + + -- @SF.CHANNEL@TSFI.RESTfulAPI @S2 @S3 + it "rejects saml responses signed with the wrong private key" $ do + (_, _, _, (_, badprivcreds)) <- registerTestIdPWithMeta + let mkareq = negotiateAuthnRequest + mkaresp _ idp spmeta authnreq = + mkAuthnResponse + badprivcreds + idp + spmeta + authnreq + True + submitaresp = submitAuthnResponse + checkresp sparresp = statusCode sparresp `shouldBe` 400 + check mkareq mkaresp submitaresp checkresp + + -- @SF.CHANNEL@TSFI.RESTfulAPI @S2 @S3 + it "rejects saml responses to requests not in cassandra:spar.authreq" $ do + let mkareq idp = do + req <- negotiateAuthnRequest idp + runSpar $ AReqIDStore.unStore (req ^. SAML.rqID) + pure req + mkaresp privcreds idp spmeta authnreq = mkAuthnResponse privcreds idp spmeta authnreq True + submitaresp = submitAuthnResponse + checkresp sparresp = do + statusCode sparresp `shouldBe` 200 + (cs . fromJust . responseBody $ sparresp) `shouldContain` "wire:sso:error:forbidden" + (cs . fromJust . responseBody $ sparresp) `shouldContain` "bad InResponseTo attribute(s)" + check mkareq mkaresp submitaresp checkresp + + -- @SF.CHANNEL@TSFI.RESTfulAPI @S2 @S3 + it "rejects saml responses already seen (and recorded in cassandra:spar.authresp)" $ do + let mkareq = negotiateAuthnRequest + mkaresp privcreds idp spmeta authnreq = mkAuthnResponse privcreds idp spmeta authnreq True + submitaresp teamid authnresp = do + _ <- submitAuthnResponse teamid authnresp + submitAuthnResponse teamid authnresp + checkresp sparresp = do + statusCode sparresp `shouldBe` 200 + (cs . fromJust . responseBody $ sparresp) `shouldContain` "wire:sso:error:forbidden" + check mkareq mkaresp submitaresp checkresp + context "IdP changes response format" $ do it "treats NameId case-insensitively" $ do (_ownerid, tid, idp, (_, privcreds)) <- registerTestIdPWithMeta diff --git a/services/spar/test-integration/Test/Spar/Scim/AuthSpec.hs b/services/spar/test-integration/Test/Spar/Scim/AuthSpec.hs index 6ac1b2bbf30..8d100b15459 100644 --- a/services/spar/test-integration/Test/Spar/Scim/AuthSpec.hs +++ b/services/spar/test-integration/Test/Spar/Scim/AuthSpec.hs @@ -52,7 +52,7 @@ spec = do specDeleteToken specListTokens describe "Miscellaneous" $ do - it "doesn't allow SCIM operations without a SCIM token" $ testAuthIsNeeded + it "doesn't allow SCIM operations with invalid or missing SCIM token" testAuthIsNeeded ---------------------------------------------------------------------------- -- Token creation @@ -60,11 +60,11 @@ spec = do -- | Tests for @POST /auth-tokens@. specCreateToken :: SpecWith TestEnv specCreateToken = describe "POST /auth-tokens" $ do - it "works" $ testCreateToken - it "respects the token limit" $ testTokenLimit - it "requires the team to have no more than one IdP" $ testNumIdPs - it "authorizes only admins and owners" $ testCreateTokenAuthorizesOnlyAdmins - it "requires a password" $ testCreateTokenRequiresPassword + it "works" testCreateToken + it "respects the token limit" testTokenLimit + it "requires the team to have no more than one IdP" testNumIdPs + it "authorizes only admins and owners" testCreateTokenAuthorizesOnlyAdmins + it "requires a password" testCreateTokenRequiresPassword -- FUTUREWORK: we should also test that for a password-less user, e.g. for an SSO user, -- reauthentication is not required. We currently (2019-03-05) can't test that because @@ -362,10 +362,15 @@ testDeletedTokensAreUnlistable = do ---------------------------------------------------------------------------- -- Miscellaneous tests --- | Test that without a token, the SCIM API can't be used. +-- @SF.PROVISIONING @TSFI.RESTfulAPI @S2 +-- This test verifies that the SCIM API responds with an authentication error +-- and can't be used if it receives an invalid secret token +-- or if no token is provided at all testAuthIsNeeded :: TestSpar () testAuthIsNeeded = do env <- ask + -- Try to do @GET /Users@ with an invalid token and check that it fails + let invalidToken = ScimToken "this-is-an-invalid-token" + listUsers_ (Just invalidToken) Nothing (env ^. teSpar) !!! checkErr 401 Nothing -- Try to do @GET /Users@ without a token and check that it fails - listUsers_ Nothing Nothing (env ^. teSpar) - !!! checkErr 401 Nothing + listUsers_ Nothing Nothing (env ^. teSpar) !!! checkErr 401 Nothing diff --git a/services/spar/test-integration/Test/Spar/Scim/UserSpec.hs b/services/spar/test-integration/Test/Spar/Scim/UserSpec.hs index e04cf943236..ef91c6fd2fb 100644 --- a/services/spar/test-integration/Test/Spar/Scim/UserSpec.hs +++ b/services/spar/test-integration/Test/Spar/Scim/UserSpec.hs @@ -203,7 +203,7 @@ specCreateUser = describe "POST /Users" $ do context "team has no SAML IdP" $ do it "creates a user with PendingInvitation, and user can follow usual invitation process" $ do testCreateUserNoIdP - it "fails if no email can be extraced from externalId" $ do + it "fails if no email can be extracted from externalId" $ do testCreateUserNoIdPNoEmail it "doesn't list users that exceed their invitation period, and allows recreating them" $ do testCreateUserTimeout From b50e9060ea4c87f0498499f18ac5667ba357d8ec Mon Sep 17 00:00:00 2001 From: fisx Date: Thu, 16 Dec 2021 17:22:28 +0100 Subject: [PATCH 14/74] Add end-tags for test tag parser (BSI audit). (#2001) * Add end-tags for test tag parser (BSI audit). * Changelog. * Fixup * Make test identifiers unique-ish. * Fixup * Fixup --- changelog.d/5-internal/sqservices-1118-2 | 1 + libs/zauth/test/ZAuth.hs | 2 + .../brig/test/integration/API/User/Account.hs | 10 +++++ .../brig/test/integration/API/User/Auth.hs | 8 ++++ .../brig/test/integration/API/User/Client.hs | 10 +++++ .../brig/test/integration/API/User/Handles.hs | 2 + services/galley/test/integration/API.hs | 16 ++++++++ .../test-integration/Test/Spar/APISpec.hs | 38 ++++++++++++------- .../Test/Spar/Scim/AuthSpec.hs | 6 ++- .../Test/Spar/Scim/UserSpec.hs | 2 + 10 files changed, 80 insertions(+), 15 deletions(-) create mode 100644 changelog.d/5-internal/sqservices-1118-2 diff --git a/changelog.d/5-internal/sqservices-1118-2 b/changelog.d/5-internal/sqservices-1118-2 new file mode 100644 index 00000000000..0d240283035 --- /dev/null +++ b/changelog.d/5-internal/sqservices-1118-2 @@ -0,0 +1 @@ +Tag integration tests for certification. diff --git a/libs/zauth/test/ZAuth.hs b/libs/zauth/test/ZAuth.hs index 67729073b45..65dbd8b4c83 100644 --- a/libs/zauth/test/ZAuth.hs +++ b/libs/zauth/test/ZAuth.hs @@ -105,6 +105,8 @@ testExpired p = do x <- liftIO $ runValidate p $ check t liftIO $ Left Expired @=? x +-- @END + testSignAndVerify :: V.Env -> Create () testSignAndVerify p = do u <- liftIO nextRandom diff --git a/services/brig/test/integration/API/User/Account.hs b/services/brig/test/integration/API/User/Account.hs index f09af13c4a9..7c4959d9b92 100644 --- a/services/brig/test/integration/API/User/Account.hs +++ b/services/brig/test/integration/API/User/Account.hs @@ -177,6 +177,8 @@ testCreateUserWithInvalidVerificationCode brig = do ] postUserRegister' regEmail brig !!! const 404 === statusCode +-- @END + testUpdateUserEmailByTeamOwner :: Brig -> Http () testUpdateUserEmailByTeamOwner brig = do (_, teamOwner, emailOwner : otherTeamMember : _) <- createPopulatedBindingTeamWithNamesAndHandles brig 2 @@ -305,6 +307,8 @@ testCreateUserEmptyName brig = do post (brig . path "/register" . contentJson . body p) !!! const 400 === statusCode +-- @END + -- The testCreateUserLongName test conforms to the following testing standards: -- @SF.Provisioning @TSFI.RESTfulAPI @S2 -- @@ -319,6 +323,8 @@ testCreateUserLongName brig = do post (brig . path "/register" . contentJson . body p) !!! const 400 === statusCode +-- @END + testCreateUserAnon :: Brig -> Galley -> Http () testCreateUserAnon brig galley = do let p = @@ -437,6 +443,8 @@ testCreateUserConflict _ brig = do const 409 === statusCode const (Just "key-exists") === fmap Error.label . responseJsonMaybe +-- @END + -- The testCreateUserInvalidEmailOrPhone test conforms to the following testing standards: -- @SF.Provisioning @TSFI.RESTfulAPI @S2 -- @@ -468,6 +476,8 @@ testCreateUserInvalidEmailOrPhone _ brig = do post (brig . path "/register" . contentJson . body reqPhone) !!! const 400 === statusCode +-- @END + testCreateUserBlacklist :: Opt.Opts -> Brig -> AWS.Env -> Http () testCreateUserBlacklist (Opt.setRestrictUserCreation . Opt.optSettings -> Just True) _ _ = pure () testCreateUserBlacklist _ brig aws = diff --git a/services/brig/test/integration/API/User/Auth.hs b/services/brig/test/integration/API/User/Auth.hs index 22ed0ea735e..f4ee97ea3f5 100644 --- a/services/brig/test/integration/API/User/Auth.hs +++ b/services/brig/test/integration/API/User/Auth.hs @@ -377,6 +377,8 @@ testLoginFailure brig = do login brig (PasswordLogin (LoginByEmail badmail) defPassword Nothing) PersistentCookie !!! const 403 === statusCode +-- @END + testThrottleLogins :: Opts.Opts -> Brig -> Http () testThrottleLogins conf b = do -- Get the maximum amount of times we are allowed to login before @@ -455,6 +457,8 @@ testLimitRetries conf brig = do liftIO $ threadDelay (1000000 * 2) login brig (defEmailLogin email) SessionCookie !!! const 200 === statusCode +-- @END + ------------------------------------------------------------------------------- -- LegalHold Login @@ -599,6 +603,8 @@ testInvalidCookie z b = do const 403 === statusCode const (Just "expired") =~= responseBody +-- @END + testInvalidToken :: Brig -> Http () testInvalidToken b = do -- Syntactically invalid @@ -967,6 +973,8 @@ testTooManyCookies config b = do ) xxx -> error ("Unexpected status code when logging in: " ++ show xxx) +-- @END + testLogout :: Brig -> Http () testLogout b = do Just email <- userEmail <$> randomUser b diff --git a/services/brig/test/integration/API/User/Client.hs b/services/brig/test/integration/API/User/Client.hs index 27a20c4c3e2..0a2e1315695 100644 --- a/services/brig/test/integration/API/User/Client.hs +++ b/services/brig/test/integration/API/User/Client.hs @@ -444,6 +444,8 @@ testTooManyClients opts brig = do const (Just "too-many-clients") === fmap Error.label . responseJsonMaybe const (Just "application/json;charset=utf-8") === getHeader "Content-Type" +-- @END + -- The testRemoveClient test conforms to the following testing standards: -- @SF.Provisioning @TSFI.RESTfulAPI @S2 -- @@ -489,6 +491,8 @@ testRemoveClient hasPwd brig cannon = do newClientCookie = Just defCookieLabel } +-- @END + -- The testRemoveClientShortPwd test conforms to the following testing standards: -- @SF.Provisioning @TSFI.RESTfulAPI @S2 -- @@ -521,6 +525,8 @@ testRemoveClientShortPwd brig = do newClientCookie = Just defCookieLabel } +-- @END + -- The testRemoveClientIncorrectPwd test conforms to the following testing standards: -- @SF.Provisioning @TSFI.RESTfulAPI @S2 -- @@ -553,6 +559,8 @@ testRemoveClientIncorrectPwd brig = do newClientCookie = Just defCookieLabel } +-- @END + testUpdateClient :: Opt.Opts -> Brig -> Http () testUpdateClient opts brig = do uid <- userId <$> randomUser brig @@ -760,6 +768,8 @@ testAddMultipleTemporary brig galley = do . zUser u return $ Vec.length <$> (preview _Array =<< responseJsonMaybe @Value r) +-- @END + testPreKeyRace :: Brig -> Http () testPreKeyRace brig = do uid <- userId <$> randomUser brig diff --git a/services/brig/test/integration/API/User/Handles.hs b/services/brig/test/integration/API/User/Handles.hs index a3391991284..5ea8fce5b74 100644 --- a/services/brig/test/integration/API/User/Handles.hs +++ b/services/brig/test/integration/API/User/Handles.hs @@ -138,6 +138,8 @@ testHandleUpdate brig cannon = do put (brig . path "/self/handle" . contentJson . zUser uid2 . zConn "c" . body update) !!! const 200 === statusCode +-- @END + testHandleRace :: Brig -> Http () testHandleRace brig = do us <- replicateM 10 (userId <$> randomUser brig) diff --git a/services/galley/test/integration/API.hs b/services/galley/test/integration/API.hs index 0186f3448d8..13d41e2b806 100644 --- a/services/galley/test/integration/API.hs +++ b/services/galley/test/integration/API.hs @@ -452,6 +452,8 @@ postCryptoMessageVerifyMsgSentAndRejectIfMissingClient = do liftIO $ assertBool "unexpected equal clients" (bc /= bc2) assertNoMsg wsB2 (wsAssertOtr qconv qalice ac bc cipher) +-- @END + -- @SF.Separation @TSFI.RESTfulAPI @S2 -- This test verifies basic mismatch behavior of the the JSON endpoint. postCryptoMessageVerifyRejectMissingClientAndRepondMissingPrekeysJson :: TestM () @@ -478,6 +480,8 @@ postCryptoMessageVerifyRejectMissingClientAndRepondMissingPrekeysJson = do Map.keys (userClientMap (getUserClientPrekeyMap p)) @=? [eve] Map.keys <$> Map.lookup eve (userClientMap (getUserClientPrekeyMap p)) @=? Just [ec] +-- @END + -- @SF.Separation @TSFI.RESTfulAPI @S2 -- This test verifies basic mismatch behaviour of the protobuf endpoint. postCryptoMessageVerifyRejectMissingClientAndRepondMissingPrekeysProto :: TestM () @@ -506,6 +510,8 @@ postCryptoMessageVerifyRejectMissingClientAndRepondMissingPrekeysProto = do Map.keys (userClientMap (getUserClientPrekeyMap p)) @=? [eve] Map.keys <$> Map.lookup eve (userClientMap (getUserClientPrekeyMap p)) @=? Just [ec] +-- @END + -- | This test verifies behaviour when an unknown client posts the message. Only -- tests the Protobuf endpoint. postCryptoMessageNotAuthorizeUnknownClient :: TestM () @@ -547,6 +553,8 @@ postMessageClientNotInGroupDoesNotReceiveMsg = do checkEveGetsMsg checkChadDoesNotGetMsg +-- @END + -- @SF.Separation @TSFI.RESTfulAPI @S2 -- This test verifies that when a client sends a message not to all clients of a group then the server should reject the message and sent a notification to the sender (412 Missing clients). -- The test is somewhat redundant because this is already tested as part of other tests already. This is a stand alone test that solely tests the behavior described above. @@ -575,6 +583,8 @@ postMessageRejectIfMissingClients = do mkMsg :: ByteString -> (UserId, ClientId) -> (UserId, ClientId, Text) mkMsg text (userId, clientId) = (userId, clientId, toBase64Text text) +-- @END + -- @SF.Separation @TSFI.RESTfulAPI @S2 -- This test verifies behaviour under various values of ignore_missing and -- report_missing. Only tests the JSON endpoint. @@ -633,6 +643,8 @@ postCryptoMessageVerifyCorrectResponseIfIgnoreAndReportMissingQueryParam = do where listToByteString = BS.intercalate "," . map toByteString' +-- @END + -- | Sets up a conversation on Backend A known as "owning backend". All user's -- on this backend have names begining with 'A'. The conversation has a couple -- of users from backend B and one user from backend C. @@ -832,6 +844,8 @@ postMessageQualifiedLocalOwningBackendMissingClients = do assertMismatchQualified mempty expectedMissing mempty mempty WS.assertNoEvent (1 # Second) [wsBob, wsChad] +-- @END + -- | Sets up a conversation on Backend A known as "owning backend". One of the -- users from Backend A will send the message, it is expected that message will -- be sent successfully. @@ -1056,6 +1070,8 @@ postMessageQualifiedLocalOwningBackendIgnoreMissingClients = do assertMismatchQualified mempty expectedMissing mempty mempty WS.assertNoEvent (1 # Second) [wsBob, wsChad] +-- @END + postMessageQualifiedLocalOwningBackendFailedToSendClients :: TestM () postMessageQualifiedLocalOwningBackendFailedToSendClients = do -- WS receive timeout diff --git a/services/spar/test-integration/Test/Spar/APISpec.hs b/services/spar/test-integration/Test/Spar/APISpec.hs index 272674dac7a..af466b1650e 100644 --- a/services/spar/test-integration/Test/Spar/APISpec.hs +++ b/services/spar/test-integration/Test/Spar/APISpec.hs @@ -200,9 +200,8 @@ specInitiateLogin = do specFinalizeLogin :: SpecWith TestEnv specFinalizeLogin = do describe "POST /sso/finalize-login" $ do - -- @SF.CHANNEL@TSFI.RESTfulAPI @S2 @S3 - -- Receiving an invalid SAML token from client should not give the user a valid access token - context "access denied" $ do + -- @SF.Channel @TSFI.RESTfulAPI @S2 @S3 + context "rejectsSAMLResponseSayingAccessNotGranted" $ do it "responds with a very peculiar 'forbidden' HTTP response" $ do (_, tid, idp, (_, privcreds)) <- registerTestIdPWithMeta authnreq <- negotiateAuthnRequest idp @@ -222,6 +221,8 @@ specFinalizeLogin = do bdy `shouldContain` "}, receiverOrigin)" hasPersistentCookieHeader sparresp `shouldBe` Left "no set-cookie header" + -- @END + context "access granted" $ do let loginSuccess :: HasCallStack => ResponseLBS -> TestSpar () loginSuccess sparresp = liftIO $ do @@ -295,9 +296,8 @@ specFinalizeLogin = do authnresp <- runSimpleSP $ mkAuthnResponse privcreds idp3 spmeta authnreq True loginSuccess =<< submitAuthnResponse tid3 authnresp - -- @SF.CHANNEL@TSFI.RESTfulAPI @S2 @S3 - -- Receiving an invalid SAML token from client should not give the user a valid access token - context "idp sends user to two teams with same issuer, nameid" $ do + -- @SF.Channel @TSFI.RESTfulAPI @S2 @S3 + context "rejectsSAMLResponseInWrongTeam" $ do it "fails" $ do skipIdPAPIVersions [ WireIdPAPIV1 @@ -321,6 +321,8 @@ specFinalizeLogin = do authnresp <- runSimpleSP $ mkAuthnResponseWithSubj subj privcreds idp2 spmeta authnreq True loginFailure =<< submitAuthnResponse tid2 authnresp + -- @END + context "user is created once, then deleted in team settings, then can login again." $ do it "responds with 'allowed'" $ do (ownerid, teamid, idp, (_, privcreds)) <- registerTestIdPWithMeta @@ -414,8 +416,8 @@ specFinalizeLogin = do g (c : s) = c : g s g "" = "" - -- @SF.CHANNEL@TSFI.RESTfulAPI @S2 @S3 - it "rejects saml responses with invalid issuer entity id" $ do + -- @SF.Channel @TSFI.RESTfulAPI @S2 @S3 + it "rejectsSAMLResponseFromWrongIssuer" $ do let mkareq = negotiateAuthnRequest mkaresp privcreds idp spmeta authnreq = mkAuthnResponse @@ -433,8 +435,10 @@ specFinalizeLogin = do (cs . fromJust . responseBody $ sparresp) `shouldContainInBase64` "Input {iName = \"SAMLResponse\"" check mkareq mkaresp submitaresp checkresp - -- @SF.CHANNEL@TSFI.RESTfulAPI @S2 @S3 - it "rejects saml responses signed with the wrong private key" $ do + -- @END + + -- @SF.Channel @TSFI.RESTfulAPI @S2 @S3 + it "rejectsSAMLResponseSignedWithWrongKey" $ do (_, _, _, (_, badprivcreds)) <- registerTestIdPWithMeta let mkareq = negotiateAuthnRequest mkaresp _ idp spmeta authnreq = @@ -448,8 +452,10 @@ specFinalizeLogin = do checkresp sparresp = statusCode sparresp `shouldBe` 400 check mkareq mkaresp submitaresp checkresp - -- @SF.CHANNEL@TSFI.RESTfulAPI @S2 @S3 - it "rejects saml responses to requests not in cassandra:spar.authreq" $ do + -- @END + + -- @SF.Channel @TSFI.RESTfulAPI @S2 @S3 + it "rejectsSAMLResponseIfRequestIsStale" $ do let mkareq idp = do req <- negotiateAuthnRequest idp runSpar $ AReqIDStore.unStore (req ^. SAML.rqID) @@ -462,8 +468,10 @@ specFinalizeLogin = do (cs . fromJust . responseBody $ sparresp) `shouldContain` "bad InResponseTo attribute(s)" check mkareq mkaresp submitaresp checkresp - -- @SF.CHANNEL@TSFI.RESTfulAPI @S2 @S3 - it "rejects saml responses already seen (and recorded in cassandra:spar.authresp)" $ do + -- @END + + -- @SF.Channel @TSFI.RESTfulAPI @S2 @S3 + it "rejectsSAMLResponseIfResponseIsStale" $ do let mkareq = negotiateAuthnRequest mkaresp privcreds idp spmeta authnreq = mkAuthnResponse privcreds idp spmeta authnreq True submitaresp teamid authnresp = do @@ -474,6 +482,8 @@ specFinalizeLogin = do (cs . fromJust . responseBody $ sparresp) `shouldContain` "wire:sso:error:forbidden" check mkareq mkaresp submitaresp checkresp + -- @END + context "IdP changes response format" $ do it "treats NameId case-insensitively" $ do (_ownerid, tid, idp, (_, privcreds)) <- registerTestIdPWithMeta diff --git a/services/spar/test-integration/Test/Spar/Scim/AuthSpec.hs b/services/spar/test-integration/Test/Spar/Scim/AuthSpec.hs index 8d100b15459..bee95ef5112 100644 --- a/services/spar/test-integration/Test/Spar/Scim/AuthSpec.hs +++ b/services/spar/test-integration/Test/Spar/Scim/AuthSpec.hs @@ -178,6 +178,8 @@ testCreateTokenAuthorizesOnlyAdmins = do (mkUser Galley.RoleAdmin >>= createToken') !!! const 200 === statusCode +-- @END + -- | Test that for a user with a password, token creation requires reauthentication (i.e. the -- field @"password"@ should be provided). -- @@ -362,7 +364,7 @@ testDeletedTokensAreUnlistable = do ---------------------------------------------------------------------------- -- Miscellaneous tests --- @SF.PROVISIONING @TSFI.RESTfulAPI @S2 +-- @SF.Provisioning @TSFI.RESTfulAPI @S2 -- This test verifies that the SCIM API responds with an authentication error -- and can't be used if it receives an invalid secret token -- or if no token is provided at all @@ -374,3 +376,5 @@ testAuthIsNeeded = do listUsers_ (Just invalidToken) Nothing (env ^. teSpar) !!! checkErr 401 Nothing -- Try to do @GET /Users@ without a token and check that it fails listUsers_ Nothing Nothing (env ^. teSpar) !!! checkErr 401 Nothing + +-- @END diff --git a/services/spar/test-integration/Test/Spar/Scim/UserSpec.hs b/services/spar/test-integration/Test/Spar/Scim/UserSpec.hs index ef91c6fd2fb..118ecc67450 100644 --- a/services/spar/test-integration/Test/Spar/Scim/UserSpec.hs +++ b/services/spar/test-integration/Test/Spar/Scim/UserSpec.hs @@ -479,6 +479,8 @@ testCreateRejectsInvalidHandle = do createUser_ (Just tok) (user {Scim.User.userName = "#invalid name"}) (env ^. teSpar) !!! const 400 === statusCode +-- @END + -- | Test that user creation fails if handle is already in use (even on different team). testCreateRejectsTakenHandle :: TestSpar () testCreateRejectsTakenHandle = do From 4e1b8fbbaa079ac909d5945d971dea427ccdbbde Mon Sep 17 00:00:00 2001 From: Sandy Maguire Date: Thu, 16 Dec 2021 12:11:20 -0800 Subject: [PATCH 15/74] Polysemy Spar: Laws for DefaultSsoCode, Now, IdP and ScimExternalIdStore (#1940) --- changelog.d/5-internal/in-mem-specs | 1 + services/spar/package.yaml | 2 +- services/spar/spar.cabal | 7 +- services/spar/src/Spar/Sem/DefaultSsoCode.hs | 2 + services/spar/src/Spar/Sem/Now.hs | 2 + .../spar/src/Spar/Sem/ScimExternalIdStore.hs | 2 + services/spar/test/Arbitrary.hs | 8 +- .../test/Test/Spar/Sem/DefaultSsoCodeSpec.hs | 173 ++++++++++ .../Test/Spar/Sem/IdPRawMetadataStoreSpec.hs | 135 +++++--- services/spar/test/Test/Spar/Sem/IdPSpec.hs | 320 ++++++++++++++++-- services/spar/test/Test/Spar/Sem/NowSpec.hs | 70 ++++ .../Test/Spar/Sem/ScimExternalIdStoreSpec.hs | 192 +++++++++++ stack.yaml | 2 +- stack.yaml.lock | 8 +- 14 files changed, 825 insertions(+), 99 deletions(-) create mode 100644 changelog.d/5-internal/in-mem-specs create mode 100644 services/spar/test/Test/Spar/Sem/DefaultSsoCodeSpec.hs create mode 100644 services/spar/test/Test/Spar/Sem/NowSpec.hs create mode 100644 services/spar/test/Test/Spar/Sem/ScimExternalIdStoreSpec.hs diff --git a/changelog.d/5-internal/in-mem-specs b/changelog.d/5-internal/in-mem-specs new file mode 100644 index 00000000000..57fd8d6b57a --- /dev/null +++ b/changelog.d/5-internal/in-mem-specs @@ -0,0 +1 @@ +Added laws for DefaultSsoCode, Now, IdP and ScimExternalIdStore diff --git a/services/spar/package.yaml b/services/spar/package.yaml index fc27e3fb8a1..935e8cbfe0f 100644 --- a/services/spar/package.yaml +++ b/services/spar/package.yaml @@ -99,7 +99,7 @@ tests: - QuickCheck - spar - uri-bytestring - - polysemy-check + - polysemy-check >= 0.9 executables: spar: diff --git a/services/spar/spar.cabal b/services/spar/spar.cabal index a71562b0f42..cc6c8bb7890 100644 --- a/services/spar/spar.cabal +++ b/services/spar/spar.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: 34138a2c7fa249191ae03bc1581a7c95c94b12080f104da084a9bc37ac54c9ad +-- hash: 4a323def34cfdc7673cea02e13fe518d53d7b04bae552bff2d0784dfb6964162 name: spar version: 0.1 @@ -514,8 +514,11 @@ test-suite spec Test.Spar.Intra.BrigSpec Test.Spar.Roundtrip.ByteString Test.Spar.ScimSpec + Test.Spar.Sem.DefaultSsoCodeSpec Test.Spar.Sem.IdPRawMetadataStoreSpec Test.Spar.Sem.IdPSpec + Test.Spar.Sem.NowSpec + Test.Spar.Sem.ScimExternalIdStoreSpec Test.Spar.TypesSpec Paths_spar hs-source-dirs: @@ -564,7 +567,7 @@ test-suite spec , network-uri , optparse-applicative , polysemy - , polysemy-check + , polysemy-check >=0.9 , polysemy-plugin , raw-strings-qq , retry diff --git a/services/spar/src/Spar/Sem/DefaultSsoCode.hs b/services/spar/src/Spar/Sem/DefaultSsoCode.hs index c18f0334b15..9f594c16b4f 100644 --- a/services/spar/src/Spar/Sem/DefaultSsoCode.hs +++ b/services/spar/src/Spar/Sem/DefaultSsoCode.hs @@ -9,4 +9,6 @@ data DefaultSsoCode m a where Store :: SAML.IdPId -> DefaultSsoCode m () Delete :: DefaultSsoCode m () +deriving instance Show (DefaultSsoCode m a) + makeSem ''DefaultSsoCode diff --git a/services/spar/src/Spar/Sem/Now.hs b/services/spar/src/Spar/Sem/Now.hs index 63d8e740ad5..f5a701b1a03 100644 --- a/services/spar/src/Spar/Sem/Now.hs +++ b/services/spar/src/Spar/Sem/Now.hs @@ -9,6 +9,8 @@ data Now m a where makeSem ''Now +deriving instance Show (Now m a) + -- | Check a time against 'Now', checking if it's still alive (hasn't occurred yet.) boolTTL :: Member Now r => diff --git a/services/spar/src/Spar/Sem/ScimExternalIdStore.hs b/services/spar/src/Spar/Sem/ScimExternalIdStore.hs index b2a43f6b327..3978108770c 100644 --- a/services/spar/src/Spar/Sem/ScimExternalIdStore.hs +++ b/services/spar/src/Spar/Sem/ScimExternalIdStore.hs @@ -10,4 +10,6 @@ data ScimExternalIdStore m a where Lookup :: TeamId -> Email -> ScimExternalIdStore m (Maybe UserId) Delete :: TeamId -> Email -> ScimExternalIdStore m () +deriving instance Show (ScimExternalIdStore m a) + makeSem ''ScimExternalIdStore diff --git a/services/spar/test/Arbitrary.hs b/services/spar/test/Arbitrary.hs index bc7ca9fae4e..a5536460c6c 100644 --- a/services/spar/test/Arbitrary.hs +++ b/services/spar/test/Arbitrary.hs @@ -23,7 +23,7 @@ module Arbitrary where import Data.Aeson -import Data.Id (TeamId) +import Data.Id (TeamId, UserId) import Data.Proxy import Data.String.Conversions (cs) import Data.Swagger hiding (Header (..)) @@ -97,6 +97,8 @@ instance Arbitrary E.Replaced where instance CoArbitrary a => CoArbitrary (E.GetIdPResult a) +-- TODO(sandy): IdPIds are unlikely to collide. Does the size parameter +-- affect them? instance CoArbitrary IdPId instance CoArbitrary WireIdP @@ -105,6 +107,10 @@ instance CoArbitrary WireIdPAPIVersion instance CoArbitrary TeamId +instance CoArbitrary UserId + +instance CoArbitrary Time + instance CoArbitrary Issuer where coarbitrary (Issuer ur) = coarbitrary $ show ur diff --git a/services/spar/test/Test/Spar/Sem/DefaultSsoCodeSpec.hs b/services/spar/test/Test/Spar/Sem/DefaultSsoCodeSpec.hs new file mode 100644 index 00000000000..9a638ba9907 --- /dev/null +++ b/services/spar/test/Test/Spar/Sem/DefaultSsoCodeSpec.hs @@ -0,0 +1,173 @@ +{-# LANGUAGE QuantifiedConstraints #-} +{-# OPTIONS_GHC -Wno-orphans #-} +{-# OPTIONS_GHC -fplugin=Polysemy.Plugin #-} + +module Test.Spar.Sem.DefaultSsoCodeSpec where + +import Arbitrary () +import Imports +import Polysemy +import Polysemy.Check +import SAML2.WebSSO.Types +import qualified Spar.Sem.DefaultSsoCode as E +import Spar.Sem.DefaultSsoCode.Mem +import Test.Hspec +import Test.Hspec.QuickCheck +import Test.QuickCheck + +deriveGenericK ''E.DefaultSsoCode + +propsForInterpreter :: + PropConstraints r f => + String -> + (forall a. Sem r a -> IO (f a)) -> + Spec +propsForInterpreter interpreter lower = do + describe interpreter $ do + prop "delete/delete" $ prop_deleteDelete Nothing lower + prop "delete/get" $ prop_deleteGet Nothing lower + prop "delete/store" $ prop_deleteStore Nothing lower + prop "get/store" $ prop_getStore Nothing lower + prop "store/delete" $ prop_storeStore Nothing lower + prop "store/get" $ prop_storeGet Nothing lower + prop "store/store" $ prop_storeStore Nothing lower + +spec :: Spec +spec = modifyMaxSuccess (const 1000) $ do + propsForInterpreter "defaultSsoCodeToMem" $ pure . run . defaultSsoCodeToMem + +-- | All the constraints we need to generalize properties in this module. +-- A regular type synonym doesn't work due to dreaded impredicative +-- polymorphism. +class + (Functor f, Member E.DefaultSsoCode r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +instance + (Functor f, Member E.DefaultSsoCode r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +prop_storeGet :: + PropConstraints r f => + Maybe (f (Maybe IdPId) -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_storeGet = + prepropLaw @'[E.DefaultSsoCode] $ do + s <- arbitrary + pure $ + simpleLaw + ( do + E.store s + E.get + ) + ( do + E.store s + pure (Just s) + ) + +prop_getStore :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_getStore = + prepropLaw @'[E.DefaultSsoCode] $ do + pure $ + simpleLaw + ( do + E.get >>= maybe (pure ()) E.store + ) + ( do + pure () + ) + +prop_storeDelete :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_storeDelete = + prepropLaw @'[E.DefaultSsoCode] $ do + s <- arbitrary + pure $ + simpleLaw + ( do + E.store s + E.delete + ) + ( do + E.delete + ) + +prop_deleteStore :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_deleteStore = + prepropLaw @'[E.DefaultSsoCode] $ do + s <- arbitrary + pure $ + simpleLaw + ( do + E.delete + E.store s + ) + ( do + E.store s + ) + +prop_storeStore :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_storeStore = + prepropLaw @'[E.DefaultSsoCode] $ do + s <- arbitrary + s' <- arbitrary + pure $ + simpleLaw + ( do + E.store s + E.store s' + ) + ( do + E.store s' + ) + +prop_deleteDelete :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_deleteDelete = + prepropLaw @'[E.DefaultSsoCode] $ do + pure $ + simpleLaw + ( do + E.delete + E.delete + ) + ( do + E.delete + ) + +prop_deleteGet :: + PropConstraints r f => + Maybe (f (Maybe IdPId) -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_deleteGet = + prepropLaw @'[E.DefaultSsoCode] $ do + pure $ + simpleLaw + ( do + E.delete + E.get + ) + ( do + E.delete + pure Nothing + ) diff --git a/services/spar/test/Test/Spar/Sem/IdPRawMetadataStoreSpec.hs b/services/spar/test/Test/Spar/Sem/IdPRawMetadataStoreSpec.hs index c1d5f6b9b81..28aa64122dd 100644 --- a/services/spar/test/Test/Spar/Sem/IdPRawMetadataStoreSpec.hs +++ b/services/spar/test/Test/Spar/Sem/IdPRawMetadataStoreSpec.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE QuantifiedConstraints #-} {-# OPTIONS_GHC -Wno-orphans #-} {-# OPTIONS_GHC -fplugin=Polysemy.Plugin #-} @@ -15,97 +16,121 @@ import Test.QuickCheck deriveGenericK ''E.IdPRawMetadataStore +class + (Functor f, Member E.IdPRawMetadataStore r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +instance + (Functor f, Member E.IdPRawMetadataStore r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + prop_storeGetRaw :: - Member E.IdPRawMetadataStore r => - (forall a. Sem r a -> IO (RawState, a)) -> + PropConstraints r f => + Maybe (f (Maybe Text) -> String) -> + (forall a. Sem r a -> IO (f a)) -> Property -prop_storeGetRaw x = +prop_storeGetRaw = prepropLaw @'[E.IdPRawMetadataStore] ( do idpid <- arbitrary t <- arbitrary - pure - ( do - E.store idpid t - E.get idpid, - do - E.store idpid t - pure (Just t) - ) + pure $ + simpleLaw + ( do + E.store idpid t + E.get idpid + ) + ( do + E.store idpid t + pure (Just t) + ) ) - x prop_storeStoreRaw :: - Member E.IdPRawMetadataStore r => - (forall a. Sem r a -> IO (RawState, a)) -> + PropConstraints r f => + Maybe (f (Maybe Text) -> String) -> + (forall a. Sem r a -> IO (f a)) -> Property -prop_storeStoreRaw x = +prop_storeStoreRaw = prepropLaw @'[E.IdPRawMetadataStore] ( do idpid <- arbitrary t1 <- arbitrary t2 <- arbitrary - pure - ( do - E.store idpid t1 - E.store idpid t2, - do - E.store idpid t2 - ) + pure $ + simpleLaw + ( do + E.store idpid t1 + E.store idpid t2 + E.get idpid + ) + ( do + E.store idpid t2 + E.get idpid + ) ) - x prop_storeDeleteRaw :: - Member E.IdPRawMetadataStore r => - (forall a. Sem r a -> IO (RawState, a)) -> + PropConstraints r f => + Maybe (f (Maybe Text) -> String) -> + (forall a. Sem r a -> IO (f a)) -> Property -prop_storeDeleteRaw x = - prepropLaw @'[E.IdPRawMetadataStore] - ( do - idpid <- arbitrary - t <- arbitrary - pure +prop_storeDeleteRaw = + prepropLaw @'[E.IdPRawMetadataStore] $ + do + idpid <- arbitrary + t <- arbitrary + pure $ + simpleLaw ( do E.store idpid t - E.delete idpid, - do E.delete idpid + E.get idpid + ) + ( do + E.delete idpid + E.get idpid ) - ) - x prop_deleteGetRaw :: - Member E.IdPRawMetadataStore r => - (forall a. Sem r a -> IO (RawState, a)) -> + PropConstraints r f => + Maybe (f (Maybe Text) -> String) -> + (forall a. Sem r a -> IO (f a)) -> Property -prop_deleteGetRaw x = +prop_deleteGetRaw = prepropLaw @'[E.IdPRawMetadataStore] ( do idpid <- arbitrary - pure - ( do - E.delete idpid - E.get idpid, - do - E.delete idpid - pure Nothing - ) + t <- arbitrary + pure $ + Law + { lawLhs = do + E.delete idpid + E.get idpid, + lawRhs = do + E.delete idpid + pure Nothing, + lawPrelude = + [ E.store idpid t + ], + lawPostlude = [] @(Sem _ ()) + } ) - x testInterpreter :: Sem '[E.IdPRawMetadataStore] a -> IO (RawState, a) testInterpreter = pure . run . idpRawMetadataStoreToMem propsForInterpreter :: - Member E.IdPRawMetadataStore r => - (forall a. Sem r a -> IO (RawState, a)) -> + PropConstraints r f => + (forall x. f x -> x) -> + (forall a. Sem r a -> IO (f a)) -> Spec -propsForInterpreter lower = do - prop "store/store" $ prop_storeStoreRaw lower - prop "store/get" $ prop_storeGetRaw lower - prop "store/deleteRawMetadata" $ prop_storeDeleteRaw lower - prop "deleteRawMetadata/get" $ prop_deleteGetRaw lower +propsForInterpreter extract lower = do + prop "store/store" $ prop_storeStoreRaw (Just $ constructorLabel . extract) lower + prop "store/get" $ prop_storeGetRaw (Just $ constructorLabel . extract) lower + prop "store/delete" $ prop_storeDeleteRaw (Just $ constructorLabel . extract) lower + prop "delete/get" $ prop_deleteGetRaw (Just $ constructorLabel . extract) lower spec :: Spec spec = modifyMaxSuccess (const 1000) $ do - propsForInterpreter testInterpreter + propsForInterpreter snd testInterpreter diff --git a/services/spar/test/Test/Spar/Sem/IdPSpec.hs b/services/spar/test/Test/Spar/Sem/IdPSpec.hs index 162f5b1625f..ef1eba1f471 100644 --- a/services/spar/test/Test/Spar/Sem/IdPSpec.hs +++ b/services/spar/test/Test/Spar/Sem/IdPSpec.hs @@ -1,72 +1,322 @@ +{-# LANGUAGE DeriveDataTypeable #-} +{-# LANGUAGE QuantifiedConstraints #-} +{-# LANGUAGE StandaloneDeriving #-} {-# OPTIONS_GHC -Wno-orphans #-} {-# OPTIONS_GHC -fplugin=Polysemy.Plugin #-} module Test.Spar.Sem.IdPSpec where import Arbitrary () +import Control.Arrow import Control.Lens +import Data.Data (Data) import Imports import Polysemy import Polysemy.Check import SAML2.WebSSO.Types +import qualified SAML2.WebSSO.Types as SAML import qualified Spar.Sem.IdP as E import Spar.Sem.IdP.Mem import Test.Hspec import Test.Hspec.QuickCheck import Test.QuickCheck +import qualified Wire.API.User.IdentityProvider as IP deriveGenericK ''E.IdP +deriving instance Data (E.GetIdPResult IdPId) + +deriving instance Data (IdPId) + +propsForInterpreter :: + (Member E.IdP r, PropConstraints r f) => + String -> + (forall x. f x -> x) -> + (forall x. Show x => Maybe (f x -> String)) -> + (forall x. Sem r x -> IO (f x)) -> + Spec +propsForInterpreter interpreter extract labeler lower = do + describe interpreter $ do + prop "deleteConfig/deleteConfig" $ prop_deleteDelete Nothing lower + prop "deleteConfig/getConfig" $ prop_deleteGet labeler lower + prop "getConfig/storeConfig" $ prop_getStore (Just $ show . (() <$) . extract) lower + prop "getConfig/getConfig" $ prop_getGet (Just $ show . ((() <$) *** (() <$)) . extract) lower + prop "setReplacedBy/clearReplacedBy" $ prop_setClear labeler lower + prop "setReplacedBy/getReplacedBy" $ prop_setGet (Just $ show . (fmap (() <$)) . extract) lower + prop "setReplacedBy/setReplacedBy" $ prop_setSet (Just $ show . (fmap (() <$)) . extract) lower + prop "storeConfig/getConfig" $ prop_storeGet (Just $ show . (() <$) . extract) lower + prop "storeConfig/getIdByIssuerWithoutTeam" $ prop_storeGetByIssuer (Just $ constructorLabel . extract) lower + prop "storeConfig/storeConfig (different keys)" $ prop_storeStoreInterleave Nothing lower + prop "storeConfig/storeConfig (same keys)" $ prop_storeStore Nothing lower + +spec :: Spec +spec = modifyMaxSuccess (const 1000) $ do + propsForInterpreter "idPToMem" snd (Just $ show . snd) $ pure . run . idPToMem + +getReplacedBy :: Member E.IdP r => SAML.IdPId -> Sem r (Maybe (Maybe SAML.IdPId)) +getReplacedBy idpid = fmap (view $ SAML.idpExtraInfo . IP.wiReplacedBy) <$> E.getConfig idpid + +-- | All the constraints we need to generalize properties in this module. +-- A regular type synonym doesn't work due to dreaded impredicative +-- polymorphism. +class + (Functor f, Member E.IdP r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +instance + (Functor f, Member E.IdP r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +prop_storeStore :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_storeStore = + prepropLaw @'[E.IdP] $ do + s <- arbitrary + s' <- arbitrary + pure $ + Law + { lawLhs = do + E.storeConfig $ s & SAML.idpId .~ s' ^. SAML.idpId + E.storeConfig s', + lawRhs = do + E.storeConfig s', + lawPrelude = [], + lawPostlude = [E.getConfig $ s' ^. SAML.idpId] + } + +prop_storeStoreInterleave :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_storeStoreInterleave = + prepropLaw @'[E.IdP] $ do + s <- arbitrary + s' <- arbitrary + !_ <- + when (s ^. SAML.idpId == s' ^. SAML.idpId) discard + pure $ + Law + { lawLhs = do + E.storeConfig s + E.storeConfig s', + lawRhs = do + E.storeConfig s' + E.storeConfig s, + lawPrelude = [], + lawPostlude = [E.getConfig $ s ^. SAML.idpId, E.getConfig $ s' ^. SAML.idpId] + } + prop_storeGet :: - Member E.IdP r => - (forall a. Sem r a -> IO (TypedState, a)) -> + PropConstraints r f => + Maybe (f (Maybe IP.IdP) -> String) -> + (forall x. Sem r x -> IO (f x)) -> Property -prop_storeGet x = - prepropLaw @'[E.IdP] - ( do - s <- arbitrary - pure +prop_storeGet = + prepropLaw @'[E.IdP] $ + do + s <- arbitrary + pure $ + simpleLaw ( do E.storeConfig s - E.getConfig $ s ^. idpId, - do + E.getConfig $ s ^. idpId + ) + ( do E.storeConfig s pure (Just s) ) - ) - x + +prop_deleteGet :: + PropConstraints r f => + Maybe (f (Maybe IP.IdP) -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_deleteGet = + prepropLaw @'[E.IdP] $ do + s <- arbitrary + pure $ + Law + { lawLhs = do + E.deleteConfig s + E.getConfig $ s ^. SAML.idpId, + lawRhs = do + E.deleteConfig s + pure Nothing, + lawPrelude = + [ E.storeConfig s + ], + lawPostlude = [] :: [Sem r ()] + } + +prop_deleteDelete :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_deleteDelete = + prepropLaw @'[E.IdP] $ do + s <- arbitrary + pure $ + simpleLaw + ( do + E.deleteConfig s + E.deleteConfig s + ) + ( do + E.deleteConfig s + ) prop_storeGetByIssuer :: - Member E.IdP r => - (forall a. Sem r a -> IO (TypedState, a)) -> + PropConstraints r f => + Maybe (f (E.GetIdPResult IdPId) -> String) -> + (forall x. Sem r x -> IO (f x)) -> Property -prop_storeGetByIssuer x = - prepropLaw @'[E.IdP] - ( do - s <- arbitrary - pure +prop_storeGetByIssuer = + prepropLaw @'[E.IdP] $ + do + s <- arbitrary + pure $ + simpleLaw ( do E.storeConfig s - E.getIdByIssuerWithoutTeam $ s ^. idpMetadata . edIssuer, - do + E.getIdByIssuerWithoutTeam $ s ^. idpMetadata . edIssuer + ) + ( do E.storeConfig s pure $ E.GetIdPFound $ s ^. idpId ) - ) - x -testInterpreter :: Sem '[E.IdP] a -> IO (TypedState, a) -testInterpreter = pure . run . idPToMem +prop_setClear :: + PropConstraints r f => + Maybe (f (Maybe (Maybe IdPId)) -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_setClear = + prepropLaw @'[E.IdP] $ + do + idp <- arbitrary + replaced_id <- arbitrary + let replaced = E.Replaced replaced_id + replacing <- arbitrary + pure $ + Law + { lawLhs = do + E.setReplacedBy replaced replacing + E.clearReplacedBy replaced + getReplacedBy replaced_id, + lawRhs = do + E.clearReplacedBy replaced + getReplacedBy replaced_id, + lawPrelude = + [ E.storeConfig $ idp & SAML.idpId .~ replaced_id + ], + lawPostlude = [] @(Sem _ ()) + } -propsForInterpreter :: - Member E.IdP r => - (forall a. Sem r a -> IO (TypedState, a)) -> - Spec -propsForInterpreter lower = do - describe "Config Actions" $ do - prop "storeConfig/getConfig" $ prop_storeGet lower - prop "storeConfig/getIdByIssuerWithoutTeam" $ prop_storeGetByIssuer lower +prop_getGet :: + forall r f. + PropConstraints r f => + Maybe (f (Maybe IP.IdP, Maybe IP.IdP) -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_getGet = + prepropLaw @'[E.IdP] $ + do + idpid <- arbitrary + idp <- arbitrary + pure $ + Law + { lawLhs = do + liftA2 (,) (E.getConfig idpid) (E.getConfig idpid), + lawRhs = do + cfg <- E.getConfig idpid + pure (cfg, cfg), + lawPrelude = + [ E.storeConfig $ idp & SAML.idpId .~ idpid + ], + lawPostlude = [] :: [Sem r ()] + } -spec :: Spec -spec = modifyMaxSuccess (const 1000) $ do - propsForInterpreter testInterpreter +prop_getStore :: + PropConstraints r f => + Maybe (f (Maybe IP.IdP) -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_getStore = + prepropLaw @'[E.IdP] $ + do + idpid <- arbitrary + s <- arbitrary + let s' = s & SAML.idpId .~ idpid + pure $ + Law + { lawLhs = do + r <- E.getConfig idpid + maybe (pure ()) E.storeConfig r + pure r, + lawRhs = do + E.getConfig idpid, + lawPrelude = + [E.storeConfig s'], + lawPostlude = + [E.getConfig idpid] + } + +prop_setSet :: + PropConstraints r f => + Maybe (f (Maybe (Maybe IdPId)) -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_setSet = + prepropLaw @'[E.IdP] $ + do + replaced_id <- arbitrary + s <- arbitrary + let s' = s & SAML.idpId .~ replaced_id + let replaced = E.Replaced replaced_id + replacing <- arbitrary + replacing' <- arbitrary + pure $ + Law + { lawLhs = do + E.setReplacedBy replaced replacing + E.setReplacedBy replaced replacing' + getReplacedBy replaced_id, + lawRhs = do + E.setReplacedBy replaced replacing' + getReplacedBy replaced_id, + lawPrelude = + [E.storeConfig s'], + lawPostlude = [] @(Sem _ ()) + } + +prop_setGet :: + PropConstraints r f => + Maybe (f (Maybe (Maybe IdPId)) -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_setGet = + prepropLaw @'[E.IdP] $ + do + idp <- arbitrary + replaced_id <- arbitrary + let replaced = E.Replaced replaced_id + replacing_id <- arbitrary + let replacing = E.Replacing replacing_id + pure $ + Law + { lawLhs = do + E.setReplacedBy replaced replacing + getReplacedBy replaced_id, + lawRhs = do + E.setReplacedBy replaced replacing + (Just replacing_id <$) <$> E.getConfig replaced_id, + lawPrelude = + [ E.storeConfig $ idp & SAML.idpId .~ replaced_id + ], + lawPostlude = [] :: [Sem r ()] + } diff --git a/services/spar/test/Test/Spar/Sem/NowSpec.hs b/services/spar/test/Test/Spar/Sem/NowSpec.hs new file mode 100644 index 00000000000..0c9fdacadfb --- /dev/null +++ b/services/spar/test/Test/Spar/Sem/NowSpec.hs @@ -0,0 +1,70 @@ +{-# LANGUAGE QuantifiedConstraints #-} +{-# OPTIONS_GHC -Wno-orphans #-} +{-# OPTIONS_GHC -fplugin=Polysemy.Plugin #-} + +module Test.Spar.Sem.NowSpec where + +import Arbitrary () +import Data.Time +import Data.Time.Calendar.Julian +import Imports +import Polysemy +import Polysemy.Check +import Polysemy.Input +import SAML2.WebSSO.Types +import qualified Spar.Sem.Now as E +import Spar.Sem.Now.IO +import Spar.Sem.Now.Input +import Test.Hspec +import Test.Hspec.QuickCheck +import Test.QuickCheck + +deriveGenericK ''E.Now + +propsForInterpreter :: + PropConstraints r f => + String -> + (forall a. Sem r a -> IO (f a)) -> + Spec +propsForInterpreter interpreter lower = do + describe interpreter $ do + prop "now/now" $ prop_nowNow Nothing lower + +someTime :: Time +someTime = Time (UTCTime (fromJulianYearAndDay 1990 209) (secondsToDiffTime 0)) + +spec :: Spec +spec = do + modifyMaxSuccess (const 1000) $ do + propsForInterpreter "nowToIO" $ fmap Identity . runM . nowToIO . runInputConst () + propsForInterpreter "nowToInput" $ pure . Identity . run . runInputConst someTime . nowToInput . runInputConst () + +-- | All the constraints we need to generalize properties in this module. +-- A regular type synonym doesn't work due to dreaded impredicative +-- polymorphism. +class + (Functor f, Member E.Now r, Member (Input ()) r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +instance + (Functor f, Member E.Now r, Member (Input ()) r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +prop_nowNow :: + PropConstraints r f => + Maybe (f Bool -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_nowNow = + -- NOTE: This @Input ()@ effect is a workaround to an oversight in + -- @polysemy-check@. 'prepropLaw' wants to synthesize some actions to run + -- before and after its generators, and check their results for equality. We + -- can't use 'Now' as this effect, because 'E.get' won't return equivalent + -- results! And we can't keep it empty, because that triggers a crash in + -- @polysemy-check@. Thus @Input ()@, which isn't beautiful, but works fine. + prepropLaw @'[Input ()] $ do + pure $ + simpleLaw + (liftA2 (<=) E.get E.get) + ( pure True + ) diff --git a/services/spar/test/Test/Spar/Sem/ScimExternalIdStoreSpec.hs b/services/spar/test/Test/Spar/Sem/ScimExternalIdStoreSpec.hs new file mode 100644 index 00000000000..e6ccad58a4a --- /dev/null +++ b/services/spar/test/Test/Spar/Sem/ScimExternalIdStoreSpec.hs @@ -0,0 +1,192 @@ +{-# LANGUAGE QuantifiedConstraints #-} +{-# OPTIONS_GHC -Wno-orphans #-} +{-# OPTIONS_GHC -fplugin=Polysemy.Plugin #-} + +module Test.Spar.Sem.ScimExternalIdStoreSpec where + +import Arbitrary () +import Data.Id +import Imports +import Polysemy +import Polysemy.Check +import qualified Spar.Sem.ScimExternalIdStore as E +import Spar.Sem.ScimExternalIdStore.Mem +import Test.Hspec +import Test.Hspec.QuickCheck +import Test.QuickCheck + +deriveGenericK ''E.ScimExternalIdStore + +propsForInterpreter :: + PropConstraints r f => + String -> + (forall a. f a -> a) -> + (forall a. Sem r a -> IO (f a)) -> + Spec +propsForInterpreter interpreter extract lower = do + describe interpreter $ do + prop "delete/delete" $ prop_deleteDelete Nothing lower + prop "delete/lookup" $ prop_deleteLookup (Just $ show . (() <$) . extract) lower + prop "delete/insert" $ prop_deleteInsert Nothing lower + prop "lookup/insert" $ prop_lookupInsert Nothing lower + prop "insert/delete" $ prop_insertDelete Nothing lower + prop "insert/lookup" $ prop_insertLookup (Just $ show . (() <$) . extract) lower + prop "insert/insert" $ prop_insertInsert (Just $ show . (() <$) . extract) lower + +spec :: Spec +spec = modifyMaxSuccess (const 1000) $ do + propsForInterpreter "scimExternalIdStoreToMem" snd $ pure . run . scimExternalIdStoreToMem + +-- | All the constraints we need to generalize properties in this module. +-- A regular type synonym doesn't work due to dreaded impredicative +-- polymorphism. +class + (Functor f, Member E.ScimExternalIdStore r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +instance + (Functor f, Member E.ScimExternalIdStore r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +prop_insertLookup :: + PropConstraints r f => + Maybe (f (Maybe UserId) -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_insertLookup = + prepropLaw @'[E.ScimExternalIdStore] $ do + tid <- arbitrary + email <- arbitrary + uid <- arbitrary + pure $ + simpleLaw + ( do + E.insert tid email uid + E.lookup tid email + ) + ( do + E.insert tid email uid + pure (Just uid) + ) + +prop_lookupInsert :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_lookupInsert = + prepropLaw @'[E.ScimExternalIdStore] $ do + tid <- arbitrary + email <- arbitrary + pure $ + simpleLaw + ( do + E.lookup tid email >>= maybe (pure ()) (E.insert tid email) + ) + ( do + pure () + ) + +prop_insertDelete :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_insertDelete = + prepropLaw @'[E.ScimExternalIdStore] $ do + tid <- arbitrary + email <- arbitrary + uid <- arbitrary + pure $ + simpleLaw + ( do + E.insert tid email uid + E.delete tid email + ) + ( do + E.delete tid email + ) + +prop_deleteInsert :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_deleteInsert = + prepropLaw @'[E.ScimExternalIdStore] $ do + tid <- arbitrary + email <- arbitrary + uid <- arbitrary + pure $ + simpleLaw + ( do + E.delete tid email + E.insert tid email uid + ) + ( do + E.insert tid email uid + ) + +prop_insertInsert :: + PropConstraints r f => + Maybe (f (Maybe UserId) -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_insertInsert = + prepropLaw @'[E.ScimExternalIdStore] $ do + tid <- arbitrary + email <- arbitrary + uid <- arbitrary + uid' <- arbitrary + pure $ + simpleLaw + ( do + E.insert tid email uid + E.insert tid email uid' + E.lookup tid email + ) + ( do + E.insert tid email uid' + E.lookup tid email + ) + +prop_deleteDelete :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_deleteDelete = + prepropLaw @'[E.ScimExternalIdStore] $ do + tid <- arbitrary + email <- arbitrary + pure $ + simpleLaw + ( do + E.delete tid email + E.delete tid email + ) + ( do + E.delete tid email + ) + +prop_deleteLookup :: + PropConstraints r f => + Maybe (f (Maybe UserId) -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_deleteLookup = + prepropLaw @'[E.ScimExternalIdStore] $ do + tid <- arbitrary + email <- arbitrary + uid <- arbitrary + pure $ + Law + { lawLhs = do + E.delete tid email + E.lookup tid email, + lawRhs = do + E.delete tid email + pure Nothing, + lawPrelude = [E.insert tid email uid], + lawPostlude = [] @(Sem _ ()) + } diff --git a/stack.yaml b/stack.yaml index 6081f77af12..d5e4cb78413 100644 --- a/stack.yaml +++ b/stack.yaml @@ -224,7 +224,7 @@ extra-deps: # Not on stackage yet - polysemy-1.7.0.0 - polysemy-plugin-0.4.2.0 -- polysemy-check-0.8.1.0 +- polysemy-check-0.9.0.0 ############################################################ # Development tools diff --git a/stack.yaml.lock b/stack.yaml.lock index c9b4d4ccfc9..1635ea076ad 100644 --- a/stack.yaml.lock +++ b/stack.yaml.lock @@ -683,12 +683,12 @@ packages: original: hackage: polysemy-plugin-0.4.2.0 - completed: - hackage: polysemy-check-0.8.1.0@sha256:5cce3ae162d2f8d8f629397daa28ec5e425f72d357afeb4fe994e102425f2383,2648 + hackage: polysemy-check-0.9.0.0@sha256:f28c23c5cbae246a049d11e06c51ee85212a2b13a069e93598cf8cdd13ad5a18,2665 pantry-tree: - size: 1027 - sha256: bc880fb3405307ed251c02358d604979d8014040b78c2ffe6319076431f93509 + size: 1086 + sha256: a473605eda27f36717e3f0cbd66651563789107daa9b8d9db59b80cc07ff60d1 original: - hackage: polysemy-check-0.8.1.0 + hackage: polysemy-check-0.9.0.0 - completed: hackage: ormolu-0.1.4.1@sha256:ed404eac6e4eb64da1ca5fb749e0f99907431a9633e6ba34e44d260e7d7728ba,6499 pantry-tree: From e4503ff778a97624de5e77c8e0cdecc4e339c536 Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Fri, 17 Dec 2021 13:42:04 +0100 Subject: [PATCH 16/74] schema-profunctor: better optional field API (#1988) * schema-profunctor: better optional field API * Add nullable combinator and tests --- .../5-internal/schema-profunctor-optional | 1 + .../src/Galley/Types/Conversations/Intra.hs | 2 +- libs/schema-profunctor/README.md | 104 ++++++----- libs/schema-profunctor/src/Data/Schema.hs | 162 ++++++++++-------- .../test/unit/Test/Data/Schema.hs | 38 ++-- libs/types-common/src/Data/Domain.hs | 2 +- libs/wire-api/src/Wire/API/Connection.hs | 74 ++++---- libs/wire-api/src/Wire/API/Conversation.hs | 51 +++--- .../src/Wire/API/Conversation/Code.hs | 4 +- .../src/Wire/API/Conversation/Member.hs | 30 ++-- .../src/Wire/API/Event/Conversation.hs | 28 +-- libs/wire-api/src/Wire/API/Message.hs | 6 +- .../API/Routes/Internal/Brig/Connection.hs | 6 +- .../src/Wire/API/Routes/MultiTablePaging.hs | 4 +- libs/wire-api/src/Wire/API/User.hs | 14 +- libs/wire-api/src/Wire/API/User/Client.hs | 89 +++++----- libs/wire-api/src/Wire/API/User/Profile.hs | 2 +- 17 files changed, 325 insertions(+), 292 deletions(-) create mode 100644 changelog.d/5-internal/schema-profunctor-optional diff --git a/changelog.d/5-internal/schema-profunctor-optional b/changelog.d/5-internal/schema-profunctor-optional new file mode 100644 index 00000000000..4e66ad936cc --- /dev/null +++ b/changelog.d/5-internal/schema-profunctor-optional @@ -0,0 +1 @@ +Improve optional field API in schema-profunctor diff --git a/libs/galley-types/src/Galley/Types/Conversations/Intra.hs b/libs/galley-types/src/Galley/Types/Conversations/Intra.hs index 0cb0ba9afd7..e41823f3cef 100644 --- a/libs/galley-types/src/Galley/Types/Conversations/Intra.hs +++ b/libs/galley-types/src/Galley/Types/Conversations/Intra.hs @@ -72,7 +72,7 @@ instance ToSchema UpsertOne2OneConversationRequest where <*> (qUntagged . uooRemoteUser) .= field "remote_user" (qTagUnsafe <$> schema) <*> uooActor .= field "actor" schema <*> uooActorDesiredMembership .= field "actor_desired_membership" schema - <*> uooConvId .= field "conversation_id" (optWithDefault A.Null schema) + <*> uooConvId .= optField "conversation_id" (maybeWithDefault A.Null schema) newtype UpsertOne2OneConversationResponse = UpsertOne2OneConversationResponse { uuorConvId :: Qualified ConvId diff --git a/libs/schema-profunctor/README.md b/libs/schema-profunctor/README.md index 282b9fdd263..9f1421f77d4 100644 --- a/libs/schema-profunctor/README.md +++ b/libs/schema-profunctor/README.md @@ -45,8 +45,8 @@ structure of lists when using the `Applicative` interface of ## Tutorial -To learn how to use `SchemaP` in practice, let us walk through two -basic examples, one for a record, and one for a sum type. +To learn how to use `SchemaP` in practice, let us walk through some +basic examples, including records and sum types. ### Records @@ -345,9 +345,8 @@ represented on the Haskell side. ### Optional fields and default values -To define a schema for a JSON object, there are multiple ways to deal -with the serialisation of optional fields, which we will illustrate -here. +To define a schema for a JSON object, there are multiple ways to deal with the +serialisation of optional fields, which we will illustrate here. The simplest (and most common) scenario is an optional field represented by a `Maybe` type, that is simply omitted from the generated JSON if it happens to @@ -365,42 +364,48 @@ data User = User userSchema = object "User" $ User <$> userName .= field "name" schema - <*> userHandle .= opt (field "handle" schema) - <*> userExpire .= opt (field "expire" schema) + <*> userHandle .= maybe_ (optField "handle" schema) + <*> userExpire .= maybe_ (optField "expire" schema) ``` -Here we apply the `opt` combinator to the optional field, to turn it from a -schema for `Text` into a schema for `Maybe Text`. The parser for `userHandle` -will return `Nothing` when the field is missing (or is `null`), and -correspondingly the serialiser will not produce the field at all when its value -is `Nothing`. +Here we use `optField` to define schemas for optional fields, and apply the +`maybe_` combinator to the result, which has the effect of making the +serialiser omit the field when the corresponding value is `Nothing`. + +In detail, `optField "handle" schema` returns a schema from `Text` to `Maybe +Text`, i.e. a schema that is able to parse an optional text value, but does not +know how to serialise `Nothing`. Wrapping it in `maybe_` changes the first type +to `Maybe Text`, and gives the serialiser the ability to serialise `Nothing` as +well. Another possibility is a field that, when missing, is assumed to have a given default value. Most likely, in this case we do not want the field to be omitted -when serialising. The schema can then be defined simply by using the -`Alternative` instance of `SchemaP` to provide the default value: +when serialising. Such a schema can be defined simply by omitting the call to +`maybe_`, and instead converting a `Nothing` value coming from the parser into +the desired default value. ```haskell userSchemaWithDefaultName :: ValueSchema NamedSwaggerDoc User userSchemaWithDefaultName = object "User" $ User - <$> userName .= (field "name" schema <|> pure "") - <*> userHandle .= opt (field "handle" schema) - <*> userExpire .= opt (field "expire" schema) + <$> userName .= (fromMaybe "" <$> optField "name" schema) + <*> userHandle .= maybe_ (optField "handle" schema) + <*> userExpire .= maybe_ (optField "expire" schema) ``` -Now the `name` field is optional, and it is set to the empty string when missing. -However, the field will still be present in the generated JSON when its value -is the empty string. If we want the field to be omitted in that case, we can -use the previous approach, and then convert back and forth from `Maybe Text`: +Now the `name` field is optional, and it is set to the empty string when +missing. However, the field will still be present in the generated JSON when +its value is the empty string. If we want the field to be omitted in that case, +we can instead use the first approach, and manually convert back and forth from +`Maybe Text`. ```haskell userSchemaWithDefaultName' :: ValueSchema NamedSwaggerDoc User userSchemaWithDefaultName' = object "User" $ User - <$> (getOptText . userName) .= (fromMaybe "" <$> opt (field "name" schema)) + <$> (getOptText . userName) .= maybe_ (fromMaybe "" <$> field "name" schema) <*> userHandle .= opt (field "handle" schema) <*> userExpire .= opt (field "expire" schema) where @@ -417,60 +422,47 @@ techniques of the previous two examples: userSchema' :: ValueSchema NamedSwaggerDoc User userSchema' = object "User" $ User <$> field "name" schema - <*> lax (field "handle" (optWithDefault Aeson.null schema)) + <*> optField "handle" (maybeWithDefault Aeson.Null schema) <*> opt (field "expire" schema) ``` Two things to note here: - - the `optWithDefault` combinator is applied to the schema value *inside* - `field`, because the value to use if the value is `Nothing` (`Aeson.null` in - this case) applies to the value of the field, and not the containing object. - - we have wrapped the whole field inside a call to the `lax` combinator. All - this does is to add a `pure Nothing` alternative for the field, which ensures - we get a `Nothing` value (as opposed to a failure) when the field is not - present at all in the JSON object. + - we are now using `maybeWithDefault` instead of `maybe_`. This is a more + general version of `maybe_` that takes as an argument the value to use when + serialising `Nothing`. Not that `maybe_` is simply `maybeWithDefault mempty`. + - the `maybeWithDefault` combinator is applied to the schema value *inside* + `field`, because the value to use when serialising `Nothing` (`Aeson.null` in + this case) applies to the value of the field, and not the containing + (one-field) object, as in the previous examples. -One might wonder why we are using the special combinator `optWithDefault` here +One might wonder why we are using the special combinator `optField` here instead of simply using the `Alternative` instance (via `optional` or -directly). The reason is that the `Alternative` instance only really affects -the parser (and its return type), whereas here we also want to encode the fact -that the serialiser should output the default when the value of the field is -`Nothing`. That means we need to also change the input type to a `Maybe`, which -is what `opt` and `optWithDefault` do. +directly), on the schema returned by the `field` combinator. The reason is that +the `Alternative` instance would result in a slightly surprising behaviour in +case of errors in the JSON value contained in a field. -There is a subtlety here related to error messages, which can sometimes result -in surprising behaviour when parsing optional fields with default values. -Namely, given a field of the form +For example, given a field of the form ```haskell -opt (field "name" schema) +optional (field "name" schema) ``` the corresponding parser will return `Nothing` not only in the case where the `"name"` field is missing, but also if it is fails to parse correctly (for example, if it has an unexpected type). This behaviour is caused by the fact -that `opt` (and the `optWithDefault` / `lax` combo described above) are -implemented in terms of the `Alternative` instance for `Aeson.Parser`, which -cannot distinguish between "recoverable" and "unrecoverable" failures. +that `optional` is implemented in terms of the `Alternative` instance for +`Aeson.Parser`, which cannot distinguish between "recoverable" and +"unrecoverable" failures. -There are plans to improve on this behaviour in the future by directly changing -the `Alternative` instance that `SchemaP` relies on, but for the moment, if -this behaviour is not desirable, then one can use the ad-hoc `optField` -combinator to introduce optional fields. - -For example, the above schema can be implemented using `optField` as follow: +In some cases, this behaviour can be acceptable (or even desired), but in most +circumstances, it is better to define the above schema using the dedicated +`optField` combinator, as in: ```haskell -userSchema'' :: ValueSchema NamedSwaggerDoc User -userSchema'' = object "User" $ User - <$> field "name" schema - <*> optField "handle" (Just Aeson.Null) schema - <*> optField "expire" Nothing schema +optField "name" schema ``` -The argument after the field name determines how the `Nothing` case is rendered in the generated JSON. If it is itself `Nothing`, that means that the field is completely omitted in that case. - ### Redundant fields Sometimes, JSON encoding of haskell types is not as straightforward as diff --git a/libs/schema-profunctor/src/Data/Schema.hs b/libs/schema-profunctor/src/Data/Schema.hs index 9369ed4f79f..d93ecf9c9ad 100644 --- a/libs/schema-profunctor/src/Data/Schema.hs +++ b/libs/schema-profunctor/src/Data/Schema.hs @@ -26,7 +26,9 @@ module Data.Schema ( SchemaP, ValueSchema, + ValueSchemaP, ObjectSchema, + ObjectSchemaP, ToSchema (..), Schema (..), mkSchema, @@ -34,6 +36,7 @@ module Data.Schema schemaIn, schemaOut, HasDoc (..), + HasSchemaRef (..), withParser, SwaggerDoc, swaggerDoc, @@ -44,25 +47,28 @@ module Data.Schema objectWithDocModifier, objectOver, jsonObject, + FieldFunctor, field, fieldWithDocModifier, fieldOver, optField, optFieldWithDocModifier, - optFieldOver, + fieldF, + fieldOverF, + fieldWithDocModifierF, array, set, nonEmptyArray, map_, enum, - opt, - optWithDefault, - lax, + maybe_, + maybeWithDefault, bind, dispatch, text, parsedText, null_, + nullable, element, tag, unnamed, @@ -247,11 +253,13 @@ withParser :: SchemaP doc v w a b -> (b -> A.Parser b') -> SchemaP doc v w a b' withParser (SchemaP (SchemaDoc d) (SchemaIn p) (SchemaOut o)) q = SchemaP (SchemaDoc d) (SchemaIn (p >=> q)) (SchemaOut o) -type SchemaP' doc v v' a = SchemaP doc v v' a a +type ObjectSchemaP doc = SchemaP doc A.Object [A.Pair] -type ObjectSchema doc a = SchemaP' doc A.Object [A.Pair] a +type ObjectSchema doc a = ObjectSchemaP doc a a -type ValueSchema doc a = SchemaP' doc A.Value A.Value a +type ValueSchemaP doc = SchemaP doc A.Value A.Value + +type ValueSchema doc a = ValueSchemaP doc a a schemaDoc :: SchemaP ss v m a b -> ss schemaDoc (SchemaP (SchemaDoc d) _ _) = d @@ -262,6 +270,18 @@ schemaIn (SchemaP _ (SchemaIn i) _) = i schemaOut :: SchemaP ss v m a b -> a -> Maybe m schemaOut (SchemaP _ _ (SchemaOut o)) = o +class Functor f => FieldFunctor doc f where + parseFieldF :: (A.Value -> A.Parser a) -> A.Object -> Text -> A.Parser (f a) + mkDocF :: doc -> doc + +instance FieldFunctor doc Identity where + parseFieldF f obj key = Identity <$> A.explicitParseField f obj key + mkDocF = id + +instance HasOpt doc => FieldFunctor doc Maybe where + parseFieldF = A.explicitParseFieldMaybe + mkDocF = mkOpt + -- | A schema for a one-field JSON object. field :: HasField doc' doc => @@ -274,13 +294,19 @@ field = fieldOver id optField :: (HasOpt doc, HasField doc' doc) => Text -> - -- | The value to use when serialising Nothing. - Maybe A.Value -> SchemaP doc' A.Value A.Value a b -> - SchemaP doc A.Object [A.Pair] (Maybe a) (Maybe b) -optField = optFieldOver id + SchemaP doc A.Object [A.Pair] a (Maybe b) +optField = fieldF -newtype Negative x y a = Negative {runNegative :: (a -> x) -> y} +-- | A schema for a JSON object with a single optional field. +fieldF :: + (HasOpt doc, HasField doc' doc, FieldFunctor doc f) => + Text -> + SchemaP doc' A.Value A.Value a b -> + SchemaP doc A.Object [A.Pair] a (f b) +fieldF = fieldOverF id + +newtype Positive x y a = Positive {runPositive :: (a -> x) -> y} deriving (Functor) -- | A version of 'field' for more general input values. @@ -288,52 +314,36 @@ newtype Negative x y a = Negative {runNegative :: (a -> x) -> y} -- This can be used when the input type 'v' of the parser is not exactly a -- 'A.Object', but it contains one. The first argument is a lens that can -- extract the 'A.Object' contained in 'v'. -fieldOver :: - forall doc' doc v v' a b. - HasField doc' doc => +fieldOverF :: + forall f doc' doc v v' a b. + (HasField doc' doc, FieldFunctor doc f) => Lens v v' A.Object A.Value -> Text -> SchemaP doc' v' A.Value a b -> - SchemaP doc v [A.Pair] a b -fieldOver l name sch = SchemaP (SchemaDoc s) (SchemaIn r) (SchemaOut w) + SchemaP doc v [A.Pair] a (f b) +fieldOverF l name sch = SchemaP (SchemaDoc s) (SchemaIn r) (SchemaOut w) where - parseField :: A.Object -> Negative (A.Parser b) (A.Parser b) A.Value - parseField obj = Negative $ \k -> A.explicitParseField k obj name + parseField :: A.Object -> Positive (A.Parser b) (A.Parser (f b)) A.Value + parseField obj = Positive $ \k -> parseFieldF @doc k obj name - r :: v -> A.Parser b - r obj = runNegative (l parseField obj) (schemaIn sch) + r :: v -> A.Parser (f b) + r obj = runPositive (l parseField obj) (schemaIn sch) w x = do v <- schemaOut sch x pure [name A..= v] - s = mkField name (schemaDoc sch) + s = mkDocF @doc @f (mkField name (schemaDoc sch)) --- | A version of 'optField' for more general input values. --- --- See documentation of 'fieldOver' for more details. -optFieldOver :: +-- | Like 'fieldOver', but specialised to the identity functor. +fieldOver :: forall doc' doc v v' a b. - (HasOpt doc, HasField doc' doc) => + (HasField doc' doc) => Lens v v' A.Object A.Value -> Text -> - Maybe A.Value -> SchemaP doc' v' A.Value a b -> - SchemaP doc v [A.Pair] (Maybe a) (Maybe b) -optFieldOver l name def sch = SchemaP (SchemaDoc s) (SchemaIn r) (SchemaOut w) - where - parseField :: A.Object -> Negative (A.Parser b) (A.Parser (Maybe b)) A.Value - parseField obj = Negative $ \k -> A.explicitParseFieldMaybe k obj name - - r :: v -> A.Parser (Maybe b) - r obj = runNegative (l parseField obj) (schemaIn sch) - - w (Just x) = do - v <- schemaOut sch x - pure [name A..= v] - w Nothing = pure (maybeToList (fmap (name A..=) def)) - - s = mkOpt (mkField name (schemaDoc sch)) + SchemaP doc v [A.Pair] a b +fieldOver l name = fmap runIdentity . fieldOverF l name -- | Like 'field', but apply an arbitrary function to the -- documentation of the field. @@ -350,11 +360,20 @@ fieldWithDocModifier name modify sch = field name (over doc modify sch) optFieldWithDocModifier :: (HasOpt doc, HasField doc' doc) => Text -> - Maybe A.Value -> (doc' -> doc') -> SchemaP doc' A.Value A.Value a b -> - SchemaP doc A.Object [A.Pair] (Maybe a) (Maybe b) -optFieldWithDocModifier name def modify sch = optField name def (over doc modify sch) + SchemaP doc A.Object [A.Pair] a (Maybe b) +optFieldWithDocModifier name modify sch = optField name (over doc modify sch) + +-- | Like 'fieldF', but apply an arbitrary function to the +-- documentation of the field. +fieldWithDocModifierF :: + (HasOpt doc, HasField doc' doc, FieldFunctor doc f) => + Text -> + (doc' -> doc') -> + SchemaP doc' A.Value A.Value a b -> + SchemaP doc A.Object [A.Pair] a (f b) +fieldWithDocModifierF name modify sch = fieldF name (over doc modify sch) -- | Change the input type of a schema. (.=) :: Profunctor p => (a -> a') -> p a' b -> p a b @@ -508,32 +527,17 @@ enum name sch = SchemaP (SchemaDoc d) (SchemaIn i) (SchemaOut o) <|> fail ("Unexpected value for enum " <> T.unpack name) o = fmap A.toJSON . (getAlt <=< schemaOut sch) --- | An optional schema. --- --- This is most commonly used for optional fields. The parser will --- return 'Nothing' if the field is missing, and conversely the --- serialiser will simply omit the field when its value is 'Nothing'. -opt :: HasOpt d => Monoid w => SchemaP d v w a b -> SchemaP d v w (Maybe a) (Maybe b) -opt = optWithDefault mempty - --- | An optional schema with a specified failure value +-- | A schema for 'Maybe' that omits a field on serialisation. -- --- This is a more general version of 'opt' that allows a custom --- serialisation 'Nothing' value. -optWithDefault :: HasOpt d => w -> SchemaP d v w a b -> SchemaP d v w (Maybe a) (Maybe b) -optWithDefault w0 sch = SchemaP (SchemaDoc d) (SchemaIn i) (SchemaOut o) - where - d = mkOpt (schemaDoc sch) - i = optional . schemaIn sch - o = maybe (pure w0) (schemaOut sch) +-- This is most commonly used for optional fields, and it will cause the field +-- to be omitted from the output of the serialiser. +maybe_ :: HasOpt d => Monoid w => SchemaP d v w a b -> SchemaP d v w (Maybe a) b +maybe_ = maybeWithDefault mempty --- | A schema that ignores failure. --- --- Given a schema @sch :: SchemaP d v w a (Maybe b)@, the parser for --- @lax sch@ is just like the one for @sch@, except that it returns --- 'Nothing' in case of failure. -lax :: Alternative f => f (Maybe a) -> f (Maybe a) -lax = (<|> pure Nothing) +-- | A schema for 'Maybe', producing the given default value on serialisation. +maybeWithDefault :: HasOpt d => w -> SchemaP d v w a b -> SchemaP d v w (Maybe a) b +maybeWithDefault w0 (SchemaP (SchemaDoc d) (SchemaIn i) (SchemaOut o)) = + SchemaP (SchemaDoc d) (SchemaIn i) (SchemaOut (maybe (pure w0) o)) -- | A schema depending on a parsed value. -- @@ -598,12 +602,28 @@ jsonObject = mkSchema mempty pure (pure . (^.. ifolded . withIndex)) -- | A schema for a null value. -null_ :: Monoid d => SchemaP d A.Value A.Value () () +null_ :: Monoid d => ValueSchemaP d () () null_ = mkSchema mempty i o where i x = guard (x == A.Null) o _ = pure A.Null +-- | A schema for a nullable value. +-- +-- The parser accepts a JSON null as a valid value, and converts it to +-- 'Nothing'. Any non-null value is parsed using the underlying schema. +-- +-- The serialiser behaves similarly, but in the other direction. +nullable :: + (Monoid d, HasOpt d) => + ValueSchema d a -> + ValueSchema d (Maybe a) +nullable s = + mconcat + [ tag _Nothing null_, + tag _Just s + ] + data WithDeclare s = WithDeclare (Declare ()) s deriving (Functor) diff --git a/libs/schema-profunctor/test/unit/Test/Data/Schema.hs b/libs/schema-profunctor/test/unit/Test/Data/Schema.hs index 356729e9a4c..484f9fb413b 100644 --- a/libs/schema-profunctor/test/unit/Test/Data/Schema.hs +++ b/libs/schema-profunctor/test/unit/Test/Data/Schema.hs @@ -65,7 +65,8 @@ tests = testRefField, testRmClientWrong, testRmClient, - testEnumType + testEnumType, + testNullable ] testFooToJSON :: TestTree @@ -338,6 +339,23 @@ testEnumType = (s2 ^. S.type_) (Just S.SwaggerInteger) +testNullable :: TestTree +testNullable = + let sch = nullable (unnamed schema) :: ValueSchema SwaggerDoc (Maybe Int) + in testGroup + "Nullable schemas" + [ testCase "Nullable schemas should parse both null and non-null values" $ do + A.parse (schemaIn sch) (A.Number 5) @?= Success (Just 5) + A.parse (schemaIn sch) A.Null @?= Success Nothing, + testCase "Nullable schemas should produce either a value or null" $ do + schemaOut sch (Just 5) @?= Just (A.Number 5) + schemaOut sch Nothing @?= Just (A.Null), + testCase "Nullable schemas should return an error when parsing invalid non-null values" $ do + case A.parse (schemaIn sch) (A.String "foo") of + Success _ -> assertFailure "fromJSON should fail" + Error _ -> pure () + ] + --- data A = A {thing :: Text, other :: Int} @@ -445,8 +463,8 @@ instance ToSchema User where object "User" $ User <$> userName .= field "name" schema - <*> userHandle .= opt (field "handle" schema) - <*> userExpire .= opt (field "expire" schema) + <*> userHandle .= maybe_ (optField "handle" schema) + <*> userExpire .= maybe_ (optField "expire" schema) exampleUser1 :: User exampleUser1 = User "Alice" (Just "alice") Nothing @@ -554,13 +572,13 @@ rmClientSchema :: ValueSchema NamedSwaggerDoc RmClient rmClientSchema = object "RmClient" $ RmClient - <$> rmPassword .= lax (field "password" (optWithDefault Null passwordSchema)) + <$> rmPassword .= optional (field "password" (maybeWithDefault Null passwordSchema)) instance ToSchema RmClient where schema = object "RmClient" $ RmClient - <$> rmPassword .= optField "password" Nothing passwordSchema + <$> rmPassword .= maybe_ (optField "password" passwordSchema) -- examples from documentation (only type-checked) @@ -601,9 +619,9 @@ userSchemaWithDefaultName' :: ValueSchema NamedSwaggerDoc User userSchemaWithDefaultName' = object "User" $ User - <$> (getOptText . userName) .= (fromMaybe "" <$> opt (field "name" schema)) - <*> userHandle .= opt (field "handle" schema) - <*> userExpire .= opt (field "expire" schema) + <$> (getOptText . userName) .= maybe_ (fromMaybe "" <$> optField "name" schema) + <*> userHandle .= maybe_ (optField "handle" schema) + <*> userExpire .= maybe_ (optField "expire" schema) where getOptText :: Text -> Maybe Text getOptText "" = Nothing @@ -614,5 +632,5 @@ userSchemaWithDefaultName = object "User" $ User <$> userName .= (field "name" schema <|> pure "") - <*> userHandle .= opt (field "handle" schema) - <*> userExpire .= opt (field "expire" schema) + <*> userHandle .= maybe_ (optField "handle" schema) + <*> userExpire .= maybe_ (optField "expire" schema) diff --git a/libs/types-common/src/Data/Domain.hs b/libs/types-common/src/Data/Domain.hs index 7656834b433..ecbac67e280 100644 --- a/libs/types-common/src/Data/Domain.hs +++ b/libs/types-common/src/Data/Domain.hs @@ -30,7 +30,7 @@ import qualified Data.ByteString as BS import qualified Data.ByteString.Builder as Builder import qualified Data.ByteString.Char8 as BS.Char8 import Data.ByteString.Conversion -import Data.Schema hiding (opt) +import Data.Schema import Data.String.Conversions (cs) import qualified Data.Swagger as S import qualified Data.Text as Text diff --git a/libs/wire-api/src/Wire/API/Connection.hs b/libs/wire-api/src/Wire/API/Connection.hs index 612c867f265..53fa98758a4 100644 --- a/libs/wire-api/src/Wire/API/Connection.hs +++ b/libs/wire-api/src/Wire/API/Connection.hs @@ -47,13 +47,13 @@ where import Control.Applicative (optional) import Control.Lens ((?~)) -import Data.Aeson as Aeson +import Data.Aeson (FromJSON (..), ToJSON (..)) import Data.Id import Data.Json.Util (UTCTimeMillis) import Data.Qualified (Qualified (qUnqualified), deprecatedSchema) import Data.Range -import qualified Data.Schema as P -import Data.Swagger as S +import Data.Schema +import qualified Data.Swagger as S import qualified Data.Swagger.Build.Api as Doc import Data.Text as Text import Imports @@ -85,14 +85,14 @@ data UserConnectionList = UserConnectionList } deriving stock (Eq, Show, Generic) deriving (Arbitrary) via (GenericUniform UserConnectionList) - deriving (FromJSON, ToJSON, S.ToSchema) via (P.Schema UserConnectionList) + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema UserConnectionList) -instance P.ToSchema UserConnectionList where +instance ToSchema UserConnectionList where schema = - P.object "UserConnectionList" $ + object "UserConnectionList" $ UserConnectionList - <$> clConnections P..= P.field "connections" (P.array P.schema) - <*> clHasMore P..= P.fieldWithDocModifier "has_more" (P.description ?~ "Indicator that the server has more connections than returned.") P.schema + <$> clConnections .= field "connections" (array schema) + <*> clHasMore .= fieldWithDocModifier "has_more" (description ?~ "Indicator that the server has more connections than returned.") schema modelConnectionList :: Doc.Model modelConnectionList = Doc.defineModel "UserConnectionList" $ do @@ -119,21 +119,21 @@ data UserConnection = UserConnection } deriving stock (Eq, Show, Generic) deriving (Arbitrary) via (GenericUniform UserConnection) - deriving (FromJSON, ToJSON, S.ToSchema) via (P.Schema UserConnection) + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema UserConnection) -instance P.ToSchema UserConnection where +instance ToSchema UserConnection where schema = - P.object "UserConnection" $ + object "UserConnection" $ UserConnection - <$> ucFrom P..= P.field "from" P.schema - <*> ucTo P..= P.field "qualified_to" P.schema + <$> ucFrom .= field "from" schema + <*> ucTo .= field "qualified_to" schema <* (qUnqualified . ucTo) - P..= optional (P.field "to" (deprecatedSchema "qualified_to" P.schema)) - <*> ucStatus P..= P.field "status" P.schema - <*> ucLastUpdate P..= P.field "last_update" P.schema - <*> ucConvId P..= P.optField "qualified_conversation" Nothing P.schema + .= optional (field "to" (deprecatedSchema "qualified_to" schema)) + <*> ucStatus .= field "status" schema + <*> ucLastUpdate .= field "last_update" schema + <*> ucConvId .= maybe_ (optField "qualified_conversation" schema) <* (fmap qUnqualified . ucConvId) - P..= P.optField "conversation" Nothing (deprecatedSchema "qualified_conversation" P.schema) + .= maybe_ (optField "conversation" (deprecatedSchema "qualified_conversation" schema)) modelConnection :: Doc.Model modelConnection = Doc.defineModel "Connection" $ do @@ -170,7 +170,7 @@ data Relation MissingLegalholdConsent deriving stock (Eq, Ord, Show, Generic) deriving (Arbitrary) via (GenericUniform Relation) - deriving (FromJSON, ToJSON, S.ToSchema) via (P.Schema Relation) + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema Relation) instance S.ToParamSchema Relation where toParamSchema _ = mempty & S.type_ ?~ S.SwaggerString @@ -234,17 +234,17 @@ typeRelation = "missing-legalhold-consent" ] -instance P.ToSchema Relation where +instance ToSchema Relation where schema = - P.enum @Text "Relation" $ + enum @Text "Relation" $ mconcat - [ P.element "accepted" Accepted, - P.element "blocked" Blocked, - P.element "pending" Pending, - P.element "ignored" Ignored, - P.element "sent" Sent, - P.element "cancelled" Cancelled, - P.element "missing-legalhold-consent" MissingLegalholdConsent + [ element "accepted" Accepted, + element "blocked" Blocked, + element "pending" Pending, + element "ignored" Ignored, + element "sent" Sent, + element "cancelled" Cancelled, + element "missing-legalhold-consent" MissingLegalholdConsent ] instance FromHttpApiData Relation where @@ -285,14 +285,14 @@ data ConnectionRequest = ConnectionRequest } deriving stock (Eq, Show, Generic) deriving (Arbitrary) via (GenericUniform ConnectionRequest) - deriving (FromJSON, ToJSON, S.ToSchema) via (P.Schema ConnectionRequest) + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema ConnectionRequest) -instance P.ToSchema ConnectionRequest where +instance ToSchema ConnectionRequest where schema = - P.object "ConnectionRequest" $ + object "ConnectionRequest" $ ConnectionRequest - <$> crUser P..= P.fieldWithDocModifier "user" (P.description ?~ "user ID of the user to request a connection with") P.schema - <*> crName P..= P.fieldWithDocModifier "name" (P.description ?~ "Name of the (pending) conversation being initiated (1 - 256) characters)") P.schema + <$> crUser .= fieldWithDocModifier "user" (description ?~ "user ID of the user to request a connection with") schema + <*> crName .= fieldWithDocModifier "name" (description ?~ "Name of the (pending) conversation being initiated (1 - 256) characters)") schema -- | Payload type for "please change the status of this connection". newtype ConnectionUpdate = ConnectionUpdate @@ -300,13 +300,13 @@ newtype ConnectionUpdate = ConnectionUpdate } deriving stock (Eq, Show, Generic) deriving (Arbitrary) via (GenericUniform ConnectionUpdate) - deriving (FromJSON, ToJSON, S.ToSchema) via (P.Schema ConnectionUpdate) + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema ConnectionUpdate) -instance P.ToSchema ConnectionUpdate where +instance ToSchema ConnectionUpdate where schema = - P.object "ConnectionUpdate" $ + object "ConnectionUpdate" $ ConnectionUpdate - <$> cuStatus P..= P.fieldWithDocModifier "status" (P.description ?~ "New relation status") P.schema + <$> cuStatus .= fieldWithDocModifier "status" (description ?~ "New relation status") schema modelConnectionUpdate :: Doc.Model modelConnectionUpdate = Doc.defineModel "ConnectionUpdate" $ do diff --git a/libs/wire-api/src/Wire/API/Conversation.hs b/libs/wire-api/src/Wire/API/Conversation.hs index f935a206049..896ac033bf2 100644 --- a/libs/wire-api/src/Wire/API/Conversation.hs +++ b/libs/wire-api/src/Wire/API/Conversation.hs @@ -148,19 +148,18 @@ conversationMetadataObjectSchema = schema <*> cnvmAccess .= field "access" (array schema) <*> cnvmAccessRole .= field "access_role" schema - <*> cnvmName .= lax (field "name" (optWithDefault A.Null schema)) + <*> cnvmName .= optField "name" (maybeWithDefault A.Null schema) <* const ("0.0" :: Text) .= optional (field "last_event" schema) <* const ("1970-01-01T00:00:00.000Z" :: Text) .= optional (field "last_event_time" schema) - <*> cnvmTeam .= lax (field "team" (optWithDefault A.Null schema)) + <*> cnvmTeam .= optField "team" (maybeWithDefault A.Null schema) <*> cnvmMessageTimer - .= lax - ( fieldWithDocModifier - "message_timer" - (description ?~ "Per-conversation message timer (can be null)") - (optWithDefault A.Null schema) - ) - <*> cnvmReceiptMode .= lax (field "receipt_mode" (optWithDefault A.Null schema)) + .= ( optFieldWithDocModifier + "message_timer" + (description ?~ "Per-conversation message timer (can be null)") + (maybeWithDefault A.Null schema) + ) + <*> cnvmReceiptMode .= optField "receipt_mode" (maybeWithDefault A.Null schema) instance ToSchema ConversationMetadata where schema = object "ConversationMetadata" conversationMetadataObjectSchema @@ -287,7 +286,7 @@ instance ToSchema ConversationCoverView where (description ?~ "Limited view of Conversation.") $ ConversationCoverView <$> cnvCoverConvId .= field "id" schema - <*> cnvCoverName .= lax (field "name" (optWithDefault A.Null schema)) + <*> cnvCoverName .= optField "name" (maybeWithDefault A.Null schema) data ConversationList a = ConversationList { convList :: [a], @@ -618,27 +617,25 @@ newConvSchema = (array schema) <|> pure [] ) - <*> newConvName .= opt (field "name" schema) + <*> newConvName .= maybe_ (optField "name" schema) <*> (Set.toList . newConvAccess) - .= ( field "access" (Set.fromList <$> array schema) - <|> pure mempty - ) - <*> newConvAccessRole .= opt (field "access_role" schema) + .= (fromMaybe mempty <$> optField "access" (Set.fromList <$> array schema)) + <*> newConvAccessRole .= maybe_ (optField "access_role" schema) <*> newConvTeam - .= opt - ( fieldWithDocModifier + .= maybe_ + ( optFieldWithDocModifier "team" (description ?~ "Team information of this conversation") schema ) <*> newConvMessageTimer - .= opt - ( fieldWithDocModifier + .= maybe_ + ( optFieldWithDocModifier "message_timer" (description ?~ "Per-conversation message timer") schema ) - <*> newConvReceiptMode .= opt (field "receipt_mode" schema) + <*> newConvReceiptMode .= maybe_ (optField "receipt_mode" schema) <*> newConvUsersRole .= ( fieldWithDocModifier "conversation_role" (description ?~ usersRoleDesc) schema <|> pure roleNameWireAdmin @@ -711,10 +708,8 @@ instance ToSchema Invite where Invite <$> (toNonEmpty . invUsers) .= fmap List1 (field "users" (nonEmptyArray schema)) - <*> (Just . invRoleName) - .= fmap - (fromMaybe roleNameWireAdmin) - (optField "conversation_role" Nothing schema) + <*> invRoleName + .= (fromMaybe roleNameWireAdmin <$> optField "conversation_role" schema) data InviteQualified = InviteQualified { invQUsers :: NonEmpty (Qualified UserId), @@ -730,10 +725,8 @@ instance ToSchema InviteQualified where object "InviteQualified" $ InviteQualified <$> invQUsers .= field "qualified_users" (nonEmptyArray schema) - <*> (Just . invQRoleName) - .= fmap - (fromMaybe roleNameWireAdmin) - (optField "conversation_role" Nothing schema) + <*> invQRoleName + .= (fromMaybe roleNameWireAdmin <$> optField "conversation_role" schema) newInvite :: List1 UserId -> Invite newInvite us = Invite us roleNameWireAdmin @@ -836,7 +829,7 @@ instance ToSchema ConversationMessageTimerUpdate where "ConversationMessageTimerUpdate" (description ?~ "Contains conversation properties to update") $ ConversationMessageTimerUpdate - <$> cupMessageTimer .= lax (field "message_timer" (optWithDefault A.Null schema)) + <$> cupMessageTimer .= optField "message_timer" (maybeWithDefault A.Null schema) modelConversationMessageTimerUpdate :: Doc.Model modelConversationMessageTimerUpdate = Doc.defineModel "ConversationMessageTimerUpdate" $ do diff --git a/libs/wire-api/src/Wire/API/Conversation/Code.hs b/libs/wire-api/src/Wire/API/Conversation/Code.hs index 7e9cb9f74db..b30525fd7dc 100644 --- a/libs/wire-api/src/Wire/API/Conversation/Code.hs +++ b/libs/wire-api/src/Wire/API/Conversation/Code.hs @@ -82,8 +82,8 @@ instance ToSchema ConversationCode where (description ?~ "Conversation code (random)") schema <*> conversationUri - .= opt - ( fieldWithDocModifier + .= maybe_ + ( optFieldWithDocModifier "uri" (description ?~ "Full URI (containing key/code) to join a conversation") schema diff --git a/libs/wire-api/src/Wire/API/Conversation/Member.hs b/libs/wire-api/src/Wire/API/Conversation/Member.hs index e2abc40d069..e72ae9455d2 100644 --- a/libs/wire-api/src/Wire/API/Conversation/Member.hs +++ b/libs/wire-api/src/Wire/API/Conversation/Member.hs @@ -111,7 +111,7 @@ instance ToSchema Member where <$> memId .= field "qualified_id" schema <* (qUnqualified . memId) .= optional (field "id" (deprecatedSchema "qualified_id" schema)) - <*> memService .= lax (field "service" (optWithDefault A.Null schema)) + <*> memService .= optField "service" (maybeWithDefault A.Null schema) -- Remove ... <* const () .= optional (field "status" (c (0 :: Int))) <* const () .= optional (field "status_ref" (c ("0.0" :: Text))) @@ -122,13 +122,13 @@ instance ToSchema Member where (c ("1970-01-01T00:00:00.000Z" :: Text)) ) -- ... until here - <*> memOtrMutedStatus .= lax (field "otr_muted_status" (optWithDefault A.Null schema)) - <*> memOtrMutedRef .= lax (field "otr_muted_ref" (optWithDefault A.Null schema)) - <*> memOtrArchived .= (field "otr_archived" schema <|> pure False) - <*> memOtrArchivedRef .= lax (field "otr_archived_ref" (optWithDefault A.Null schema)) + <*> memOtrMutedStatus .= optField "otr_muted_status" (maybeWithDefault A.Null schema) + <*> memOtrMutedRef .= optField "otr_muted_ref" (maybeWithDefault A.Null schema) + <*> memOtrArchived .= (fromMaybe False <$> optField "otr_archived" schema) + <*> memOtrArchivedRef .= optField "otr_archived_ref" (maybeWithDefault A.Null schema) <*> memHidden .= (field "hidden" schema <|> pure False) - <*> memHiddenRef .= lax (field "hidden_ref" (optWithDefault A.Null schema)) - <*> memConvRoleName .= (field "conversation_role" schema <|> pure roleNameWireAdmin) + <*> memHiddenRef .= optField "hidden_ref" (maybeWithDefault A.Null schema) + <*> memConvRoleName .= (fromMaybe roleNameWireAdmin <$> optField "conversation_role" schema) where c :: ToJSON a => a -> ValueSchema SwaggerDoc () c val = mkSchema mempty (const (pure ())) (const (pure (toJSON val))) @@ -178,7 +178,7 @@ instance ToSchema OtherMember where OtherMember <$> omQualifiedId .= field "qualified_id" schema <* (qUnqualified . omQualifiedId) .= optional (field "id" schema) - <*> omService .= opt (fieldWithDocModifier "service" (description ?~ desc) schema) + <*> omService .= maybe_ (optFieldWithDocModifier "service" (description ?~ desc) schema) <*> omConvRoleName .= (field "conversation_role" schema <|> pure roleNameWireAdmin) <* const (0 :: Int) .= optional (fieldWithDocModifier "status" (description ?~ "deprecated") schema) -- TODO: remove where @@ -238,12 +238,12 @@ instance ToSchema MemberUpdate where (`withParser` (either fail pure . validateMemberUpdate)) . object "MemberUpdate" $ MemberUpdate - <$> mupOtrMuteStatus .= opt (field "otr_muted_status" schema) - <*> mupOtrMuteRef .= opt (field "otr_muted_ref" schema) - <*> mupOtrArchive .= opt (field "otr_archived" schema) - <*> mupOtrArchiveRef .= opt (field "otr_archived_ref" schema) - <*> mupHidden .= opt (field "hidden" schema) - <*> mupHiddenRef .= opt (field "hidden_ref" schema) + <$> mupOtrMuteStatus .= maybe_ (optField "otr_muted_status" schema) + <*> mupOtrMuteRef .= maybe_ (optField "otr_muted_ref" schema) + <*> mupOtrArchive .= maybe_ (optField "otr_archived" schema) + <*> mupOtrArchiveRef .= maybe_ (optField "otr_archived_ref" schema) + <*> mupHidden .= maybe_ (optField "hidden" schema) + <*> mupHiddenRef .= maybe_ (optField "hidden_ref" schema) instance Arbitrary MemberUpdate where arbitrary = @@ -290,7 +290,7 @@ instance ToSchema OtherMemberUpdate where "OtherMemberUpdate" (description ?~ "Update user properties of other members relative to a conversation") $ OtherMemberUpdate - <$> omuConvRoleName .= optField "conversation_role" Nothing schema + <$> omuConvRoleName .= maybe_ (optField "conversation_role" schema) validateOtherMemberUpdate :: OtherMemberUpdate -> Either String OtherMemberUpdate validateOtherMemberUpdate u diff --git a/libs/wire-api/src/Wire/API/Event/Conversation.hs b/libs/wire-api/src/Wire/API/Event/Conversation.hs index d681659225f..de7d732d848 100644 --- a/libs/wire-api/src/Wire/API/Event/Conversation.hs +++ b/libs/wire-api/src/Wire/API/Event/Conversation.hs @@ -371,10 +371,10 @@ connectObjectSchema :: ObjectSchema SwaggerDoc Connect connectObjectSchema = Connect <$> cRecipient .= field "qualified_recipient" schema - <* (Just . qUnqualified . cRecipient) .= optField "recipient" Nothing schema - <*> cMessage .= lax (field "message" (optWithDefault A.Null schema)) - <*> cName .= lax (field "name" (optWithDefault A.Null schema)) - <*> cEmail .= lax (field "email" (optWithDefault A.Null schema)) + <* (qUnqualified . cRecipient) .= optional (field "recipient" schema) + <*> cMessage .= optField "message" (maybeWithDefault A.Null schema) + <*> cName .= optField "name" (maybeWithDefault A.Null schema) + <*> cEmail .= optField "email" (maybeWithDefault A.Null schema) modelConnect :: Doc.Model modelConnect = Doc.defineModel "Connect" $ do @@ -416,14 +416,14 @@ memberUpdateDataObjectSchema :: ObjectSchema SwaggerDoc MemberUpdateData memberUpdateDataObjectSchema = MemberUpdateData <$> misTarget .= field "qualified_target" schema - <* (Just . qUnqualified . misTarget) .= optField "target" Nothing schema - <*> misOtrMutedStatus .= opt (field "otr_muted_status" schema) - <*> misOtrMutedRef .= opt (field "otr_muted_ref" schema) - <*> misOtrArchived .= opt (field "otr_archived" schema) - <*> misOtrArchivedRef .= opt (field "otr_archived_ref" schema) - <*> misHidden .= opt (field "hidden" schema) - <*> misHiddenRef .= opt (field "hidden_ref" schema) - <*> misConvRoleName .= opt (field "conversation_role" schema) + <* (qUnqualified . misTarget) .= optional (field "target" schema) + <*> misOtrMutedStatus .= maybe_ (optField "otr_muted_status" schema) + <*> misOtrMutedRef .= maybe_ (optField "otr_muted_ref" schema) + <*> misOtrArchived .= maybe_ (optField "otr_archived" schema) + <*> misOtrArchivedRef .= maybe_ (optField "otr_archived_ref" schema) + <*> misHidden .= maybe_ (optField "hidden" schema) + <*> misHiddenRef .= maybe_ (optField "hidden_ref" schema) + <*> misConvRoleName .= maybe_ (optField "conversation_role" schema) modelMemberUpdateData :: Doc.Model modelMemberUpdateData = Doc.defineModel "MemberUpdateData" $ do @@ -478,8 +478,8 @@ otrMessageObjectSchema = (description ?~ textDesc) schema <*> otrData - .= opt - ( fieldWithDocModifier + .= maybe_ + ( optFieldWithDocModifier "data" (description ?~ dataDesc) schema diff --git a/libs/wire-api/src/Wire/API/Message.hs b/libs/wire-api/src/Wire/API/Message.hs index b9ee8acf812..e5b1c33cf9b 100644 --- a/libs/wire-api/src/Wire/API/Message.hs +++ b/libs/wire-api/src/Wire/API/Message.hs @@ -152,9 +152,9 @@ instance ToSchema NewOtrMessage where <*> newOtrRecipients .= field "recipients" schema <*> newOtrNativePush .= (field "native_push" schema <|> pure True) <*> newOtrTransient .= (field "transient" schema <|> pure False) - <*> newOtrNativePriority .= opt (field "native_priority" schema) - <*> newOtrData .= opt (field "data" schema) - <*> newOtrReportMissing .= opt (field "report_missing" (array schema)) + <*> newOtrNativePriority .= maybe_ (optField "native_priority" schema) + <*> newOtrData .= maybe_ (optField "data" schema) + <*> newOtrReportMissing .= maybe_ (optField "report_missing" (array schema)) instance FromProto NewOtrMessage where fromProto bs = protoToNewOtrMessage <$> runGetLazy Protobuf.decodeMessage bs diff --git a/libs/wire-api/src/Wire/API/Routes/Internal/Brig/Connection.hs b/libs/wire-api/src/Wire/API/Routes/Internal/Brig/Connection.hs index 1132c6f920f..cf410a00f8a 100644 --- a/libs/wire-api/src/Wire/API/Routes/Internal/Brig/Connection.hs +++ b/libs/wire-api/src/Wire/API/Routes/Internal/Brig/Connection.hs @@ -22,7 +22,7 @@ instance ToSchema ConnectionsStatusRequest where object "ConnectionsStatusRequest" $ ConnectionsStatusRequest <$> csrFrom .= field "from" (array schema) - <*> csrTo .= optField "to" Nothing (array schema) + <*> csrTo .= maybe_ (optField "to" (array schema)) data ConnectionsStatusRequestV2 = ConnectionsStatusRequestV2 { csrv2From :: ![UserId], @@ -37,8 +37,8 @@ instance ToSchema ConnectionsStatusRequestV2 where object "ConnectionsStatusRequestV2" $ ConnectionsStatusRequestV2 <$> csrv2From .= field "from" (array schema) - <*> csrv2To .= optField "to" Nothing (array schema) - <*> csrv2Relation .= optField "relation" Nothing schema + <*> csrv2To .= maybe_ (optField "to" (array schema)) + <*> csrv2Relation .= maybe_ (optField "relation" schema) data ConnectionStatus = ConnectionStatus { csFrom :: !UserId, diff --git a/libs/wire-api/src/Wire/API/Routes/MultiTablePaging.hs b/libs/wire-api/src/Wire/API/Routes/MultiTablePaging.hs index e05cbf6e987..0fd21bd3a21 100644 --- a/libs/wire-api/src/Wire/API/Routes/MultiTablePaging.hs +++ b/libs/wire-api/src/Wire/API/Routes/MultiTablePaging.hs @@ -90,8 +90,8 @@ instance RequestSchemaConstraint name tables max def => ToSchema (GetMultiTableP ("GetPaginated_" <> textFromSymbol @name) (description ?~ "A request to list some or all of a user's " <> textFromSymbol @name <> ", including remote ones") $ GetMultiTablePageRequest - <$> gmtprSize .= (fieldWithDocModifier "size" addSizeDoc schema <|> pure (toRange (Proxy @def))) - <*> gmtprState .= optFieldWithDocModifier "paging_state" Nothing addPagingStateDoc schema + <$> gmtprSize .= (fromMaybe (toRange (Proxy @def)) <$> optFieldWithDocModifier "size" addSizeDoc schema) + <*> gmtprState .= maybe_ (optFieldWithDocModifier "paging_state" addPagingStateDoc schema) textFromNat :: forall n. KnownNat n => Text textFromNat = Text.pack . show . natVal $ Proxy @n diff --git a/libs/wire-api/src/Wire/API/User.hs b/libs/wire-api/src/Wire/API/User.hs index 30bbe2933e8..18464c0b185 100644 --- a/libs/wire-api/src/Wire/API/User.hs +++ b/libs/wire-api/src/Wire/API/User.hs @@ -234,12 +234,12 @@ instance ToSchema UserProfile where <*> profileAssets .= (field "assets" (array schema) <|> pure []) <*> profileAccentId .= field "accent_id" schema <*> ((\del -> if del then Just True else Nothing) . profileDeleted) - .= fmap (fromMaybe False) (opt (field "deleted" schema)) - <*> profileService .= opt (field "service" schema) - <*> profileHandle .= opt (field "handle" schema) - <*> profileExpire .= opt (field "expires_at" schema) - <*> profileTeam .= opt (field "team" schema) - <*> profileEmail .= opt (field "email" schema) + .= maybe_ (fromMaybe False <$> optField "deleted" schema) + <*> profileService .= maybe_ (optField "service" schema) + <*> profileHandle .= maybe_ (optField "handle" schema) + <*> profileExpire .= maybe_ (optField "expires_at" schema) + <*> profileTeam .= maybe_ (optField "team" schema) + <*> profileEmail .= maybe_ (optField "email" schema) <*> profileLegalholdStatus .= field "legalhold_status" schema modelUser :: Doc.Model @@ -986,7 +986,7 @@ instance ToSchema DeleteUser where schema = object "DeleteUser" $ DeleteUser - <$> deleteUserPassword .= opt (field "password" schema) + <$> deleteUserPassword .= maybe_ (optField "password" schema) mkDeleteUser :: Maybe PlainTextPassword -> DeleteUser mkDeleteUser = DeleteUser diff --git a/libs/wire-api/src/Wire/API/User/Client.hs b/libs/wire-api/src/Wire/API/User/Client.hs index 6e601ecbb81..7437a6e74dd 100644 --- a/libs/wire-api/src/Wire/API/User/Client.hs +++ b/libs/wire-api/src/Wire/API/User/Client.hs @@ -74,7 +74,8 @@ module Wire.API.User.Client where import qualified Cassandra as Cql -import Control.Lens (view, (?~), (^.)) +import Control.Applicative +import Control.Lens (over, view, (?~), (^.)) import Data.Aeson (FromJSON (..), ToJSON (..)) import qualified Data.Aeson as A import Data.Bifunctor (second) @@ -180,12 +181,14 @@ newtype ClientCapabilityList = ClientCapabilityList {fromClientCapabilityList :: instance ToSchema ClientCapabilityList where schema = object "ClientCapabilityList" $ - ClientCapabilityList <$> fromClientCapabilityList .= capabilitiesFieldSchema + ClientCapabilityList <$> fromClientCapabilityList .= fmap runIdentity capabilitiesFieldSchema -capabilitiesFieldSchema :: ObjectSchema SwaggerDoc (Set ClientCapability) +capabilitiesFieldSchema :: + FieldFunctor SwaggerDoc f => + ObjectSchemaP SwaggerDoc (Set ClientCapability) (f (Set ClientCapability)) capabilitiesFieldSchema = Set.toList - .= fieldWithDocModifier "capabilities" mods (Set.fromList <$> array schema) + .= fieldWithDocModifierF "capabilities" mods (Set.fromList <$> array schema) where mods = description @@ -218,14 +221,22 @@ modelOtrClientMap = Doc.defineModel "OtrClientMap" $ do instance ToSchema a => ToSchema (UserClientMap a) where schema = userClientMapSchema schema +class WrapName doc where + wrapName :: doc -> (Text -> Text) -> SwaggerDoc -> doc + +instance WrapName SwaggerDoc where + wrapName _ _ = id + +instance WrapName NamedSwaggerDoc where + wrapName d f = fmap (Swagger.NamedSchema (Just (f (maybe "" ("_" <>) (getName d))))) + userClientMapSchema :: - ValueSchema NamedSwaggerDoc a -> - ValueSchema NamedSwaggerDoc (UserClientMap a) + (WrapName doc, HasSchemaRef doc) => + ValueSchema doc a -> + ValueSchema doc (UserClientMap a) userClientMapSchema sch = - named nm $ + over doc (wrapName (schemaDoc sch) ("UserClientMap" <>)) $ UserClientMap <$> userClientMap .= map_ (map_ sch) - where - nm = "UserClientMap" <> maybe "" (" " <>) (getName (schemaDoc sch)) newtype UserClientPrekeyMap = UserClientPrekeyMap {getUserClientPrekeyMap :: UserClientMap (Maybe Prekey)} @@ -240,8 +251,8 @@ instance ToSchema UserClientPrekeyMap where schema = UserClientPrekeyMap <$> getUserClientPrekeyMap .= addDoc sch where sch = - named "UserClientPrekeyMap" . unnamed $ - userClientMapSchema (optWithDefault A.Null schema) + named "UserClientPrekeyMap" $ + userClientMapSchema (nullable (unnamed schema)) addDoc = Swagger.schema . Swagger.example ?~ toJSON @@ -442,12 +453,12 @@ instance ToSchema Client where <$> clientId .= field "id" schema <*> clientType .= field "type" schema <*> clientTime .= field "time" schema - <*> clientClass .= opt (field "class" schema) - <*> clientLabel .= opt (field "label" schema) - <*> clientCookie .= opt (field "cookie" schema) - <*> clientLocation .= opt (field "location" schema) - <*> clientModel .= opt (field "model" schema) - <*> clientCapabilities .= (field "capabilities" schema <|> pure mempty) + <*> clientClass .= maybe_ (optField "class" schema) + <*> clientLabel .= maybe_ (optField "label" schema) + <*> clientCookie .= maybe_ (optField "cookie" schema) + <*> clientLocation .= maybe_ (optField "location" schema) + <*> clientModel .= maybe_ (optField "model" schema) + <*> clientCapabilities .= (fromMaybe mempty <$> optField "capabilities" schema) modelClient :: Doc.Model modelClient = Doc.defineModel "Client" $ do @@ -654,10 +665,10 @@ instance ToSchema NewClient where \When a temporary client already exists, it is replaced." ) schema - <*> newClientLabel .= opt (field "label" schema) + <*> newClientLabel .= maybe_ (optField "label" schema) <*> newClientClass - .= opt - ( fieldWithDocModifier + .= maybe_ + ( optFieldWithDocModifier "class" ( description ?~ "The device class this client belongs to. \ @@ -666,15 +677,15 @@ instance ToSchema NewClient where schema ) <*> newClientCookie - .= opt - ( fieldWithDocModifier + .= maybe_ + ( optFieldWithDocModifier "cookie" (description ?~ "The cookie label, i.e. the label used when logging in.") schema ) <*> newClientPassword - .= opt - ( fieldWithDocModifier + .= maybe_ + ( optFieldWithDocModifier "password" ( description ?~ "The password of the authenticated user for verification. \ @@ -682,8 +693,8 @@ instance ToSchema NewClient where ) schema ) - <*> newClientModel .= opt (field "model" schema) - <*> newClientCapabilities .= opt capabilitiesFieldSchema + <*> newClientModel .= maybe_ (optField "model" schema) + <*> newClientCapabilities .= maybe_ capabilitiesFieldSchema newClient :: ClientType -> LastPrekey -> NewClient newClient t k = @@ -717,30 +728,29 @@ instance ToSchema UpdateClient where schema = object "UpdateClient" $ UpdateClient - <$> (Just . updateClientPrekeys) + <$> updateClientPrekeys .= ( fromMaybe [] - <$> opt - ( fieldWithDocModifier - "prekeys" - (description ?~ "New prekeys for other clients to establish OTR sessions.") - (array schema) - ) + <$> ( optFieldWithDocModifier + "prekeys" + (description ?~ "New prekeys for other clients to establish OTR sessions.") + (array schema) + ) ) <*> updateClientLastKey - .= opt - ( fieldWithDocModifier + .= maybe_ + ( optFieldWithDocModifier "lastkey" (description ?~ "New last-resort prekey.") schema ) <*> updateClientLabel - .= opt - ( fieldWithDocModifier + .= maybe_ + ( optFieldWithDocModifier "label" (description ?~ "A new name for this client.") schema ) - <*> updateClientCapabilities .= opt capabilitiesFieldSchema + <*> updateClientCapabilities .= maybe_ capabilitiesFieldSchema modelUpdateClient :: Doc.Model modelUpdateClient = Doc.defineModel "UpdateClient" $ do @@ -781,12 +791,11 @@ instance ToSchema RmClient where <$> rmPassword .= optFieldWithDocModifier "password" - (Just A.Null) ( description ?~ "The password of the authenticated user for verification. \ \The password is not required for deleting temporary clients." ) - schema + (maybeWithDefault A.Null schema) modelDeleteClient :: Doc.Model modelDeleteClient = Doc.defineModel "DeleteClient" $ do diff --git a/libs/wire-api/src/Wire/API/User/Profile.hs b/libs/wire-api/src/Wire/API/User/Profile.hs index 7bea3c03def..0c4a7a389cb 100644 --- a/libs/wire-api/src/Wire/API/User/Profile.hs +++ b/libs/wire-api/src/Wire/API/User/Profile.hs @@ -126,7 +126,7 @@ instance ToSchema Asset where object "UserAsset" $ ImageAsset <$> assetKey .= field "key" schema - <*> assetSize .= opt (field "size" schema) + <*> assetSize .= maybe_ (optField "size" schema) <* const () .= field "type" typeSchema where typeSchema :: ValueSchema NamedSwaggerDoc () From 6cff0a961b7da7021f05c05f1ef5fa6969739cbd Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Fri, 17 Dec 2021 17:24:13 +0100 Subject: [PATCH 17/74] SQSERVICES-1169 New internal endpoint to configure the guest links team feature (#1993) Internal endpoints for: * set guest link team feature status * get guest link team feature status * set guest link team feature lock status --- changelog.d/5-internal/sqservices-1169 | 1 + services/galley/galley.cabal | 1 + services/galley/schema/src/Main.hs | 4 +- .../schema/src/V57_GuestLinksLockStatus.hs | 33 +++++++++++ services/galley/src/Galley/API/Internal.hs | 12 ++++ .../galley/src/Galley/API/Teams/Features.hs | 29 ++++----- services/galley/src/Galley/Cassandra.hs | 2 +- .../src/Galley/Cassandra/TeamFeatures.hs | 3 - .../galley/src/Galley/Data/TeamFeatures.hs | 3 +- .../test/integration/API/Teams/Feature.hs | 59 +++++++++++++++---- 10 files changed, 112 insertions(+), 35 deletions(-) create mode 100644 changelog.d/5-internal/sqservices-1169 create mode 100644 services/galley/schema/src/V57_GuestLinksLockStatus.hs diff --git a/changelog.d/5-internal/sqservices-1169 b/changelog.d/5-internal/sqservices-1169 new file mode 100644 index 00000000000..cc2869e0494 --- /dev/null +++ b/changelog.d/5-internal/sqservices-1169 @@ -0,0 +1 @@ +New internal endpoint to configure the guest links team feature. diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal index 680261a3317..f743ae99946 100644 --- a/services/galley/galley.cabal +++ b/services/galley/galley.cabal @@ -438,6 +438,7 @@ executable galley-schema V54_TeamFeatureSelfDeletingMessages V55_SelfDeletingMessagesLockStatus V56_GuestLinksTeamFeatureStatus + V57_GuestLinksLockStatus Paths_galley hs-source-dirs: schema/src diff --git a/services/galley/schema/src/Main.hs b/services/galley/schema/src/Main.hs index 63cda90e089..e67775ed7a6 100644 --- a/services/galley/schema/src/Main.hs +++ b/services/galley/schema/src/Main.hs @@ -59,6 +59,7 @@ import qualified V53_AddRemoteConvStatus import qualified V54_TeamFeatureSelfDeletingMessages import qualified V55_SelfDeletingMessagesLockStatus import qualified V56_GuestLinksTeamFeatureStatus +import qualified V57_GuestLinksLockStatus main :: IO () main = do @@ -103,7 +104,8 @@ main = do V53_AddRemoteConvStatus.migration, V54_TeamFeatureSelfDeletingMessages.migration, V55_SelfDeletingMessagesLockStatus.migration, - V56_GuestLinksTeamFeatureStatus.migration + V56_GuestLinksTeamFeatureStatus.migration, + V57_GuestLinksLockStatus.migration -- When adding migrations here, don't forget to update -- 'schemaVersion' in Galley.Cassandra -- (see also docs/developer/cassandra-interaction.md) diff --git a/services/galley/schema/src/V57_GuestLinksLockStatus.hs b/services/galley/schema/src/V57_GuestLinksLockStatus.hs new file mode 100644 index 00000000000..6afa380236e --- /dev/null +++ b/services/galley/schema/src/V57_GuestLinksLockStatus.hs @@ -0,0 +1,33 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2020 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module V57_GuestLinksLockStatus + ( migration, + ) +where + +import Cassandra.Schema +import Imports +import Text.RawString.QQ + +migration :: Migration +migration = Migration 57 "Add lock status for guest links team feature" $ do + schema' + [r| ALTER TABLE team_features ADD ( + guest_links_lock_status int + ) + |] diff --git a/services/galley/src/Galley/API/Internal.hs b/services/galley/src/Galley/API/Internal.hs index ff2b743f92f..85234b2efd8 100644 --- a/services/galley/src/Galley/API/Internal.hs +++ b/services/galley/src/Galley/API/Internal.hs @@ -198,6 +198,15 @@ data InternalApi routes = InternalApi iTeamFeatureLockStatusSelfDeletingMessagesPut :: routes :- IFeatureStatusLockStatusPut 'Public.TeamFeatureSelfDeletingMessages, + iTeamFeatureStatusGuestLinksGet :: + routes + :- IFeatureStatusGet 'Public.WithLockStatus 'Public.TeamFeatureGuestLinks, + iTeamFeatureStatusGuestLinksPut :: + routes + :- IFeatureStatusPut 'Public.TeamFeatureGuestLinks, + iTeamFeatureLockStatusGuestLinksPut :: + routes + :- IFeatureStatusLockStatusPut 'Public.TeamFeatureGuestLinks, -- This endpoint can lead to the following events being sent: -- - MemberLeave event to members for all conversations the user was in iDeleteUser :: @@ -318,6 +327,9 @@ servantSitemap = iTeamFeatureStatusSelfDeletingMessagesPut = iPutTeamFeature @'Public.TeamFeatureSelfDeletingMessages Features.setSelfDeletingMessagesInternal, iTeamFeatureStatusSelfDeletingMessagesGet = iGetTeamFeature @'Public.WithLockStatus @'Public.TeamFeatureSelfDeletingMessages Features.getSelfDeletingMessagesInternal, iTeamFeatureLockStatusSelfDeletingMessagesPut = Features.setLockStatus @'Public.TeamFeatureSelfDeletingMessages, + iTeamFeatureStatusGuestLinksGet = iGetTeamFeature @'Public.WithLockStatus @'Public.TeamFeatureGuestLinks Features.getGuestLinkInternal, + iTeamFeatureStatusGuestLinksPut = iPutTeamFeature @'Public.TeamFeatureGuestLinks Features.setGuestLinkInternal, + iTeamFeatureLockStatusGuestLinksPut = Features.setLockStatus @'Public.TeamFeatureGuestLinks, iDeleteUser = rmUser, iConnect = Create.createConnectConversation, iUpsertOne2OneConversation = One2One.iUpsertOne2OneConversation diff --git a/services/galley/src/Galley/API/Teams/Features.hs b/services/galley/src/Galley/API/Teams/Features.hs index 63ef91cd19a..0eb176aee28 100644 --- a/services/galley/src/Galley/API/Teams/Features.hs +++ b/services/galley/src/Galley/API/Teams/Features.hs @@ -590,8 +590,8 @@ getSelfDeletingMessagesInternal = \case Right tid -> do cfgDefault <- getCfgDefault let defLockStatus = Public.tfwcapsLockStatus cfgDefault - (maybeFeatureStatus, fromMaybe defLockStatus -> lockStatus) <- TeamFeatures.getSelfDeletingMessagesStatus tid - pure $ case (lockStatus, maybeFeatureStatus) of + (mbFeatureStatus, fromMaybe defLockStatus -> lockStatus) <- TeamFeatures.getSelfDeletingMessagesStatus tid + pure $ case (lockStatus, mbFeatureStatus) of (Public.Unlocked, Just featureStatus) -> Public.TeamFeatureStatusWithConfigAndLockStatus (Public.tfwcStatus featureStatus) @@ -616,15 +616,14 @@ setSelfDeletingMessagesInternal :: Public.TeamFeatureStatus 'Public.WithoutLockStatus 'Public.TeamFeatureSelfDeletingMessages -> Sem r (Public.TeamFeatureStatus 'Public.WithoutLockStatus 'Public.TeamFeatureSelfDeletingMessages) setSelfDeletingMessagesInternal tid st = do - dftLockStatus <- Public.tfwcapsLockStatus <$> getCfgDefault - guardLockStatus @'Public.TeamFeatureSelfDeletingMessages tid dftLockStatus + getDftLockStatus >>= guardLockStatus @'Public.TeamFeatureSelfDeletingMessages tid let pushEvent = pushFeatureConfigEvent tid $ Event.Event Event.Update Public.TeamFeatureSelfDeletingMessages (EdFeatureSelfDeletingMessagesChanged st) TeamFeatures.setSelfDeletingMessagesStatus tid st <* pushEvent where - getCfgDefault :: Sem r (Public.TeamFeatureStatusWithConfigAndLockStatus Public.TeamFeatureSelfDeletingMessagesConfig) - getCfgDefault = input <&> view (optSettings . setFeatureFlags . flagSelfDeletingMessages . unDefaults) + getDftLockStatus :: Sem r Public.LockStatusValue + getDftLockStatus = input <&> view (optSettings . setFeatureFlags . flagSelfDeletingMessages . unDefaults . to Public.tfwcapsLockStatus) getGuestLinkInternal :: forall r. @@ -635,15 +634,14 @@ getGuestLinkInternal = \case Left _ -> getCfgDefault Right tid -> do cfgDefault <- getCfgDefault - let defLockStatus = Public.tfwoapsLockStatus cfgDefault - maybeFeatureStatus <- TeamFeatures.getFeatureStatusNoConfig @'Public.TeamFeatureGuestLinks tid - pure $ case (defLockStatus, maybeFeatureStatus) of + (mbFeatureStatus, fromMaybe (Public.tfwoapsLockStatus cfgDefault) -> lockStatus) <- TeamFeatures.getFeatureStatusNoConfigAndLockStatus @'Public.TeamFeatureGuestLinks tid + pure $ case (lockStatus, mbFeatureStatus) of (Public.Unlocked, Just featureStatus) -> Public.TeamFeatureStatusNoConfigAndLockStatus (Public.tfwoStatus featureStatus) - Public.Unlocked - (Public.Unlocked, Nothing) -> cfgDefault {Public.tfwoapsLockStatus = Public.Unlocked} - (Public.Locked, _) -> cfgDefault {Public.tfwoapsLockStatus = Public.Locked} + lockStatus + (Public.Unlocked, Nothing) -> cfgDefault {Public.tfwoapsLockStatus = lockStatus} + (Public.Locked, _) -> cfgDefault {Public.tfwoapsLockStatus = lockStatus} where getCfgDefault :: Sem r (Public.TeamFeatureStatus 'Public.WithLockStatus 'Public.TeamFeatureGuestLinks) getCfgDefault = input <&> view (optSettings . setFeatureFlags . flagConversationGuestLinks . unDefaults) @@ -661,8 +659,7 @@ setGuestLinkInternal :: Public.TeamFeatureStatus 'Public.WithoutLockStatus 'Public.TeamFeatureGuestLinks -> Sem r (Public.TeamFeatureStatus 'Public.WithoutLockStatus 'Public.TeamFeatureGuestLinks) setGuestLinkInternal tid status = do - cfgDefault <- Public.tfwoapsLockStatus <$> getCfgDefault - guardLockStatus @'Public.TeamFeatureGuestLinks tid cfgDefault + getDftLockStatus >>= guardLockStatus @'Public.TeamFeatureGuestLinks tid let pushEvent = pushFeatureConfigEvent tid $ Event.Event @@ -673,8 +670,8 @@ setGuestLinkInternal tid status = do ) TeamFeatures.setFeatureStatusNoConfig @'Public.TeamFeatureGuestLinks tid status <* pushEvent where - getCfgDefault :: Sem r (Public.TeamFeatureStatus 'Public.WithLockStatus 'Public.TeamFeatureGuestLinks) - getCfgDefault = input <&> view (optSettings . setFeatureFlags . flagConversationGuestLinks . unDefaults) + getDftLockStatus :: Sem r Public.LockStatusValue + getDftLockStatus = input <&> view (optSettings . setFeatureFlags . flagConversationGuestLinks . unDefaults . to Public.tfwoapsLockStatus) -- TODO(fisx): move this function to a more suitable place / module. guardLockStatus :: diff --git a/services/galley/src/Galley/Cassandra.hs b/services/galley/src/Galley/Cassandra.hs index a5477dd16d9..2840865d44c 100644 --- a/services/galley/src/Galley/Cassandra.hs +++ b/services/galley/src/Galley/Cassandra.hs @@ -20,4 +20,4 @@ module Galley.Cassandra (schemaVersion) where import Imports schemaVersion :: Int32 -schemaVersion = 56 +schemaVersion = 57 diff --git a/services/galley/src/Galley/Cassandra/TeamFeatures.hs b/services/galley/src/Galley/Cassandra/TeamFeatures.hs index 53a4065f874..4bd104cbe9e 100644 --- a/services/galley/src/Galley/Cassandra/TeamFeatures.hs +++ b/services/galley/src/Galley/Cassandra/TeamFeatures.hs @@ -29,9 +29,6 @@ import Polysemy import Polysemy.Input import Wire.API.Team.Feature --- TODO(leif): according to the specs it should only be supported to read the lock status via the api --- changes can only be made in the server configuration file --- we can probably remove the lock status from the db? getFeatureStatusNoConfigAndLockStatus :: forall (a :: TeamFeatureName) m. (MonadClient m, FeatureHasNoConfig 'WithoutLockStatus a, HasStatusCol a, HasLockStatusCol a) => diff --git a/services/galley/src/Galley/Data/TeamFeatures.hs b/services/galley/src/Galley/Data/TeamFeatures.hs index 12cd3dc28b6..8d694625601 100644 --- a/services/galley/src/Galley/Data/TeamFeatures.hs +++ b/services/galley/src/Galley/Data/TeamFeatures.hs @@ -65,7 +65,8 @@ instance {-# OVERLAPPABLE #-} HasLockStatusCol a => MaybeHasLockStatusCol a wher instance HasLockStatusCol 'TeamFeatureSelfDeletingMessages where lockStatusCol = "self_deleting_messages_lock_status" -instance MaybeHasLockStatusCol 'TeamFeatureGuestLinks where maybeLockStatusCol = Nothing +instance HasLockStatusCol 'TeamFeatureGuestLinks where + lockStatusCol = "guest_links_lock_status" instance MaybeHasLockStatusCol 'TeamFeatureLegalHold where maybeLockStatusCol = Nothing diff --git a/services/galley/test/integration/API/Teams/Feature.hs b/services/galley/test/integration/API/Teams/Feature.hs index f3b1989bd91..04c283203de 100644 --- a/services/galley/test/integration/API/Teams/Feature.hs +++ b/services/galley/test/integration/API/Teams/Feature.hs @@ -70,7 +70,8 @@ tests s = test s "Feature Configs / Team Features Consistency" testFeatureConfigConsistency, test s "ConferenceCalling" $ testSimpleFlag @'Public.TeamFeatureConferenceCalling Public.TeamFeatureEnabled, test s "SelfDeletingMessages" testSelfDeletingMessages, - test s "ConversationGuestLinks" testGuestLinks + test s "ConversationGuestLinks - public API" testGuestLinksPublic, + test s "ConversationGuestLinks - internal API" testGuestLinksInternal ] testSSO :: TestM () @@ -474,26 +475,58 @@ testSelfDeletingMessages = do checkSetLockStatus Public.Unlocked checkGet TeamFeatureDisabled 30 Public.Unlocked -testGuestLinks :: TestM () -testGuestLinks = do +testGuestLinksInternal :: TestM () +testGuestLinksInternal = do galley <- view tsGalley + testGuestLinks + (const $ Util.getTeamFeatureFlagInternal Public.TeamFeatureGuestLinks) + (const $ Util.putTeamFeatureFlagInternal @'Public.TeamFeatureGuestLinks galley) + (Util.setLockStatusInternal @'Public.TeamFeatureGuestLinks galley) + +testGuestLinksPublic :: TestM () +testGuestLinksPublic = do + galley <- view tsGalley + testGuestLinks + (Util.getTeamFeatureFlagWithGalley Public.TeamFeatureGuestLinks galley) + (Util.putTeamFeatureFlagWithGalley @'Public.TeamFeatureGuestLinks galley) + (Util.setLockStatusInternal @'Public.TeamFeatureGuestLinks galley) + +testGuestLinks :: + (UserId -> TeamId -> TestM ResponseLBS) -> + (UserId -> TeamId -> Public.TeamFeatureStatusNoConfig -> TestM ResponseLBS) -> + (TeamId -> Public.LockStatusValue -> TestM ResponseLBS) -> + TestM () +testGuestLinks getStatus putStatus setLockStatusInternal = do (owner, tid, []) <- Util.createBindingTeamWithNMembers 0 let checkGet :: HasCallStack => Public.TeamFeatureStatusValue -> Public.LockStatusValue -> TestM () checkGet status lock = - do - Util.getTeamFeatureFlagWithGalley Public.TeamFeatureGuestLinks galley owner tid - !!! responseJsonEither === const (Right (Public.TeamFeatureStatusNoConfigAndLockStatus status lock)) - checkSet :: HasCallStack => Public.TeamFeatureStatusValue -> TestM () - checkSet status = - do - Util.putTeamFeatureFlagWithGalley @'Public.TeamFeatureGuestLinks galley owner tid (Public.TeamFeatureStatusNoConfig status) - !!! statusCode === const 200 + getStatus owner tid !!! do + statusCode === const 200 + responseJsonEither === const (Right (Public.TeamFeatureStatusNoConfigAndLockStatus status lock)) + + checkSet :: HasCallStack => Public.TeamFeatureStatusValue -> Int -> TestM () + checkSet status expectedStatusCode = + putStatus owner tid (Public.TeamFeatureStatusNoConfig status) !!! statusCode === const expectedStatusCode + + checkSetLockStatusInternal :: HasCallStack => Public.LockStatusValue -> TestM () + checkSetLockStatusInternal lockStatus = + setLockStatusInternal tid lockStatus !!! statusCode === const 200 checkGet Public.TeamFeatureEnabled Public.Unlocked - checkSet Public.TeamFeatureDisabled + checkSet Public.TeamFeatureDisabled 200 checkGet Public.TeamFeatureDisabled Public.Unlocked - checkSet Public.TeamFeatureEnabled + checkSet Public.TeamFeatureEnabled 200 checkGet Public.TeamFeatureEnabled Public.Unlocked + checkSet Public.TeamFeatureDisabled 200 + checkGet Public.TeamFeatureDisabled Public.Unlocked + -- when locks status is locked the team default feature status should be returned + -- and the team feature status can not be changed + checkSetLockStatusInternal Public.Locked + checkGet Public.TeamFeatureEnabled Public.Locked + checkSet Public.TeamFeatureDisabled 409 + -- when lock status is unlocked again the previously set feature status is restored + checkSetLockStatusInternal Public.Unlocked + checkGet Public.TeamFeatureDisabled Public.Unlocked -- | Call 'GET /teams/:tid/features' and 'GET /feature-configs', and check if all -- features are there. From f99f1e48292f884c74e1c6b1c59262e0970ea3f3 Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Fri, 17 Dec 2021 17:26:00 +0100 Subject: [PATCH 18/74] SQSERVICES-547 Enforce disabling guest links when conversation is joined (#1976) --- changelog.d/2-features/pr-1976 | 1 + .../wire-api/src/Wire/API/ErrorDescription.hs | 2 + .../src/Wire/API/Routes/Public/Galley.hs | 1 + services/galley/src/Galley/API/Error.hs | 5 ++ services/galley/src/Galley/API/Query.hs | 40 +++++++++---- .../galley/src/Galley/API/Teams/Features.hs | 1 + services/galley/test/integration/API.hs | 59 +++++++++++++++++++ services/galley/test/integration/API/Util.hs | 8 ++- 8 files changed, 103 insertions(+), 14 deletions(-) create mode 100644 changelog.d/2-features/pr-1976 diff --git a/changelog.d/2-features/pr-1976 b/changelog.d/2-features/pr-1976 new file mode 100644 index 00000000000..e67dd816a08 --- /dev/null +++ b/changelog.d/2-features/pr-1976 @@ -0,0 +1 @@ +If the guest links team feature is disabled guest links will be revoked. diff --git a/libs/wire-api/src/Wire/API/ErrorDescription.hs b/libs/wire-api/src/Wire/API/ErrorDescription.hs index f02a37cc0f0..1cb53ce4a38 100644 --- a/libs/wire-api/src/Wire/API/ErrorDescription.hs +++ b/libs/wire-api/src/Wire/API/ErrorDescription.hs @@ -261,6 +261,8 @@ type HandleNotFound = ErrorDescription 404 "not-found" "Handle not found" type TooManyClients = ErrorDescription 403 "too-many-clients" "Too many clients" +type GuestLinksDisabled = ErrorDescription 409 "guest-links-disabled" "The guest link feature is disabled and all guest links have been revoked." + type MissingAuth = ErrorDescription 403 diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs b/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs index 158fddcca3f..30e402c0590 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs @@ -195,6 +195,7 @@ data Api routes = Api :> CanThrow CodeNotFound :> CanThrow ConvNotFound :> CanThrow ConvAccessDenied + :> CanThrow GuestLinksDisabled :> ZLocalUser :> "conversations" :> "join" diff --git a/services/galley/src/Galley/API/Error.hs b/services/galley/src/Galley/API/Error.hs index a4569e788c9..2e5a8a9afd5 100644 --- a/services/galley/src/Galley/API/Error.hs +++ b/services/galley/src/Galley/API/Error.hs @@ -122,6 +122,7 @@ data ConversationError | ConvMemberNotFound | NoBindingTeamMembers | NoManagedTeamConv + | GuestLinksDisabled instance APIError ConversationError where toWai ConvAccessDenied = errorDescriptionTypeToWai @ConvAccessDenied @@ -130,6 +131,7 @@ instance APIError ConversationError where toWai ConvMemberNotFound = errorDescriptionTypeToWai @ConvMemberNotFound toWai NoBindingTeamMembers = noBindingTeamMembers toWai NoManagedTeamConv = noManagedTeamConv + toWai GuestLinksDisabled = guestLinksDisabled data TeamError = NoBindingTeam @@ -396,6 +398,9 @@ teamMemberNotFound = mkError status404 "no-team-member" "team member not found" noManagedTeamConv :: Error noManagedTeamConv = mkError status400 "no-managed-team-conv" "Managed team conversations have been deprecated." +guestLinksDisabled :: Error +guestLinksDisabled = mkError status409 "guest-links-disabled" "The guest link feature is disabled and all guest links have been revoked." + userBindingExists :: Error userBindingExists = mkError status403 "binding-exists" "User already bound to a different team." diff --git a/services/galley/src/Galley/API/Query.hs b/services/galley/src/Galley/API/Query.hs index b2c678d8371..ace30d6eec7 100644 --- a/services/galley/src/Galley/API/Query.hs +++ b/services/galley/src/Galley/API/Query.hs @@ -34,6 +34,7 @@ module Galley.API.Query where import qualified Cassandra as C +import Control.Lens import qualified Data.ByteString.Lazy as LBS import Data.Code import Data.CommaSeparatedList @@ -54,9 +55,12 @@ import qualified Galley.Effects.ConversationStore as E import qualified Galley.Effects.FederatorAccess as E import qualified Galley.Effects.ListItems as E import qualified Galley.Effects.MemberStore as E +import qualified Galley.Effects.TeamFeatureStore as TeamFeatures +import Galley.Options import Galley.Types import Galley.Types.Conversations.Members import Galley.Types.Conversations.Roles +import Galley.Types.Teams import Imports import Network.HTTP.Types import Network.Wai @@ -77,6 +81,7 @@ import qualified Wire.API.Federation.API.Galley as F import Wire.API.Federation.Error import qualified Wire.API.Provider.Bot as Public import qualified Wire.API.Routes.MultiTablePaging as Public +import Wire.API.Team.Feature as Public getBotConversationH :: Members '[ConversationStore, Error ConversationError, Input (Local ())] r => @@ -490,16 +495,17 @@ getConversationMeta cnv = do pure Nothing getConversationByReusableCode :: - Members - '[ BrigAccess, - CodeStore, - ConversationStore, - Error CodeError, - Error ConversationError, - Error NotATeamMember, - TeamStore - ] - r => + forall r. + ( Member BrigAccess r, + Member CodeStore r, + Member ConversationStore r, + Member (Error CodeError) r, + Member (Error ConversationError) r, + Member (Error NotATeamMember) r, + Member TeamStore r, + Member TeamFeatureStore r, + Member (Input Opts) r + ) => Local UserId -> Key -> Value -> @@ -507,7 +513,9 @@ getConversationByReusableCode :: getConversationByReusableCode lusr key value = do c <- verifyReusableCode (ConversationCode key value Nothing) conv <- ensureConversationAccess (tUnqualified lusr) (Data.codeConversation c) CodeAccess - pure $ coverView conv + getFeatureStatus conv >>= \case + TeamFeatureEnabled -> pure $ coverView conv + TeamFeatureDisabled -> throw GuestLinksDisabled where coverView :: Data.Conversation -> ConversationCoverView coverView conv = @@ -515,3 +523,13 @@ getConversationByReusableCode lusr key value = do { cnvCoverConvId = Data.convId conv, cnvCoverName = Data.convName conv } + + getDefaultFeatureStatus :: Sem r TeamFeatureStatusValue + getDefaultFeatureStatus = + input <&> view (optSettings . setFeatureFlags . flagConversationGuestLinks . unDefaults . to tfwoapsStatus) + + getFeatureStatus :: Data.Conversation -> Sem r TeamFeatureStatusValue + getFeatureStatus conv = do + defaultStatus <- getDefaultFeatureStatus + maybeFeatureStatus <- join <$> TeamFeatures.getFeatureStatusNoConfig @'TeamFeatureGuestLinks `traverse` Data.convTeam conv + pure $ maybe defaultStatus tfwoStatus maybeFeatureStatus diff --git a/services/galley/src/Galley/API/Teams/Features.hs b/services/galley/src/Galley/API/Teams/Features.hs index 0eb176aee28..4593501051f 100644 --- a/services/galley/src/Galley/API/Teams/Features.hs +++ b/services/galley/src/Galley/API/Teams/Features.hs @@ -17,6 +17,7 @@ module Galley.API.Teams.Features ( getFeatureStatus, + getFeatureStatusNoConfig, setFeatureStatus, getFeatureConfig, getAllFeatureConfigs, diff --git a/services/galley/test/integration/API.hs b/services/galley/test/integration/API.hs index 13d41e2b806..3d2706dd4e5 100644 --- a/services/galley/test/integration/API.hs +++ b/services/galley/test/integration/API.hs @@ -33,6 +33,8 @@ import qualified API.Teams.Feature as TeamFeature import qualified API.Teams.LegalHold as Teams.LegalHold import qualified API.Teams.LegalHold.DisabledByDefault import API.Util +import qualified API.Util as Util +import API.Util.TeamFeature as TeamFeatures import Bilge hiding (timeout) import Bilge.Assert import Brig.Types @@ -93,6 +95,7 @@ import Wire.API.Federation.API.Galley import qualified Wire.API.Federation.API.Galley as F import qualified Wire.API.Message as Message import Wire.API.Routes.MultiTablePaging +import qualified Wire.API.Team.Feature as Public import Wire.API.User.Client import Wire.API.UserMap (UserMap (..)) @@ -218,6 +221,8 @@ tests s = test s "convert code to team-access conversation" postConvertTeamConv, test s "local and remote guests are removed when access changes" testAccessUpdateGuestRemoved, test s "cannot join private conversation" postJoinConvFail, + test s "revoke guest links for team conversation" testJoinTeamConvGuestLinksDisabled, + test s "revoke guest links for non-team conversation" testJoinNonTeamConvGuestLinksDisabled, test s "remove user with only local convs" removeUserNoFederation, test s "remove user with local and remote convs" removeUser, test s "iUpsertOne2OneConversation" testAllOne2OneConversationRequests, @@ -1236,6 +1241,60 @@ testJoinCodeConv = do getJoinCodeConv eve (conversationKey cCode) (conversationCode cCode) !!! do const 403 === statusCode +testJoinTeamConvGuestLinksDisabled :: TestM () +testJoinTeamConvGuestLinksDisabled = do + galley <- view tsGalley + let convName = "testConversation" + (owner, teamId, []) <- Util.createBindingTeamWithNMembers 0 + userNotInTeam <- randomUser + convId <- decodeConvId <$> postTeamConv teamId owner [] (Just convName) [CodeAccess] (Just ActivatedAccessRole) Nothing + cCode <- decodeConvCodeEvent <$> postConvCode owner convId + + -- works by default + getJoinCodeConv userNotInTeam (conversationKey cCode) (conversationCode cCode) !!! do + const (Right (ConversationCoverView convId (Just convName))) === responseJsonEither + const 200 === statusCode + + -- fails if disabled + let tfStatus = Public.TeamFeatureStatusNoConfig Public.TeamFeatureDisabled + TeamFeatures.putTeamFeatureFlagWithGalley @'Public.TeamFeatureGuestLinks galley owner teamId tfStatus !!! do + const 200 === statusCode + + getJoinCodeConv userNotInTeam (conversationKey cCode) (conversationCode cCode) !!! do + const 409 === statusCode + + -- after re-enabling, the old link is still valid + let tfStatus' = Public.TeamFeatureStatusNoConfig Public.TeamFeatureEnabled + TeamFeatures.putTeamFeatureFlagWithGalley @'Public.TeamFeatureGuestLinks galley owner teamId tfStatus' !!! do + const 200 === statusCode + + getJoinCodeConv userNotInTeam (conversationKey cCode) (conversationCode cCode) !!! do + const (Right (ConversationCoverView convId (Just convName))) === responseJsonEither + const 200 === statusCode + +testJoinNonTeamConvGuestLinksDisabled :: TestM () +testJoinNonTeamConvGuestLinksDisabled = do + galley <- view tsGalley + let convName = "testConversation" + (owner, teamId, []) <- Util.createBindingTeamWithNMembers 0 + userNotInTeam <- randomUser + convId <- decodeConvId <$> postConv owner [] (Just convName) [CodeAccess] (Just ActivatedAccessRole) Nothing + cCode <- decodeConvCodeEvent <$> postConvCode owner convId + + -- works by default + getJoinCodeConv userNotInTeam (conversationKey cCode) (conversationCode cCode) !!! do + const (Right (ConversationCoverView convId (Just convName))) === responseJsonEither + const 200 === statusCode + + -- for non-team conversations it still works if status is disabled for the team but not server wide + let tfStatus = Public.TeamFeatureStatusNoConfig Public.TeamFeatureDisabled + TeamFeatures.putTeamFeatureFlagWithGalley @'Public.TeamFeatureGuestLinks galley owner teamId tfStatus !!! do + const 200 === statusCode + + getJoinCodeConv userNotInTeam (conversationKey cCode) (conversationCode cCode) !!! do + const (Right (ConversationCoverView convId (Just convName))) === responseJsonEither + const 200 === statusCode + postJoinCodeConvOk :: TestM () postJoinCodeConvOk = do c <- view tsCannon diff --git a/services/galley/test/integration/API/Util.hs b/services/galley/test/integration/API/Util.hs index b1cb862d558..31af5c651a5 100644 --- a/services/galley/test/integration/API/Util.hs +++ b/services/galley/test/integration/API/Util.hs @@ -1738,7 +1738,10 @@ randomTeamCreator :: HasCallStack => TestM UserId randomTeamCreator = qUnqualified <$> randomUser' True True True randomUser' :: HasCallStack => Bool -> Bool -> Bool -> TestM (Qualified UserId) -randomUser' isCreator hasPassword hasEmail = do +randomUser' isCreator hasPassword hasEmail = userQualifiedId . selfUser <$> randomUserProfile' isCreator hasPassword hasEmail + +randomUserProfile' :: HasCallStack => Bool -> Bool -> Bool -> TestM SelfProfile +randomUserProfile' isCreator hasPassword hasEmail = do b <- view tsBrig e <- liftIO randomEmail let p = @@ -1747,8 +1750,7 @@ randomUser' isCreator hasPassword hasEmail = do <> ["password" .= defPassword | hasPassword] <> ["email" .= fromEmail e | hasEmail] <> ["team" .= Team.BindingNewTeam (Team.newNewTeam (unsafeRange "teamName") (unsafeRange "defaultIcon")) | isCreator] - selfProfile <- responseJsonUnsafe <$> (post (b . path "/i/users" . json p) (post (b . path "/i/users" . json p) TestM UserId ephemeralUser = do From e1866b3f6184bb177332191ffb2af57c91146abf Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Mon, 20 Dec 2021 10:19:50 +0100 Subject: [PATCH 19/74] Servantify cargohold and add `federationDomain` option (#1990) * Schema instances for Asset and related types * Initial stub of cargohold servant API * Servantify asset upload endpoint * Add federation domain configuration to Cargohold * Add examples to Asset swagger schema * Formatting fixes * Document postAsset errors in Swagger * Add federation domain conf to kube-ci * Migrate v3 dowload assets endpoint to Servant * Migrate v3 delete asset endpoint to Servant * Simplify implementation of ZLocalUser * Simplify implementation of MultiVerb headers * Migrate v3 token endpoints to Servant * Add bot and provider asset APIs * Specialise asset API for bots and providers * Check Z-Type header for bot and provider requests * Migrate legacy asset API to Servant * Migrate internal status endpoints to Servant * Use dependent kinds to define `PrincipalPrefix` Thanks @isovector for suggesting this approach. * Add servant prometheus metrics and remove wai * Document new federationDomain option * Add CHANGELOG entry about cargohold domain * Improve federationDomain docs Co-authored-by: jschaul * Formatting fixes * Fix and simplify API paths by principal * Support asset_token query parameter Co-authored-by: Sven Tennie Co-authored-by: jschaul --- changelog.d/0-release-notes/cargohold-domain | 1 + charts/cargohold/templates/configmap.yaml | 3 + .../conf/cargohold.demo-docker.yaml | 1 + deploy/services-demo/conf/cargohold.demo.yaml | 1 + docs/reference/config-options.md | 17 +- hack/helmfile.yaml | 4 + libs/metrics-wai/src/Data/Metrics/Servant.hs | 6 + libs/schema-profunctor/src/Data/Schema.hs | 4 + libs/types-common/src/Data/Id.hs | 1 + libs/wire-api/package.yaml | 2 + libs/wire-api/src/Wire/API/Asset/V3.hs | 122 ++++--- .../wire-api/src/Wire/API/ErrorDescription.hs | 8 + .../wire-api/src/Wire/API/Routes/AssetBody.hs | 63 ++++ .../wire-api/src/Wire/API/Routes/MultiVerb.hs | 99 +++-- libs/wire-api/src/Wire/API/Routes/Public.hs | 82 ++++- .../src/Wire/API/Routes/Public/Cargohold.hs | 179 +++++++++ .../src/Wire/API/Routes/Public/Galley.hs | 8 +- libs/wire-api/wire-api.cabal | 6 +- services/brig/src/Brig/API/Public.hs | 8 +- services/cargohold/cargohold.cabal | 9 +- services/cargohold/cargohold.integration.yaml | 1 + services/cargohold/package.yaml | 6 +- services/cargohold/src/CargoHold/API.hs | 40 -- services/cargohold/src/CargoHold/API/Error.hs | 34 +- .../cargohold/src/CargoHold/API/Public.hs | 345 +++++------------- services/cargohold/src/CargoHold/Options.hs | 20 +- services/cargohold/src/CargoHold/Run.hs | 30 +- services/cargohold/test/integration/API/V3.hs | 9 + services/cargohold/test/integration/Main.hs | 12 +- 29 files changed, 665 insertions(+), 456 deletions(-) create mode 100644 changelog.d/0-release-notes/cargohold-domain create mode 100644 libs/wire-api/src/Wire/API/Routes/AssetBody.hs create mode 100644 libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs delete mode 100644 services/cargohold/src/CargoHold/API.hs diff --git a/changelog.d/0-release-notes/cargohold-domain b/changelog.d/0-release-notes/cargohold-domain new file mode 100644 index 00000000000..33e8403cf87 --- /dev/null +++ b/changelog.d/0-release-notes/cargohold-domain @@ -0,0 +1 @@ +This release introduces a mandatory `federationDomain` configuration setting to cargohold. Please update your `values/wire-server/values.yaml` to set `cargohold.settings.federationDomain` to the same value as the corresponding option in galley (and brig). diff --git a/charts/cargohold/templates/configmap.yaml b/charts/cargohold/templates/configmap.yaml index 3e1f4ce9b2b..52f3ad7c0c6 100644 --- a/charts/cargohold/templates/configmap.yaml +++ b/charts/cargohold/templates/configmap.yaml @@ -30,5 +30,8 @@ data: {{- end }} settings: + {{- with .Values.config.settings }} maxTotalBytes: 5368709120 downloadLinkTTL: 300 # Seconds + federationDomain: {{ .federationDomain }} + {{- end }} diff --git a/deploy/services-demo/conf/cargohold.demo-docker.yaml b/deploy/services-demo/conf/cargohold.demo-docker.yaml index 0290f1fc0e3..c9dda863aad 100644 --- a/deploy/services-demo/conf/cargohold.demo-docker.yaml +++ b/deploy/services-demo/conf/cargohold.demo-docker.yaml @@ -11,6 +11,7 @@ aws: settings: maxTotalBytes: 27262976 downloadLinkTTL: 300 # Seconds + federationDomain: example.com logLevel: Info logNetStrings: false diff --git a/deploy/services-demo/conf/cargohold.demo.yaml b/deploy/services-demo/conf/cargohold.demo.yaml index b4200096b44..0f1228dcbaa 100644 --- a/deploy/services-demo/conf/cargohold.demo.yaml +++ b/deploy/services-demo/conf/cargohold.demo.yaml @@ -11,6 +11,7 @@ aws: settings: maxTotalBytes: 27262976 downloadLinkTTL: 300 # Seconds + federationDomain: example.com logLevel: Info logNetStrings: false diff --git a/docs/reference/config-options.md b/docs/reference/config-options.md index ce3535eb736..03630aee07a 100644 --- a/docs/reference/config-options.md +++ b/docs/reference/config-options.md @@ -169,18 +169,25 @@ This record should have entries which lead to the federator. **IMPORTANT** Once this option is set, it cannot be changed without breaking experience for all the users which are already using the backend. -This configuration needs to be made in brig and in galley. (note the slighly different spelling of the config options) +This configuration needs to be made in brig, cargohold and galley (note the +slighly different spelling of the config options). ```yaml -# galley.yaml +# brig.yaml +optSettings: + setFederationDomain: example.com +``` + +```yaml +# cargohold.yaml settings: federationDomain: example.com ``` ```yaml -# brig.yaml -optSettings: - setFederationDomain: example.com +# galley.yaml +settings: + federationDomain: example.com ``` ### Federation allow list diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml index 578cef0d9a4..e345085bc1d 100644 --- a/hack/helmfile.yaml +++ b/hack/helmfile.yaml @@ -108,6 +108,8 @@ releases: value: {{ .Values.federationDomain }} - name: galley.config.settings.federationDomain value: {{ .Values.federationDomain }} + - name: cargohold.config.settings.federationDomain + value: {{ .Values.federationDomain }} - name: '{{ .Values.namespace }}-wire-server-2' namespace: '{{ .Values.namespaceFed2 }}' @@ -120,3 +122,5 @@ releases: value: {{ .Values.federationDomainFed2 }} - name: galley.config.settings.federationDomain value: {{ .Values.federationDomainFed2 }} + - name: cargohold.config.settings.federationDomain + value: {{ .Values.federationDomainFed2 }} diff --git a/libs/metrics-wai/src/Data/Metrics/Servant.hs b/libs/metrics-wai/src/Data/Metrics/Servant.hs index 1bcac5d86e1..02ac26f11ac 100644 --- a/libs/metrics-wai/src/Data/Metrics/Servant.hs +++ b/libs/metrics-wai/src/Data/Metrics/Servant.hs @@ -104,6 +104,12 @@ instance where getRoutes = getRoutes @rest +instance + (RoutesToPaths rest) => + RoutesToPaths (StreamBody' opts framing ct a :> rest) + where + getRoutes = getRoutes @rest + instance (RoutesToPaths rest) => RoutesToPaths (Summary summary :> rest) diff --git a/libs/schema-profunctor/src/Data/Schema.hs b/libs/schema-profunctor/src/Data/Schema.hs index d93ecf9c9ad..398d4caaabd 100644 --- a/libs/schema-profunctor/src/Data/Schema.hs +++ b/libs/schema-profunctor/src/Data/Schema.hs @@ -36,6 +36,7 @@ module Data.Schema schemaIn, schemaOut, HasDoc (..), + doc', HasSchemaRef (..), withParser, SwaggerDoc, @@ -249,6 +250,9 @@ instance Choice (SchemaP doc v v') where instance HasDoc (SchemaP doc v v' a b) (SchemaP doc' v v' a b) doc doc' where doc = lens schemaDoc $ \(SchemaP d i o) d' -> SchemaP (Lens.set doc d' d) i o +doc' :: Lens' (SchemaP doc v w a b) doc +doc' = doc + withParser :: SchemaP doc v w a b -> (b -> A.Parser b') -> SchemaP doc v w a b' withParser (SchemaP (SchemaDoc d) (SchemaIn p) (SchemaOut o)) q = SchemaP (SchemaDoc d) (SchemaIn (p >=> q)) (SchemaOut o) diff --git a/libs/types-common/src/Data/Id.hs b/libs/types-common/src/Data/Id.hs index 60f418eba18..44df9477eb8 100644 --- a/libs/types-common/src/Data/Id.hs +++ b/libs/types-common/src/Data/Id.hs @@ -325,6 +325,7 @@ newtype BotId = BotId Ord, FromByteString, ToByteString, + FromHttpApiData, Hashable, NFData, FromJSON, diff --git a/libs/wire-api/package.yaml b/libs/wire-api/package.yaml index b69330dc9ff..d4e8cd1a362 100644 --- a/libs/wire-api/package.yaml +++ b/libs/wire-api/package.yaml @@ -35,6 +35,7 @@ library: - cassava >= 0.5 - cereal - comonad + - conduit - cookie - cryptonite - currency-codes >=2.0 @@ -68,6 +69,7 @@ library: - resourcet - servant-client - servant-client-core + - servant-conduit - servant-multipart - servant-server - servant-swagger diff --git a/libs/wire-api/src/Wire/API/Asset/V3.hs b/libs/wire-api/src/Wire/API/Asset/V3.hs index db55ca96908..7890b020ab9 100644 --- a/libs/wire-api/src/Wire/API/Asset/V3.hs +++ b/libs/wire-api/src/Wire/API/Asset/V3.hs @@ -57,19 +57,26 @@ module Wire.API.Asset.V3 where import qualified Codec.MIME.Type as MIME -import Control.Lens (makeLenses) -import Data.Aeson +import Control.Lens (makeLenses, (?~)) +import Data.Aeson (FromJSON (..), ToJSON (..)) +import qualified Data.Aeson as Aeson import Data.Attoparsec.ByteString.Char8 +import Data.Bifunctor import Data.ByteString.Builder import Data.ByteString.Conversion import qualified Data.ByteString.Lazy as LBS import Data.Id -import Data.Json.Util (UTCTimeMillis (fromUTCTimeMillis), toUTCTimeMillis, (#)) +import Data.Json.Util (UTCTimeMillis (fromUTCTimeMillis), toUTCTimeMillis) +import Data.Proxy +import Data.Schema +import qualified Data.Swagger as S +import qualified Data.Text as T import Data.Text.Ascii (AsciiBase64Url) import qualified Data.Text.Encoding as T import Data.Time.Clock import qualified Data.UUID as UUID import Imports +import Servant import Wire.API.Arbitrary (Arbitrary (..), GenericUniform (..)) -------------------------------------------------------------------------------- @@ -82,6 +89,7 @@ data Asset = Asset _assetToken :: Maybe AssetToken } deriving stock (Eq, Show, Generic) + deriving (FromJSON, ToJSON, S.ToSchema) via Schema Asset -- Generate expiry time with millisecond precision instance Arbitrary Asset where @@ -92,20 +100,15 @@ instance Arbitrary Asset where mkAsset :: AssetKey -> Asset mkAsset k = Asset k Nothing Nothing -instance ToJSON Asset where - toJSON a = - object $ - "key" .= _assetKey a - # "expires" .= fmap toUTCTimeMillis (_assetExpires a) - # "token" .= _assetToken a - # [] - -instance FromJSON Asset where - parseJSON = withObject "Asset" $ \o -> - Asset - <$> o .: "key" - <*> o .:? "expires" - <*> o .:? "token" +instance ToSchema Asset where + schema = + object "Asset" $ + Asset + <$> _assetKey .= field "key" schema + <*> (fmap toUTCTimeMillis . _assetExpires) + .= maybe_ + (optField "expires" (fromUTCTimeMillis <$> schema)) + <*> _assetToken .= maybe_ (optField "token" schema) -------------------------------------------------------------------------------- -- AssetKey @@ -116,6 +119,7 @@ instance FromJSON Asset where data AssetKey = AssetKeyV3 AssetId AssetRetention deriving stock (Eq, Show, Generic) deriving (Arbitrary) via (GenericUniform AssetKey) + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema AssetKey) instance FromByteString AssetKey where parser = do @@ -143,13 +147,17 @@ instance ToByteString AssetKey where <> builder '-' <> builder (UUID.toASCIIBytes (toUUID i)) -instance ToJSON AssetKey where - toJSON = String . T.decodeUtf8 . toByteString' +instance ToSchema AssetKey where + schema = + (T.decodeUtf8 . toByteString') + .= parsedText "AssetKey" (runParser parser . T.encodeUtf8) + & doc' . S.schema . S.example ?~ toJSON ("3-1-47de4580-ae51-4650-acbb-d10c028cb0ac" :: Text) -instance FromJSON AssetKey where - parseJSON = - withText "AssetKey" $ - either fail pure . runParser parser . T.encodeUtf8 +instance S.ToParamSchema AssetKey where + toParamSchema _ = S.toParamSchema (Proxy @Text) + +instance FromHttpApiData AssetKey where + parseUrlPiece = first T.pack . runParser parser . T.encodeUtf8 -------------------------------------------------------------------------------- -- AssetToken @@ -157,21 +165,32 @@ instance FromJSON AssetKey where -- | Asset tokens are bearer tokens that grant access to a single asset. newtype AssetToken = AssetToken {assetTokenAscii :: AsciiBase64Url} deriving stock (Eq, Show) - deriving newtype (FromByteString, ToByteString, FromJSON, ToJSON, Arbitrary) + deriving newtype (FromByteString, ToByteString, Arbitrary) + deriving (FromJSON, ToJSON) via (Schema AssetToken) + +instance ToSchema AssetToken where + schema = + AssetToken <$> assetTokenAscii + .= schema + & doc' . S.schema . S.example ?~ toJSON ("aGVsbG8" :: Text) + +instance S.ToParamSchema AssetToken where + toParamSchema _ = S.toParamSchema (Proxy @Text) + +instance FromHttpApiData AssetToken where + parseUrlPiece = first T.pack . runParser parser . T.encodeUtf8 -- | A newly (re)generated token for an existing asset. newtype NewAssetToken = NewAssetToken {newAssetToken :: AssetToken} deriving stock (Eq, Show) deriving newtype (Arbitrary) + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema NewAssetToken) -instance FromJSON NewAssetToken where - parseJSON = withObject "NewAssetToken" $ \o -> - NewAssetToken <$> o .: "token" - -instance ToJSON NewAssetToken where - toJSON (NewAssetToken tok) = - object ["token" .= tok] +instance ToSchema NewAssetToken where + schema = + object "NewAssetToken" $ + NewAssetToken <$> newAssetToken .= field "token" schema -------------------------------------------------------------------------------- -- Body Construction @@ -208,7 +227,7 @@ beginMultipartBody sets (AssetHeaders t l) = <> "\r\n\ \\r\n" where - settingsJson = encode sets + settingsJson = Aeson.encode (schemaToJSON sets) -- | The trailer of a non-resumable @multipart/mixed@ request body initiated -- via 'beginMultipartBody'. @@ -237,22 +256,17 @@ data AssetSettings = AssetSettings } deriving stock (Eq, Show, Generic) deriving (Arbitrary) via (GenericUniform AssetSettings) + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema AssetSettings) defAssetSettings :: AssetSettings defAssetSettings = AssetSettings False Nothing -instance ToJSON AssetSettings where - toJSON s = - object $ - "public" .= _setAssetPublic s - # "retention" .= _setAssetRetention s - # [] - -instance FromJSON AssetSettings where - parseJSON = withObject "AssetSettings" $ \o -> - AssetSettings - <$> o .:? "public" .!= False - <*> o .:? "retention" +instance ToSchema AssetSettings where + schema = + object "AssetSettings" $ + AssetSettings + <$> _setAssetPublic .= (fromMaybe False <$> optField "public" schema) + <*> _setAssetRetention .= maybe_ (optField "retention" schema) -------------------------------------------------------------------------------- -- AssetRetention @@ -275,6 +289,7 @@ data AssetRetention AssetExpiring deriving stock (Eq, Show, Enum, Bounded, Generic) deriving (Arbitrary) via (GenericUniform AssetRetention) + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema AssetRetention) -- | The minimum TTL in seconds corresponding to a chosen retention. assetRetentionSeconds :: AssetRetention -> Maybe NominalDiffTime @@ -308,9 +323,6 @@ instance FromByteString AssetRetention where 5 -> return AssetExpiring _ -> fail $ "Invalid asset retention: " ++ show d -instance ToJSON AssetRetention where - toJSON = String . retentionToTextRep - retentionToTextRep :: AssetRetention -> Text retentionToTextRep AssetEternal = "eternal" retentionToTextRep AssetPersistent = "persistent" @@ -318,16 +330,12 @@ retentionToTextRep AssetVolatile = "volatile" retentionToTextRep AssetEternalInfrequentAccess = "eternal-infrequent_access" retentionToTextRep AssetExpiring = "expiring" --- | JSON representation, used by AssetSettings are -instance FromJSON AssetRetention where - parseJSON = withText "AssetRetention" $ \t -> - case t of - "eternal" -> pure AssetEternal - "persistent" -> pure AssetPersistent - "volatile" -> pure AssetVolatile - "eternal-infrequent_access" -> pure AssetEternalInfrequentAccess - "expiring" -> pure AssetExpiring - _ -> fail $ "Invalid asset retention: " ++ show t +instance ToSchema AssetRetention where + schema = + enum @Text "AssetRetention" $ + foldMap + (\value -> element (retentionToTextRep value) value) + [minBound .. maxBound] makeLenses ''Asset makeLenses ''AssetSettings diff --git a/libs/wire-api/src/Wire/API/ErrorDescription.hs b/libs/wire-api/src/Wire/API/ErrorDescription.hs index 1cb53ce4a38..3b3b194ca03 100644 --- a/libs/wire-api/src/Wire/API/ErrorDescription.hs +++ b/libs/wire-api/src/Wire/API/ErrorDescription.hs @@ -240,6 +240,8 @@ operationDenied = operationDeniedSpecialized . show type NotATeamMember = ErrorDescription 403 "no-team-member" "Requesting user is not a team member" +type Unauthorised = ErrorDescription 403 "unauthorised" "Unauthorised operation" + type ActionDenied = ErrorDescription 403 "action-denied" "Insufficient authorization" actionDenied :: Show a => a -> ActionDenied @@ -312,3 +314,9 @@ type InvalidOpOne2OneConv = InvalidOp "invalid operation for 1:1 conversations" type InvalidOpConnectConv = InvalidOp "invalid operation for connect conversation" type InvalidTargetAccess = InvalidOp "invalid target access" + +type AssetTooLarge = ErrorDescription 413 "client-error" "Asset too large" + +type InvalidLength = ErrorDescription 400 "invalid-length" "Invalid content length" + +type AssetNotFound = ErrorDescription 404 "not-found" "Asset not found" diff --git a/libs/wire-api/src/Wire/API/Routes/AssetBody.hs b/libs/wire-api/src/Wire/API/Routes/AssetBody.hs new file mode 100644 index 00000000000..fb9f6661093 --- /dev/null +++ b/libs/wire-api/src/Wire/API/Routes/AssetBody.hs @@ -0,0 +1,63 @@ +{-# LANGUAGE GeneralizedNewtypeDeriving #-} + +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2021 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Wire.API.Routes.AssetBody + ( AssetBody, + AssetSource (..), + ) +where + +import Conduit +import qualified Data.ByteString.Lazy as LBS +import Data.Swagger +import Data.Swagger.Internal.Schema +import Imports +import Network.HTTP.Media ((//)) +import Servant +import Servant.Conduit () +import Servant.Swagger.Internal.Orphans () + +data MultipartMixed + +instance Accept MultipartMixed where + contentType _ = "multipart" // "mixed" + +instance MimeUnrender MultipartMixed ByteString where + mimeUnrender _ = pure . LBS.toStrict + +newtype AssetSource = AssetSource + { getAssetSource :: + ConduitT () ByteString (ResourceT IO) () + } + deriving newtype (FromSourceIO ByteString) + +instance ToSchema AssetSource where + declareNamedSchema _ = pure $ named "AssetSource" $ mempty + +type AssetBody = + StreamBody' + '[ Description + "A body with content type `multipart/mixed body`. The first section's \ + \content type should be `application/json`. The second section's content \ + \type should be always be `application/octet-stream`. Other content types \ + \will be ignored by the server." + ] + NoFraming + MultipartMixed + AssetSource diff --git a/libs/wire-api/src/Wire/API/Routes/MultiVerb.hs b/libs/wire-api/src/Wire/API/Routes/MultiVerb.hs index 79b9f508253..3c500512c88 100644 --- a/libs/wire-api/src/Wire/API/Routes/MultiVerb.hs +++ b/libs/wire-api/src/Wire/API/Routes/MultiVerb.hs @@ -44,6 +44,7 @@ where import Control.Applicative import Control.Lens hiding (Context) import qualified Data.ByteString.Lazy as LBS +import qualified Data.CaseInsensitive as CI import Data.Containers.ListUtils import Data.HashMap.Strict.InsOrd (InsOrdHashMap) import qualified Data.HashMap.Strict.InsOrd as InsOrdHashMap @@ -54,19 +55,20 @@ import qualified Data.Sequence as Seq import qualified Data.Swagger as S import qualified Data.Swagger.Declare as S import qualified Data.Text as Text +import qualified Data.Text.Encoding as Text import GHC.TypeLits import Generics.SOP as GSOP import Imports import qualified Network.HTTP.Media as M import Network.HTTP.Types (HeaderName, hContentType) +import qualified Network.HTTP.Types as HTTP import Network.HTTP.Types.Status import qualified Network.Wai as Wai import Servant.API import Servant.API.ContentTypes -import Servant.API.ResponseHeaders import Servant.API.Status (KnownStatus (..)) import Servant.Client -import Servant.Client.Core +import Servant.Client.Core hiding (addHeader) import Servant.Server import Servant.Server.Internal import Servant.Swagger as S @@ -210,28 +212,59 @@ data WithHeaders (hs :: [*]) (a :: *) (r :: *) -- | This is used to convert a response containing headers to a custom type -- including the information in the headers. -class AsHeaders hs a b where - fromHeaders :: Headers hs a -> b - toHeaders :: b -> Headers hs a +class AsHeaders xs a b where + fromHeaders :: (NP I xs, a) -> b + toHeaders :: b -> (NP I xs, a) -instance AsHeaders hs a (Headers hs a) where - fromHeaders = id - toHeaders = id +-- single-header empty response +instance AsHeaders '[a] () a where + toHeaders a = (I a :* Nil, ()) + fromHeaders = unI . hd . fst data DescHeader (name :: Symbol) (desc :: Symbol) (a :: *) --- convert a list of 'Header's and 'HeaderDesc' to a list of 'Header's -type family ServantHeaders (hs :: [*]) :: [*] +class ServantHeaders hs xs | hs -> xs where + constructHeaders :: NP I xs -> [HTTP.Header] + extractHeaders :: [HTTP.Header] -> Maybe (NP I xs) -type instance ServantHeaders '[] = '[] +instance ServantHeaders '[] '[] where + constructHeaders Nil = [] + extractHeaders _ = Just Nil -type instance - ServantHeaders (DescHeader name desc a ': hs) = - Header name a ': ServantHeaders hs +headerName :: forall name. KnownSymbol name => HTTP.HeaderName +headerName = + CI.mk + . Text.encodeUtf8 + . Text.pack + $ symbolVal (Proxy @name) -type instance - ServantHeaders (Header name a ': hs) = - Header name a ': ServantHeaders hs +instance + ( KnownSymbol name, + ServantHeader h name x, + ToHttpApiData x, + FromHttpApiData x, + ServantHeaders hs xs + ) => + ServantHeaders (h ': hs) (x ': xs) + where + constructHeaders (I x :* xs) = + (headerName @name, toHeader x) : + constructHeaders @hs xs + + extractHeaders hs = do + let name = headerName @name + (hs0, hs1) = partition (\(h, _) -> h == name) hs + x <- case hs0 of + [] -> empty + ((_, h) : _) -> either (const empty) pure (parseHeader h) + xs <- extractHeaders @hs hs1 + pure (I x :* xs) + +class ServantHeader h (name :: Symbol) x | h -> name x + +instance ServantHeader (Header' mods name x) name x + +instance ServantHeader (DescHeader name desc x) name x instance (KnownSymbol name, KnownSymbol desc, S.ToParamSchema a) => @@ -246,29 +279,24 @@ instance type instance ResponseType (WithHeaders hs a r) = a instance - ( AsHeaders (ServantHeaders hs) (ResponseType r) a, - GetHeaders' (ServantHeaders hs), - BuildHeadersTo (ServantHeaders hs), - AllToResponseHeader hs, + ( AsHeaders xs (ResponseType r) a, + ServantHeaders hs xs, IsResponse cs r ) => IsResponse cs (WithHeaders hs a r) where type ResponseStatus (WithHeaders hs a r) = ResponseStatus r - responseRender acc x = - fmap addHeaders - . responseRender @cs @r acc - . getResponse - $ h + responseRender acc x = fmap addHeaders $ responseRender @cs @r acc y where - h = toHeaders @(ServantHeaders hs) x - addHeaders r = r {roHeaders = roHeaders r ++ getHeaders h} + (hs, y) = toHeaders @xs x + addHeaders r = r {roHeaders = roHeaders r ++ constructHeaders @hs hs} responseUnrender c output = do x <- responseUnrender @cs @r c output - let headers = Headers x (buildHeadersTo @(ServantHeaders hs) (roHeaders output)) - pure (fromHeaders headers) + case extractHeaders @hs (roHeaders output) of + Nothing -> UnrenderError "Failed to parse headers" + Just hs -> pure $ fromHeaders @xs (hs, x) instance (AllToResponseHeader hs, IsSwaggerResponse r) => @@ -375,10 +403,13 @@ instance rs ~ ResponseTypes as => AsUnion as (Union rs) where toUnion = id fromUnion = id -instance AsUnion '[RespondEmpty code desc] () where - toUnion () = Z (I ()) - fromUnion (Z (I ())) = () - fromUnion (S x) = case x of +-- | A handler with a single response. +instance (ResponseType r ~ a) => AsUnion '[r] a where + toUnion = Z . I + fromUnion = unI . unZ + +_foo :: Union '[Int] +_foo = toUnion @'[Respond 200 "test" Int] @Int 3 class InjectAfter as bs where injectAfter :: Union bs -> Union (as .++ bs) diff --git a/libs/wire-api/src/Wire/API/Routes/Public.hs b/libs/wire-api/src/Wire/API/Routes/Public.hs index 9fd26ba7fe5..0b64129d31d 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public.hs @@ -25,6 +25,8 @@ module Wire.API.Routes.Public ZConn, ZOptUser, ZOptConn, + ZBot, + ZProvider, -- * Swagger combinators OmitDocs, @@ -35,15 +37,17 @@ import Control.Lens ((<>~)) import Data.Domain import qualified Data.HashMap.Strict.InsOrd as InsOrdHashMap import Data.Id as Id -import Data.Kind import Data.Metrics.Servant import Data.Qualified import Data.Swagger import GHC.Base (Symbol) import GHC.TypeLits (KnownSymbol) import Imports hiding (All, head) +import qualified Network.Wai as Wai import Servant hiding (Handler, JSON, addHeader, respond) import Servant.API.Modifiers +import Servant.Server.Internal.Delayed +import Servant.Server.Internal.DelayedIO import Servant.Swagger (HasSwagger (toSwagger)) mapRequestArgument :: @@ -70,42 +74,67 @@ data ZType ZLocalAuthUser | -- | Get a 'ConnId' from the Z-Conn header ZAuthConn + | ZAuthBot + | ZAuthProvider class (KnownSymbol (ZHeader ztype), FromHttpApiData (ZParam ztype)) => - IsZType (ztype :: ZType) + IsZType (ztype :: ZType) ctx where type ZHeader ztype :: Symbol type ZParam ztype :: * type ZQualifiedParam ztype :: * - type ZConstraint ztype (ctx :: [*]) :: Constraint - qualifyZParam :: ZConstraint ztype ctx => Context ctx -> ZParam ztype -> ZQualifiedParam ztype + qualifyZParam :: Context ctx -> ZParam ztype -> ZQualifiedParam ztype -instance IsZType 'ZLocalAuthUser where +class HasTokenType ztype where + -- | The expected value of the "Z-Type" header. + tokenType :: Maybe ByteString + +instance {-# OVERLAPPABLE #-} HasTokenType ztype where + tokenType = Nothing + +instance HasContextEntry ctx Domain => IsZType 'ZLocalAuthUser ctx where type ZHeader 'ZLocalAuthUser = "Z-User" type ZParam 'ZLocalAuthUser = UserId type ZQualifiedParam 'ZLocalAuthUser = Local UserId - type ZConstraint 'ZLocalAuthUser ctx = HasContextEntry ctx Domain qualifyZParam ctx = toLocalUnsafe (getContextEntry ctx) -instance IsZType 'ZAuthUser where +instance IsZType 'ZAuthUser ctx where type ZHeader 'ZAuthUser = "Z-User" type ZParam 'ZAuthUser = UserId type ZQualifiedParam 'ZAuthUser = UserId - type ZConstraint 'ZAuthUser ctx = () qualifyZParam _ = id -instance IsZType 'ZAuthConn where +instance IsZType 'ZAuthConn ctx where type ZHeader 'ZAuthConn = "Z-Connection" type ZParam 'ZAuthConn = ConnId type ZQualifiedParam 'ZAuthConn = ConnId - type ZConstraint 'ZAuthConn ctx = () qualifyZParam _ = id +instance IsZType 'ZAuthBot ctx where + type ZHeader 'ZAuthBot = "Z-Bot" + type ZParam 'ZAuthBot = BotId + type ZQualifiedParam 'ZAuthBot = BotId + + qualifyZParam _ = id + +instance HasTokenType 'ZAuthBot where + tokenType = Just "bot" + +instance IsZType 'ZAuthProvider ctx where + type ZHeader 'ZAuthProvider = "Z-Provider" + type ZParam 'ZAuthProvider = ProviderId + type ZQualifiedParam 'ZAuthProvider = ProviderId + + qualifyZParam _ = id + +instance HasTokenType 'ZAuthProvider where + tokenType = Just "provider" + data ZAuthServant (ztype :: ZType) (opts :: [*]) type InternalAuthDefOpts = '[Servant.Required, Servant.Strict] @@ -122,6 +151,10 @@ type ZUser = ZAuthServant 'ZAuthUser InternalAuthDefOpts type ZConn = ZAuthServant 'ZAuthConn InternalAuthDefOpts +type ZBot = ZAuthServant 'ZAuthBot InternalAuthDefOpts + +type ZProvider = ZAuthServant 'ZAuthProvider InternalAuthDefOpts + type ZOptUser = ZAuthServant 'ZAuthUser '[Servant.Optional, Servant.Strict] type ZOptConn = ZAuthServant 'ZAuthConn '[Servant.Optional, Servant.Strict] @@ -141,13 +174,16 @@ instance HasSwagger api => HasSwagger (ZAuthServant 'ZAuthUser _opts :> api) whe instance HasSwagger api => HasSwagger (ZAuthServant 'ZLocalAuthUser opts :> api) where toSwagger _ = toSwagger (Proxy @(ZAuthServant 'ZAuthUser opts :> api)) -instance HasSwagger api => HasSwagger (ZAuthServant 'ZAuthConn _opts :> api) where +instance + {-# OVERLAPPABLE #-} + HasSwagger api => + HasSwagger (ZAuthServant ztype _opts :> api) + where toSwagger _ = toSwagger (Proxy @api) instance - ( IsZType ztype, + ( IsZType ztype ctx, HasContextEntry (ctx .++ DefaultErrorFormatters) ErrorFormatters, - ZConstraint ztype ctx, SBoolI (FoldLenient opts), SBoolI (FoldRequired opts), HasServer api ctx @@ -158,11 +194,27 @@ instance ServerT (ZAuthServant ztype opts :> api) m = RequestArgument opts (ZQualifiedParam ztype) -> ServerT api m - route _ ctx subserver = + route _ ctx subserver = do Servant.route (Proxy @(InternalAuth ztype opts :> api)) ctx - (fmap (. mapRequestArgument @opts (qualifyZParam @ztype ctx)) subserver) + ( fmap + (. mapRequestArgument @opts (qualifyZParam @ztype ctx)) + (addAcceptCheck subserver (withRequest (checkType (tokenType @ztype)))) + ) + where + checkType :: Maybe ByteString -> Wai.Request -> DelayedIO () + checkType token req = case (token, lookup "Z-Type" (Wai.requestHeaders req)) of + (Just t, value) + | value /= Just t -> + delayedFail + ServerError + { errHTTPCode = 403, + errReasonPhrase = "Access denied", + errBody = "", + errHeaders = [] + } + _ -> pure () hoistServerWithContext _ pc nt s = hoistServerWithContext (Proxy :: Proxy api) pc nt . s diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs b/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs new file mode 100644 index 00000000000..f73dc764a2a --- /dev/null +++ b/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs @@ -0,0 +1,179 @@ +{-# LANGUAGE GeneralizedNewtypeDeriving #-} + +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2021 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Wire.API.Routes.Public.Cargohold where + +import Data.Id +import Data.Metrics.Servant +import Data.Qualified +import Data.SOP +import qualified Data.Swagger as Swagger +import Imports +import Servant +import Servant.Swagger.Internal +import Servant.Swagger.Internal.Orphans () +import Wire.API.Asset +import Wire.API.ErrorDescription +import Wire.API.Routes.AssetBody +import Wire.API.Routes.MultiVerb +import Wire.API.Routes.Public + +data PrincipalTag = UserPrincipalTag | BotPrincipalTag | ProviderPrincipalTag + deriving (Eq, Show) + +type family PrincipalId (tag :: PrincipalTag) = (id :: *) | id -> tag where + PrincipalId 'UserPrincipalTag = Local UserId + PrincipalId 'BotPrincipalTag = BotId + PrincipalId 'ProviderPrincipalTag = ProviderId + +type family ApplyPrincipalPath (tag :: PrincipalTag) api + +type instance ApplyPrincipalPath 'UserPrincipalTag api = ZLocalUser :> "assets" :> "v3" :> api + +type instance ApplyPrincipalPath 'BotPrincipalTag api = ZBot :> "bot" :> "assets" :> api + +type instance ApplyPrincipalPath 'ProviderPrincipalTag api = ZProvider :> "provider" :> "assets" :> api + +instance HasSwagger (ApplyPrincipalPath tag api) => HasSwagger (tag :> api) where + toSwagger _ = toSwagger (Proxy @(ApplyPrincipalPath tag api)) + +instance HasServer (ApplyPrincipalPath tag api) ctx => HasServer (tag :> api) ctx where + type ServerT (tag :> api) m = ServerT (ApplyPrincipalPath tag api) m + route _ = route (Proxy @(ApplyPrincipalPath tag api)) + hoistServerWithContext _ = hoistServerWithContext (Proxy @(ApplyPrincipalPath tag api)) + +instance RoutesToPaths (ApplyPrincipalPath tag api) => RoutesToPaths (tag :> api) where + getRoutes = getRoutes @(ApplyPrincipalPath tag api) + +newtype AssetLocation = AssetLocation {getAssetLocation :: Text} + deriving newtype + ( ToHttpApiData, + FromHttpApiData, + Swagger.ToParamSchema + ) + +instance AsHeaders '[AssetLocation] Asset (Asset, AssetLocation) where + toHeaders (asset, loc) = (I loc :* Nil, asset) + fromHeaders (I loc :* Nil, asset) = (asset, loc) + +type GetAsset = + MultiVerb + 'GET + '[JSON] + '[ AssetNotFound, + WithHeaders + '[DescHeader "Location" "Asset location" AssetLocation] + AssetLocation + (RespondEmpty 302 "Asset found") + ] + (Maybe AssetLocation) + +type ServantAPI = + ( Summary "Renew an asset token" + :> CanThrow AssetNotFound + :> CanThrow Unauthorised + :> ZLocalUser + :> "assets" + :> "v3" + :> Capture "key" AssetKey + :> "token" + :> Post '[JSON] NewAssetToken + ) + :<|> ( Summary "Delete an asset token" + :> Description "**Note**: deleting the token makes the asset public." + :> ZLocalUser + :> "assets" + :> "v3" + :> Capture "key" AssetKey + :> "token" + :> MultiVerb + 'DELETE + '[JSON] + '[RespondEmpty 200 "Asset token deleted"] + () + ) + :<|> BaseAPI 'UserPrincipalTag + :<|> BaseAPI 'BotPrincipalTag + :<|> BaseAPI 'ProviderPrincipalTag + :<|> LegacyAPI + :<|> InternalAPI + +type BaseAPI (tag :: PrincipalTag) = + ( Summary "Upload an asset" + :> CanThrow AssetTooLarge + :> CanThrow InvalidLength + :> tag + :> AssetBody + :> MultiVerb + 'POST + '[JSON] + '[ WithHeaders + '[DescHeader "Location" "Asset location" AssetLocation] + (Asset, AssetLocation) + (Respond 201 "Asset posted" Asset) + ] + (Asset, AssetLocation) + ) + :<|> ( Summary "Download an asset" + :> tag + :> Capture "key" AssetKey + :> Header "Asset-Token" AssetToken + :> QueryParam "asset_token" AssetToken + :> GetAsset + ) + :<|> ( Summary "Delete an asset" + :> CanThrow AssetNotFound + :> CanThrow Unauthorised + :> tag + :> Capture "key" AssetKey + :> MultiVerb + 'DELETE + '[JSON] + '[RespondEmpty 200 "Asset deleted"] + () + ) + +type LegacyAPI = + ( ZLocalUser + :> "assets" + :> QueryParam' [Required, Strict] "conv_id" ConvId + :> Capture "id" AssetId + :> GetAsset + ) + :<|> ( ZLocalUser + :> "conversations" + :> Capture "cnv" ConvId + :> "assets" + :> Capture "id" AssetId + :> GetAsset + ) + :<|> ( ZLocalUser + :> "conversations" + :> Capture "cnv" ConvId + :> "otr" + :> "assets" + :> Capture "id" AssetId + :> GetAsset + ) + +type InternalAPI = + "i" :> "status" :> MultiVerb 'GET '[PlainText] '[RespondEmpty 200 "OK"] () + +swaggerDoc :: Swagger.Swagger +swaggerDoc = toSwagger (Proxy @ServantAPI) diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs b/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs index 30e402c0590..63949b1d841 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs @@ -25,6 +25,7 @@ import Data.CommaSeparatedList import Data.Id (ConvId, TeamId, UserId) import Data.Qualified (Qualified (..)) import Data.Range +import Data.SOP import qualified Data.Swagger as Swagger import GHC.TypeLits (AppendSymbol) import Imports hiding (head) @@ -45,10 +46,9 @@ import Wire.API.ServantProto (Proto, RawProto) import Wire.API.Team.Conversation import Wire.API.Team.Feature -instance AsHeaders '[Header "Location" ConvId] Conversation Conversation where - -- FUTUREWORK: use addHeader - toHeaders c = Headers c (HCons (Header (qUnqualified (cnvQualifiedId c))) HNil) - fromHeaders = getResponse +instance AsHeaders '[ConvId] Conversation Conversation where + toHeaders c = (I (qUnqualified (cnvQualifiedId c)) :* Nil, c) + fromHeaders = snd type ConversationResponse = ResponseForExistedCreated Conversation diff --git a/libs/wire-api/wire-api.cabal b/libs/wire-api/wire-api.cabal index 6d643e1c4d8..f84b7fddf85 100644 --- a/libs/wire-api/wire-api.cabal +++ b/libs/wire-api/wire-api.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: bcda0d57293162140cc50d149ad068df7975f9526cf1621c7aa4f753b45e36a6 +-- hash: 80606059441fcf0f725386078aef563bf04549ce657002575a6559ae37544964 name: wire-api version: 0.1.0 @@ -48,6 +48,7 @@ library Wire.API.Provider.Service.Tag Wire.API.Push.Token Wire.API.Push.V2.Token + Wire.API.Routes.AssetBody Wire.API.Routes.Internal.Brig Wire.API.Routes.Internal.Brig.Connection Wire.API.Routes.Internal.Brig.EJPD @@ -56,6 +57,7 @@ library Wire.API.Routes.MultiVerb Wire.API.Routes.Public Wire.API.Routes.Public.Brig + Wire.API.Routes.Public.Cargohold Wire.API.Routes.Public.Galley Wire.API.Routes.Public.LegalHold Wire.API.Routes.Public.Spar @@ -113,6 +115,7 @@ library , cassava >=0.5 , cereal , comonad + , conduit , containers >=0.5 , cookie , cryptonite @@ -153,6 +156,7 @@ library , servant , servant-client , servant-client-core + , servant-conduit , servant-multipart , servant-server , servant-swagger diff --git a/services/brig/src/Brig/API/Public.hs b/services/brig/src/Brig/API/Public.hs index 081c96f8009..7bf438b4186 100644 --- a/services/brig/src/Brig/API/Public.hs +++ b/services/brig/src/Brig/API/Public.hs @@ -104,6 +104,7 @@ import qualified Wire.API.Properties as Public import qualified Wire.API.Routes.MultiTablePaging as Public import Wire.API.Routes.Public.Brig (Api (updateConnectionUnqualified)) import qualified Wire.API.Routes.Public.Brig as BrigAPI +import qualified Wire.API.Routes.Public.Cargohold as CargoholdAPI import qualified Wire.API.Routes.Public.Galley as GalleyAPI import qualified Wire.API.Routes.Public.LegalHold as LegalHoldAPI import qualified Wire.API.Routes.Public.Spar as SparAPI @@ -131,7 +132,12 @@ type ServantAPI = BrigAPI.ServantAPI swaggerDocsAPI :: Servant.Server SwaggerDocsAPI swaggerDocsAPI = swaggerSchemaUIServer $ - (BrigAPI.swagger <> GalleyAPI.swaggerDoc <> LegalHoldAPI.swaggerDoc <> SparAPI.swaggerDoc) + ( BrigAPI.swagger + <> GalleyAPI.swaggerDoc + <> LegalHoldAPI.swaggerDoc + <> SparAPI.swaggerDoc + <> CargoholdAPI.swaggerDoc + ) & S.info . S.title .~ "Wire-Server API" & S.info . S.description ?~ Brig.Docs.Swagger.contents <> mempty & S.security %~ nub diff --git a/services/cargohold/cargohold.cabal b/services/cargohold/cargohold.cabal index 94c6b8635f6..778c25303a1 100644 --- a/services/cargohold/cargohold.cabal +++ b/services/cargohold/cargohold.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: e1d35804687900338efb4ea42019d902f778d6b03ea61e3e8a1d3e0fcba0115e +-- hash: b8ba2ed196939ab2b035138d3e428c84b88ff77ff219741840e47ae8dac9de0a name: cargohold version: 1.5.0 @@ -25,7 +25,6 @@ flag static library exposed-modules: - CargoHold.API CargoHold.API.Error CargoHold.API.Federation CargoHold.API.Legacy @@ -92,10 +91,7 @@ library , uri-bytestring >=0.2 , uuid >=1.3.5 , wai >=3.0 - , wai-conduit >=3.0 - , wai-extra >=3.0 - , wai-predicates >=0.8 - , wai-routing >=0.12 + , wai-extra , wai-utilities >=0.16.1 , wire-api , wire-api-federation @@ -165,7 +161,6 @@ executable cargohold-integration , http-types >=0.8 , imports , lens >=3.8 - , metrics-wai , mime >=0.4 , optparse-applicative , safe >=0.3 diff --git a/services/cargohold/cargohold.integration.yaml b/services/cargohold/cargohold.integration.yaml index 5254cf9368d..2773a838499 100644 --- a/services/cargohold/cargohold.integration.yaml +++ b/services/cargohold/cargohold.integration.yaml @@ -18,6 +18,7 @@ aws: settings: maxTotalBytes: 27262976 downloadLinkTTL: 300 # Seconds + federationDomain: example.com logLevel: Info logNetStrings: false diff --git a/services/cargohold/package.yaml b/services/cargohold/package.yaml index c73d7934df3..d4bd3dfbcdc 100644 --- a/services/cargohold/package.yaml +++ b/services/cargohold/package.yaml @@ -59,10 +59,7 @@ library: - uri-bytestring >=0.2 - uuid >=1.3.5 - wai >=3.0 - - wai-conduit >=3.0 - - wai-extra >=3.0 - - wai-predicates >=0.8 - - wai-routing >=0.12 + - wai-extra - wai-utilities >=0.16.1 - wire-api - wire-api-federation @@ -76,7 +73,6 @@ executables: - cargohold-types - http-client-tls >=0.2 - lens >=3.8 - - metrics-wai - optparse-applicative - tagged >=0.8 - tasty >=1.0 diff --git a/services/cargohold/src/CargoHold/API.hs b/services/cargohold/src/CargoHold/API.hs deleted file mode 100644 index ce5bb5fc32e..00000000000 --- a/services/cargohold/src/CargoHold/API.hs +++ /dev/null @@ -1,40 +0,0 @@ --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2020 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module CargoHold.API - ( sitemap, - ) -where - -import qualified CargoHold.API.Public as Public -import CargoHold.App (Handler) -import Data.Predicate (true) -import qualified Data.Swagger.Build.Api as Doc -import Imports hiding (head) -import Network.Wai.Routing (Routes, continue, get, head) -import Network.Wai.Utilities (empty) - -sitemap :: Routes Doc.ApiBuilder Handler () -sitemap = do - Public.sitemap - Public.apiDocs - routesInternal - -routesInternal :: Routes a Handler () -routesInternal = do - get "/i/status" (continue $ const $ return empty) true - head "/i/status" (continue $ const $ return empty) true diff --git a/services/cargohold/src/CargoHold/API/Error.hs b/services/cargohold/src/CargoHold/API/Error.hs index bffd8ba433c..12d469439bf 100644 --- a/services/cargohold/src/CargoHold/API/Error.hs +++ b/services/cargohold/src/CargoHold/API/Error.hs @@ -17,21 +17,47 @@ module CargoHold.API.Error where +import Data.Proxy +import qualified Data.Text.Lazy as LT +import GHC.TypeLits import Imports import Network.HTTP.Types.Status import Network.Wai.Utilities.Error +import Servant.API.Status +import Wire.API.ErrorDescription + +errorDescriptionToWai :: + forall (code :: Nat) (lbl :: Symbol) (desc :: Symbol). + (KnownStatus code, KnownSymbol lbl) => + ErrorDescription code lbl desc -> + Error +errorDescriptionToWai (ErrorDescription msg) = + mkError + (statusVal (Proxy @code)) + (LT.pack (symbolVal (Proxy @lbl))) + (LT.fromStrict msg) + +errorDescriptionTypeToWai :: + forall e (code :: Nat) (lbl :: Symbol) (desc :: Symbol). + ( KnownStatus code, + KnownSymbol lbl, + KnownSymbol desc, + e ~ ErrorDescription code lbl desc + ) => + Error +errorDescriptionTypeToWai = errorDescriptionToWai (mkErrorDescription :: e) assetTooLarge :: Error -assetTooLarge = mkError status413 "client-error" "Asset too large." +assetTooLarge = errorDescriptionTypeToWai @AssetTooLarge unauthorised :: Error -unauthorised = mkError status403 "unauthorised" "Unauthorised operation." +unauthorised = errorDescriptionTypeToWai @Unauthorised invalidLength :: Error -invalidLength = mkError status400 "invalid-length" "Invalid content length." +invalidLength = errorDescriptionTypeToWai @InvalidLength assetNotFound :: Error -assetNotFound = mkError status404 "not-found" "Asset not found." +assetNotFound = errorDescriptionTypeToWai @AssetNotFound invalidMD5 :: Error invalidMD5 = mkError status400 "client-error" "Invalid MD5." diff --git a/services/cargohold/src/CargoHold/API/Public.hs b/services/cargohold/src/CargoHold/API/Public.hs index b2b4660e5be..1c588ce99b0 100644 --- a/services/cargohold/src/CargoHold/API/Public.hs +++ b/services/cargohold/src/CargoHold/API/Public.hs @@ -1,6 +1,6 @@ -- This file is part of the Wire Server implementation. -- --- Copyright (C) 2020 Wire Swiss GmbH +-- Copyright (C) 2021 Wire Swiss GmbH -- -- This program is free software: you can redistribute it and/or modify it under -- the terms of the GNU Affero General Public License as published by the Free @@ -15,269 +15,98 @@ -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . -module CargoHold.API.Public - ( sitemap, - apiDocs, - ) -where +module CargoHold.API.Public (servantSitemap) where -import qualified CargoHold.API.Error as Error import qualified CargoHold.API.Legacy as LegacyAPI import qualified CargoHold.API.V3 as V3 import CargoHold.App -import qualified CargoHold.Types.V3 as V3 (Principal (..)) -import Control.Error -import Control.Lens ((^.)) +import qualified CargoHold.Types.V3 as V3 +import Control.Lens import Data.ByteString.Conversion import Data.Id -import Data.Predicate -import qualified Data.Swagger.Build.Api as Doc -import Data.Text.Encoding (decodeLatin1) +import Data.Qualified +import qualified Data.Text.Encoding as Text +import qualified Data.Text.Encoding.Error as Text import Imports hiding (head) -import Network.HTTP.Types.Status -import Network.Wai (Request, Response) -import Network.Wai.Conduit (sourceRequestBody) -import Network.Wai.Predicate hiding (Error, setStatus) -import Network.Wai.Routing -import Network.Wai.Utilities hiding (message) -import Network.Wai.Utilities.Swagger (document, mkSwaggerApi) -import qualified Network.Wai.Utilities.Swagger as Doc -import Network.Wai.Utilities.ZAuth +import Servant ((:<|>) (..)) +import Servant.Server hiding (Handler) import URI.ByteString -import qualified Wire.API.Asset as Public - --- FUTUREWORK: restore (and servantify) resumable upload functionality, removed --- in https://github.com/wireapp/wire-server/pull/1998 - -sitemap :: Routes Doc.ApiBuilder Handler () -sitemap = do - --------------------------------------------------------------------------- - -- User API - - -- Simple (one-step) Upload - - post "/assets/v3" (continue uploadAssetV3) $ - header "Z-User" - .&. contentType "multipart" "mixed" - .&. request - document "POST" "uploadAsset" $ do - Doc.summary "Upload an asset. In the multipart/mixed body, the first section's content type should be application/json. The second section's content type should be always application/octet-stream. Other content types will be ignored by the server." - Doc.consumes "multipart/mixed" - Doc.errorResponse Error.assetTooLarge - Doc.errorResponse Error.invalidLength - Doc.response 201 "Asset posted" Doc.end - - --- Download - - get "/assets/v3/:key" (continue downloadAssetV3) $ - header "Z-User" - .&. capture "key" - .&. opt (header "Asset-Token" .|. query "asset_token") - document "GET" "downloadAsset" $ do - Doc.summary "Download an asset" - Doc.parameter Doc.Path "key" Doc.bytes' $ - Doc.description "Asset key" - Doc.parameter Doc.Header "Asset-Token" Doc.bytes' $ do - Doc.description "Asset token" - Doc.optional - Doc.errorResponse Error.assetNotFound - Doc.response 302 "Asset found" Doc.end - - --- Token Management - - post "/assets/v3/:key/token" (continue renewTokenV3) $ - header "Z-User" - .&. capture "key" - document "POST" "renewAssetToken" $ do - Doc.summary "Renew an asset token" - Doc.parameter Doc.Path "key" Doc.bytes' $ - Doc.description "Asset key" - Doc.response 200 "Asset token renewed" Doc.end - Doc.errorResponse Error.assetNotFound - Doc.errorResponse Error.unauthorised - - delete "/assets/v3/:key/token" (continue deleteTokenV3) $ - header "Z-User" - .&. capture "key" - document "DELETE" "deleteAssetToken" $ do - Doc.summary "Delete an asset token" - Doc.notes "Deleting the token makes the asset public." - Doc.parameter Doc.Path "key" Doc.bytes' $ - Doc.description "Asset key" - Doc.response 200 "Asset token deleted" Doc.end - - --- Deletion - - delete "/assets/v3/:key" (continue deleteAssetV3) $ - header "Z-User" - .&. capture "key" - document "DELETE" "deleteAsset" $ do - Doc.summary "Delete an asset" - Doc.parameter Doc.Path "key" Doc.bytes' $ - Doc.description "Asset key" - Doc.response 200 "Asset deleted" Doc.end - Doc.errorResponse Error.assetNotFound - Doc.errorResponse Error.unauthorised - - --------------------------------------------------------------------------- - -- Provider API - - post "/provider/assets" (continue providerUploadV3) $ - zauth ZAuthProvider - .&> contentType "multipart" "mixed" - .&> zauthProviderId - .&. request - - get "/provider/assets/:key" (continue providerDownloadV3) $ - zauth ZAuthProvider - .&> zauthProviderId - .&. capture "key" - .&. opt (header "Asset-Token" .|. query "asset_token") - - delete "/provider/assets/:key" (continue providerDeleteV3) $ - zauth ZAuthProvider - .&> zauthProviderId - .&. capture "key" - - --------------------------------------------------------------------------- - -- Bot API - - post "/bot/assets" (continue botUploadV3) $ - zauth ZAuthBot - .&> contentType "multipart" "mixed" - .&> zauthBotId - .&. request - - get "/bot/assets/:key" (continue botDownloadV3) $ - zauth ZAuthBot - .&> zauthBotId - .&. capture "key" - .&. opt (header "Asset-Token" .|. query "asset_token") - - delete "/bot/assets/:key" (continue botDeleteV3) $ - zauth ZAuthBot - .&> zauthBotId - .&. capture "key" - - -- Legacy - - get "/assets/:id" (continue legacyDownloadPlain) $ - header "Z-User" - .&. param "conv_id" - .&. capture "id" - - get "/conversations/:cnv/assets/:id" (continue legacyDownloadPlain) $ - header "Z-User" - .&. capture "cnv" - .&. capture "id" - - get "/conversations/:cnv/otr/assets/:id" (continue legacyDownloadOtr) $ - header "Z-User" - .&. capture "cnv" - .&. capture "id" - -apiDocs :: Routes Doc.ApiBuilder Handler () -apiDocs = do - get - "/assets/api-docs" - ( \(_ ::: url) k -> - let doc = mkSwaggerApi (decodeLatin1 url) [] sitemap - in k $ json doc - ) - $ accept "application" "json" - .&. query "base_url" - ------------------------------------------------------------------------------ --- User API Handlers - --- FUTUREWORK: make these types more descriptive than 'Request' -> 'Response' -uploadAssetV3 :: UserId ::: Media "multipart" "mixed" ::: Request -> Handler Response -uploadAssetV3 (usr ::: _ ::: req) = do - let principal = V3.UserPrincipal usr - assetResponse principal <$> V3.upload principal (sourceRequestBody req) - -downloadAssetV3 :: UserId ::: Public.AssetKey ::: Maybe Public.AssetToken -> Handler Response -downloadAssetV3 (usr ::: key ::: tok) = do - url <- V3.download (V3.UserPrincipal usr) key tok - redirect url - -deleteAssetV3 :: UserId ::: Public.AssetKey -> Handler Response -deleteAssetV3 (usr ::: key) = do - V3.delete (V3.UserPrincipal usr) key - return empty - -renewTokenV3 :: UserId ::: Public.AssetKey -> Handler Response -renewTokenV3 (usr ::: key) = do - tok <- V3.renewToken (V3.UserPrincipal usr) key - return $ json (Public.NewAssetToken tok) - -deleteTokenV3 :: UserId ::: Public.AssetKey -> Handler Response -deleteTokenV3 (usr ::: key) = do - V3.deleteToken (V3.UserPrincipal usr) key - return empty - --------------------------------------------------------------------------------- --- Provider API Handlers - -providerUploadV3 :: ProviderId ::: Request -> Handler Response -providerUploadV3 (prv ::: req) = do - let principal = V3.ProviderPrincipal prv - assetResponse principal <$> V3.upload principal (sourceRequestBody req) - -providerDownloadV3 :: ProviderId ::: Public.AssetKey ::: Maybe Public.AssetToken -> Handler Response -providerDownloadV3 (prv ::: key ::: tok) = do - url <- V3.download (V3.ProviderPrincipal prv) key tok - redirect url - -providerDeleteV3 :: ProviderId ::: Public.AssetKey -> Handler Response -providerDeleteV3 (prv ::: key) = do - V3.delete (V3.ProviderPrincipal prv) key - return empty - --------------------------------------------------------------------------------- --- Bot API Handlers - -botUploadV3 :: BotId ::: Request -> Handler Response -botUploadV3 (bot ::: req) = do - let principal = V3.BotPrincipal bot - assetResponse principal <$> V3.upload principal (sourceRequestBody req) - -botDownloadV3 :: BotId ::: Public.AssetKey ::: Maybe Public.AssetToken -> Handler Response -botDownloadV3 (bot ::: key ::: tok) = do - url <- V3.download (V3.BotPrincipal bot) key tok - redirect url - -botDeleteV3 :: BotId ::: Public.AssetKey -> Handler Response -botDeleteV3 (bot ::: key) = do - V3.delete (V3.BotPrincipal bot) key - return empty - --------------------------------------------------------------------------------- --- Helpers - -assetResponse :: V3.Principal -> Public.Asset -> Response -assetResponse prc asset = - setStatus status201 . loc (asset ^. Public.assetKey) $ json asset +import Wire.API.Asset +import Wire.API.Routes.AssetBody +import Wire.API.Routes.Public.Cargohold + +servantSitemap :: ServerT ServantAPI Handler +servantSitemap = + renewTokenV3 :<|> deleteTokenV3 + :<|> userAPI + :<|> botAPI + :<|> providerAPI + :<|> legacyAPI + :<|> internalAPI where - loc k = location $ case prc of - V3.UserPrincipal {} -> "/assets/v3/" <> toByteString k - V3.BotPrincipal {} -> "/bot/assets/" <> toByteString k - V3.ProviderPrincipal {} -> "/provider/assets/" <> toByteString k - -redirect :: Maybe URI -> Handler Response -redirect (Just url) = return . setStatus status302 $ location (serializeURIRef url) empty -redirect Nothing = throwE Error.assetNotFound -{-# INLINE redirect #-} - -location :: ToByteString a => a -> Response -> Response -location = addHeader "Location" . toByteString' -{-# INLINE location #-} - --------------------------------------------------------------------------------- --- Legacy - -legacyDownloadPlain :: UserId ::: ConvId ::: AssetId -> Handler Response -legacyDownloadPlain (usr ::: cnv ::: ast) = LegacyAPI.download usr cnv ast >>= redirect - -legacyDownloadOtr :: UserId ::: ConvId ::: AssetId -> Handler Response -legacyDownloadOtr (usr ::: cnv ::: ast) = LegacyAPI.downloadOtr usr cnv ast >>= redirect + userAPI :: forall tag. tag ~ 'UserPrincipalTag => ServerT (BaseAPI tag) Handler + userAPI = uploadAssetV3 @tag :<|> downloadAssetV3 @tag :<|> deleteAssetV3 @tag + botAPI :: forall tag. tag ~ 'BotPrincipalTag => ServerT (BaseAPI tag) Handler + botAPI = uploadAssetV3 @tag :<|> downloadAssetV3 @tag :<|> deleteAssetV3 @tag + providerAPI :: forall tag. tag ~ 'ProviderPrincipalTag => ServerT (BaseAPI tag) Handler + providerAPI = uploadAssetV3 @tag :<|> downloadAssetV3 @tag :<|> deleteAssetV3 @tag + legacyAPI = legacyDownloadPlain :<|> legacyDownloadPlain :<|> legacyDownloadOtr + internalAPI = pure () + +class MakePrincipal (tag :: PrincipalTag) (id :: *) | id -> tag, tag -> id where + mkPrincipal :: id -> V3.Principal + +instance MakePrincipal 'UserPrincipalTag (Local UserId) where + mkPrincipal = V3.UserPrincipal . tUnqualified + +instance MakePrincipal 'BotPrincipalTag BotId where + mkPrincipal = V3.BotPrincipal + +instance MakePrincipal 'ProviderPrincipalTag ProviderId where + mkPrincipal = V3.ProviderPrincipal + +uploadAssetV3 :: + MakePrincipal tag id => + id -> + AssetSource -> + Handler (Asset, AssetLocation) +uploadAssetV3 pid req = do + let principal = mkPrincipal pid + asset <- V3.upload principal (getAssetSource req) + let key = Text.decodeUtf8With Text.lenientDecode (toByteString' (asset ^. assetKey)) + let loc = case principal of + V3.UserPrincipal {} -> "/assets/v3/" <> key + V3.BotPrincipal {} -> "/bot/assets/" <> key + V3.ProviderPrincipal {} -> "/provider/assets/" <> key + pure (asset, AssetLocation loc) + +downloadAssetV3 :: + MakePrincipal tag id => + id -> + AssetKey -> + Maybe AssetToken -> + Maybe AssetToken -> + Handler (Maybe AssetLocation) +downloadAssetV3 usr key tok1 tok2 = do + url <- V3.download (mkPrincipal usr) key (tok1 <|> tok2) + pure $ fmap (AssetLocation . Text.decodeUtf8With Text.lenientDecode . serializeURIRef') url + +deleteAssetV3 :: MakePrincipal tag id => id -> AssetKey -> Handler () +deleteAssetV3 usr key = V3.delete (mkPrincipal usr) key + +renewTokenV3 :: Local UserId -> AssetKey -> Handler NewAssetToken +renewTokenV3 (tUnqualified -> usr) key = + NewAssetToken <$> V3.renewToken (V3.UserPrincipal usr) key + +deleteTokenV3 :: Local UserId -> AssetKey -> Handler () +deleteTokenV3 (tUnqualified -> usr) key = V3.deleteToken (V3.UserPrincipal usr) key + +legacyDownloadPlain :: Local UserId -> ConvId -> AssetId -> Handler (Maybe AssetLocation) +legacyDownloadPlain (tUnqualified -> usr) cnv ast = do + url <- LegacyAPI.download usr cnv ast + pure $ fmap (AssetLocation . Text.decodeUtf8With Text.lenientDecode . serializeURIRef') url + +legacyDownloadOtr :: Local UserId -> ConvId -> AssetId -> Handler (Maybe AssetLocation) +legacyDownloadOtr (tUnqualified -> usr) cnv ast = do + url <- LegacyAPI.downloadOtr usr cnv ast + pure $ fmap (AssetLocation . Text.decodeUtf8With Text.lenientDecode . serializeURIRef') url diff --git a/services/cargohold/src/CargoHold/Options.hs b/services/cargohold/src/CargoHold/Options.hs index 31992d462af..302bcc68977 100644 --- a/services/cargohold/src/CargoHold/Options.hs +++ b/services/cargohold/src/CargoHold/Options.hs @@ -19,10 +19,11 @@ module CargoHold.Options where -import CargoHold.CloudFront (Domain (..), KeyPairId (..)) +import qualified CargoHold.CloudFront as CF import Control.Lens hiding (Level) import Data.Aeson (FromJSON (..), withText) import Data.Aeson.TH +import Data.Domain import Imports import System.Logger.Extended (Level, LogFormat) import Util.Options @@ -31,9 +32,9 @@ import Util.Options.Common -- | AWS CloudFront settings. data CloudFrontOpts = CloudFrontOpts { -- | Domain - _cfDomain :: Domain, + _cfDomain :: CF.Domain, -- | Keypair ID - _cfKeyPairId :: KeyPairId, + _cfKeyPairId :: CF.KeyPairId, -- | Path to private key _cfPrivateKey :: FilePath } @@ -78,7 +79,18 @@ data Settings = Settings { -- | Maximum allowed size for uploads, in bytes _setMaxTotalBytes :: !Int, -- | TTL for download links, in seconds - _setDownloadLinkTTL :: !Word + _setDownloadLinkTTL :: !Word, + -- | FederationDomain is required, even when not wanting to federate with other backends + -- (in that case the 'setFederationAllowedDomains' can be set to empty in Federator) + -- Federation domain is used to qualify local IDs and handles, + -- e.g. 0c4d8944-70fa-480e-a8b7-9d929862d18c@wire.com and somehandle@wire.com. + -- It should also match the SRV DNS records under which other wire-server installations can find this backend: + -- _wire-server-federator._tcp. + -- Once set, DO NOT change it: if you do, existing users may have a broken experience and/or stop working + -- Remember to keep it the same in Galley and in Brig. + -- This is referred to as the 'backend domain' in the public documentation; See + -- https://docs.wire.com/how-to/install/configure-federation.html#choose-a-backend-domain-name + _setFederationDomain :: !Domain } deriving (Show, Generic) diff --git a/services/cargohold/src/CargoHold/Run.hs b/services/cargohold/src/CargoHold/Run.hs index bf305eb2d06..08a98d085a0 100644 --- a/services/cargohold/src/CargoHold/Run.hs +++ b/services/cargohold/src/CargoHold/Run.hs @@ -20,15 +20,16 @@ module CargoHold.Run ) where -import CargoHold.API (sitemap) import CargoHold.API.Federation +import CargoHold.API.Public import CargoHold.App import CargoHold.Options import Control.Lens (set, (^.)) import Control.Monad.Catch (finally) import Data.Default +import Data.Domain import Data.Id -import Data.Metrics.Middleware.Prometheus (waiPrometheusMiddleware) +import Data.Metrics.Servant import Data.Proxy import Data.Text (unpack) import Imports @@ -37,12 +38,13 @@ import qualified Network.Wai.Middleware.Gzip as GZip import Network.Wai.Utilities.Request import Network.Wai.Utilities.Server import qualified Network.Wai.Utilities.Server as Server -import Servant (hoistServer) import qualified Servant import Servant.API +import Servant.Server hiding (Handler, runHandler) import Util.Options +import qualified Wire.API.Routes.Public.Cargohold as Public -type CombinedAPI = FederationAPI :<|> Servant.Raw +type CombinedAPI = FederationAPI :<|> Public.ServantAPI run :: Opts -> IO () run o = do @@ -51,22 +53,30 @@ run o = do runSettingsWithShutdown s (middleware e $ servantApp e) 5 `finally` closeEnv e where - rtree = compile sitemap server e = defaultServer (unpack $ o ^. optCargohold . epHost) (o ^. optCargohold . epPort) (e ^. appLogger) (e ^. metrics) middleware :: Env -> Wai.Middleware middleware e = - waiPrometheusMiddleware sitemap + servantPrometheusMiddleware (Proxy @CombinedAPI) . GZip.gzip GZip.def . catchErrors (e ^. appLogger) [Right $ e ^. metrics] - serve e r k = runHandler e (Server.route rtree r k) servantApp e0 r = let e = set requestId (maybe def RequestId (lookupRequestId r)) e0 - in Servant.serve + in Servant.serveWithContext (Proxy @CombinedAPI) - ( hoistServer (Proxy @FederationAPI) (toServantHandler e) federationSitemap - :<|> Servant.Tagged (serve e) + ((o ^. optSettings . setFederationDomain) :. Servant.EmptyContext) + ( hoistServer' @FederationAPI (toServantHandler e) federationSitemap + :<|> hoistServer' @Public.ServantAPI (toServantHandler e) servantSitemap ) r toServantHandler :: Env -> Handler a -> Servant.Handler a toServantHandler env = liftIO . runHandler env + +-- | See 'Galley.Run' for an explanation of this function. +hoistServer' :: + forall api m n. + HasServer api '[Domain] => + (forall x. m x -> n x) -> + ServerT api m -> + ServerT api n +hoistServer' = hoistServerWithContext (Proxy @api) (Proxy @'[Domain]) diff --git a/services/cargohold/test/integration/API/V3.hs b/services/cargohold/test/integration/API/V3.hs index a6a34f6ef58..f3d2a780011 100644 --- a/services/cargohold/test/integration/API/V3.hs +++ b/services/cargohold/test/integration/API/V3.hs @@ -125,9 +125,15 @@ testSimpleTokens c = do -- No access without token from other user (opaque 404) get (c . path loc . zUser uid2 . noRedirect) !!! const 404 === statusCode + -- No access with empty token query parameter from other user (opaque 404) + get (c . path loc . zUser uid2 . queryItem' "asset_token" Nothing . noRedirect) + !!! const 404 === statusCode -- No access with wrong token (opaque 404) get (c . path loc . zUser uid2 . header "Asset-Token" "acb123" . noRedirect) !!! const 404 === statusCode + -- No access with wrong token as query parameter (opaque 404) + get (c . path loc . zUser uid2 . queryItem "asset_token" "acb123" . noRedirect) + !!! const 404 === statusCode -- Token renewal fails if not done by owner post (c . paths ["assets", "v3", toByteString' key, "token"] . zUser uid2) !!! do const 403 === statusCode @@ -156,6 +162,9 @@ testSimpleTokens c = do -- Verify access with new token from a different user. get (c . path loc . header "Asset-Token" (toByteString' tok') . zUser uid2 . noRedirect) !!! const 302 === statusCode + -- Verify access with new token as query parameter from a different user + get (c . path loc . queryItem "asset_token" (toByteString' tok') . zUser uid2 . noRedirect) + !!! const 302 === statusCode -- Delete Token fails if not done by owner delete (c . paths ["assets", "v3", toByteString' key, "token"] . zUser uid2) !!! do const 403 === statusCode diff --git a/services/cargohold/test/integration/Main.hs b/services/cargohold/test/integration/Main.hs index ea5156ec567..21e1e336102 100644 --- a/services/cargohold/test/integration/Main.hs +++ b/services/cargohold/test/integration/Main.hs @@ -22,9 +22,6 @@ where import qualified API.V3 import Bilge hiding (body, header) -import qualified CargoHold.API (sitemap) -import Data.Metrics.Test (pathsConsistencyCheck) -import Data.Metrics.WaiRoute (treeToPaths) import Data.Proxy import Data.Tagged import Data.Text.Encoding (encodeUtf8) @@ -33,10 +30,8 @@ import Imports hiding (local) import qualified Metrics import Network.HTTP.Client (responseTimeoutMicro) import Network.HTTP.Client.TLS -import Network.Wai.Utilities.Server (compile) import Options.Applicative import Test.Tasty -import Test.Tasty.HUnit import Test.Tasty.Options import TestSetup import Util.Options @@ -86,12 +81,7 @@ main = runTests go go c i = withResource (getOpts c i) releaseOpts $ \opts -> testGroup "Cargohold" - [ testCase "sitemap" $ - assertEqual - "inconcistent sitemap" - mempty - (pathsConsistencyCheck . treeToPaths . compile $ CargoHold.API.sitemap), - API.V3.tests opts, + [ API.V3.tests opts, Metrics.tests opts ] getOpts _ i = do From a176340595fd32b7fbb1aac625c35ac53545dd9d Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Mon, 20 Dec 2021 10:31:05 +0100 Subject: [PATCH 20/74] Update polysemy-check version in cabal.freeze (#2006) --- cabal.project.freeze | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cabal.project.freeze b/cabal.project.freeze index 6e81f6e4022..fda8eeef868 100644 --- a/cabal.project.freeze +++ b/cabal.project.freeze @@ -1690,7 +1690,7 @@ constraints: any.AC-Angle ==1.0, any.polynomials-bernstein ==1.1.2, any.polyparse ==1.13, any.polysemy ==1.7.0.0, - any.polysemy-check ==0.8.1.0, + any.polysemy-check ==0.9.0.0, any.polysemy-plugin ==0.4.2.0, any.pooled-io ==0.0.2.2, any.port-utils ==0.2.1.0, From dd36838727fe712a5ad2450fcc6f7292ec2c6fcb Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Mon, 20 Dec 2021 13:49:46 +0100 Subject: [PATCH 21/74] post/get conversation/:cnv/code returns error if feature disabled (#1980) --- changelog.d/2-features/pr-1980 | 1 + services/galley/src/Galley/API/Query.hs | 33 ++++++---- services/galley/src/Galley/API/Update.hs | 78 +++++++++++++----------- services/galley/test/integration/API.hs | 38 ++++++++++++ 4 files changed, 102 insertions(+), 48 deletions(-) create mode 100644 changelog.d/2-features/pr-1980 diff --git a/changelog.d/2-features/pr-1980 b/changelog.d/2-features/pr-1980 new file mode 100644 index 00000000000..1eafba32ca6 --- /dev/null +++ b/changelog.d/2-features/pr-1980 @@ -0,0 +1 @@ +Revoke guest links if feature is disabled. If the guest links team feature is disabled `get /conversations/join`, `post /conversations/:cnv/code`, and `get /conversations/:cnv/code` will return an error. diff --git a/services/galley/src/Galley/API/Query.hs b/services/galley/src/Galley/API/Query.hs index ace30d6eec7..736e401397e 100644 --- a/services/galley/src/Galley/API/Query.hs +++ b/services/galley/src/Galley/API/Query.hs @@ -30,6 +30,7 @@ module Galley.API.Query internalGetMemberH, getConversationMetaH, getConversationByReusableCode, + ensureGuestLinksEnabled, ) where @@ -513,9 +514,8 @@ getConversationByReusableCode :: getConversationByReusableCode lusr key value = do c <- verifyReusableCode (ConversationCode key value Nothing) conv <- ensureConversationAccess (tUnqualified lusr) (Data.codeConversation c) CodeAccess - getFeatureStatus conv >>= \case - TeamFeatureEnabled -> pure $ coverView conv - TeamFeatureDisabled -> throw GuestLinksDisabled + ensureGuestLinksEnabled conv + pure $ coverView conv where coverView :: Data.Conversation -> ConversationCoverView coverView conv = @@ -524,12 +524,23 @@ getConversationByReusableCode lusr key value = do cnvCoverName = Data.convName conv } +-- FUTUREWORK(leif): refactor and make it consistent for all team features +ensureGuestLinksEnabled :: + forall r. + ( Member (Error ConversationError) r, + Member TeamFeatureStore r, + Member (Input Opts) r + ) => + Data.Conversation -> + Sem r () +ensureGuestLinksEnabled conv = do + defaultStatus <- getDefaultFeatureStatus + maybeFeatureStatus <- join <$> TeamFeatures.getFeatureStatusNoConfig @'TeamFeatureGuestLinks `traverse` Data.convTeam conv + case maybe defaultStatus tfwoStatus maybeFeatureStatus of + TeamFeatureEnabled -> pure () + TeamFeatureDisabled -> throw GuestLinksDisabled + where getDefaultFeatureStatus :: Sem r TeamFeatureStatusValue - getDefaultFeatureStatus = - input <&> view (optSettings . setFeatureFlags . flagConversationGuestLinks . unDefaults . to tfwoapsStatus) - - getFeatureStatus :: Data.Conversation -> Sem r TeamFeatureStatusValue - getFeatureStatus conv = do - defaultStatus <- getDefaultFeatureStatus - maybeFeatureStatus <- join <$> TeamFeatures.getFeatureStatusNoConfig @'TeamFeatureGuestLinks `traverse` Data.convTeam conv - pure $ maybe defaultStatus tfwoStatus maybeFeatureStatus + getDefaultFeatureStatus = do + status <- input <&> view (optSettings . setFeatureFlags . flagConversationGuestLinks . unDefaults) + pure $ tfwoapsStatus status diff --git a/services/galley/src/Galley/API/Update.hs b/services/galley/src/Galley/API/Update.hs index 26af1fabcb2..aec91abde61 100644 --- a/services/galley/src/Galley/API/Update.hs +++ b/services/galley/src/Galley/API/Update.hs @@ -82,6 +82,7 @@ import Galley.API.Error import Galley.API.LegalHold.Conflicts import Galley.API.Mapping import Galley.API.Message +import qualified Galley.API.Query as Query import Galley.API.Util import qualified Galley.Data.Conversation as Data import Galley.Data.Services as Data @@ -522,16 +523,17 @@ getUpdateResult :: Sem (Error NoChanges ': r) a -> Sem r (UpdateResult a) getUpdateResult = fmap (either (const Unchanged) Updated) . runError addCodeH :: - Members - '[ CodeStore, - ConversationStore, - Error ConversationError, - ExternalAccess, - GundeckAccess, - Input (Local ()), - Input UTCTime - ] - r => + forall r. + ( Member CodeStore r, + Member ConversationStore r, + Member (Error ConversationError) r, + Member ExternalAccess r, + Member GundeckAccess r, + Member (Input (Local ())) r, + Member (Input UTCTime) r, + Member (Input Opts) r, + Member TeamFeatureStore r + ) => UserId ::: ConnId ::: ConvId -> Sem r Response addCodeH (usr ::: zcon ::: cnv) = do @@ -547,21 +549,22 @@ data AddCodeResult addCode :: forall r. - Members - '[ CodeStore, - ConversationStore, - Error ConversationError, - ExternalAccess, - GundeckAccess, - Input UTCTime - ] - r => + ( Member CodeStore r, + Member ConversationStore r, + Member (Error ConversationError) r, + Member ExternalAccess r, + Member GundeckAccess r, + Member (Input UTCTime) r, + Member (Input Opts) r, + Member TeamFeatureStore r + ) => Local UserId -> ConnId -> Local ConvId -> Sem r AddCodeResult addCode lusr zcon lcnv = do conv <- E.getConversation (tUnqualified lcnv) >>= note ConvNotFound + Query.ensureGuestLinksEnabled conv ensureConvMember (Data.convLocalMembers conv) (tUnqualified lusr) ensureAccess conv CodeAccess let (bots, users) = localBotsAndUsers $ Data.convLocalMembers conv @@ -582,8 +585,7 @@ addCode lusr zcon lcnv = do where createCode :: Code -> Sem r ConversationCode createCode code = do - urlPrefix <- E.getConversationCodeURI - return $ mkConversationCode (codeKey code) (codeValue code) urlPrefix + mkConversationCode (codeKey code) (codeValue code) <$> E.getConversationCodeURI rmCodeH :: Members @@ -631,32 +633,35 @@ rmCode lusr zcon lcnv = do pure event getCodeH :: - Members - '[ CodeStore, - ConversationStore, - Error CodeError, - Error ConversationError - ] - r => + forall r. + ( Member CodeStore r, + Member ConversationStore r, + Member (Error CodeError) r, + Member (Error ConversationError) r, + Member (Input Opts) r, + Member TeamFeatureStore r + ) => UserId ::: ConvId -> Sem r Response getCodeH (usr ::: cnv) = setStatus status200 . json <$> getCode usr cnv getCode :: - Members - '[ CodeStore, - ConversationStore, - Error CodeError, - Error ConversationError - ] - r => + forall r. + ( Member CodeStore r, + Member ConversationStore r, + Member (Error CodeError) r, + Member (Error ConversationError) r, + Member (Input Opts) r, + Member TeamFeatureStore r + ) => UserId -> ConvId -> Sem r Public.ConversationCode getCode usr cnv = do conv <- E.getConversation cnv >>= note ConvNotFound + Query.ensureGuestLinksEnabled conv ensureAccess conv CodeAccess ensureConvMember (Data.convLocalMembers conv) usr key <- E.makeKey cnv @@ -665,8 +670,7 @@ getCode usr cnv = do returnCode :: Member CodeStore r => Code -> Sem r Public.ConversationCode returnCode c = do - urlPrefix <- E.getConversationCodeURI - pure $ Public.mkConversationCode (codeKey c) (codeValue c) urlPrefix + Public.mkConversationCode (codeKey c) (codeValue c) <$> E.getConversationCodeURI checkReusableCodeH :: Members '[CodeStore, Error CodeError, WaiRoutes] r => diff --git a/services/galley/test/integration/API.hs b/services/galley/test/integration/API.hs index 3d2706dd4e5..5a29641a606 100644 --- a/services/galley/test/integration/API.hs +++ b/services/galley/test/integration/API.hs @@ -223,6 +223,8 @@ tests s = test s "cannot join private conversation" postJoinConvFail, test s "revoke guest links for team conversation" testJoinTeamConvGuestLinksDisabled, test s "revoke guest links for non-team conversation" testJoinNonTeamConvGuestLinksDisabled, + test s "get code rejected if guest links disabled" testGetCodeRejectedIfGuestLinksDisabled, + test s "post code rejected if guest links disabled" testPostCodeRejectedIfGuestLinksDisabled, test s "remove user with only local convs" removeUserNoFederation, test s "remove user with local and remote convs" removeUser, test s "iUpsertOne2OneConversation" testAllOne2OneConversationRequests, @@ -1241,6 +1243,42 @@ testJoinCodeConv = do getJoinCodeConv eve (conversationKey cCode) (conversationCode cCode) !!! do const 403 === statusCode +testGetCodeRejectedIfGuestLinksDisabled :: TestM () +testGetCodeRejectedIfGuestLinksDisabled = do + galley <- view tsGalley + (owner, teamId, []) <- Util.createBindingTeamWithNMembers 0 + let createConvWithGuestLink = do + convId <- decodeConvId <$> postTeamConv teamId owner [] (Just "testConversation") [CodeAccess] (Just ActivatedAccessRole) Nothing + void $ decodeConvCodeEvent <$> postConvCode owner convId + pure convId + convId <- createConvWithGuestLink + let checkGetCode expectedStatus = getConvCode owner convId !!! statusCode === const expectedStatus + let setStatus tfStatus = + TeamFeatures.putTeamFeatureFlagWithGalley @'Public.TeamFeatureGuestLinks galley owner teamId (Public.TeamFeatureStatusNoConfig tfStatus) !!! do + const 200 === statusCode + + checkGetCode 200 + setStatus Public.TeamFeatureDisabled + checkGetCode 409 + setStatus Public.TeamFeatureEnabled + checkGetCode 200 + +testPostCodeRejectedIfGuestLinksDisabled :: TestM () +testPostCodeRejectedIfGuestLinksDisabled = do + galley <- view tsGalley + (owner, teamId, []) <- Util.createBindingTeamWithNMembers 0 + convId <- decodeConvId <$> postTeamConv teamId owner [] (Just "testConversation") [CodeAccess] (Just ActivatedAccessRole) Nothing + let checkPostCode expectedStatus = postConvCode owner convId !!! statusCode === const expectedStatus + let setStatus tfStatus = + TeamFeatures.putTeamFeatureFlagWithGalley @'Public.TeamFeatureGuestLinks galley owner teamId (Public.TeamFeatureStatusNoConfig tfStatus) !!! do + const 200 === statusCode + + checkPostCode 201 + setStatus Public.TeamFeatureDisabled + checkPostCode 409 + setStatus Public.TeamFeatureEnabled + checkPostCode 200 + testJoinTeamConvGuestLinksDisabled :: TestM () testJoinTeamConvGuestLinksDisabled = do galley <- view tsGalley From b3730cf8e4b01bf41b769549d7a1408464041f32 Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Mon, 20 Dec 2021 15:43:30 +0000 Subject: [PATCH 22/74] wip --- .../src/Wire/API/Routes/Public/Galley.hs | 22 +++++++++- libs/wire-api/src/Wire/API/Team.hs | 6 +++ services/galley/src/Galley/API/Public.hs | 15 +------ services/galley/src/Galley/API/Teams.hs | 41 +++++-------------- 4 files changed, 40 insertions(+), 44 deletions(-) diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs b/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs index 63949b1d841..54b95c0c154 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs @@ -1,4 +1,5 @@ {-# LANGUAGE DerivingVia #-} +{-# LANGUAGE GeneralizedNewtypeDeriving #-} {-# OPTIONS_GHC -Wno-orphans #-} -- This file is part of the Wire Server implementation. @@ -43,6 +44,7 @@ import Wire.API.Routes.Public import Wire.API.Routes.Public.Util import Wire.API.Routes.QualifiedCapture import Wire.API.ServantProto (Proto, RawProto) +import Wire.API.Team import Wire.API.Team.Conversation import Wire.API.Team.Feature @@ -705,7 +707,25 @@ data Api routes = Api :- FeatureConfigGet 'WithLockStatus 'TeamFeatureSelfDeletingMessages, featureConfigGuestLinksGet :: routes - :- FeatureConfigGet 'WithLockStatus 'TeamFeatureGuestLinks + :- FeatureConfigGet 'WithLockStatus 'TeamFeatureGuestLinks, + -- teams + createNonBindingTeam :: + routes + :- Summary "Create a new non binding team" + :> ZUser + :> ZConn + :> "teams" + :> ReqBody '[Servant.JSON] NonBindingNewTeam + :> MultiVerb + 'POST + '[JSON] + '[ NotConnected, + WithHeaders + '[DescHeader "Location" "Team ID" TeamId] + TeamId + (RespondEmpty 201 "Team ID as `Location` header value") + ] + TeamId } deriving (Generic) diff --git a/libs/wire-api/src/Wire/API/Team.hs b/libs/wire-api/src/Wire/API/Team.hs index b1307bea120..2f96d6c6a49 100644 --- a/libs/wire-api/src/Wire/API/Team.hs +++ b/libs/wire-api/src/Wire/API/Team.hs @@ -77,6 +77,8 @@ import Data.Id (TeamId, UserId) import Data.Json.Util import Data.Misc (PlainTextPassword (..)) import Data.Range +import Data.Schema (Schema (..), ToSchema, schema) +import qualified Data.Swagger as S import qualified Data.Swagger.Build.Api as Doc import Imports import Test.QuickCheck.Gen (suchThat) @@ -227,6 +229,10 @@ instance Arbitrary BindingNewTeam where -- | FUTUREWORK: this is dead code! remove! newtype NonBindingNewTeam = NonBindingNewTeam (NewTeam (Range 1 127 [TeamMember])) deriving stock (Eq, Show, Generic) + deriving (S.ToSchema) via (Schema NonBindingNewTeam) + +instance ToSchema NonBindingNewTeam where + schema = error "todo" modelNewNonBindingTeam :: Doc.Model modelNewNonBindingTeam = Doc.defineModel "newNonBindingTeam" $ do diff --git a/services/galley/src/Galley/API/Public.hs b/services/galley/src/Galley/API/Public.hs index 31d41ae0531..42db3104f91 100644 --- a/services/galley/src/Galley/API/Public.hs +++ b/services/galley/src/Galley/API/Public.hs @@ -187,25 +187,14 @@ servantSitemap = GalleyAPI.featureConfigClassifiedDomainsGet = Features.getFeatureConfig @'Public.WithoutLockStatus @'Public.TeamFeatureClassifiedDomains Features.getClassifiedDomainsInternal, GalleyAPI.featureConfigConferenceCallingGet = Features.getFeatureConfig @'Public.WithoutLockStatus @'Public.TeamFeatureConferenceCalling Features.getConferenceCallingInternal, GalleyAPI.featureConfigSelfDeletingMessagesGet = Features.getFeatureConfig @'Public.WithLockStatus @'Public.TeamFeatureSelfDeletingMessages Features.getSelfDeletingMessagesInternal, - GalleyAPI.featureConfigGuestLinksGet = Features.getFeatureConfig @'Public.WithLockStatus @'Public.TeamFeatureGuestLinks Features.getGuestLinkInternal + GalleyAPI.featureConfigGuestLinksGet = Features.getFeatureConfig @'Public.WithLockStatus @'Public.TeamFeatureGuestLinks Features.getGuestLinkInternal, + GalleyAPI.createNonBindingTeam = Teams.createNonBindingTeamH } sitemap :: Routes ApiBuilder (Sem GalleyEffects) () sitemap = do -- Team API ----------------------------------------------------------- - post "/teams" (continue Teams.createNonBindingTeamH) $ - zauthUserId - .&. zauthConnId - .&. jsonRequest @Public.NonBindingNewTeam - .&. accept "application" "json" - document "POST" "createNonBindingTeam" $ do - summary "Create a new non binding team" - body (ref Public.modelNewNonBindingTeam) $ - description "JSON body" - response 201 "Team ID as `Location` header value" end - errorResponse (Error.errorDescriptionTypeToWai @Error.NotConnected) - put "/teams/:tid" (continue Teams.updateTeamH) $ zauthUserId .&. zauthConnId diff --git a/services/galley/src/Galley/API/Teams.hs b/services/galley/src/Galley/API/Teams.hs index 9d46c2fcd7a..d85390e7fd5 100644 --- a/services/galley/src/Galley/API/Teams.hs +++ b/services/galley/src/Galley/API/Teams.hs @@ -203,40 +203,21 @@ lookupTeam zusr tid = do else pure Nothing createNonBindingTeamH :: - Members - '[ BrigAccess, - Error ActionError, - Error TeamError, - GundeckAccess, - Input UTCTime, - P.TinyLog, - TeamStore, - WaiRoutes - ] - r => - UserId ::: ConnId ::: JsonRequest Public.NonBindingNewTeam ::: JSON -> - Sem r Response -createNonBindingTeamH (zusr ::: zcon ::: req ::: _) = do - newTeam <- fromJsonBody req - newTeamId <- createNonBindingTeam zusr zcon newTeam - pure (empty & setStatus status201 . location newTeamId) - -createNonBindingTeam :: - Members - '[ BrigAccess, - Error ActionError, - Error TeamError, - GundeckAccess, - Input UTCTime, - TeamStore, - P.TinyLog - ] - r => + forall r. + ( Member BrigAccess r, + Member (Error ActionError) r, + Member (Error TeamError) r, + Member GundeckAccess r, + Member (Input UTCTime) r, + Member P.TinyLog r, + Member TeamStore r, + Member WaiRoutes r + ) => UserId -> ConnId -> Public.NonBindingNewTeam -> Sem r TeamId -createNonBindingTeam zusr zcon (Public.NonBindingNewTeam body) = do +createNonBindingTeamH zusr zcon (Public.NonBindingNewTeam body) = do let owner = Public.TeamMember zusr fullPermissions Nothing LH.defUserLegalHoldStatus let others = filter ((zusr /=) . view userId) From dcd754eb939a77b7f39daf51aafd6e09fb032ca2 Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Mon, 20 Dec 2021 15:45:30 +0000 Subject: [PATCH 23/74] Revert "wip" This reverts commit b3730cf8e4b01bf41b769549d7a1408464041f32. --- .../src/Wire/API/Routes/Public/Galley.hs | 22 +--------- libs/wire-api/src/Wire/API/Team.hs | 6 --- services/galley/src/Galley/API/Public.hs | 15 ++++++- services/galley/src/Galley/API/Teams.hs | 41 ++++++++++++++----- 4 files changed, 44 insertions(+), 40 deletions(-) diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs b/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs index 54b95c0c154..63949b1d841 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs @@ -1,5 +1,4 @@ {-# LANGUAGE DerivingVia #-} -{-# LANGUAGE GeneralizedNewtypeDeriving #-} {-# OPTIONS_GHC -Wno-orphans #-} -- This file is part of the Wire Server implementation. @@ -44,7 +43,6 @@ import Wire.API.Routes.Public import Wire.API.Routes.Public.Util import Wire.API.Routes.QualifiedCapture import Wire.API.ServantProto (Proto, RawProto) -import Wire.API.Team import Wire.API.Team.Conversation import Wire.API.Team.Feature @@ -707,25 +705,7 @@ data Api routes = Api :- FeatureConfigGet 'WithLockStatus 'TeamFeatureSelfDeletingMessages, featureConfigGuestLinksGet :: routes - :- FeatureConfigGet 'WithLockStatus 'TeamFeatureGuestLinks, - -- teams - createNonBindingTeam :: - routes - :- Summary "Create a new non binding team" - :> ZUser - :> ZConn - :> "teams" - :> ReqBody '[Servant.JSON] NonBindingNewTeam - :> MultiVerb - 'POST - '[JSON] - '[ NotConnected, - WithHeaders - '[DescHeader "Location" "Team ID" TeamId] - TeamId - (RespondEmpty 201 "Team ID as `Location` header value") - ] - TeamId + :- FeatureConfigGet 'WithLockStatus 'TeamFeatureGuestLinks } deriving (Generic) diff --git a/libs/wire-api/src/Wire/API/Team.hs b/libs/wire-api/src/Wire/API/Team.hs index 2f96d6c6a49..b1307bea120 100644 --- a/libs/wire-api/src/Wire/API/Team.hs +++ b/libs/wire-api/src/Wire/API/Team.hs @@ -77,8 +77,6 @@ import Data.Id (TeamId, UserId) import Data.Json.Util import Data.Misc (PlainTextPassword (..)) import Data.Range -import Data.Schema (Schema (..), ToSchema, schema) -import qualified Data.Swagger as S import qualified Data.Swagger.Build.Api as Doc import Imports import Test.QuickCheck.Gen (suchThat) @@ -229,10 +227,6 @@ instance Arbitrary BindingNewTeam where -- | FUTUREWORK: this is dead code! remove! newtype NonBindingNewTeam = NonBindingNewTeam (NewTeam (Range 1 127 [TeamMember])) deriving stock (Eq, Show, Generic) - deriving (S.ToSchema) via (Schema NonBindingNewTeam) - -instance ToSchema NonBindingNewTeam where - schema = error "todo" modelNewNonBindingTeam :: Doc.Model modelNewNonBindingTeam = Doc.defineModel "newNonBindingTeam" $ do diff --git a/services/galley/src/Galley/API/Public.hs b/services/galley/src/Galley/API/Public.hs index 42db3104f91..31d41ae0531 100644 --- a/services/galley/src/Galley/API/Public.hs +++ b/services/galley/src/Galley/API/Public.hs @@ -187,14 +187,25 @@ servantSitemap = GalleyAPI.featureConfigClassifiedDomainsGet = Features.getFeatureConfig @'Public.WithoutLockStatus @'Public.TeamFeatureClassifiedDomains Features.getClassifiedDomainsInternal, GalleyAPI.featureConfigConferenceCallingGet = Features.getFeatureConfig @'Public.WithoutLockStatus @'Public.TeamFeatureConferenceCalling Features.getConferenceCallingInternal, GalleyAPI.featureConfigSelfDeletingMessagesGet = Features.getFeatureConfig @'Public.WithLockStatus @'Public.TeamFeatureSelfDeletingMessages Features.getSelfDeletingMessagesInternal, - GalleyAPI.featureConfigGuestLinksGet = Features.getFeatureConfig @'Public.WithLockStatus @'Public.TeamFeatureGuestLinks Features.getGuestLinkInternal, - GalleyAPI.createNonBindingTeam = Teams.createNonBindingTeamH + GalleyAPI.featureConfigGuestLinksGet = Features.getFeatureConfig @'Public.WithLockStatus @'Public.TeamFeatureGuestLinks Features.getGuestLinkInternal } sitemap :: Routes ApiBuilder (Sem GalleyEffects) () sitemap = do -- Team API ----------------------------------------------------------- + post "/teams" (continue Teams.createNonBindingTeamH) $ + zauthUserId + .&. zauthConnId + .&. jsonRequest @Public.NonBindingNewTeam + .&. accept "application" "json" + document "POST" "createNonBindingTeam" $ do + summary "Create a new non binding team" + body (ref Public.modelNewNonBindingTeam) $ + description "JSON body" + response 201 "Team ID as `Location` header value" end + errorResponse (Error.errorDescriptionTypeToWai @Error.NotConnected) + put "/teams/:tid" (continue Teams.updateTeamH) $ zauthUserId .&. zauthConnId diff --git a/services/galley/src/Galley/API/Teams.hs b/services/galley/src/Galley/API/Teams.hs index d85390e7fd5..9d46c2fcd7a 100644 --- a/services/galley/src/Galley/API/Teams.hs +++ b/services/galley/src/Galley/API/Teams.hs @@ -203,21 +203,40 @@ lookupTeam zusr tid = do else pure Nothing createNonBindingTeamH :: - forall r. - ( Member BrigAccess r, - Member (Error ActionError) r, - Member (Error TeamError) r, - Member GundeckAccess r, - Member (Input UTCTime) r, - Member P.TinyLog r, - Member TeamStore r, - Member WaiRoutes r - ) => + Members + '[ BrigAccess, + Error ActionError, + Error TeamError, + GundeckAccess, + Input UTCTime, + P.TinyLog, + TeamStore, + WaiRoutes + ] + r => + UserId ::: ConnId ::: JsonRequest Public.NonBindingNewTeam ::: JSON -> + Sem r Response +createNonBindingTeamH (zusr ::: zcon ::: req ::: _) = do + newTeam <- fromJsonBody req + newTeamId <- createNonBindingTeam zusr zcon newTeam + pure (empty & setStatus status201 . location newTeamId) + +createNonBindingTeam :: + Members + '[ BrigAccess, + Error ActionError, + Error TeamError, + GundeckAccess, + Input UTCTime, + TeamStore, + P.TinyLog + ] + r => UserId -> ConnId -> Public.NonBindingNewTeam -> Sem r TeamId -createNonBindingTeamH zusr zcon (Public.NonBindingNewTeam body) = do +createNonBindingTeam zusr zcon (Public.NonBindingNewTeam body) = do let owner = Public.TeamMember zusr fullPermissions Nothing LH.defUserLegalHoldStatus let others = filter ((zusr /=) . view userId) From 04fde5ad909ea5c50d0b403c6fcd1f571df78a2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Dimja=C5=A1evi=C4=87?= Date: Tue, 21 Dec 2021 13:51:54 +0100 Subject: [PATCH 24/74] Documentation for Federation Test Cases (#2000) Co-authored-by: Leif Battermann Co-authored-by: Matthias Fischmann --- changelog.d/5-internal/map-federation-tests | 1 + services/brig/docs/swagger.md | 2 +- services/federator/src/Federator/Discovery.hs | 2 +- .../federator/src/Federator/Validation.hs | 3 +- .../integration/Test/Federator/IngressSpec.hs | 17 ++++++++- .../integration/Test/Federator/InwardSpec.hs | 38 ++++++++++++++++--- .../unit/Test/Federator/InternalServer.hs | 5 +++ .../test/unit/Test/Federator/Options.hs | 4 +- .../test/unit/Test/Federator/Response.hs | 2 +- .../test/unit/Test/Federator/Validation.hs | 14 +++++++ .../test-integration/Test/Spar/APISpec.hs | 2 + 11 files changed, 78 insertions(+), 12 deletions(-) create mode 100644 changelog.d/5-internal/map-federation-tests diff --git a/changelog.d/5-internal/map-federation-tests b/changelog.d/5-internal/map-federation-tests new file mode 100644 index 00000000000..b53c1449dde --- /dev/null +++ b/changelog.d/5-internal/map-federation-tests @@ -0,0 +1 @@ +Tag integration tests for security audit. \ No newline at end of file diff --git a/services/brig/docs/swagger.md b/services/brig/docs/swagger.md index 63ec978afb9..59e6c1c3b8f 100644 --- a/services/brig/docs/swagger.md +++ b/services/brig/docs/swagger.md @@ -54,7 +54,7 @@ An error in this category likely indicates an issue with the configuration of fe - **Federation unavailable** (status: 500, label: `federation-not-available`): Federation is configured for this backend, but the local federator cannot be reached. This can be transient, so clients should retry the request. - **Federation not implemented** (status: 500, label: `federation-not-implemented`): Federated behaviour for a certain endpoint is not yet implemented. - - **Federator discovery failed** (status: 500, label: `discovery-failure`): A DNS error occurred during discovery of a remote backend. This can be transient, so clients should retry the request. + - **Federator discovery failed** (status: 400, label: `discovery-failure`): A DNS error occurred during discovery of a remote backend. This can be transient, so clients should retry the request. - **Local federation error** (status: 500, label: `federation-local-error`): An error occurred in the communication between this backend and its local federator. These errors are most likely caused by bugs in the backend, and should be reported as such. ### Remote federation errors diff --git a/services/federator/src/Federator/Discovery.hs b/services/federator/src/Federator/Discovery.hs index 65844e33d86..4148e67c136 100644 --- a/services/federator/src/Federator/Discovery.hs +++ b/services/federator/src/Federator/Discovery.hs @@ -49,7 +49,7 @@ instance AsWai DiscoveryFailure where where (status, label) = case e of DiscoveryFailureSrvNotAvailable _ -> (HTTP.status422, "invalid-domain") - DiscoveryFailureDNSError _ -> (HTTP.status500, "discovery-failure") + DiscoveryFailureDNSError _ -> (HTTP.status400, "discovery-failure") waiErrorDescription :: DiscoveryFailure -> Text waiErrorDescription (DiscoveryFailureSrvNotAvailable msg) = "srv record not found: " <> Text.decodeUtf8 msg diff --git a/services/federator/src/Federator/Validation.hs b/services/federator/src/Federator/Validation.hs index 4d7f35867d6..6dc9e21ebd8 100644 --- a/services/federator/src/Federator/Validation.hs +++ b/services/federator/src/Federator/Validation.hs @@ -150,6 +150,7 @@ validateDomain :: validateDomain Nothing _ = throw NoClientCertificate validateDomain (Just encodedCertificate) unparsedDomain = do targetDomain <- parseDomain unparsedDomain + ensureCanFederateWith targetDomain -- run discovery to find the hostname of the client federator certificate <- @@ -160,7 +161,7 @@ validateDomain (Just encodedCertificate) unparsedDomain = do unless (any null validationErrors) $ throw $ AuthenticationFailure validationErrors - ensureCanFederateWith targetDomain $> targetDomain + pure targetDomain -- | Match a hostname against the domain names of a certificate. -- diff --git a/services/federator/test/integration/Test/Federator/IngressSpec.hs b/services/federator/test/integration/Test/Federator/IngressSpec.hs index 08d989b647a..34b6af1d327 100644 --- a/services/federator/test/integration/Test/Federator/IngressSpec.hs +++ b/services/federator/test/integration/Test/Federator/IngressSpec.hs @@ -46,6 +46,8 @@ import Wire.API.Federation.Domain import Wire.API.User import Wire.Network.DNS.SRV +-- | This module contains tests for the interface between federator and ingress. Ingress is +-- mocked with nginz. spec :: TestEnv -> Spec spec env = do describe "Ingress" $ do @@ -68,7 +70,16 @@ spec env = do responseStatusCode resp `shouldBe` HTTP.status200 actualProfile `shouldBe` (Just expectedProfile) - it "should not be accessible without a client certificate" $ + -- @SF.Federation @TSFI.RESTfulAPI @S2 @S3 @S7 + -- + -- This test was primarily intended to test that federator is using the API right (header + -- name etc.), but it is also effectively testing that federator rejects clients without + -- certificates that have been validated by ingress. + -- + -- We can't test end-to-end here: the TLS termination happens in k8s, and would have to be + -- tested there (and with a good emulation of the concrete configuration of the prod + -- system). + it "rejectRequestsWithoutClientCertIngress" $ runTestFederator env $ do brig <- view teBrig <$> ask user <- randomUser brig @@ -93,6 +104,10 @@ spec env = do expectationFailure "Expected client certificate error, got remote error" Left (RemoteErrorResponse _ status _) -> status `shouldBe` HTTP.status400 +-- FUTUREWORK: ORMOLU_DISABLE +-- @END +-- ORMOLU_ENABLE + runTestSem :: Sem '[Input TestEnv, Embed IO] a -> TestFederator IO a runTestSem action = do e <- ask diff --git a/services/federator/test/integration/Test/Federator/InwardSpec.hs b/services/federator/test/integration/Test/Federator/InwardSpec.hs index da02ec6f459..3ef28b069d8 100644 --- a/services/federator/test/integration/Test/Federator/InwardSpec.hs +++ b/services/federator/test/integration/Test/Federator/InwardSpec.hs @@ -39,8 +39,11 @@ import Wire.API.Federation.Domain import Wire.API.User -- FUTUREWORK(federation): move these tests to brig-integration (benefit: avoid duplicating all of the brig helper code) +-- FUTUREWORK(fisx): better yet, reorganize integration tests (or at least the helpers) so +-- they don't spread out over the different sevices. --- | Path covered by this test +-- | This module contains tests for the interface between federator and brig. The tests call +-- federator directly, circumnventing ingress: -- -- +----------+ -- |federator-| +------+--+ @@ -72,6 +75,14 @@ spec env = view teTstOpts hdl <- randomHandle @@ -110,6 +123,10 @@ spec env = (encode hdl) !!! const 403 === statusCode +-- TODO: ORMOLU_DISABLE +-- @END +-- ORMOLU_ENABLE + inwardCallWithHeaders :: (MonadIO m, MonadHttp m, MonadReader TestEnv m, HasCallStack) => ByteString -> @@ -132,8 +149,17 @@ inwardCall :: LBS.ByteString -> m (Response (Maybe LByteString)) inwardCall requestPath payload = do + originDomain :: Text <- cfgOriginDomain <$> view teTstOpts + inwardCallWithOriginDomain (toByteString' originDomain) requestPath payload + +inwardCallWithOriginDomain :: + (MonadIO m, MonadHttp m, MonadReader TestEnv m, HasCallStack) => + ByteString -> + ByteString -> + LBS.ByteString -> + m (Response (Maybe LByteString)) +inwardCallWithOriginDomain originDomain requestPath payload = do Endpoint fedHost fedPort <- cfgFederatorExternal <$> view teTstOpts - originDomain <- cfgOriginDomain <$> view teTstOpts clientCertFilename <- clientCertificate . optSettings . view teOpts <$> ask clientCert <- liftIO $ BS.readFile clientCertFilename post @@ -141,6 +167,6 @@ inwardCall requestPath payload = do . port fedPort . path requestPath . header "X-SSL-Certificate" (HTTP.urlEncode True clientCert) - . header originDomainHeaderName (toByteString' originDomain) + . header originDomainHeaderName originDomain . bytes (toByteString' payload) ) diff --git a/services/federator/test/unit/Test/Federator/InternalServer.hs b/services/federator/test/unit/Test/Federator/InternalServer.hs index db3393c5290..b384d0b88fc 100644 --- a/services/federator/test/unit/Test/Federator/InternalServer.hs +++ b/services/federator/test/unit/Test/Federator/InternalServer.hs @@ -99,6 +99,9 @@ federatedRequestSuccess = body <- Wai.lazyResponseBody res body @?= "\"bar\"" +-- @SF.Federation @TSFI.RESTfulAPI @S2 @S3 @S7 +-- +-- Refuse to send outgoing request to non-included domain when allowlist is configured. federatedRequestFailureAllowList :: TestTree federatedRequestFailureAllowList = testCase "should not make a call when target domain not in the allowList" $ do @@ -134,3 +137,5 @@ federatedRequestFailureAllowList = . runInputConst settings $ callOutward request eith @?= Left (FederationDenied targetDomain) + +-- @END diff --git a/services/federator/test/unit/Test/Federator/Options.hs b/services/federator/test/unit/Test/Federator/Options.hs index 3bf5930a2dd..90345862734 100644 --- a/services/federator/test/unit/Test/Federator/Options.hs +++ b/services/federator/test/unit/Test/Federator/Options.hs @@ -171,7 +171,8 @@ testSettings = assertFailure $ "expected failure for non-existing client certificate, got: " <> show (tlsSettings ^. creds), - testCase "fail on invalid certificate" $ do + -- @SF.Federation @TSFI.RESTfulAPI @S3 @S7 + testCase "failToStartWithInvalidServerCredentials" $ do let settings = defRunSettings "test/resources/unit/invalid.pem" @@ -193,6 +194,7 @@ testSettings = assertFailure $ "expected failure for invalid client certificate, got: " <> show (tlsSettings ^. creds), + -- @END testCase "fail on invalid private key" $ do let settings = defRunSettings diff --git a/services/federator/test/unit/Test/Federator/Response.hs b/services/federator/test/unit/Test/Federator/Response.hs index 8413b9b95a2..b8addefb8de 100644 --- a/services/federator/test/unit/Test/Federator/Response.hs +++ b/services/federator/test/unit/Test/Federator/Response.hs @@ -72,7 +72,7 @@ testDiscoveryFailure = throw (DiscoveryFailureDNSError "mock error") body <- Wai.lazyResponseBody resp let merr = Aeson.decode body - Wai.responseStatus resp @?= HTTP.status500 + Wai.responseStatus resp @?= HTTP.status400 fmap Wai.label merr @?= Just "discovery-failure" testRemoteError :: TestTree diff --git a/services/federator/test/unit/Test/Federator/Validation.hs b/services/federator/test/unit/Test/Federator/Validation.hs index 4c4597a6c51..785757927fb 100644 --- a/services/federator/test/unit/Test/Federator/Validation.hs +++ b/services/federator/test/unit/Test/Federator/Validation.hs @@ -113,6 +113,9 @@ validateDomainAllowListFailSemantic = $ validateDomain (Just exampleCert) "invalid//.><-semantic-&@-domain" res @?= Left (DomainParseError "invalid//.><-semantic-&@-domain") +-- @SF.Federation @TSFI.RESTfulAPI @S2 @S3 @S7 +-- +-- Refuse to send outgoing request to non-included domain when allowlist is configured. validateDomainAllowListFail :: TestTree validateDomainAllowListFail = testCase "allow list validation" $ do @@ -127,6 +130,8 @@ validateDomainAllowListFail = $ validateDomain (Just exampleCert) "localhost.example.com" res @?= Left (FederationDenied (Domain "localhost.example.com")) +-- @END + validateDomainAllowListSuccess :: TestTree validateDomainAllowListSuccess = testCase "should give parsed domain if in the allow list" $ do @@ -153,6 +158,7 @@ validateDomainCertMissing = $ validateDomain Nothing "foo.example.com" res @?= Left NoClientCertificate +-- @SF.Federation @TSFI.RESTfulAPI @S2 @S3 @S7 validateDomainCertInvalid :: TestTree validateDomainCertInvalid = testCase "should fail if the client certificate is invalid" $ do @@ -164,6 +170,12 @@ validateDomainCertInvalid = $ validateDomain (Just "not a certificate") "foo.example.com" res @?= Left (CertificateParseError "no certificate found") +-- @END + +-- @SF.Federation @TSFI.RESTfulAPI @S3 @S7 +-- +-- Reject request if the infrastructure domain in the client cert does not match the backend +-- domain in the `Wire-origin-domain` header. validateDomainCertWrongDomain :: TestTree validateDomainCertWrongDomain = testCase "should fail if the client certificate has a wrong domain" $ do @@ -176,6 +188,8 @@ validateDomainCertWrongDomain = $ validateDomain (Just exampleCert) "foo.example.com" res @?= Left (AuthenticationFailure (pure [X509.NameMismatch "foo.example.com"])) +-- @END + validateDomainCertCN :: TestTree validateDomainCertCN = testCase "should succeed if the certificate has subject CN but no SAN" $ do diff --git a/services/spar/test-integration/Test/Spar/APISpec.hs b/services/spar/test-integration/Test/Spar/APISpec.hs index af466b1650e..7d73042cf0d 100644 --- a/services/spar/test-integration/Test/Spar/APISpec.hs +++ b/services/spar/test-integration/Test/Spar/APISpec.hs @@ -482,7 +482,9 @@ specFinalizeLogin = do (cs . fromJust . responseBody $ sparresp) `shouldContain` "wire:sso:error:forbidden" check mkareq mkaresp submitaresp checkresp + -- {- ORMOLU_DISABLE -} -- FUTUREWORK: try a newer release of ormolu? -- @END + -- {- ORMOLU_ENABLE -} context "IdP changes response format" $ do it "treats NameId case-insensitively" $ do From 9285b7412b8b931b852e7284374dbfad7b4585f9 Mon Sep 17 00:00:00 2001 From: Sandy Maguire Date: Tue, 21 Dec 2021 06:21:11 -0800 Subject: [PATCH 25/74] Move Spar Polysemy Specs into spar:lib (#2005) * Move NowSpec into spar:lb * Move DefaultSsoCodeSpec into spar:lib * Move IdPRawMetadataStoreSpec to spar:lib * Move ScimExternalIdStoreSpec to spar:lib * Move IdPSpec into spar:lib * Pend the flakey test * CI * make format * Changelog --- .../5-internal/integration-spar-polysemy | 1 + services/spar/package.yaml | 6 +- services/spar/spar.cabal | 20 +- services/spar/src/Spar/Sem/DefaultSsoCode.hs | 2 + .../spar/src/Spar/Sem/DefaultSsoCode/Spec.hs | 164 +++++++++ services/spar/src/Spar/Sem/IdP.hs | 2 + services/spar/src/Spar/Sem/IdP/Spec.hs | 315 ++++++++++++++++++ .../spar/src/Spar/Sem/IdPRawMetadataStore.hs | 2 + .../src/Spar/Sem/IdPRawMetadataStore/Spec.hs | 125 +++++++ services/spar/src/Spar/Sem/Now.hs | 2 + services/spar/src/Spar/Sem/Now/Spec.hs | 52 +++ .../spar/src/Spar/Sem/ScimExternalIdStore.hs | 2 + .../src/Spar/Sem/ScimExternalIdStore/Spec.hs | 183 ++++++++++ .../test/Test/Spar/Sem/DefaultSsoCodeSpec.hs | 158 +-------- .../Test/Spar/Sem/IdPRawMetadataStoreSpec.hs | 119 +------ services/spar/test/Test/Spar/Sem/IdPSpec.hs | 305 +---------------- services/spar/test/Test/Spar/Sem/NowSpec.hs | 45 +-- .../Test/Spar/Sem/ScimExternalIdStoreSpec.hs | 179 +--------- 18 files changed, 877 insertions(+), 805 deletions(-) create mode 100644 changelog.d/5-internal/integration-spar-polysemy create mode 100644 services/spar/src/Spar/Sem/DefaultSsoCode/Spec.hs create mode 100644 services/spar/src/Spar/Sem/IdP/Spec.hs create mode 100644 services/spar/src/Spar/Sem/IdPRawMetadataStore/Spec.hs create mode 100644 services/spar/src/Spar/Sem/Now/Spec.hs create mode 100644 services/spar/src/Spar/Sem/ScimExternalIdStore/Spec.hs diff --git a/changelog.d/5-internal/integration-spar-polysemy b/changelog.d/5-internal/integration-spar-polysemy new file mode 100644 index 00000000000..f12d69de1cf --- /dev/null +++ b/changelog.d/5-internal/integration-spar-polysemy @@ -0,0 +1 @@ +Moved specifications for Spar effects out of the test suite and into the library diff --git a/services/spar/package.yaml b/services/spar/package.yaml index 935e8cbfe0f..5efbb34c319 100644 --- a/services/spar/package.yaml +++ b/services/spar/package.yaml @@ -39,6 +39,7 @@ dependencies: - galley-types - ghc-prim - hscim + - hspec - HsOpenSSL - http-api-data - http-client @@ -53,7 +54,9 @@ dependencies: - network-uri - optparse-applicative - polysemy + - polysemy-check >= 0.9 - polysemy-plugin + - QuickCheck - raw-strings-qq - retry - saml2-web-sso >= 0.18 @@ -94,12 +97,9 @@ tests: - hspec-discover:hspec-discover dependencies: - lens-aeson - - hspec - metrics-wai - - QuickCheck - spar - uri-bytestring - - polysemy-check >= 0.9 executables: spar: diff --git a/services/spar/spar.cabal b/services/spar/spar.cabal index cc6c8bb7890..f596bfc9ecd 100644 --- a/services/spar/spar.cabal +++ b/services/spar/spar.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: 4a323def34cfdc7673cea02e13fe518d53d7b04bae552bff2d0784dfb6964162 +-- hash: c721f9a156f292d2f25f9189e2250e105c48a5a3a67d8fdbf3069c9b3a55f921 name: spar version: 0.1 @@ -50,19 +50,23 @@ library Spar.Sem.DefaultSsoCode Spar.Sem.DefaultSsoCode.Cassandra Spar.Sem.DefaultSsoCode.Mem + Spar.Sem.DefaultSsoCode.Spec Spar.Sem.GalleyAccess Spar.Sem.GalleyAccess.Http Spar.Sem.IdP Spar.Sem.IdP.Cassandra Spar.Sem.IdP.Mem + Spar.Sem.IdP.Spec Spar.Sem.IdPRawMetadataStore Spar.Sem.IdPRawMetadataStore.Cassandra Spar.Sem.IdPRawMetadataStore.Mem + Spar.Sem.IdPRawMetadataStore.Spec Spar.Sem.Logger Spar.Sem.Logger.TinyLog Spar.Sem.Now Spar.Sem.Now.Input Spar.Sem.Now.IO + Spar.Sem.Now.Spec Spar.Sem.Random Spar.Sem.Random.IO Spar.Sem.Reporter @@ -77,6 +81,7 @@ library Spar.Sem.ScimExternalIdStore Spar.Sem.ScimExternalIdStore.Cassandra Spar.Sem.ScimExternalIdStore.Mem + Spar.Sem.ScimExternalIdStore.Spec Spar.Sem.ScimTokenStore Spar.Sem.ScimTokenStore.Cassandra Spar.Sem.ScimTokenStore.Mem @@ -94,6 +99,7 @@ library ghc-options: -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates -Wpartial-fields -fwarn-tabs -optP-Wno-nonportable-include-path -j -Wno-redundant-constraints -Werror build-depends: HsOpenSSL + , QuickCheck , aeson , aeson-qq , attoparsec @@ -116,6 +122,7 @@ library , galley-types , ghc-prim , hscim + , hspec , http-api-data , http-client , http-media @@ -129,6 +136,7 @@ library , network-uri , optparse-applicative , polysemy + , polysemy-check >=0.9 , polysemy-plugin , raw-strings-qq , retry @@ -167,6 +175,7 @@ executable spar ghc-options: -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates -Wpartial-fields -fwarn-tabs -optP-Wno-nonportable-include-path -j -Wno-redundant-constraints -Werror -threaded -rtsopts -with-rtsopts=-N -with-rtsopts=-T build-depends: HsOpenSSL + , QuickCheck , aeson , aeson-qq , attoparsec @@ -189,6 +198,7 @@ executable spar , galley-types , ghc-prim , hscim + , hspec , http-api-data , http-client , http-media @@ -202,6 +212,7 @@ executable spar , network-uri , optparse-applicative , polysemy + , polysemy-check >=0.9 , polysemy-plugin , raw-strings-qq , retry @@ -298,6 +309,7 @@ executable spar-integration , network-uri , optparse-applicative , polysemy + , polysemy-check >=0.9 , polysemy-plugin , random , raw-strings-qq @@ -349,6 +361,7 @@ executable spar-migrate-data ghc-options: -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates -Wpartial-fields -fwarn-tabs -optP-Wno-nonportable-include-path -j -Wno-redundant-constraints -Werror -threaded -rtsopts -with-rtsopts=-N build-depends: HsOpenSSL + , QuickCheck , aeson , aeson-qq , attoparsec @@ -372,6 +385,7 @@ executable spar-migrate-data , galley-types , ghc-prim , hscim + , hspec , http-api-data , http-client , http-media @@ -385,6 +399,7 @@ executable spar-migrate-data , network-uri , optparse-applicative , polysemy + , polysemy-check >=0.9 , polysemy-plugin , raw-strings-qq , retry @@ -440,6 +455,7 @@ executable spar-schema ghc-options: -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates -Wpartial-fields -fwarn-tabs -optP-Wno-nonportable-include-path -j -Wno-redundant-constraints -Werror -threaded -rtsopts -with-rtsopts=-N build-depends: HsOpenSSL + , QuickCheck , aeson , aeson-qq , attoparsec @@ -462,6 +478,7 @@ executable spar-schema , galley-types , ghc-prim , hscim + , hspec , http-api-data , http-client , http-media @@ -475,6 +492,7 @@ executable spar-schema , network-uri , optparse-applicative , polysemy + , polysemy-check >=0.9 , polysemy-plugin , raw-strings-qq , retry diff --git a/services/spar/src/Spar/Sem/DefaultSsoCode.hs b/services/spar/src/Spar/Sem/DefaultSsoCode.hs index 9f594c16b4f..af9a49bda4c 100644 --- a/services/spar/src/Spar/Sem/DefaultSsoCode.hs +++ b/services/spar/src/Spar/Sem/DefaultSsoCode.hs @@ -2,6 +2,7 @@ module Spar.Sem.DefaultSsoCode where import Imports import Polysemy +import Polysemy.Check (deriveGenericK) import qualified SAML2.WebSSO as SAML data DefaultSsoCode m a where @@ -12,3 +13,4 @@ data DefaultSsoCode m a where deriving instance Show (DefaultSsoCode m a) makeSem ''DefaultSsoCode +deriveGenericK ''DefaultSsoCode diff --git a/services/spar/src/Spar/Sem/DefaultSsoCode/Spec.hs b/services/spar/src/Spar/Sem/DefaultSsoCode/Spec.hs new file mode 100644 index 00000000000..e9aac14b42a --- /dev/null +++ b/services/spar/src/Spar/Sem/DefaultSsoCode/Spec.hs @@ -0,0 +1,164 @@ +{-# LANGUAGE QuantifiedConstraints #-} +{-# OPTIONS_GHC -fplugin=Polysemy.Plugin #-} + +module Spar.Sem.DefaultSsoCode.Spec (propsForInterpreter) where + +import Imports +import Polysemy +import Polysemy.Check +import SAML2.WebSSO.Types +import qualified Spar.Sem.DefaultSsoCode as E +import Test.Hspec +import Test.Hspec.QuickCheck +import Test.QuickCheck + +propsForInterpreter :: + PropConstraints r f => + String -> + (forall a. Sem r a -> IO (f a)) -> + Spec +propsForInterpreter interpreter lower = do + describe interpreter $ do + prop "delete/delete" $ prop_deleteDelete Nothing lower + prop "delete/get" $ prop_deleteGet Nothing lower + prop "delete/store" $ prop_deleteStore Nothing lower + prop "get/store" $ prop_getStore Nothing lower + prop "store/delete" $ prop_storeDelete Nothing lower + prop "store/get" $ prop_storeGet Nothing lower + prop "store/store" $ prop_storeStore Nothing lower + +-- | All the constraints we need to generalize properties in this module. +-- A regular type synonym doesn't work due to dreaded impredicative +-- polymorphism. +class + (Arbitrary IdPId, CoArbitrary IdPId, Functor f, Member E.DefaultSsoCode r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +instance + (Arbitrary IdPId, CoArbitrary IdPId, Functor f, Member E.DefaultSsoCode r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +prop_storeGet :: + PropConstraints r f => + Maybe (f (Maybe IdPId) -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_storeGet = + prepropLaw @'[E.DefaultSsoCode] $ do + s <- arbitrary + pure $ + simpleLaw + ( do + E.store s + E.get + ) + ( do + E.store s + pure (Just s) + ) + +prop_getStore :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_getStore = + prepropLaw @'[E.DefaultSsoCode] $ do + pure $ + simpleLaw + ( do + E.get >>= maybe (pure ()) E.store + ) + ( do + pure () + ) + +prop_storeDelete :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_storeDelete = + prepropLaw @'[E.DefaultSsoCode] $ do + s <- arbitrary + pure $ + simpleLaw + ( do + E.store s + E.delete + ) + ( do + E.delete + ) + +prop_deleteStore :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_deleteStore = + prepropLaw @'[E.DefaultSsoCode] $ do + s <- arbitrary + pure $ + simpleLaw + ( do + E.delete + E.store s + ) + ( do + E.store s + ) + +prop_storeStore :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_storeStore = + prepropLaw @'[E.DefaultSsoCode] $ do + s <- arbitrary + s' <- arbitrary + pure $ + simpleLaw + ( do + E.store s + E.store s' + ) + ( do + E.store s' + ) + +prop_deleteDelete :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_deleteDelete = + prepropLaw @'[E.DefaultSsoCode] $ do + pure $ + simpleLaw + ( do + E.delete + E.delete + ) + ( do + E.delete + ) + +prop_deleteGet :: + PropConstraints r f => + Maybe (f (Maybe IdPId) -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_deleteGet = + prepropLaw @'[E.DefaultSsoCode] $ do + pure $ + simpleLaw + ( do + E.delete + E.get + ) + ( do + E.delete + pure Nothing + ) diff --git a/services/spar/src/Spar/Sem/IdP.hs b/services/spar/src/Spar/Sem/IdP.hs index 53e94faed03..81674e06e6e 100644 --- a/services/spar/src/Spar/Sem/IdP.hs +++ b/services/spar/src/Spar/Sem/IdP.hs @@ -3,6 +3,7 @@ module Spar.Sem.IdP where import Data.Id import Imports import Polysemy +import Polysemy.Check (deriveGenericK) import qualified SAML2.WebSSO as SAML import qualified Wire.API.User.IdentityProvider as IP @@ -41,3 +42,4 @@ deriving stock instance Show (IdP m a) -- TODO(sandy): Inline this definition --- no TH makeSem ''IdP +deriveGenericK ''IdP diff --git a/services/spar/src/Spar/Sem/IdP/Spec.hs b/services/spar/src/Spar/Sem/IdP/Spec.hs new file mode 100644 index 00000000000..d561ebc28ff --- /dev/null +++ b/services/spar/src/Spar/Sem/IdP/Spec.hs @@ -0,0 +1,315 @@ +{-# LANGUAGE DeriveDataTypeable #-} +{-# LANGUAGE QuantifiedConstraints #-} +{-# LANGUAGE StandaloneDeriving #-} +{-# OPTIONS_GHC -Wno-orphans #-} +{-# OPTIONS_GHC -fplugin=Polysemy.Plugin #-} + +module Spar.Sem.IdP.Spec (propsForInterpreter) where + +import Control.Arrow +import Control.Lens +import Data.Data (Data) +import Imports +import Polysemy +import Polysemy.Check +import SAML2.WebSSO.Types +import qualified SAML2.WebSSO.Types as SAML +import qualified Spar.Sem.IdP as E +import Test.Hspec +import Test.Hspec.QuickCheck +import Test.QuickCheck +import qualified Wire.API.User.IdentityProvider as IP + +deriving instance Data IdPId + +deriving instance Data (E.GetIdPResult IdPId) + +propsForInterpreter :: + (Member E.IdP r, PropConstraints r f) => + String -> + (forall x. f x -> x) -> + (forall x. Show x => Maybe (f x -> String)) -> + (forall x. Sem r x -> IO (f x)) -> + Spec +propsForInterpreter interpreter extract labeler lower = do + describe interpreter $ do + prop "deleteConfig/deleteConfig" $ prop_deleteDelete Nothing lower + prop "deleteConfig/getConfig" $ prop_deleteGet labeler lower + prop "getConfig/storeConfig" $ prop_getStore (Just $ show . (() <$) . extract) lower + prop "getConfig/getConfig" $ prop_getGet (Just $ show . ((() <$) *** (() <$)) . extract) lower + prop "setReplacedBy/clearReplacedBy" $ prop_setClear labeler lower + prop "setReplacedBy/getReplacedBy" $ prop_setGet (Just $ show . (fmap (() <$)) . extract) lower + prop "setReplacedBy/setReplacedBy" $ prop_setSet (Just $ show . (fmap (() <$)) . extract) lower + prop "storeConfig/getConfig" $ prop_storeGet (Just $ show . (() <$) . extract) lower + xit "storeConfig/getIdByIssuerWithoutTeam" $ property $ prop_storeGetByIssuer (Just $ constructorLabel . extract) lower + prop "storeConfig/storeConfig (different keys)" $ prop_storeStoreInterleave Nothing lower + prop "storeConfig/storeConfig (same keys)" $ prop_storeStore Nothing lower + +getReplacedBy :: Member E.IdP r => SAML.IdPId -> Sem r (Maybe (Maybe SAML.IdPId)) +getReplacedBy idpid = fmap (view $ SAML.idpExtraInfo . IP.wiReplacedBy) <$> E.getConfig idpid + +-- | All the constraints we need to generalize properties in this module. +-- A regular type synonym doesn't work due to dreaded impredicative +-- polymorphism. +class + (Arbitrary Issuer, CoArbitrary Issuer, Arbitrary E.Replaced, Arbitrary E.Replaced, Arbitrary E.Replacing, Arbitrary IdPId, CoArbitrary IdPId, Arbitrary IP.IdP, CoArbitrary IP.IdP, CoArbitrary (E.GetIdPResult IdPId), Functor f, Member E.IdP r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +instance + (Arbitrary Issuer, CoArbitrary Issuer, Arbitrary E.Replaced, Arbitrary E.Replaced, Arbitrary E.Replacing, Arbitrary IdPId, CoArbitrary IdPId, Arbitrary IP.IdP, CoArbitrary IP.IdP, CoArbitrary (E.GetIdPResult IdPId), Functor f, Member E.IdP r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +prop_storeStore :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_storeStore = + prepropLaw @'[E.IdP] $ do + s <- arbitrary + s' <- arbitrary + pure $ + Law + { lawLhs = do + E.storeConfig $ s & SAML.idpId .~ s' ^. SAML.idpId + E.storeConfig s', + lawRhs = do + E.storeConfig s', + lawPrelude = [], + lawPostlude = [E.getConfig $ s' ^. SAML.idpId] + } + +prop_storeStoreInterleave :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_storeStoreInterleave = + prepropLaw @'[E.IdP] $ do + s <- arbitrary + s' <- arbitrary + !_ <- + when (s ^. SAML.idpId == s' ^. SAML.idpId) discard + pure $ + Law + { lawLhs = do + E.storeConfig s + E.storeConfig s', + lawRhs = do + E.storeConfig s' + E.storeConfig s, + lawPrelude = [], + lawPostlude = [E.getConfig $ s ^. SAML.idpId, E.getConfig $ s' ^. SAML.idpId] + } + +prop_storeGet :: + PropConstraints r f => + Maybe (f (Maybe IP.IdP) -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_storeGet = + prepropLaw @'[E.IdP] $ + do + s <- arbitrary + pure $ + simpleLaw + ( do + E.storeConfig s + E.getConfig $ s ^. idpId + ) + ( do + E.storeConfig s + pure (Just s) + ) + +prop_deleteGet :: + PropConstraints r f => + Maybe (f (Maybe IP.IdP) -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_deleteGet = + prepropLaw @'[E.IdP] $ do + s <- arbitrary + pure $ + Law + { lawLhs = do + E.deleteConfig s + E.getConfig $ s ^. SAML.idpId, + lawRhs = do + E.deleteConfig s + pure Nothing, + lawPrelude = + [ E.storeConfig s + ], + lawPostlude = [] :: [Sem r ()] + } + +prop_deleteDelete :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_deleteDelete = + prepropLaw @'[E.IdP] $ do + s <- arbitrary + pure $ + simpleLaw + ( do + E.deleteConfig s + E.deleteConfig s + ) + ( do + E.deleteConfig s + ) + +prop_storeGetByIssuer :: + PropConstraints r f => + Maybe (f (E.GetIdPResult IdPId) -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_storeGetByIssuer = + prepropLaw @'[E.IdP] $ + do + s <- arbitrary + pure $ + simpleLaw + ( do + E.storeConfig s + E.getIdByIssuerWithoutTeam $ s ^. idpMetadata . edIssuer + ) + ( do + E.storeConfig s + -- NOT TRUE! This can also return E.GetIdPNonUnique with nonzero probability! + pure $ E.GetIdPFound $ s ^. idpId + ) + +prop_setClear :: + PropConstraints r f => + Maybe (f (Maybe (Maybe IdPId)) -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_setClear = + prepropLaw @'[E.IdP] $ + do + idp <- arbitrary + replaced_id <- arbitrary + let replaced = E.Replaced replaced_id + replacing <- arbitrary + pure $ + Law + { lawLhs = do + E.setReplacedBy replaced replacing + E.clearReplacedBy replaced + getReplacedBy replaced_id, + lawRhs = do + E.clearReplacedBy replaced + getReplacedBy replaced_id, + lawPrelude = + [ E.storeConfig $ idp & SAML.idpId .~ replaced_id + ], + lawPostlude = [] @(Sem _ ()) + } + +prop_getGet :: + forall r f. + PropConstraints r f => + Maybe (f (Maybe IP.IdP, Maybe IP.IdP) -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_getGet = + prepropLaw @'[E.IdP] $ + do + idpid <- arbitrary + idp <- arbitrary + pure $ + Law + { lawLhs = do + liftA2 (,) (E.getConfig idpid) (E.getConfig idpid), + lawRhs = do + cfg <- E.getConfig idpid + pure (cfg, cfg), + lawPrelude = + [ E.storeConfig $ idp & SAML.idpId .~ idpid + ], + lawPostlude = [] :: [Sem r ()] + } + +prop_getStore :: + PropConstraints r f => + Maybe (f (Maybe IP.IdP) -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_getStore = + prepropLaw @'[E.IdP] $ + do + idpid <- arbitrary + s <- arbitrary + let s' = s & SAML.idpId .~ idpid + pure $ + Law + { lawLhs = do + r <- E.getConfig idpid + maybe (pure ()) E.storeConfig r + pure r, + lawRhs = do + E.getConfig idpid, + lawPrelude = + [E.storeConfig s'], + lawPostlude = + [E.getConfig idpid] + } + +prop_setSet :: + PropConstraints r f => + Maybe (f (Maybe (Maybe IdPId)) -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_setSet = + prepropLaw @'[E.IdP] $ + do + replaced_id <- arbitrary + s <- arbitrary + let s' = s & SAML.idpId .~ replaced_id + let replaced = E.Replaced replaced_id + replacing <- arbitrary + replacing' <- arbitrary + pure $ + Law + { lawLhs = do + E.setReplacedBy replaced replacing + E.setReplacedBy replaced replacing' + getReplacedBy replaced_id, + lawRhs = do + E.setReplacedBy replaced replacing' + getReplacedBy replaced_id, + lawPrelude = + [E.storeConfig s'], + lawPostlude = [] @(Sem _ ()) + } + +prop_setGet :: + PropConstraints r f => + Maybe (f (Maybe (Maybe IdPId)) -> String) -> + (forall x. Sem r x -> IO (f x)) -> + Property +prop_setGet = + prepropLaw @'[E.IdP] $ + do + idp <- arbitrary + replaced_id <- arbitrary + let replaced = E.Replaced replaced_id + replacing_id <- arbitrary + let replacing = E.Replacing replacing_id + pure $ + Law + { lawLhs = do + E.setReplacedBy replaced replacing + getReplacedBy replaced_id, + lawRhs = do + E.setReplacedBy replaced replacing + (Just replacing_id <$) <$> E.getConfig replaced_id, + lawPrelude = + [ E.storeConfig $ idp & SAML.idpId .~ replaced_id + ], + lawPostlude = [] :: [Sem r ()] + } diff --git a/services/spar/src/Spar/Sem/IdPRawMetadataStore.hs b/services/spar/src/Spar/Sem/IdPRawMetadataStore.hs index 4cee44c4c80..5b000899378 100644 --- a/services/spar/src/Spar/Sem/IdPRawMetadataStore.hs +++ b/services/spar/src/Spar/Sem/IdPRawMetadataStore.hs @@ -2,6 +2,7 @@ module Spar.Sem.IdPRawMetadataStore where import Imports import Polysemy +import Polysemy.Check (deriveGenericK) import qualified SAML2.WebSSO as SAML data IdPRawMetadataStore m a where @@ -13,3 +14,4 @@ deriving stock instance Show (IdPRawMetadataStore m a) -- TODO(sandy): Inline this definition --- no TH makeSem ''IdPRawMetadataStore +deriveGenericK ''IdPRawMetadataStore diff --git a/services/spar/src/Spar/Sem/IdPRawMetadataStore/Spec.hs b/services/spar/src/Spar/Sem/IdPRawMetadataStore/Spec.hs new file mode 100644 index 00000000000..181aec23304 --- /dev/null +++ b/services/spar/src/Spar/Sem/IdPRawMetadataStore/Spec.hs @@ -0,0 +1,125 @@ +{-# LANGUAGE QuantifiedConstraints #-} +{-# OPTIONS_GHC -fplugin=Polysemy.Plugin #-} + +module Spar.Sem.IdPRawMetadataStore.Spec (propsForInterpreter) where + +import Imports +import Polysemy +import Polysemy.Check +import SAML2.WebSSO.Types (IdPId) +import qualified Spar.Sem.IdPRawMetadataStore as E +import Test.Hspec +import Test.Hspec.QuickCheck +import Test.QuickCheck + +class + (Arbitrary IdPId, CoArbitrary IdPId, Arbitrary Text, CoArbitrary Text, Functor f, Member E.IdPRawMetadataStore r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +instance + (Arbitrary IdPId, CoArbitrary IdPId, Arbitrary Text, CoArbitrary Text, Functor f, Member E.IdPRawMetadataStore r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +prop_storeGetRaw :: + PropConstraints r f => + Maybe (f (Maybe Text) -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_storeGetRaw = + prepropLaw @'[E.IdPRawMetadataStore] + ( do + idpid <- arbitrary + t <- arbitrary + pure $ + simpleLaw + ( do + E.store idpid t + E.get idpid + ) + ( do + E.store idpid t + pure (Just t) + ) + ) + +prop_storeStoreRaw :: + PropConstraints r f => + Maybe (f (Maybe Text) -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_storeStoreRaw = + prepropLaw @'[E.IdPRawMetadataStore] + ( do + idpid <- arbitrary + t1 <- arbitrary + t2 <- arbitrary + pure $ + simpleLaw + ( do + E.store idpid t1 + E.store idpid t2 + E.get idpid + ) + ( do + E.store idpid t2 + E.get idpid + ) + ) + +prop_storeDeleteRaw :: + PropConstraints r f => + Maybe (f (Maybe Text) -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_storeDeleteRaw = + prepropLaw @'[E.IdPRawMetadataStore] $ + do + idpid <- arbitrary + t <- arbitrary + pure $ + simpleLaw + ( do + E.store idpid t + E.delete idpid + E.get idpid + ) + ( do + E.delete idpid + E.get idpid + ) + +prop_deleteGetRaw :: + PropConstraints r f => + Maybe (f (Maybe Text) -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_deleteGetRaw = + prepropLaw @'[E.IdPRawMetadataStore] + ( do + idpid <- arbitrary + t <- arbitrary + pure $ + Law + { lawLhs = do + E.delete idpid + E.get idpid, + lawRhs = do + E.delete idpid + pure Nothing, + lawPrelude = + [ E.store idpid t + ], + lawPostlude = [] @(Sem _ ()) + } + ) + +propsForInterpreter :: + PropConstraints r f => + (forall x. f x -> x) -> + (forall a. Sem r a -> IO (f a)) -> + Spec +propsForInterpreter extract lower = do + prop "store/store" $ prop_storeStoreRaw (Just $ constructorLabel . extract) lower + prop "store/get" $ prop_storeGetRaw (Just $ constructorLabel . extract) lower + prop "store/delete" $ prop_storeDeleteRaw (Just $ constructorLabel . extract) lower + prop "delete/get" $ prop_deleteGetRaw (Just $ constructorLabel . extract) lower diff --git a/services/spar/src/Spar/Sem/Now.hs b/services/spar/src/Spar/Sem/Now.hs index f5a701b1a03..883eb8d44cf 100644 --- a/services/spar/src/Spar/Sem/Now.hs +++ b/services/spar/src/Spar/Sem/Now.hs @@ -2,12 +2,14 @@ module Spar.Sem.Now where import Imports import Polysemy +import Polysemy.Check (deriveGenericK) import qualified SAML2.WebSSO as SAML data Now m a where Get :: Now m SAML.Time makeSem ''Now +deriveGenericK ''Now deriving instance Show (Now m a) diff --git a/services/spar/src/Spar/Sem/Now/Spec.hs b/services/spar/src/Spar/Sem/Now/Spec.hs new file mode 100644 index 00000000000..f7fab8c9d1f --- /dev/null +++ b/services/spar/src/Spar/Sem/Now/Spec.hs @@ -0,0 +1,52 @@ +{-# LANGUAGE QuantifiedConstraints #-} +{-# OPTIONS_GHC -fplugin=Polysemy.Plugin #-} + +module Spar.Sem.Now.Spec (propsForInterpreter) where + +import Imports +import Polysemy +import Polysemy.Check +import Polysemy.Input +import qualified Spar.Sem.Now as E +import Test.Hspec +import Test.Hspec.QuickCheck +import Test.QuickCheck + +propsForInterpreter :: + PropConstraints r f => + String -> + (forall a. Sem r a -> IO (f a)) -> + Spec +propsForInterpreter interpreter lower = do + describe interpreter $ do + prop "now/now" $ prop_nowNow Nothing lower + +-- | All the constraints we need to generalize properties in this module. +-- A regular type synonym doesn't work due to dreaded impredicative +-- polymorphism. +class + (Functor f, Member E.Now r, Member (Input ()) r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +instance + (Functor f, Member E.Now r, Member (Input ()) r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +prop_nowNow :: + PropConstraints r f => + Maybe (f Bool -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_nowNow = + -- NOTE: This @Input ()@ effect is a workaround to an oversight in + -- @polysemy-check@. 'prepropLaw' wants to synthesize some actions to run + -- before and after its generators, and check their results for equality. We + -- can't use 'Now' as this effect, because 'E.get' won't return equivalent + -- results! And we can't keep it empty, because that triggers a crash in + -- @polysemy-check@. Thus @Input ()@, which isn't beautiful, but works fine. + prepropLaw @'[Input ()] $ do + pure $ + simpleLaw + (liftA2 (<=) E.get E.get) + ( pure True + ) diff --git a/services/spar/src/Spar/Sem/ScimExternalIdStore.hs b/services/spar/src/Spar/Sem/ScimExternalIdStore.hs index 3978108770c..717daa58b8a 100644 --- a/services/spar/src/Spar/Sem/ScimExternalIdStore.hs +++ b/services/spar/src/Spar/Sem/ScimExternalIdStore.hs @@ -3,6 +3,7 @@ module Spar.Sem.ScimExternalIdStore where import Data.Id (TeamId, UserId) import Imports import Polysemy +import Polysemy.Check (deriveGenericK) import Wire.API.User.Identity (Email) data ScimExternalIdStore m a where @@ -13,3 +14,4 @@ data ScimExternalIdStore m a where deriving instance Show (ScimExternalIdStore m a) makeSem ''ScimExternalIdStore +deriveGenericK ''ScimExternalIdStore diff --git a/services/spar/src/Spar/Sem/ScimExternalIdStore/Spec.hs b/services/spar/src/Spar/Sem/ScimExternalIdStore/Spec.hs new file mode 100644 index 00000000000..57c2a1742e9 --- /dev/null +++ b/services/spar/src/Spar/Sem/ScimExternalIdStore/Spec.hs @@ -0,0 +1,183 @@ +{-# LANGUAGE QuantifiedConstraints #-} +{-# OPTIONS_GHC -fplugin=Polysemy.Plugin #-} + +module Spar.Sem.ScimExternalIdStore.Spec (propsForInterpreter) where + +import Data.Id +import Imports +import Polysemy +import Polysemy.Check +import qualified Spar.Sem.ScimExternalIdStore as E +import Test.Hspec +import Test.Hspec.QuickCheck +import Test.QuickCheck + +propsForInterpreter :: + PropConstraints r f => + String -> + (forall a. f a -> a) -> + (forall a. Sem r a -> IO (f a)) -> + Spec +propsForInterpreter interpreter extract lower = do + describe interpreter $ do + prop "delete/delete" $ prop_deleteDelete Nothing lower + prop "delete/lookup" $ prop_deleteLookup (Just $ show . (() <$) . extract) lower + prop "delete/insert" $ prop_deleteInsert Nothing lower + prop "lookup/insert" $ prop_lookupInsert Nothing lower + prop "insert/delete" $ prop_insertDelete Nothing lower + prop "insert/lookup" $ prop_insertLookup (Just $ show . (() <$) . extract) lower + prop "insert/insert" $ prop_insertInsert (Just $ show . (() <$) . extract) lower + +-- | All the constraints we need to generalize properties in this module. +-- A regular type synonym doesn't work due to dreaded impredicative +-- polymorphism. +class + (Arbitrary UserId, CoArbitrary UserId, Functor f, Member E.ScimExternalIdStore r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +instance + (Arbitrary UserId, CoArbitrary UserId, Functor f, Member E.ScimExternalIdStore r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => + PropConstraints r f + +prop_insertLookup :: + PropConstraints r f => + Maybe (f (Maybe UserId) -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_insertLookup = + prepropLaw @'[E.ScimExternalIdStore] $ do + tid <- arbitrary + email <- arbitrary + uid <- arbitrary + pure $ + simpleLaw + ( do + E.insert tid email uid + E.lookup tid email + ) + ( do + E.insert tid email uid + pure (Just uid) + ) + +prop_lookupInsert :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_lookupInsert = + prepropLaw @'[E.ScimExternalIdStore] $ do + tid <- arbitrary + email <- arbitrary + pure $ + simpleLaw + ( do + E.lookup tid email >>= maybe (pure ()) (E.insert tid email) + ) + ( do + pure () + ) + +prop_insertDelete :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_insertDelete = + prepropLaw @'[E.ScimExternalIdStore] $ do + tid <- arbitrary + email <- arbitrary + uid <- arbitrary + pure $ + simpleLaw + ( do + E.insert tid email uid + E.delete tid email + ) + ( do + E.delete tid email + ) + +prop_deleteInsert :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_deleteInsert = + prepropLaw @'[E.ScimExternalIdStore] $ do + tid <- arbitrary + email <- arbitrary + uid <- arbitrary + pure $ + simpleLaw + ( do + E.delete tid email + E.insert tid email uid + ) + ( do + E.insert tid email uid + ) + +prop_insertInsert :: + PropConstraints r f => + Maybe (f (Maybe UserId) -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_insertInsert = + prepropLaw @'[E.ScimExternalIdStore] $ do + tid <- arbitrary + email <- arbitrary + uid <- arbitrary + uid' <- arbitrary + pure $ + simpleLaw + ( do + E.insert tid email uid + E.insert tid email uid' + E.lookup tid email + ) + ( do + E.insert tid email uid' + E.lookup tid email + ) + +prop_deleteDelete :: + PropConstraints r f => + Maybe (f () -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_deleteDelete = + prepropLaw @'[E.ScimExternalIdStore] $ do + tid <- arbitrary + email <- arbitrary + pure $ + simpleLaw + ( do + E.delete tid email + E.delete tid email + ) + ( do + E.delete tid email + ) + +prop_deleteLookup :: + PropConstraints r f => + Maybe (f (Maybe UserId) -> String) -> + (forall a. Sem r a -> IO (f a)) -> + Property +prop_deleteLookup = + prepropLaw @'[E.ScimExternalIdStore] $ do + tid <- arbitrary + email <- arbitrary + uid <- arbitrary + pure $ + Law + { lawLhs = do + E.delete tid email + E.lookup tid email, + lawRhs = do + E.delete tid email + pure Nothing, + lawPrelude = [E.insert tid email uid], + lawPostlude = [] @(Sem _ ()) + } diff --git a/services/spar/test/Test/Spar/Sem/DefaultSsoCodeSpec.hs b/services/spar/test/Test/Spar/Sem/DefaultSsoCodeSpec.hs index 9a638ba9907..bc55fa5e410 100644 --- a/services/spar/test/Test/Spar/Sem/DefaultSsoCodeSpec.hs +++ b/services/spar/test/Test/Spar/Sem/DefaultSsoCodeSpec.hs @@ -7,167 +7,11 @@ module Test.Spar.Sem.DefaultSsoCodeSpec where import Arbitrary () import Imports import Polysemy -import Polysemy.Check -import SAML2.WebSSO.Types -import qualified Spar.Sem.DefaultSsoCode as E import Spar.Sem.DefaultSsoCode.Mem +import Spar.Sem.DefaultSsoCode.Spec import Test.Hspec import Test.Hspec.QuickCheck -import Test.QuickCheck - -deriveGenericK ''E.DefaultSsoCode - -propsForInterpreter :: - PropConstraints r f => - String -> - (forall a. Sem r a -> IO (f a)) -> - Spec -propsForInterpreter interpreter lower = do - describe interpreter $ do - prop "delete/delete" $ prop_deleteDelete Nothing lower - prop "delete/get" $ prop_deleteGet Nothing lower - prop "delete/store" $ prop_deleteStore Nothing lower - prop "get/store" $ prop_getStore Nothing lower - prop "store/delete" $ prop_storeStore Nothing lower - prop "store/get" $ prop_storeGet Nothing lower - prop "store/store" $ prop_storeStore Nothing lower spec :: Spec spec = modifyMaxSuccess (const 1000) $ do propsForInterpreter "defaultSsoCodeToMem" $ pure . run . defaultSsoCodeToMem - --- | All the constraints we need to generalize properties in this module. --- A regular type synonym doesn't work due to dreaded impredicative --- polymorphism. -class - (Functor f, Member E.DefaultSsoCode r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => - PropConstraints r f - -instance - (Functor f, Member E.DefaultSsoCode r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => - PropConstraints r f - -prop_storeGet :: - PropConstraints r f => - Maybe (f (Maybe IdPId) -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_storeGet = - prepropLaw @'[E.DefaultSsoCode] $ do - s <- arbitrary - pure $ - simpleLaw - ( do - E.store s - E.get - ) - ( do - E.store s - pure (Just s) - ) - -prop_getStore :: - PropConstraints r f => - Maybe (f () -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_getStore = - prepropLaw @'[E.DefaultSsoCode] $ do - pure $ - simpleLaw - ( do - E.get >>= maybe (pure ()) E.store - ) - ( do - pure () - ) - -prop_storeDelete :: - PropConstraints r f => - Maybe (f () -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_storeDelete = - prepropLaw @'[E.DefaultSsoCode] $ do - s <- arbitrary - pure $ - simpleLaw - ( do - E.store s - E.delete - ) - ( do - E.delete - ) - -prop_deleteStore :: - PropConstraints r f => - Maybe (f () -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_deleteStore = - prepropLaw @'[E.DefaultSsoCode] $ do - s <- arbitrary - pure $ - simpleLaw - ( do - E.delete - E.store s - ) - ( do - E.store s - ) - -prop_storeStore :: - PropConstraints r f => - Maybe (f () -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_storeStore = - prepropLaw @'[E.DefaultSsoCode] $ do - s <- arbitrary - s' <- arbitrary - pure $ - simpleLaw - ( do - E.store s - E.store s' - ) - ( do - E.store s' - ) - -prop_deleteDelete :: - PropConstraints r f => - Maybe (f () -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_deleteDelete = - prepropLaw @'[E.DefaultSsoCode] $ do - pure $ - simpleLaw - ( do - E.delete - E.delete - ) - ( do - E.delete - ) - -prop_deleteGet :: - PropConstraints r f => - Maybe (f (Maybe IdPId) -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_deleteGet = - prepropLaw @'[E.DefaultSsoCode] $ do - pure $ - simpleLaw - ( do - E.delete - E.get - ) - ( do - E.delete - pure Nothing - ) diff --git a/services/spar/test/Test/Spar/Sem/IdPRawMetadataStoreSpec.hs b/services/spar/test/Test/Spar/Sem/IdPRawMetadataStoreSpec.hs index 28aa64122dd..2f8ce187cc8 100644 --- a/services/spar/test/Test/Spar/Sem/IdPRawMetadataStoreSpec.hs +++ b/services/spar/test/Test/Spar/Sem/IdPRawMetadataStoreSpec.hs @@ -1,5 +1,3 @@ -{-# LANGUAGE QuantifiedConstraints #-} -{-# OPTIONS_GHC -Wno-orphans #-} {-# OPTIONS_GHC -fplugin=Polysemy.Plugin #-} module Test.Spar.Sem.IdPRawMetadataStoreSpec where @@ -7,130 +5,15 @@ module Test.Spar.Sem.IdPRawMetadataStoreSpec where import Arbitrary () import Imports import Polysemy -import Polysemy.Check import qualified Spar.Sem.IdPRawMetadataStore as E import Spar.Sem.IdPRawMetadataStore.Mem +import Spar.Sem.IdPRawMetadataStore.Spec import Test.Hspec import Test.Hspec.QuickCheck -import Test.QuickCheck - -deriveGenericK ''E.IdPRawMetadataStore - -class - (Functor f, Member E.IdPRawMetadataStore r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => - PropConstraints r f - -instance - (Functor f, Member E.IdPRawMetadataStore r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => - PropConstraints r f - -prop_storeGetRaw :: - PropConstraints r f => - Maybe (f (Maybe Text) -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_storeGetRaw = - prepropLaw @'[E.IdPRawMetadataStore] - ( do - idpid <- arbitrary - t <- arbitrary - pure $ - simpleLaw - ( do - E.store idpid t - E.get idpid - ) - ( do - E.store idpid t - pure (Just t) - ) - ) - -prop_storeStoreRaw :: - PropConstraints r f => - Maybe (f (Maybe Text) -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_storeStoreRaw = - prepropLaw @'[E.IdPRawMetadataStore] - ( do - idpid <- arbitrary - t1 <- arbitrary - t2 <- arbitrary - pure $ - simpleLaw - ( do - E.store idpid t1 - E.store idpid t2 - E.get idpid - ) - ( do - E.store idpid t2 - E.get idpid - ) - ) - -prop_storeDeleteRaw :: - PropConstraints r f => - Maybe (f (Maybe Text) -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_storeDeleteRaw = - prepropLaw @'[E.IdPRawMetadataStore] $ - do - idpid <- arbitrary - t <- arbitrary - pure $ - simpleLaw - ( do - E.store idpid t - E.delete idpid - E.get idpid - ) - ( do - E.delete idpid - E.get idpid - ) - -prop_deleteGetRaw :: - PropConstraints r f => - Maybe (f (Maybe Text) -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_deleteGetRaw = - prepropLaw @'[E.IdPRawMetadataStore] - ( do - idpid <- arbitrary - t <- arbitrary - pure $ - Law - { lawLhs = do - E.delete idpid - E.get idpid, - lawRhs = do - E.delete idpid - pure Nothing, - lawPrelude = - [ E.store idpid t - ], - lawPostlude = [] @(Sem _ ()) - } - ) testInterpreter :: Sem '[E.IdPRawMetadataStore] a -> IO (RawState, a) testInterpreter = pure . run . idpRawMetadataStoreToMem -propsForInterpreter :: - PropConstraints r f => - (forall x. f x -> x) -> - (forall a. Sem r a -> IO (f a)) -> - Spec -propsForInterpreter extract lower = do - prop "store/store" $ prop_storeStoreRaw (Just $ constructorLabel . extract) lower - prop "store/get" $ prop_storeGetRaw (Just $ constructorLabel . extract) lower - prop "store/delete" $ prop_storeDeleteRaw (Just $ constructorLabel . extract) lower - prop "delete/get" $ prop_deleteGetRaw (Just $ constructorLabel . extract) lower - spec :: Spec spec = modifyMaxSuccess (const 1000) $ do propsForInterpreter snd testInterpreter diff --git a/services/spar/test/Test/Spar/Sem/IdPSpec.hs b/services/spar/test/Test/Spar/Sem/IdPSpec.hs index ef1eba1f471..43232daa1b2 100644 --- a/services/spar/test/Test/Spar/Sem/IdPSpec.hs +++ b/services/spar/test/Test/Spar/Sem/IdPSpec.hs @@ -7,316 +7,13 @@ module Test.Spar.Sem.IdPSpec where import Arbitrary () -import Control.Arrow -import Control.Lens -import Data.Data (Data) import Imports import Polysemy -import Polysemy.Check -import SAML2.WebSSO.Types -import qualified SAML2.WebSSO.Types as SAML -import qualified Spar.Sem.IdP as E import Spar.Sem.IdP.Mem +import Spar.Sem.IdP.Spec import Test.Hspec import Test.Hspec.QuickCheck -import Test.QuickCheck -import qualified Wire.API.User.IdentityProvider as IP - -deriveGenericK ''E.IdP - -deriving instance Data (E.GetIdPResult IdPId) - -deriving instance Data (IdPId) - -propsForInterpreter :: - (Member E.IdP r, PropConstraints r f) => - String -> - (forall x. f x -> x) -> - (forall x. Show x => Maybe (f x -> String)) -> - (forall x. Sem r x -> IO (f x)) -> - Spec -propsForInterpreter interpreter extract labeler lower = do - describe interpreter $ do - prop "deleteConfig/deleteConfig" $ prop_deleteDelete Nothing lower - prop "deleteConfig/getConfig" $ prop_deleteGet labeler lower - prop "getConfig/storeConfig" $ prop_getStore (Just $ show . (() <$) . extract) lower - prop "getConfig/getConfig" $ prop_getGet (Just $ show . ((() <$) *** (() <$)) . extract) lower - prop "setReplacedBy/clearReplacedBy" $ prop_setClear labeler lower - prop "setReplacedBy/getReplacedBy" $ prop_setGet (Just $ show . (fmap (() <$)) . extract) lower - prop "setReplacedBy/setReplacedBy" $ prop_setSet (Just $ show . (fmap (() <$)) . extract) lower - prop "storeConfig/getConfig" $ prop_storeGet (Just $ show . (() <$) . extract) lower - prop "storeConfig/getIdByIssuerWithoutTeam" $ prop_storeGetByIssuer (Just $ constructorLabel . extract) lower - prop "storeConfig/storeConfig (different keys)" $ prop_storeStoreInterleave Nothing lower - prop "storeConfig/storeConfig (same keys)" $ prop_storeStore Nothing lower spec :: Spec spec = modifyMaxSuccess (const 1000) $ do propsForInterpreter "idPToMem" snd (Just $ show . snd) $ pure . run . idPToMem - -getReplacedBy :: Member E.IdP r => SAML.IdPId -> Sem r (Maybe (Maybe SAML.IdPId)) -getReplacedBy idpid = fmap (view $ SAML.idpExtraInfo . IP.wiReplacedBy) <$> E.getConfig idpid - --- | All the constraints we need to generalize properties in this module. --- A regular type synonym doesn't work due to dreaded impredicative --- polymorphism. -class - (Functor f, Member E.IdP r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => - PropConstraints r f - -instance - (Functor f, Member E.IdP r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => - PropConstraints r f - -prop_storeStore :: - PropConstraints r f => - Maybe (f () -> String) -> - (forall x. Sem r x -> IO (f x)) -> - Property -prop_storeStore = - prepropLaw @'[E.IdP] $ do - s <- arbitrary - s' <- arbitrary - pure $ - Law - { lawLhs = do - E.storeConfig $ s & SAML.idpId .~ s' ^. SAML.idpId - E.storeConfig s', - lawRhs = do - E.storeConfig s', - lawPrelude = [], - lawPostlude = [E.getConfig $ s' ^. SAML.idpId] - } - -prop_storeStoreInterleave :: - PropConstraints r f => - Maybe (f () -> String) -> - (forall x. Sem r x -> IO (f x)) -> - Property -prop_storeStoreInterleave = - prepropLaw @'[E.IdP] $ do - s <- arbitrary - s' <- arbitrary - !_ <- - when (s ^. SAML.idpId == s' ^. SAML.idpId) discard - pure $ - Law - { lawLhs = do - E.storeConfig s - E.storeConfig s', - lawRhs = do - E.storeConfig s' - E.storeConfig s, - lawPrelude = [], - lawPostlude = [E.getConfig $ s ^. SAML.idpId, E.getConfig $ s' ^. SAML.idpId] - } - -prop_storeGet :: - PropConstraints r f => - Maybe (f (Maybe IP.IdP) -> String) -> - (forall x. Sem r x -> IO (f x)) -> - Property -prop_storeGet = - prepropLaw @'[E.IdP] $ - do - s <- arbitrary - pure $ - simpleLaw - ( do - E.storeConfig s - E.getConfig $ s ^. idpId - ) - ( do - E.storeConfig s - pure (Just s) - ) - -prop_deleteGet :: - PropConstraints r f => - Maybe (f (Maybe IP.IdP) -> String) -> - (forall x. Sem r x -> IO (f x)) -> - Property -prop_deleteGet = - prepropLaw @'[E.IdP] $ do - s <- arbitrary - pure $ - Law - { lawLhs = do - E.deleteConfig s - E.getConfig $ s ^. SAML.idpId, - lawRhs = do - E.deleteConfig s - pure Nothing, - lawPrelude = - [ E.storeConfig s - ], - lawPostlude = [] :: [Sem r ()] - } - -prop_deleteDelete :: - PropConstraints r f => - Maybe (f () -> String) -> - (forall x. Sem r x -> IO (f x)) -> - Property -prop_deleteDelete = - prepropLaw @'[E.IdP] $ do - s <- arbitrary - pure $ - simpleLaw - ( do - E.deleteConfig s - E.deleteConfig s - ) - ( do - E.deleteConfig s - ) - -prop_storeGetByIssuer :: - PropConstraints r f => - Maybe (f (E.GetIdPResult IdPId) -> String) -> - (forall x. Sem r x -> IO (f x)) -> - Property -prop_storeGetByIssuer = - prepropLaw @'[E.IdP] $ - do - s <- arbitrary - pure $ - simpleLaw - ( do - E.storeConfig s - E.getIdByIssuerWithoutTeam $ s ^. idpMetadata . edIssuer - ) - ( do - E.storeConfig s - pure $ E.GetIdPFound $ s ^. idpId - ) - -prop_setClear :: - PropConstraints r f => - Maybe (f (Maybe (Maybe IdPId)) -> String) -> - (forall x. Sem r x -> IO (f x)) -> - Property -prop_setClear = - prepropLaw @'[E.IdP] $ - do - idp <- arbitrary - replaced_id <- arbitrary - let replaced = E.Replaced replaced_id - replacing <- arbitrary - pure $ - Law - { lawLhs = do - E.setReplacedBy replaced replacing - E.clearReplacedBy replaced - getReplacedBy replaced_id, - lawRhs = do - E.clearReplacedBy replaced - getReplacedBy replaced_id, - lawPrelude = - [ E.storeConfig $ idp & SAML.idpId .~ replaced_id - ], - lawPostlude = [] @(Sem _ ()) - } - -prop_getGet :: - forall r f. - PropConstraints r f => - Maybe (f (Maybe IP.IdP, Maybe IP.IdP) -> String) -> - (forall x. Sem r x -> IO (f x)) -> - Property -prop_getGet = - prepropLaw @'[E.IdP] $ - do - idpid <- arbitrary - idp <- arbitrary - pure $ - Law - { lawLhs = do - liftA2 (,) (E.getConfig idpid) (E.getConfig idpid), - lawRhs = do - cfg <- E.getConfig idpid - pure (cfg, cfg), - lawPrelude = - [ E.storeConfig $ idp & SAML.idpId .~ idpid - ], - lawPostlude = [] :: [Sem r ()] - } - -prop_getStore :: - PropConstraints r f => - Maybe (f (Maybe IP.IdP) -> String) -> - (forall x. Sem r x -> IO (f x)) -> - Property -prop_getStore = - prepropLaw @'[E.IdP] $ - do - idpid <- arbitrary - s <- arbitrary - let s' = s & SAML.idpId .~ idpid - pure $ - Law - { lawLhs = do - r <- E.getConfig idpid - maybe (pure ()) E.storeConfig r - pure r, - lawRhs = do - E.getConfig idpid, - lawPrelude = - [E.storeConfig s'], - lawPostlude = - [E.getConfig idpid] - } - -prop_setSet :: - PropConstraints r f => - Maybe (f (Maybe (Maybe IdPId)) -> String) -> - (forall x. Sem r x -> IO (f x)) -> - Property -prop_setSet = - prepropLaw @'[E.IdP] $ - do - replaced_id <- arbitrary - s <- arbitrary - let s' = s & SAML.idpId .~ replaced_id - let replaced = E.Replaced replaced_id - replacing <- arbitrary - replacing' <- arbitrary - pure $ - Law - { lawLhs = do - E.setReplacedBy replaced replacing - E.setReplacedBy replaced replacing' - getReplacedBy replaced_id, - lawRhs = do - E.setReplacedBy replaced replacing' - getReplacedBy replaced_id, - lawPrelude = - [E.storeConfig s'], - lawPostlude = [] @(Sem _ ()) - } - -prop_setGet :: - PropConstraints r f => - Maybe (f (Maybe (Maybe IdPId)) -> String) -> - (forall x. Sem r x -> IO (f x)) -> - Property -prop_setGet = - prepropLaw @'[E.IdP] $ - do - idp <- arbitrary - replaced_id <- arbitrary - let replaced = E.Replaced replaced_id - replacing_id <- arbitrary - let replacing = E.Replacing replacing_id - pure $ - Law - { lawLhs = do - E.setReplacedBy replaced replacing - getReplacedBy replaced_id, - lawRhs = do - E.setReplacedBy replaced replacing - (Just replacing_id <$) <$> E.getConfig replaced_id, - lawPrelude = - [ E.storeConfig $ idp & SAML.idpId .~ replaced_id - ], - lawPostlude = [] :: [Sem r ()] - } diff --git a/services/spar/test/Test/Spar/Sem/NowSpec.hs b/services/spar/test/Test/Spar/Sem/NowSpec.hs index 0c9fdacadfb..b31dae84d42 100644 --- a/services/spar/test/Test/Spar/Sem/NowSpec.hs +++ b/services/spar/test/Test/Spar/Sem/NowSpec.hs @@ -9,26 +9,13 @@ import Data.Time import Data.Time.Calendar.Julian import Imports import Polysemy -import Polysemy.Check import Polysemy.Input import SAML2.WebSSO.Types -import qualified Spar.Sem.Now as E import Spar.Sem.Now.IO import Spar.Sem.Now.Input +import Spar.Sem.Now.Spec import Test.Hspec import Test.Hspec.QuickCheck -import Test.QuickCheck - -deriveGenericK ''E.Now - -propsForInterpreter :: - PropConstraints r f => - String -> - (forall a. Sem r a -> IO (f a)) -> - Spec -propsForInterpreter interpreter lower = do - describe interpreter $ do - prop "now/now" $ prop_nowNow Nothing lower someTime :: Time someTime = Time (UTCTime (fromJulianYearAndDay 1990 209) (secondsToDiffTime 0)) @@ -38,33 +25,3 @@ spec = do modifyMaxSuccess (const 1000) $ do propsForInterpreter "nowToIO" $ fmap Identity . runM . nowToIO . runInputConst () propsForInterpreter "nowToInput" $ pure . Identity . run . runInputConst someTime . nowToInput . runInputConst () - --- | All the constraints we need to generalize properties in this module. --- A regular type synonym doesn't work due to dreaded impredicative --- polymorphism. -class - (Functor f, Member E.Now r, Member (Input ()) r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => - PropConstraints r f - -instance - (Functor f, Member E.Now r, Member (Input ()) r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => - PropConstraints r f - -prop_nowNow :: - PropConstraints r f => - Maybe (f Bool -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_nowNow = - -- NOTE: This @Input ()@ effect is a workaround to an oversight in - -- @polysemy-check@. 'prepropLaw' wants to synthesize some actions to run - -- before and after its generators, and check their results for equality. We - -- can't use 'Now' as this effect, because 'E.get' won't return equivalent - -- results! And we can't keep it empty, because that triggers a crash in - -- @polysemy-check@. Thus @Input ()@, which isn't beautiful, but works fine. - prepropLaw @'[Input ()] $ do - pure $ - simpleLaw - (liftA2 (<=) E.get E.get) - ( pure True - ) diff --git a/services/spar/test/Test/Spar/Sem/ScimExternalIdStoreSpec.hs b/services/spar/test/Test/Spar/Sem/ScimExternalIdStoreSpec.hs index e6ccad58a4a..964eb9a74f6 100644 --- a/services/spar/test/Test/Spar/Sem/ScimExternalIdStoreSpec.hs +++ b/services/spar/test/Test/Spar/Sem/ScimExternalIdStoreSpec.hs @@ -1,192 +1,15 @@ -{-# LANGUAGE QuantifiedConstraints #-} -{-# OPTIONS_GHC -Wno-orphans #-} {-# OPTIONS_GHC -fplugin=Polysemy.Plugin #-} module Test.Spar.Sem.ScimExternalIdStoreSpec where import Arbitrary () -import Data.Id import Imports import Polysemy -import Polysemy.Check -import qualified Spar.Sem.ScimExternalIdStore as E import Spar.Sem.ScimExternalIdStore.Mem +import Spar.Sem.ScimExternalIdStore.Spec import Test.Hspec import Test.Hspec.QuickCheck -import Test.QuickCheck - -deriveGenericK ''E.ScimExternalIdStore - -propsForInterpreter :: - PropConstraints r f => - String -> - (forall a. f a -> a) -> - (forall a. Sem r a -> IO (f a)) -> - Spec -propsForInterpreter interpreter extract lower = do - describe interpreter $ do - prop "delete/delete" $ prop_deleteDelete Nothing lower - prop "delete/lookup" $ prop_deleteLookup (Just $ show . (() <$) . extract) lower - prop "delete/insert" $ prop_deleteInsert Nothing lower - prop "lookup/insert" $ prop_lookupInsert Nothing lower - prop "insert/delete" $ prop_insertDelete Nothing lower - prop "insert/lookup" $ prop_insertLookup (Just $ show . (() <$) . extract) lower - prop "insert/insert" $ prop_insertInsert (Just $ show . (() <$) . extract) lower spec :: Spec spec = modifyMaxSuccess (const 1000) $ do propsForInterpreter "scimExternalIdStoreToMem" snd $ pure . run . scimExternalIdStoreToMem - --- | All the constraints we need to generalize properties in this module. --- A regular type synonym doesn't work due to dreaded impredicative --- polymorphism. -class - (Functor f, Member E.ScimExternalIdStore r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => - PropConstraints r f - -instance - (Functor f, Member E.ScimExternalIdStore r, forall z. Show z => Show (f z), forall z. Eq z => Eq (f z)) => - PropConstraints r f - -prop_insertLookup :: - PropConstraints r f => - Maybe (f (Maybe UserId) -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_insertLookup = - prepropLaw @'[E.ScimExternalIdStore] $ do - tid <- arbitrary - email <- arbitrary - uid <- arbitrary - pure $ - simpleLaw - ( do - E.insert tid email uid - E.lookup tid email - ) - ( do - E.insert tid email uid - pure (Just uid) - ) - -prop_lookupInsert :: - PropConstraints r f => - Maybe (f () -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_lookupInsert = - prepropLaw @'[E.ScimExternalIdStore] $ do - tid <- arbitrary - email <- arbitrary - pure $ - simpleLaw - ( do - E.lookup tid email >>= maybe (pure ()) (E.insert tid email) - ) - ( do - pure () - ) - -prop_insertDelete :: - PropConstraints r f => - Maybe (f () -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_insertDelete = - prepropLaw @'[E.ScimExternalIdStore] $ do - tid <- arbitrary - email <- arbitrary - uid <- arbitrary - pure $ - simpleLaw - ( do - E.insert tid email uid - E.delete tid email - ) - ( do - E.delete tid email - ) - -prop_deleteInsert :: - PropConstraints r f => - Maybe (f () -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_deleteInsert = - prepropLaw @'[E.ScimExternalIdStore] $ do - tid <- arbitrary - email <- arbitrary - uid <- arbitrary - pure $ - simpleLaw - ( do - E.delete tid email - E.insert tid email uid - ) - ( do - E.insert tid email uid - ) - -prop_insertInsert :: - PropConstraints r f => - Maybe (f (Maybe UserId) -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_insertInsert = - prepropLaw @'[E.ScimExternalIdStore] $ do - tid <- arbitrary - email <- arbitrary - uid <- arbitrary - uid' <- arbitrary - pure $ - simpleLaw - ( do - E.insert tid email uid - E.insert tid email uid' - E.lookup tid email - ) - ( do - E.insert tid email uid' - E.lookup tid email - ) - -prop_deleteDelete :: - PropConstraints r f => - Maybe (f () -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_deleteDelete = - prepropLaw @'[E.ScimExternalIdStore] $ do - tid <- arbitrary - email <- arbitrary - pure $ - simpleLaw - ( do - E.delete tid email - E.delete tid email - ) - ( do - E.delete tid email - ) - -prop_deleteLookup :: - PropConstraints r f => - Maybe (f (Maybe UserId) -> String) -> - (forall a. Sem r a -> IO (f a)) -> - Property -prop_deleteLookup = - prepropLaw @'[E.ScimExternalIdStore] $ do - tid <- arbitrary - email <- arbitrary - uid <- arbitrary - pure $ - Law - { lawLhs = do - E.delete tid email - E.lookup tid email, - lawRhs = do - E.delete tid email - pure Nothing, - lawPrelude = [E.insert tid email uid], - lawPostlude = [] @(Sem _ ()) - } From 1e6843a9ef9730956adbdb575bdbeaff4ec3d054 Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Wed, 22 Dec 2021 08:30:26 +0100 Subject: [PATCH 26/74] Fix cabal install script (#2007) * Fix cabal install script The script should create the `dist` directory, if it does not exist. --- changelog.d/5-internal/fix-cabal-install | 1 + hack/bin/cabal-install-artefacts.sh | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 changelog.d/5-internal/fix-cabal-install diff --git a/changelog.d/5-internal/fix-cabal-install b/changelog.d/5-internal/fix-cabal-install new file mode 100644 index 00000000000..f09f1ea0b38 --- /dev/null +++ b/changelog.d/5-internal/fix-cabal-install @@ -0,0 +1 @@ +The `cabal-install-artefacts.sh` script now creates the `dist` directory if it does not exist diff --git a/hack/bin/cabal-install-artefacts.sh b/hack/bin/cabal-install-artefacts.sh index c80ef37de18..5307fd30931 100755 --- a/hack/bin/cabal-install-artefacts.sh +++ b/hack/bin/cabal-install-artefacts.sh @@ -6,6 +6,8 @@ TOP_LEVEL="$(cd "$DIR/../.." && pwd)" DIST="$TOP_LEVEL/dist" +mkdir -p "$DIST" + if [[ "$1" == "all" ]]; then pattern='*' else From b6311910b6f14cad3abdf3a168ee873caab591ec Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Thu, 16 Dec 2021 11:59:42 +0100 Subject: [PATCH 27/74] Add streaming support to MultiVerb Note that streaming is not yet supported for the client. Therefore, when a client for an endpoint that includes a streaming response is invoked, and a streaming response is received, the client will just fail. --- .../wire-api/src/Wire/API/ErrorDescription.hs | 22 +- .../wire-api/src/Wire/API/Routes/MultiVerb.hs | 194 +++++++++++++----- 2 files changed, 152 insertions(+), 64 deletions(-) diff --git a/libs/wire-api/src/Wire/API/ErrorDescription.hs b/libs/wire-api/src/Wire/API/ErrorDescription.hs index 3b3b194ca03..30f7ebf71ba 100644 --- a/libs/wire-api/src/Wire/API/ErrorDescription.hs +++ b/libs/wire-api/src/Wire/API/ErrorDescription.hs @@ -2,7 +2,6 @@ module Wire.API.ErrorDescription where import Control.Lens (at, (%~), (.~), (<>~), (?~)) import qualified Data.Aeson as A -import qualified Data.ByteString.Lazy as LBS import Data.Metrics.Servant import Data.SOP (I (..), NP (..), NS (..)) import Data.Schema @@ -13,10 +12,12 @@ import qualified Data.Text as Text import GHC.TypeLits (KnownSymbol, Symbol, natVal, symbolVal) import GHC.TypeNats (Nat) import Imports hiding (head) +import Network.HTTP.Types as HTTP import Servant hiding (Handler, addHeader, contentType, respond) import Servant.API (contentType) import Servant.API.ContentTypes (AllMimeRender, AllMimeUnrender) import Servant.API.Status (KnownStatus, statusVal) +import Servant.Client.Core import Servant.Swagger.Internal import Wire.API.Routes.MultiVerb @@ -113,6 +114,7 @@ instance IsResponse cs (ErrorDescription s label desc) where type ResponseStatus (ErrorDescription s label desc) = s + type ResponseBody (ErrorDescription s label desc) = LByteString responseRender = responseRender @cs @(RespondWithErrorDescription s label desc) responseUnrender = responseUnrender @cs @(RespondWithErrorDescription s label desc) @@ -160,18 +162,20 @@ instance IsResponse cs (EmptyErrorForLegacyReasons s desc) where type ResponseStatus (EmptyErrorForLegacyReasons s desc) = s + type ResponseBody (EmptyErrorForLegacyReasons s desc) = () responseRender _ () = pure $ - roAddContentType + addContentType (contentType (Proxy @PlainText)) - (RenderOutput (statusVal (Proxy @s)) mempty mempty) - - responseUnrender _ output = - guard - ( LBS.null (roBody output) - && roStatus output == statusVal (Proxy @s) - ) + Response + { responseStatusCode = statusVal (Proxy @s), + responseHeaders = mempty, + responseBody = (), + responseHttpVersion = HTTP.http11 + } + + responseUnrender _ output = guard (responseStatusCode output == statusVal (Proxy @s)) instance (KnownStatus s, KnownSymbol desc) => diff --git a/libs/wire-api/src/Wire/API/Routes/MultiVerb.hs b/libs/wire-api/src/Wire/API/Routes/MultiVerb.hs index 3c500512c88..0ce00f642ec 100644 --- a/libs/wire-api/src/Wire/API/Routes/MultiVerb.hs +++ b/libs/wire-api/src/Wire/API/Routes/MultiVerb.hs @@ -1,3 +1,6 @@ +{-# LANGUAGE ApplicativeDo #-} +{-# LANGUAGE RecordWildCards #-} + -- This file is part of the Wire Server implementation. -- -- Copyright (C) 2021 Wire Swiss GmbH @@ -20,6 +23,7 @@ module Wire.API.Routes.MultiVerb MultiVerb, Respond, RespondEmpty, + RespondStreaming, WithHeaders, DescHeader, AsHeaders (..), @@ -33,16 +37,15 @@ module Wire.API.Routes.MultiVerb IsResponse (..), IsSwaggerResponse (..), combineResponseSwagger, - RenderOutput (..), - roAddContentType, - roResponse, ResponseTypes, IsResponseList (..), + addContentType, ) where import Control.Applicative -import Control.Lens hiding (Context) +import Control.Lens hiding (Context, (<|)) +import Data.ByteString.Builder import qualified Data.ByteString.Lazy as LBS import qualified Data.CaseInsensitive as CI import Data.Containers.ListUtils @@ -51,16 +54,18 @@ import qualified Data.HashMap.Strict.InsOrd as InsOrdHashMap import Data.Metrics.Servant import Data.Proxy import Data.SOP +import Data.Sequence (Seq, (<|), pattern (:<|)) import qualified Data.Sequence as Seq import qualified Data.Swagger as S import qualified Data.Swagger.Declare as S import qualified Data.Text as Text import qualified Data.Text.Encoding as Text +import Data.Typeable import GHC.TypeLits import Generics.SOP as GSOP import Imports import qualified Network.HTTP.Media as M -import Network.HTTP.Types (HeaderName, hContentType) +import Network.HTTP.Types (hContentType) import qualified Network.HTTP.Types as HTTP import Network.HTTP.Types.Status import qualified Network.Wai as Wai @@ -73,6 +78,7 @@ import Servant.Server import Servant.Server.Internal import Servant.Swagger as S import Servant.Swagger.Internal as S +import Servant.Types.SourceT type Declare = S.Declare (S.Definitions S.Schema) @@ -86,11 +92,11 @@ data Respond (s :: Nat) (desc :: Symbol) (a :: *) -- Includes status code and description. data RespondEmpty (s :: Nat) (desc :: Symbol) -data RenderOutput = RenderOutput - { roStatus :: Status, - roBody :: LByteString, - roHeaders :: [(HeaderName, ByteString)] - } +-- | A type to describe a streaming 'MultiVerb' response. +-- +-- Includes status code, description, framing strategy and content type. Note +-- that the handler return type is hardcoded to be 'SourceIO ByteString'. +data RespondStreaming (s :: Nat) (desc :: Symbol) (framing :: *) (ct :: *) -- | The result of parsing a response as a union alternative of type 'a'. -- @@ -132,16 +138,18 @@ class IsSwaggerResponse a where type family ResponseType a :: * -class IsResponse cs a where +class IsWaiBody (ResponseBody a) => IsResponse cs a where type ResponseStatus a :: Nat + type ResponseBody a :: * - responseRender :: AcceptHeader -> ResponseType a -> Maybe RenderOutput - responseUnrender :: M.MediaType -> RenderOutput -> UnrenderResult (ResponseType a) + responseRender :: AcceptHeader -> ResponseType a -> Maybe (ResponseF (ResponseBody a)) + responseUnrender :: M.MediaType -> ResponseF (ResponseBody a) -> UnrenderResult (ResponseType a) type instance ResponseType (Respond s desc a) = a instance (AllMimeRender cs a, AllMimeUnrender cs a, KnownStatus s) => IsResponse cs (Respond s desc a) where type ResponseStatus (Respond s desc a) = s + type ResponseBody (Respond s desc a) = LByteString -- Note: here it seems like we are rendering for all possible content types, -- only to choose the correct one afterwards. However, render results besides the @@ -150,21 +158,22 @@ instance (AllMimeRender cs a, AllMimeUnrender cs a, KnownStatus s) => IsResponse responseRender (AcceptHeader acc) x = M.mapAcceptMedia (map (uncurry mkRenderOutput) (allMimeRender (Proxy @cs) x)) acc where - mkRenderOutput :: M.MediaType -> LByteString -> (M.MediaType, RenderOutput) + mkRenderOutput :: M.MediaType -> LByteString -> (M.MediaType, Response) mkRenderOutput c body = - (c,) . roAddContentType c $ - RenderOutput - { roStatus = statusVal (Proxy @s), - roBody = body, - roHeaders = [] + (c,) . addContentType c $ + Response + { responseStatusCode = statusVal (Proxy @s), + responseBody = body, + responseHeaders = mempty, + responseHttpVersion = HTTP.http11 } responseUnrender c output = do - guard (roStatus output == statusVal (Proxy @s)) + guard (responseStatusCode output == statusVal (Proxy @s)) let results = allMimeUnrender (Proxy @cs) case lookup c results of Nothing -> empty - Just f -> either UnrenderError UnrenderSuccess (f (roBody output)) + Just f -> either UnrenderError UnrenderSuccess (f (responseBody output)) instance (KnownStatus s, KnownSymbol desc, S.ToSchema a) => @@ -181,20 +190,19 @@ type instance ResponseType (RespondEmpty s desc) = () instance KnownStatus s => IsResponse cs (RespondEmpty s desc) where type ResponseStatus (RespondEmpty s desc) = s + type ResponseBody (RespondEmpty s desc) = () responseRender _ _ = - Just - RenderOutput - { roStatus = statusVal (Proxy @s), - roBody = mempty, - roHeaders = [] + Just $ + Response + { responseStatusCode = statusVal (Proxy @s), + responseBody = (), + responseHeaders = mempty, + responseHttpVersion = HTTP.http11 } responseUnrender _ output = - guard - ( roStatus output == statusVal (Proxy @s) - && LBS.null (roBody output) - ) + guard (responseStatusCode output == statusVal (Proxy @s)) instance (KnownStatus s, KnownSymbol desc) => IsSwaggerResponse (RespondEmpty s desc) where responseSwagger = @@ -202,6 +210,33 @@ instance (KnownStatus s, KnownSymbol desc) => IsSwaggerResponse (RespondEmpty s mempty & S.description .~ Text.pack (symbolVal (Proxy @desc)) +type instance ResponseType (RespondStreaming s desc framing ct) = SourceIO ByteString + +instance + (Accept ct, KnownStatus s) => + IsResponse cs (RespondStreaming s desc framing ct) + where + type ResponseStatus (RespondStreaming s desc framing ct) = s + type ResponseBody (RespondStreaming s desc framing ct) = SourceIO ByteString + responseRender _ x = + pure . addContentType (contentType (Proxy @ct)) $ + Response + { responseStatusCode = statusVal (Proxy @s), + responseBody = x, + responseHeaders = mempty, + responseHttpVersion = HTTP.http11 + } + + responseUnrender _ resp = do + guard (responseStatusCode resp == statusVal (Proxy @s)) + pure $ responseBody resp + +instance (KnownStatus s, KnownSymbol desc) => IsSwaggerResponse (RespondStreaming s desc framing ct) where + responseSwagger = + pure $ + mempty + & S.description .~ Text.pack (symbolVal (Proxy @desc)) + -- | This type adds response headers to a 'MultiVerb' response. -- -- Type variables: @@ -225,7 +260,7 @@ data DescHeader (name :: Symbol) (desc :: Symbol) (a :: *) class ServantHeaders hs xs | hs -> xs where constructHeaders :: NP I xs -> [HTTP.Header] - extractHeaders :: [HTTP.Header] -> Maybe (NP I xs) + extractHeaders :: Seq HTTP.Header -> Maybe (NP I xs) instance ServantHeaders '[] '[] where constructHeaders Nil = [] @@ -251,12 +286,14 @@ instance (headerName @name, toHeader x) : constructHeaders @hs xs + -- FUTUREWORK: should we concatenate all the matching headers instead of just + -- taking the first one? extractHeaders hs = do let name = headerName @name - (hs0, hs1) = partition (\(h, _) -> h == name) hs + (hs0, hs1) = Seq.partition (\(h, _) -> h == name) hs x <- case hs0 of - [] -> empty - ((_, h) : _) -> either (const empty) pure (parseHeader h) + Seq.Empty -> empty + ((_, h) :<| _) -> either (const empty) pure (parseHeader h) xs <- extractHeaders @hs hs1 pure (I x :* xs) @@ -286,15 +323,19 @@ instance IsResponse cs (WithHeaders hs a r) where type ResponseStatus (WithHeaders hs a r) = ResponseStatus r + type ResponseBody (WithHeaders hs a r) = ResponseBody r responseRender acc x = fmap addHeaders $ responseRender @cs @r acc y where (hs, y) = toHeaders @xs x - addHeaders r = r {roHeaders = roHeaders r ++ constructHeaders @hs hs} + addHeaders r = + r + { responseHeaders = responseHeaders r <> Seq.fromList (constructHeaders @hs hs) + } responseUnrender c output = do x <- responseUnrender @cs @r c output - case extractHeaders @hs (roHeaders output) of + case extractHeaders @hs (responseHeaders output) of Nothing -> UnrenderError "Failed to parse headers" Just hs -> pure $ fromHeaders @xs (hs, x) @@ -315,8 +356,8 @@ type family ResponseTypes (as :: [*]) where ResponseTypes (a ': as) = ResponseType a ': ResponseTypes as class IsResponseList cs as where - responseListRender :: AcceptHeader -> Union (ResponseTypes as) -> Maybe RenderOutput - responseListUnrender :: M.MediaType -> RenderOutput -> UnrenderResult (Union (ResponseTypes as)) + responseListRender :: AcceptHeader -> Union (ResponseTypes as) -> Maybe SomeResponse + responseListUnrender :: M.MediaType -> SomeResponse -> UnrenderResult (Union (ResponseTypes as)) responseListStatuses :: [Status] @@ -335,11 +376,11 @@ instance ) => IsResponseList cs (a ': as) where - responseListRender acc (Z (I x)) = responseRender @cs @a acc x + responseListRender acc (Z (I x)) = fmap SomeResponse (responseRender @cs @a acc x) responseListRender acc (S x) = responseListRender @cs @as acc x responseListUnrender c output = - Z . I <$> responseUnrender @cs @a c output + Z . I <$> (responseUnrender @cs @a c =<< fromSomeResponse output) <|> S <$> responseListUnrender @cs @as c output responseListStatuses = statusVal (Proxy @(ResponseStatus a)) : responseListStatuses @cs @as @@ -580,11 +621,56 @@ instance cs = allMime (Proxy @cs) (defs, responses) = S.runDeclare (responseListSwagger @as) mempty -roResponse :: RenderOutput -> Wai.Response -roResponse ro = Wai.responseLBS (roStatus ro) (roHeaders ro) (roBody ro) +class Typeable a => IsWaiBody a where + responseToWai :: ResponseF a -> Wai.Response + +instance IsWaiBody LByteString where + responseToWai r = + Wai.responseLBS + (responseStatusCode r) + (toList (responseHeaders r)) + (responseBody r) + +instance IsWaiBody () where + responseToWai r = + Wai.responseLBS + (responseStatusCode r) + (toList (responseHeaders r)) + mempty + +instance IsWaiBody (SourceIO ByteString) where + responseToWai r = + Wai.responseStream + (responseStatusCode r) + (toList (responseHeaders r)) + $ \output flush -> do + foreach + (const (pure ())) + (\chunk -> output (byteString chunk) *> flush) + (responseBody r) + +data SomeResponse = forall a. IsWaiBody a => SomeResponse (ResponseF a) + +addContentType :: M.MediaType -> ResponseF a -> ResponseF a +addContentType c r = r {responseHeaders = (hContentType, M.renderHeader c) <| responseHeaders r} -roAddContentType :: M.MediaType -> RenderOutput -> RenderOutput -roAddContentType c ro = ro {roHeaders = (hContentType, M.renderHeader c) : roHeaders ro} +setEmptyBody :: SomeResponse -> SomeResponse +setEmptyBody (SomeResponse r) = SomeResponse (go r) + where + go :: ResponseF a -> ResponseF LByteString + go Response {..} = Response {responseBody = mempty, ..} + +someResponseToWai :: SomeResponse -> Wai.Response +someResponseToWai (SomeResponse r) = responseToWai r + +fromSomeResponse :: (Alternative m, Typeable a) => SomeResponse -> m (ResponseF a) +fromSomeResponse (SomeResponse Response {..}) = do + body <- maybe empty pure $ cast responseBody + pure $ + Response + { responseBody = body, + .. + } instance (AllMime cs, IsResponseList cs as, AsUnion as r, ReflectMethod method) => @@ -607,12 +693,11 @@ instance `addAcceptCheck` acceptCheck (Proxy @cs) acc runAction action' env req k $ \output -> do let mresp = responseListRender @cs @as acc (toUnion @as output) - resp' <- case mresp of + someResponseToWai <$> case mresp of Nothing -> FailFatal err406 Just resp - | allowedMethodHead method req -> pure $ resp {roBody = mempty} + | allowedMethodHead method req -> pure (setEmptyBody resp) | otherwise -> pure resp - pure (roResponse resp') where method = reflectMethod (Proxy @method) @@ -647,16 +732,15 @@ instance } c <- getResponseContentType response - let output = - RenderOutput - { roBody = responseBody response, - roHeaders = toList (responseHeaders response), - roStatus = responseStatusCode response - } - unless (any (M.matches c) accept) $ do throwClientError $ UnsupportedContentType c response - case responseListUnrender @cs @as c output of + + -- FUTUREWORK: support streaming + let sresp = + if LBS.null (responseBody response) + then SomeResponse response {responseBody = ()} + else SomeResponse response + case responseListUnrender @cs @as c sresp of StatusMismatch -> throwClientError (DecodeFailure "Status mismatch" response) UnrenderError e -> throwClientError (DecodeFailure (Text.pack e) response) UnrenderSuccess x -> pure (fromUnion @as x) From 3e4d728dc11a77f82154919ceeca33c35606d20d Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Thu, 16 Dec 2021 13:19:18 +0100 Subject: [PATCH 28/74] Implement qualified asset download endpoint --- libs/wire-api/src/Wire/API/Asset.hs | 45 +++++++++++++++ .../src/Wire/API/Routes/Public/Cargohold.hs | 57 ++++++++++++------- services/cargohold/cargohold.cabal | 3 +- .../cargohold/src/CargoHold/API/Public.hs | 18 +++++- services/cargohold/src/CargoHold/API/Util.hs | 27 +++++++++ 5 files changed, 127 insertions(+), 23 deletions(-) create mode 100644 services/cargohold/src/CargoHold/API/Util.hs diff --git a/libs/wire-api/src/Wire/API/Asset.hs b/libs/wire-api/src/Wire/API/Asset.hs index 94522eba44e..e2a3e4de056 100644 --- a/libs/wire-api/src/Wire/API/Asset.hs +++ b/libs/wire-api/src/Wire/API/Asset.hs @@ -1,3 +1,5 @@ +{-# LANGUAGE GeneralizedNewtypeDeriving #-} + -- This file is part of the Wire Server implementation. -- -- Copyright (C) 2020 Wire Swiss GmbH @@ -17,7 +19,50 @@ module Wire.API.Asset ( module V3, + AssetLocation (..), + LocalOrRemoteAsset (..), ) where +import Data.SOP +import qualified Data.Swagger as Swagger +import GHC.TypeLits +import Imports +import Servant import Wire.API.Asset.V3 as V3 +import Wire.API.ErrorDescription +import Wire.API.Routes.MultiVerb + +newtype AssetLocation = AssetLocation {getAssetLocation :: Text} + deriving newtype + ( ToHttpApiData, + FromHttpApiData, + Swagger.ToParamSchema + ) + +instance AsHeaders '[AssetLocation] Asset (Asset, AssetLocation) where + toHeaders (asset, loc) = (I loc :* Nil, asset) + fromHeaders (I loc :* Nil, asset) = (asset, loc) + +-- | An asset as returned by the download API: if the asset is local, only a +-- URL is returned, and if it is remote the content of the asset is streamed. +data LocalOrRemoteAsset + = LocalAsset AssetLocation + | RemoteAsset (SourceIO ByteString) + +instance + ( ResponseType r0 ~ ErrorDescription code label desc, + ResponseType r1 ~ AssetLocation, + ResponseType r2 ~ SourceIO ByteString, + KnownSymbol desc + ) => + AsUnion '[r0, r1, r2] (Maybe LocalOrRemoteAsset) + where + toUnion Nothing = Z (I mkErrorDescription) + toUnion (Just (LocalAsset loc)) = S (Z (I loc)) + toUnion (Just (RemoteAsset asset)) = S (S (Z (I asset))) + + fromUnion (Z (I _)) = Nothing + fromUnion (S (Z (I loc))) = Just (LocalAsset loc) + fromUnion (S (S (Z (I asset)))) = Just (RemoteAsset asset) + fromUnion (S (S (S x))) = case x of diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs b/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs index f73dc764a2a..d21207b702f 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs @@ -33,6 +33,7 @@ import Wire.API.ErrorDescription import Wire.API.Routes.AssetBody import Wire.API.Routes.MultiVerb import Wire.API.Routes.Public +import Wire.API.Routes.QualifiedCapture data PrincipalTag = UserPrincipalTag | BotPrincipalTag | ProviderPrincipalTag deriving (Eq, Show) @@ -61,27 +62,24 @@ instance HasServer (ApplyPrincipalPath tag api) ctx => HasServer (tag :> api) ct instance RoutesToPaths (ApplyPrincipalPath tag api) => RoutesToPaths (tag :> api) where getRoutes = getRoutes @(ApplyPrincipalPath tag api) -newtype AssetLocation = AssetLocation {getAssetLocation :: Text} - deriving newtype - ( ToHttpApiData, - FromHttpApiData, - Swagger.ToParamSchema - ) +type AssetRedirect = + WithHeaders + '[DescHeader "Location" "Asset location" AssetLocation] + AssetLocation + (RespondEmpty 302 "Asset found") -instance AsHeaders '[AssetLocation] Asset (Asset, AssetLocation) where - toHeaders (asset, loc) = (I loc :* Nil, asset) - fromHeaders (I loc :* Nil, asset) = (asset, loc) +type AssetStreaming = + RespondStreaming + 200 + "Asset returned directly with content type `application/octet-stream`" + NoFraming + OctetStream type GetAsset = MultiVerb 'GET '[JSON] - '[ AssetNotFound, - WithHeaders - '[DescHeader "Location" "Asset location" AssetLocation] - AssetLocation - (RespondEmpty 302 "Asset found") - ] + '[AssetNotFound, AssetRedirect] (Maybe AssetLocation) type ServantAPI = @@ -108,13 +106,14 @@ type ServantAPI = '[RespondEmpty 200 "Asset token deleted"] () ) - :<|> BaseAPI 'UserPrincipalTag - :<|> BaseAPI 'BotPrincipalTag - :<|> BaseAPI 'ProviderPrincipalTag + :<|> BaseAPIv3 'UserPrincipalTag + :<|> BaseAPIv3 'BotPrincipalTag + :<|> BaseAPIv3 'ProviderPrincipalTag + :<|> QualifiedAPI :<|> LegacyAPI :<|> InternalAPI -type BaseAPI (tag :: PrincipalTag) = +type BaseAPIv3 (tag :: PrincipalTag) = ( Summary "Upload an asset" :> CanThrow AssetTooLarge :> CanThrow InvalidLength @@ -149,6 +148,26 @@ type BaseAPI (tag :: PrincipalTag) = () ) +type QualifiedAPI = + ( Summary "Download an asset" + :> Description + "**Note**: local assets result in a redirect, \ + \while remote assets are streamed directly." + :> ZLocalUser + :> "assets" + :> "v4" + :> QualifiedCapture "key" AssetKey + :> Header "Asset-Token" AssetToken + :> MultiVerb + 'GET + '[JSON] + '[ AssetNotFound, + AssetRedirect, + AssetStreaming + ] + (Maybe LocalOrRemoteAsset) + ) + type LegacyAPI = ( ZLocalUser :> "assets" diff --git a/services/cargohold/cargohold.cabal b/services/cargohold/cargohold.cabal index 778c25303a1..d254bedd2a6 100644 --- a/services/cargohold/cargohold.cabal +++ b/services/cargohold/cargohold.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: b8ba2ed196939ab2b035138d3e428c84b88ff77ff219741840e47ae8dac9de0a +-- hash: cd2a03f62587d5c87a0542f95b7124ff553f9ac6ec40dd1ea1cbfe2fd8ace570 name: cargohold version: 1.5.0 @@ -29,6 +29,7 @@ library CargoHold.API.Federation CargoHold.API.Legacy CargoHold.API.Public + CargoHold.API.Util CargoHold.API.V3 CargoHold.App CargoHold.AWS diff --git a/services/cargohold/src/CargoHold/API/Public.hs b/services/cargohold/src/CargoHold/API/Public.hs index 1c588ce99b0..913ec34c83c 100644 --- a/services/cargohold/src/CargoHold/API/Public.hs +++ b/services/cargohold/src/CargoHold/API/Public.hs @@ -18,6 +18,7 @@ module CargoHold.API.Public (servantSitemap) where import qualified CargoHold.API.Legacy as LegacyAPI +import CargoHold.API.Util import qualified CargoHold.API.V3 as V3 import CargoHold.App import qualified CargoHold.Types.V3 as V3 @@ -41,16 +42,18 @@ servantSitemap = :<|> userAPI :<|> botAPI :<|> providerAPI + :<|> qualifiedAPI :<|> legacyAPI :<|> internalAPI where - userAPI :: forall tag. tag ~ 'UserPrincipalTag => ServerT (BaseAPI tag) Handler + userAPI :: forall tag. tag ~ 'UserPrincipalTag => ServerT (BaseAPIv3 tag) Handler userAPI = uploadAssetV3 @tag :<|> downloadAssetV3 @tag :<|> deleteAssetV3 @tag - botAPI :: forall tag. tag ~ 'BotPrincipalTag => ServerT (BaseAPI tag) Handler + botAPI :: forall tag. tag ~ 'BotPrincipalTag => ServerT (BaseAPIv3 tag) Handler botAPI = uploadAssetV3 @tag :<|> downloadAssetV3 @tag :<|> deleteAssetV3 @tag - providerAPI :: forall tag. tag ~ 'ProviderPrincipalTag => ServerT (BaseAPI tag) Handler + providerAPI :: forall tag. tag ~ 'ProviderPrincipalTag => ServerT (BaseAPIv3 tag) Handler providerAPI = uploadAssetV3 @tag :<|> downloadAssetV3 @tag :<|> deleteAssetV3 @tag legacyAPI = legacyDownloadPlain :<|> legacyDownloadPlain :<|> legacyDownloadOtr + qualifiedAPI = downloadAssetV4 internalAPI = pure () class MakePrincipal (tag :: PrincipalTag) (id :: *) | id -> tag, tag -> id where @@ -91,6 +94,15 @@ downloadAssetV3 usr key tok1 tok2 = do url <- V3.download (mkPrincipal usr) key (tok1 <|> tok2) pure $ fmap (AssetLocation . Text.decodeUtf8With Text.lenientDecode . serializeURIRef') url +downloadAssetV4 :: + Local UserId -> + Qualified AssetKey -> + Maybe AssetToken -> + Handler (Maybe LocalOrRemoteAsset) +downloadAssetV4 usr qkey tok = do + key <- tUnqualified <$> ensureLocal usr qkey + LocalAsset <$$> downloadAssetV3 usr key tok + deleteAssetV3 :: MakePrincipal tag id => id -> AssetKey -> Handler () deleteAssetV3 usr key = V3.delete (mkPrincipal usr) key diff --git a/services/cargohold/src/CargoHold/API/Util.hs b/services/cargohold/src/CargoHold/API/Util.hs new file mode 100644 index 00000000000..d52407b847b --- /dev/null +++ b/services/cargohold/src/CargoHold/API/Util.hs @@ -0,0 +1,27 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2021 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module CargoHold.API.Util (ensureLocal) where + +import CargoHold.App +import Control.Error +import Data.Qualified +import Imports +import Wire.API.Federation.Error + +ensureLocal :: Local x -> Qualified a -> Handler (Local a) +ensureLocal loc = foldQualified loc pure (\_ -> throwE federationNotImplemented) From 254ec8f3aeaad4c7e005f6a666500f0393ed55ee Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Thu, 16 Dec 2021 14:00:01 +0100 Subject: [PATCH 29/74] Add qualified asset delete endpoint --- .../src/Wire/API/Routes/Public/Cargohold.hs | 14 ++++++++++++++ services/cargohold/src/CargoHold/API/Public.hs | 7 ++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs b/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs index d21207b702f..8d14b8a57c6 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs @@ -167,6 +167,20 @@ type QualifiedAPI = ] (Maybe LocalOrRemoteAsset) ) + :<|> ( Summary "Delete an asset" + :> Description "**Note**: only local assets can be deleted." + :> CanThrow AssetNotFound + :> CanThrow Unauthorised + :> ZLocalUser + :> "assets" + :> "v4" + :> QualifiedCapture "key" AssetKey + :> MultiVerb + 'DELETE + '[JSON] + '[RespondEmpty 200 "Asset deleted"] + () + ) type LegacyAPI = ( ZLocalUser diff --git a/services/cargohold/src/CargoHold/API/Public.hs b/services/cargohold/src/CargoHold/API/Public.hs index 913ec34c83c..212e6b84ef7 100644 --- a/services/cargohold/src/CargoHold/API/Public.hs +++ b/services/cargohold/src/CargoHold/API/Public.hs @@ -53,7 +53,7 @@ servantSitemap = providerAPI :: forall tag. tag ~ 'ProviderPrincipalTag => ServerT (BaseAPIv3 tag) Handler providerAPI = uploadAssetV3 @tag :<|> downloadAssetV3 @tag :<|> deleteAssetV3 @tag legacyAPI = legacyDownloadPlain :<|> legacyDownloadPlain :<|> legacyDownloadOtr - qualifiedAPI = downloadAssetV4 + qualifiedAPI = downloadAssetV4 :<|> deleteAssetV4 internalAPI = pure () class MakePrincipal (tag :: PrincipalTag) (id :: *) | id -> tag, tag -> id where @@ -106,6 +106,11 @@ downloadAssetV4 usr qkey tok = do deleteAssetV3 :: MakePrincipal tag id => id -> AssetKey -> Handler () deleteAssetV3 usr key = V3.delete (mkPrincipal usr) key +deleteAssetV4 :: Local UserId -> Qualified AssetKey -> Handler () +deleteAssetV4 usr qkey = do + key <- tUnqualified <$> ensureLocal usr qkey + V3.delete (mkPrincipal usr) key + renewTokenV3 :: Local UserId -> AssetKey -> Handler NewAssetToken renewTokenV3 (tUnqualified -> usr) key = NewAssetToken <$> V3.renewToken (V3.UserPrincipal usr) key From 5bfeed7877fc941275eee62ff0fd8162afdff0b3 Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Thu, 16 Dec 2021 15:25:12 +0100 Subject: [PATCH 30/74] Add domain field to newly created assets --- .../cargohold-types/src/CargoHold/Types/V3.hs | 2 +- libs/wire-api/src/Wire/API/Asset.hs | 333 ++++++++++++++++- libs/wire-api/src/Wire/API/Asset/V3.hs | 341 ------------------ libs/wire-api/wire-api.cabal | 3 +- .../brig/test/integration/API/User/Account.hs | 13 +- .../brig/test/integration/API/User/Util.hs | 10 +- .../cargohold/src/CargoHold/API/Public.hs | 10 +- services/cargohold/src/CargoHold/API/Util.hs | 18 +- services/cargohold/src/CargoHold/API/V3.hs | 9 +- services/cargohold/src/CargoHold/App.hs | 8 +- services/cargohold/test/integration/API/V3.hs | 13 +- 11 files changed, 390 insertions(+), 370 deletions(-) delete mode 100644 libs/wire-api/src/Wire/API/Asset/V3.hs diff --git a/libs/cargohold-types/src/CargoHold/Types/V3.hs b/libs/cargohold-types/src/CargoHold/Types/V3.hs index b4c3f2531ef..e0b8ee3fe1f 100644 --- a/libs/cargohold-types/src/CargoHold/Types/V3.hs +++ b/libs/cargohold-types/src/CargoHold/Types/V3.hs @@ -58,7 +58,7 @@ where import Data.ByteString.Conversion import Data.Id import Imports -import Wire.API.Asset.V3 +import Wire.API.Asset -------------------------------------------------------------------------------- -- Principal diff --git a/libs/wire-api/src/Wire/API/Asset.hs b/libs/wire-api/src/Wire/API/Asset.hs index e2a3e4de056..1e142ffe86c 100644 --- a/libs/wire-api/src/Wire/API/Asset.hs +++ b/libs/wire-api/src/Wire/API/Asset.hs @@ -1,4 +1,6 @@ {-# LANGUAGE GeneralizedNewtypeDeriving #-} +{-# LANGUAGE StrictData #-} +{-# LANGUAGE TemplateHaskell #-} -- This file is part of the Wire Server implementation. -- @@ -18,21 +20,345 @@ -- with this program. If not, see . module Wire.API.Asset - ( module V3, + ( -- * Asset + Asset, + Asset', + mkAsset, + assetKey, + assetExpires, + assetToken, + + -- * AssetKey + AssetKey (..), + + -- * AssetToken + AssetToken (..), + NewAssetToken (..), + + -- * Body Construction + buildMultipartBody, + beginMultipartBody, + endMultipartBody, + + -- * AssetHeaders + AssetHeaders (..), + mkHeaders, + + -- * AssetSettings + AssetSettings, + defAssetSettings, + setAssetPublic, + setAssetRetention, + AssetRetention (..), + assetRetentionSeconds, + assetExpiringSeconds, + assetVolatileSeconds, + retentionToTextRep, + + -- * Streaming AssetLocation (..), LocalOrRemoteAsset (..), ) where +import qualified Codec.MIME.Type as MIME +import Control.Lens (makeLenses, (?~)) +import Data.Aeson (FromJSON (..), ToJSON (..)) +import qualified Data.Aeson as Aeson +import Data.Attoparsec.ByteString.Char8 hiding (I) +import Data.Bifunctor +import Data.ByteString.Builder +import Data.ByteString.Conversion +import qualified Data.ByteString.Lazy as LBS +import Data.Id +import Data.Json.Util (UTCTimeMillis (fromUTCTimeMillis), toUTCTimeMillis) +import Data.Proxy +import Data.Qualified import Data.SOP +import Data.Schema +import qualified Data.Swagger as S import qualified Data.Swagger as Swagger +import qualified Data.Text as T +import Data.Text.Ascii (AsciiBase64Url) +import qualified Data.Text.Encoding as T +import Data.Time.Clock +import qualified Data.UUID as UUID import GHC.TypeLits import Imports import Servant -import Wire.API.Asset.V3 as V3 +import Wire.API.Arbitrary (Arbitrary (..), GenericUniform (..)) import Wire.API.ErrorDescription import Wire.API.Routes.MultiVerb +-------------------------------------------------------------------------------- +-- Asset + +type Asset = Asset' (Qualified AssetKey) + +-- | A newly uploaded asset. +data Asset' key = Asset + { _assetKey :: key, + _assetExpires :: Maybe UTCTime, + _assetToken :: Maybe AssetToken + } + deriving stock (Eq, Show, Generic, Functor) + +deriving via Schema (Asset' key) instance ToSchema (Asset' key) => (ToJSON (Asset' key)) + +deriving via Schema (Asset' key) instance ToSchema (Asset' key) => (FromJSON (Asset' key)) + +deriving via Schema (Asset' key) instance ToSchema (Asset' key) => (S.ToSchema (Asset' key)) + +-- Generate expiry time with millisecond precision +instance Arbitrary key => Arbitrary (Asset' key) where + arbitrary = Asset <$> arbitrary <*> (fmap milli <$> arbitrary) <*> arbitrary + where + milli = fromUTCTimeMillis . toUTCTimeMillis + +mkAsset :: key -> Asset' key +mkAsset k = Asset k Nothing Nothing + +instance ToSchema Asset where + schema = + object "Asset" $ + Asset + <$> _assetKey + .= ( Qualified + <$> qUnqualified .= field "key" schema + <*> qDomain .= field "domain" schema + ) + <*> (fmap toUTCTimeMillis . _assetExpires) + .= maybe_ + (optField "expires" (fromUTCTimeMillis <$> schema)) + <*> _assetToken .= maybe_ (optField "token" schema) + +-------------------------------------------------------------------------------- +-- AssetKey + +-- | A unique, versioned asset identifier. +-- Note: Can be turned into a sum type with additional constructors +-- for future versions. +data AssetKey = AssetKeyV3 AssetId AssetRetention + deriving stock (Eq, Show, Generic) + deriving (Arbitrary) via (GenericUniform AssetKey) + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema AssetKey) + +instance FromByteString AssetKey where + parser = do + v <- decimal + _ <- char '-' + case (v :: Word) of + 3 -> parseV3 + _ -> fail $ "Invalid asset version: " ++ show v + where + -- AssetKeyV3 ::= Retention "-" uuid + -- Retention ::= decimal + parseV3 = do + r <- parser + _ <- char '-' + b <- takeByteString + case UUID.fromASCIIBytes b of + Just i -> return $! AssetKeyV3 (Id i) r + Nothing -> fail "Invalid asset ID" + +instance ToByteString AssetKey where + builder (AssetKeyV3 i r) = + builder '3' + <> builder '-' + <> builder r + <> builder '-' + <> builder (UUID.toASCIIBytes (toUUID i)) + +instance ToSchema AssetKey where + schema = + (T.decodeUtf8 . toByteString') + .= parsedText "AssetKey" (runParser parser . T.encodeUtf8) + & doc' . S.schema . S.example ?~ toJSON ("3-1-47de4580-ae51-4650-acbb-d10c028cb0ac" :: Text) + +instance S.ToParamSchema AssetKey where + toParamSchema _ = S.toParamSchema (Proxy @Text) + +instance FromHttpApiData AssetKey where + parseUrlPiece = first T.pack . runParser parser . T.encodeUtf8 + +-------------------------------------------------------------------------------- +-- AssetToken + +-- | Asset tokens are bearer tokens that grant access to a single asset. +newtype AssetToken = AssetToken {assetTokenAscii :: AsciiBase64Url} + deriving stock (Eq, Show) + deriving newtype (FromByteString, ToByteString, Arbitrary) + deriving (FromJSON, ToJSON) via (Schema AssetToken) + +instance ToSchema AssetToken where + schema = + AssetToken <$> assetTokenAscii + .= schema + & doc' . S.schema . S.example ?~ toJSON ("aGVsbG8" :: Text) + +instance S.ToParamSchema AssetToken where + toParamSchema _ = S.toParamSchema (Proxy @Text) + +instance FromHttpApiData AssetToken where + parseUrlPiece = first T.pack . runParser parser . T.encodeUtf8 + +-- | A newly (re)generated token for an existing asset. +newtype NewAssetToken = NewAssetToken + {newAssetToken :: AssetToken} + deriving stock (Eq, Show) + deriving newtype (Arbitrary) + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema NewAssetToken) + +instance ToSchema NewAssetToken where + schema = + object "NewAssetToken" $ + NewAssetToken <$> newAssetToken .= field "token" schema + +-------------------------------------------------------------------------------- +-- Body Construction + +-- | Build a complete @multipart/mixed@ request body for a one-shot, +-- non-resumable asset upload. +buildMultipartBody :: AssetSettings -> MIME.Type -> LByteString -> Builder +buildMultipartBody sets typ bs = + let hdrs = mkHeaders typ bs + in beginMultipartBody sets hdrs <> lazyByteString bs <> endMultipartBody + +-- | Begin building a @multipart/mixed@ request body for a non-resumable upload. +-- The returned 'Builder' can be immediately followed by the actual asset bytes. +beginMultipartBody :: AssetSettings -> AssetHeaders -> Builder +beginMultipartBody sets (AssetHeaders t l) = + byteString + "--frontier\r\n\ + \Content-Type: application/json\r\n\ + \Content-Length: " + <> int64Dec (LBS.length settingsJson) + <> byteString + "\r\n\ + \\r\n" + <> lazyByteString settingsJson + <> byteString + "\r\n\ + \--frontier\r\n\ + \Content-Type: " + <> byteString (T.encodeUtf8 (MIME.showType t)) + <> byteString + "\r\n\ + \Content-Length: " + <> wordDec l + <> "\r\n\ + \\r\n" + where + settingsJson = Aeson.encode (schemaToJSON sets) + +-- | The trailer of a non-resumable @multipart/mixed@ request body initiated +-- via 'beginMultipartBody'. +endMultipartBody :: Builder +endMultipartBody = byteString "\r\n--frontier--\r\n" + +-------------------------------------------------------------------------------- +-- AssetHeaders + +-- | Headers provided during upload. +data AssetHeaders = AssetHeaders + { hdrType :: MIME.Type, + hdrLength :: Word + } + +mkHeaders :: MIME.Type -> LByteString -> AssetHeaders +mkHeaders t b = AssetHeaders t (fromIntegral (LBS.length b)) + +-------------------------------------------------------------------------------- +-- AssetSettings + +-- | Settings provided during upload. +data AssetSettings = AssetSettings + { _setAssetPublic :: Bool, + _setAssetRetention :: Maybe AssetRetention + } + deriving stock (Eq, Show, Generic) + deriving (Arbitrary) via (GenericUniform AssetSettings) + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema AssetSettings) + +defAssetSettings :: AssetSettings +defAssetSettings = AssetSettings False Nothing + +instance ToSchema AssetSettings where + schema = + object "AssetSettings" $ + AssetSettings + <$> _setAssetPublic .= (fromMaybe False <$> optField "public" schema) + <*> _setAssetRetention .= maybe_ (optField "retention" schema) + +-------------------------------------------------------------------------------- +-- AssetRetention + +-- | The desired asset retention. +data AssetRetention + = -- | The asset is retained indefinitely. Typically used + -- for profile pictures / assets frequently accessed. + AssetEternal + | -- | DEPRECATED: should not be used by clients for new assets + -- The asset is retained indefinitely. + AssetPersistent + | -- | The asset is retained for a short period of time. + AssetVolatile + | -- | The asset is retained indefinitely, storage is optimised + -- for infrequent access + AssetEternalInfrequentAccess + | -- | The asset is retained for an extended period of time, + -- but not indefinitely. + AssetExpiring + deriving stock (Eq, Show, Enum, Bounded, Generic) + deriving (Arbitrary) via (GenericUniform AssetRetention) + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema AssetRetention) + +-- | The minimum TTL in seconds corresponding to a chosen retention. +assetRetentionSeconds :: AssetRetention -> Maybe NominalDiffTime +assetRetentionSeconds AssetEternal = Nothing +assetRetentionSeconds AssetPersistent = Nothing +assetRetentionSeconds AssetVolatile = Just assetVolatileSeconds +assetRetentionSeconds AssetEternalInfrequentAccess = Nothing +assetRetentionSeconds AssetExpiring = Just assetExpiringSeconds + +assetVolatileSeconds :: NominalDiffTime +assetVolatileSeconds = 28 * 24 * 3600 -- 28 days + +assetExpiringSeconds :: NominalDiffTime +assetExpiringSeconds = 365 * 24 * 3600 -- 365 days + +instance ToByteString AssetRetention where + builder AssetEternal = builder '1' + builder AssetPersistent = builder '2' + builder AssetVolatile = builder '3' + builder AssetEternalInfrequentAccess = builder '4' + builder AssetExpiring = builder '5' + +-- | ByteString representation is used in AssetKey +instance FromByteString AssetRetention where + parser = + decimal >>= \d -> case (d :: Word) of + 1 -> return AssetEternal + 2 -> return AssetPersistent + 3 -> return AssetVolatile + 4 -> return AssetEternalInfrequentAccess + 5 -> return AssetExpiring + _ -> fail $ "Invalid asset retention: " ++ show d + +retentionToTextRep :: AssetRetention -> Text +retentionToTextRep AssetEternal = "eternal" +retentionToTextRep AssetPersistent = "persistent" +retentionToTextRep AssetVolatile = "volatile" +retentionToTextRep AssetEternalInfrequentAccess = "eternal-infrequent_access" +retentionToTextRep AssetExpiring = "expiring" + +instance ToSchema AssetRetention where + schema = + enum @Text "AssetRetention" $ + foldMap + (\value -> element (retentionToTextRep value) value) + [minBound .. maxBound] + newtype AssetLocation = AssetLocation {getAssetLocation :: Text} deriving newtype ( ToHttpApiData, @@ -66,3 +392,6 @@ instance fromUnion (S (Z (I loc))) = Just (LocalAsset loc) fromUnion (S (S (Z (I asset)))) = Just (RemoteAsset asset) fromUnion (S (S (S x))) = case x of + +makeLenses ''Asset' +makeLenses ''AssetSettings diff --git a/libs/wire-api/src/Wire/API/Asset/V3.hs b/libs/wire-api/src/Wire/API/Asset/V3.hs deleted file mode 100644 index 7890b020ab9..00000000000 --- a/libs/wire-api/src/Wire/API/Asset/V3.hs +++ /dev/null @@ -1,341 +0,0 @@ -{-# LANGUAGE GeneralizedNewtypeDeriving #-} -{-# LANGUAGE StrictData #-} -{-# LANGUAGE TemplateHaskell #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2020 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Wire.API.Asset.V3 - ( -- * Asset - Asset, - mkAsset, - assetKey, - assetExpires, - assetToken, - - -- * AssetKey - AssetKey (..), - - -- * AssetToken - AssetToken (..), - NewAssetToken (..), - - -- * Body Construction - buildMultipartBody, - beginMultipartBody, - endMultipartBody, - - -- * AssetHeaders - AssetHeaders (..), - mkHeaders, - - -- * AssetSettings - AssetSettings, - defAssetSettings, - setAssetPublic, - setAssetRetention, - AssetRetention (..), - assetRetentionSeconds, - assetExpiringSeconds, - assetVolatileSeconds, - retentionToTextRep, - ) -where - -import qualified Codec.MIME.Type as MIME -import Control.Lens (makeLenses, (?~)) -import Data.Aeson (FromJSON (..), ToJSON (..)) -import qualified Data.Aeson as Aeson -import Data.Attoparsec.ByteString.Char8 -import Data.Bifunctor -import Data.ByteString.Builder -import Data.ByteString.Conversion -import qualified Data.ByteString.Lazy as LBS -import Data.Id -import Data.Json.Util (UTCTimeMillis (fromUTCTimeMillis), toUTCTimeMillis) -import Data.Proxy -import Data.Schema -import qualified Data.Swagger as S -import qualified Data.Text as T -import Data.Text.Ascii (AsciiBase64Url) -import qualified Data.Text.Encoding as T -import Data.Time.Clock -import qualified Data.UUID as UUID -import Imports -import Servant -import Wire.API.Arbitrary (Arbitrary (..), GenericUniform (..)) - --------------------------------------------------------------------------------- --- Asset - --- | A newly uploaded asset. -data Asset = Asset - { _assetKey :: AssetKey, - _assetExpires :: Maybe UTCTime, - _assetToken :: Maybe AssetToken - } - deriving stock (Eq, Show, Generic) - deriving (FromJSON, ToJSON, S.ToSchema) via Schema Asset - --- Generate expiry time with millisecond precision -instance Arbitrary Asset where - arbitrary = Asset <$> arbitrary <*> (fmap milli <$> arbitrary) <*> arbitrary - where - milli = fromUTCTimeMillis . toUTCTimeMillis - -mkAsset :: AssetKey -> Asset -mkAsset k = Asset k Nothing Nothing - -instance ToSchema Asset where - schema = - object "Asset" $ - Asset - <$> _assetKey .= field "key" schema - <*> (fmap toUTCTimeMillis . _assetExpires) - .= maybe_ - (optField "expires" (fromUTCTimeMillis <$> schema)) - <*> _assetToken .= maybe_ (optField "token" schema) - --------------------------------------------------------------------------------- --- AssetKey - --- | A unique, versioned asset identifier. --- Note: Can be turned into a sum type with additional constructors --- for future versions. -data AssetKey = AssetKeyV3 AssetId AssetRetention - deriving stock (Eq, Show, Generic) - deriving (Arbitrary) via (GenericUniform AssetKey) - deriving (FromJSON, ToJSON, S.ToSchema) via (Schema AssetKey) - -instance FromByteString AssetKey where - parser = do - v <- decimal - _ <- char '-' - case (v :: Word) of - 3 -> parseV3 - _ -> fail $ "Invalid asset version: " ++ show v - where - -- AssetKeyV3 ::= Retention "-" uuid - -- Retention ::= decimal - parseV3 = do - r <- parser - _ <- char '-' - b <- takeByteString - case UUID.fromASCIIBytes b of - Just i -> return $! AssetKeyV3 (Id i) r - Nothing -> fail "Invalid asset ID" - -instance ToByteString AssetKey where - builder (AssetKeyV3 i r) = - builder '3' - <> builder '-' - <> builder r - <> builder '-' - <> builder (UUID.toASCIIBytes (toUUID i)) - -instance ToSchema AssetKey where - schema = - (T.decodeUtf8 . toByteString') - .= parsedText "AssetKey" (runParser parser . T.encodeUtf8) - & doc' . S.schema . S.example ?~ toJSON ("3-1-47de4580-ae51-4650-acbb-d10c028cb0ac" :: Text) - -instance S.ToParamSchema AssetKey where - toParamSchema _ = S.toParamSchema (Proxy @Text) - -instance FromHttpApiData AssetKey where - parseUrlPiece = first T.pack . runParser parser . T.encodeUtf8 - --------------------------------------------------------------------------------- --- AssetToken - --- | Asset tokens are bearer tokens that grant access to a single asset. -newtype AssetToken = AssetToken {assetTokenAscii :: AsciiBase64Url} - deriving stock (Eq, Show) - deriving newtype (FromByteString, ToByteString, Arbitrary) - deriving (FromJSON, ToJSON) via (Schema AssetToken) - -instance ToSchema AssetToken where - schema = - AssetToken <$> assetTokenAscii - .= schema - & doc' . S.schema . S.example ?~ toJSON ("aGVsbG8" :: Text) - -instance S.ToParamSchema AssetToken where - toParamSchema _ = S.toParamSchema (Proxy @Text) - -instance FromHttpApiData AssetToken where - parseUrlPiece = first T.pack . runParser parser . T.encodeUtf8 - --- | A newly (re)generated token for an existing asset. -newtype NewAssetToken = NewAssetToken - {newAssetToken :: AssetToken} - deriving stock (Eq, Show) - deriving newtype (Arbitrary) - deriving (FromJSON, ToJSON, S.ToSchema) via (Schema NewAssetToken) - -instance ToSchema NewAssetToken where - schema = - object "NewAssetToken" $ - NewAssetToken <$> newAssetToken .= field "token" schema - --------------------------------------------------------------------------------- --- Body Construction - --- | Build a complete @multipart/mixed@ request body for a one-shot, --- non-resumable asset upload. -buildMultipartBody :: AssetSettings -> MIME.Type -> LByteString -> Builder -buildMultipartBody sets typ bs = - let hdrs = mkHeaders typ bs - in beginMultipartBody sets hdrs <> lazyByteString bs <> endMultipartBody - --- | Begin building a @multipart/mixed@ request body for a non-resumable upload. --- The returned 'Builder' can be immediately followed by the actual asset bytes. -beginMultipartBody :: AssetSettings -> AssetHeaders -> Builder -beginMultipartBody sets (AssetHeaders t l) = - byteString - "--frontier\r\n\ - \Content-Type: application/json\r\n\ - \Content-Length: " - <> int64Dec (LBS.length settingsJson) - <> byteString - "\r\n\ - \\r\n" - <> lazyByteString settingsJson - <> byteString - "\r\n\ - \--frontier\r\n\ - \Content-Type: " - <> byteString (T.encodeUtf8 (MIME.showType t)) - <> byteString - "\r\n\ - \Content-Length: " - <> wordDec l - <> "\r\n\ - \\r\n" - where - settingsJson = Aeson.encode (schemaToJSON sets) - --- | The trailer of a non-resumable @multipart/mixed@ request body initiated --- via 'beginMultipartBody'. -endMultipartBody :: Builder -endMultipartBody = byteString "\r\n--frontier--\r\n" - --------------------------------------------------------------------------------- --- AssetHeaders - --- | Headers provided during upload. -data AssetHeaders = AssetHeaders - { hdrType :: MIME.Type, - hdrLength :: Word - } - -mkHeaders :: MIME.Type -> LByteString -> AssetHeaders -mkHeaders t b = AssetHeaders t (fromIntegral (LBS.length b)) - --------------------------------------------------------------------------------- --- AssetSettings - --- | Settings provided during upload. -data AssetSettings = AssetSettings - { _setAssetPublic :: Bool, - _setAssetRetention :: Maybe AssetRetention - } - deriving stock (Eq, Show, Generic) - deriving (Arbitrary) via (GenericUniform AssetSettings) - deriving (FromJSON, ToJSON, S.ToSchema) via (Schema AssetSettings) - -defAssetSettings :: AssetSettings -defAssetSettings = AssetSettings False Nothing - -instance ToSchema AssetSettings where - schema = - object "AssetSettings" $ - AssetSettings - <$> _setAssetPublic .= (fromMaybe False <$> optField "public" schema) - <*> _setAssetRetention .= maybe_ (optField "retention" schema) - --------------------------------------------------------------------------------- --- AssetRetention - --- | The desired asset retention. -data AssetRetention - = -- | The asset is retained indefinitely. Typically used - -- for profile pictures / assets frequently accessed. - AssetEternal - | -- | DEPRECATED: should not be used by clients for new assets - -- The asset is retained indefinitely. - AssetPersistent - | -- | The asset is retained for a short period of time. - AssetVolatile - | -- | The asset is retained indefinitely, storage is optimised - -- for infrequent access - AssetEternalInfrequentAccess - | -- | The asset is retained for an extended period of time, - -- but not indefinitely. - AssetExpiring - deriving stock (Eq, Show, Enum, Bounded, Generic) - deriving (Arbitrary) via (GenericUniform AssetRetention) - deriving (FromJSON, ToJSON, S.ToSchema) via (Schema AssetRetention) - --- | The minimum TTL in seconds corresponding to a chosen retention. -assetRetentionSeconds :: AssetRetention -> Maybe NominalDiffTime -assetRetentionSeconds AssetEternal = Nothing -assetRetentionSeconds AssetPersistent = Nothing -assetRetentionSeconds AssetVolatile = Just assetVolatileSeconds -assetRetentionSeconds AssetEternalInfrequentAccess = Nothing -assetRetentionSeconds AssetExpiring = Just assetExpiringSeconds - -assetVolatileSeconds :: NominalDiffTime -assetVolatileSeconds = 28 * 24 * 3600 -- 28 days - -assetExpiringSeconds :: NominalDiffTime -assetExpiringSeconds = 365 * 24 * 3600 -- 365 days - -instance ToByteString AssetRetention where - builder AssetEternal = builder '1' - builder AssetPersistent = builder '2' - builder AssetVolatile = builder '3' - builder AssetEternalInfrequentAccess = builder '4' - builder AssetExpiring = builder '5' - --- | ByteString representation is used in AssetKey -instance FromByteString AssetRetention where - parser = - decimal >>= \d -> case (d :: Word) of - 1 -> return AssetEternal - 2 -> return AssetPersistent - 3 -> return AssetVolatile - 4 -> return AssetEternalInfrequentAccess - 5 -> return AssetExpiring - _ -> fail $ "Invalid asset retention: " ++ show d - -retentionToTextRep :: AssetRetention -> Text -retentionToTextRep AssetEternal = "eternal" -retentionToTextRep AssetPersistent = "persistent" -retentionToTextRep AssetVolatile = "volatile" -retentionToTextRep AssetEternalInfrequentAccess = "eternal-infrequent_access" -retentionToTextRep AssetExpiring = "expiring" - -instance ToSchema AssetRetention where - schema = - enum @Text "AssetRetention" $ - foldMap - (\value -> element (retentionToTextRep value) value) - [minBound .. maxBound] - -makeLenses ''Asset -makeLenses ''AssetSettings diff --git a/libs/wire-api/wire-api.cabal b/libs/wire-api/wire-api.cabal index f84b7fddf85..bc4d4c1a684 100644 --- a/libs/wire-api/wire-api.cabal +++ b/libs/wire-api/wire-api.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: 80606059441fcf0f725386078aef563bf04549ce657002575a6559ae37544964 +-- hash: f2aacb81c413e8753829ba09eb08d1d2524c973346739389dfeb4284d4a16ebd name: wire-api version: 0.1.0 @@ -21,7 +21,6 @@ library exposed-modules: Wire.API.Arbitrary Wire.API.Asset - Wire.API.Asset.V3 Wire.API.Call.Config Wire.API.Connection Wire.API.Conversation diff --git a/services/brig/test/integration/API/User/Account.hs b/services/brig/test/integration/API/User/Account.hs index 7c4959d9b92..935ef9e7af5 100644 --- a/services/brig/test/integration/API/User/Account.hs +++ b/services/brig/test/integration/API/User/Account.hs @@ -34,7 +34,6 @@ import Brig.Types import Brig.Types.Intra import Brig.Types.User.Auth hiding (user) import qualified Brig.Types.User.Auth as Auth -import qualified CargoHold.Types.V3 as CHV3 import Control.Arrow ((&&&)) import Control.Exception (throw) import Control.Lens (ix, preview, (^.), (^?)) @@ -79,6 +78,7 @@ import UnliftIO (mapConcurrently_) import Util as Util import Util.AWS as Util import Web.Cookie (parseSetCookie) +import qualified Wire.API.Asset as Asset import Wire.API.Federation.API.Brig (UserDeletedConnectionsNotification (..)) import qualified Wire.API.Federation.API.Brig as FedBrig import Wire.API.Federation.API.Common (EmptyResponse (EmptyResponse)) @@ -1321,8 +1321,13 @@ testDeleteWithProfilePic brig cargohold = do uid <- userId <$> createAnonUser "anon" brig ast <- uploadAsset cargohold uid "this is my profile pic" -- Ensure that the asset is there - downloadAsset cargohold uid (toByteString' (ast ^. CHV3.assetKey)) !!! const 200 === statusCode - let newAssets = Just [ImageAsset (T.decodeLatin1 $ toByteString' (ast ^. CHV3.assetKey)) (Just AssetComplete)] + downloadAsset cargohold uid (ast ^. Asset.assetKey) !!! const 200 === statusCode + let newAssets = + Just + [ ImageAsset + (T.decodeLatin1 $ toByteString' (qUnqualified (ast ^. Asset.assetKey))) + (Just AssetComplete) + ] userUpdate = UserUpdate Nothing Nothing newAssets Nothing update = RequestBodyLBS . encode $ userUpdate -- Update profile with the uploaded asset @@ -1330,7 +1335,7 @@ testDeleteWithProfilePic brig cargohold = do !!! const 200 === statusCode deleteUser uid Nothing brig !!! const 200 === statusCode -- Check that the asset gets deleted - downloadAsset cargohold uid (toByteString' (ast ^. CHV3.assetKey)) !!! const 404 === statusCode + downloadAsset cargohold uid (ast ^. Asset.assetKey) !!! const 404 === statusCode testDeleteWithRemotes :: Opt.Opts -> Brig -> Http () testDeleteWithRemotes opts brig = do diff --git a/services/brig/test/integration/API/User/Util.hs b/services/brig/test/integration/API/User/Util.hs index 6e8087c4961..665a4351e90 100644 --- a/services/brig/test/integration/API/User/Util.hs +++ b/services/brig/test/integration/API/User/Util.hs @@ -53,6 +53,7 @@ import Imports import qualified Test.Tasty.Cannon as WS import Test.Tasty.HUnit import Util +import Wire.API.Asset import qualified Wire.API.Event.Conversation as Conv import qualified Wire.API.Federation.API.Brig as F import Wire.API.Federation.Component @@ -426,11 +427,16 @@ uploadAsset c usr dat = do === statusCode responseJsonError rsp -downloadAsset :: CargoHold -> UserId -> ByteString -> (MonadIO m, MonadHttp m) => m (Response (Maybe LB.ByteString)) +downloadAsset :: + (MonadIO m, MonadHttp m) => + CargoHold -> + UserId -> + Qualified AssetKey -> + m (Response (Maybe LB.ByteString)) downloadAsset c usr ast = get ( c - . paths ["/assets/v3", ast] + . paths ["/assets/v4", toByteString' (qDomain ast), toByteString' (qUnqualified ast)] . zUser usr . zConn "conn" ) diff --git a/services/cargohold/src/CargoHold/API/Public.hs b/services/cargohold/src/CargoHold/API/Public.hs index 212e6b84ef7..50b80f595ca 100644 --- a/services/cargohold/src/CargoHold/API/Public.hs +++ b/services/cargohold/src/CargoHold/API/Public.hs @@ -76,12 +76,14 @@ uploadAssetV3 :: uploadAssetV3 pid req = do let principal = mkPrincipal pid asset <- V3.upload principal (getAssetSource req) - let key = Text.decodeUtf8With Text.lenientDecode (toByteString' (asset ^. assetKey)) + let key = + Text.decodeUtf8With Text.lenientDecode $ + toByteString' (tUnqualified (asset ^. assetKey)) let loc = case principal of V3.UserPrincipal {} -> "/assets/v3/" <> key V3.BotPrincipal {} -> "/bot/assets/" <> key V3.ProviderPrincipal {} -> "/provider/assets/" <> key - pure (asset, AssetLocation loc) + pure (fmap qUntagged asset, AssetLocation loc) downloadAssetV3 :: MakePrincipal tag id => @@ -100,7 +102,7 @@ downloadAssetV4 :: Maybe AssetToken -> Handler (Maybe LocalOrRemoteAsset) downloadAssetV4 usr qkey tok = do - key <- tUnqualified <$> ensureLocal usr qkey + key <- tUnqualified <$> ensureLocal qkey LocalAsset <$$> downloadAssetV3 usr key tok deleteAssetV3 :: MakePrincipal tag id => id -> AssetKey -> Handler () @@ -108,7 +110,7 @@ deleteAssetV3 usr key = V3.delete (mkPrincipal usr) key deleteAssetV4 :: Local UserId -> Qualified AssetKey -> Handler () deleteAssetV4 usr qkey = do - key <- tUnqualified <$> ensureLocal usr qkey + key <- tUnqualified <$> ensureLocal qkey V3.delete (mkPrincipal usr) key renewTokenV3 :: Local UserId -> AssetKey -> Handler NewAssetToken diff --git a/services/cargohold/src/CargoHold/API/Util.hs b/services/cargohold/src/CargoHold/API/Util.hs index d52407b847b..33c30e480ff 100644 --- a/services/cargohold/src/CargoHold/API/Util.hs +++ b/services/cargohold/src/CargoHold/API/Util.hs @@ -15,13 +15,25 @@ -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . -module CargoHold.API.Util (ensureLocal) where +module CargoHold.API.Util + ( ensureLocal, + qualifyLocal, + ) +where import CargoHold.App import Control.Error +import Control.Lens import Data.Qualified import Imports import Wire.API.Federation.Error -ensureLocal :: Local x -> Qualified a -> Handler (Local a) -ensureLocal loc = foldQualified loc pure (\_ -> throwE federationNotImplemented) +ensureLocal :: Qualified a -> Handler (Local a) +ensureLocal value = do + loc <- view localUnit + foldQualified loc pure (\_ -> throwE federationNotImplemented) value + +qualifyLocal :: a -> Handler (Local a) +qualifyLocal x = do + loc <- view localUnit + pure (qualifyAs loc x) diff --git a/services/cargohold/src/CargoHold/API/V3.hs b/services/cargohold/src/CargoHold/API/V3.hs index fcbe55d5530..07917d86b08 100644 --- a/services/cargohold/src/CargoHold/API/V3.hs +++ b/services/cargohold/src/CargoHold/API/V3.hs @@ -26,6 +26,7 @@ module CargoHold.API.V3 where import CargoHold.API.Error +import CargoHold.API.Util import CargoHold.App import qualified CargoHold.Metrics as Metrics import CargoHold.Options @@ -48,6 +49,7 @@ import Data.Conduit import qualified Data.Conduit.Attoparsec as Conduit import Data.Id import qualified Data.List as List +import Data.Qualified import qualified Data.Text.Ascii as Ascii import Data.Text.Encoding (decodeLatin1) import qualified Data.Text.Lazy as LT @@ -57,8 +59,9 @@ import Imports hiding (take) import Network.HTTP.Types.Header import Network.Wai.Utilities (Error (..)) import URI.ByteString +import Wire.API.Asset -upload :: V3.Principal -> ConduitM () ByteString (ResourceT IO) () -> Handler V3.Asset +upload :: V3.Principal -> ConduitM () ByteString (ResourceT IO) () -> Handler (Asset' (Local AssetKey)) upload own bdy = do (rsrc, sets) <- parseMetadata bdy assetSettings (src, hdrs) <- parseHeaders rsrc assetHeaders @@ -71,8 +74,8 @@ upload own bdy = do ast <- liftIO $ Id <$> nextRandom tok <- if sets ^. V3.setAssetPublic then return Nothing else Just <$> randToken let ret = fromMaybe V3.AssetPersistent (sets ^. V3.setAssetRetention) - let key = V3.AssetKeyV3 ast ret - void $ S3.uploadV3 own key hdrs tok src + key <- qualifyLocal (V3.AssetKeyV3 ast ret) + void $ S3.uploadV3 own (tUnqualified key) hdrs tok src Metrics.s3UploadOk Metrics.s3UploadSize cl expires <- case V3.assetRetentionSeconds ret of diff --git a/services/cargohold/src/CargoHold/App.hs b/services/cargohold/src/CargoHold/App.hs index 79aa9dddfa5..faeeb21b9fe 100644 --- a/services/cargohold/src/CargoHold/App.hs +++ b/services/cargohold/src/CargoHold/App.hs @@ -30,6 +30,7 @@ module CargoHold.App appLogger, requestId, settings, + localUnit, -- * App Monad AppT, @@ -56,6 +57,7 @@ import Control.Monad.Trans.Resource (ResourceT, runResourceT, transResourceT) import Data.Default (def) import Data.Metrics.Middleware (Metrics) import qualified Data.Metrics.Middleware as Metrics +import Data.Qualified import Imports hiding (log) import Network.HTTP.Client (ManagerSettings (..), requestHeaders, responseTimeoutMicro) import Network.HTTP.Client.OpenSSL @@ -75,7 +77,8 @@ data Env = Env _appLogger :: Logger, _httpManager :: Manager, _requestId :: RequestId, - _settings :: Opt.Settings + _settings :: Opt.Settings, + _localUnit :: Local () } makeLenses ''Env @@ -86,7 +89,8 @@ newEnv o = do lgr <- Log.mkLogger (o ^. optLogLevel) (o ^. optLogNetStrings) (o ^. optLogFormat) mgr <- initHttpManager (o ^. optAws . awsS3Compatibility) ama <- initAws (o ^. optAws) lgr mgr - return $ Env ama met lgr mgr def (o ^. optSettings) + let loc = toLocalUnsafe (o ^. optSettings . Opt.setFederationDomain) () + return $ Env ama met lgr mgr def (o ^. optSettings) loc initAws :: AWSOpts -> Logger -> Manager -> IO AWS.Env initAws o l m = diff --git a/services/cargohold/test/integration/API/V3.hs b/services/cargohold/test/integration/API/V3.hs index f3d2a780011..750cb2d0264 100644 --- a/services/cargohold/test/integration/API/V3.hs +++ b/services/cargohold/test/integration/API/V3.hs @@ -30,6 +30,7 @@ import qualified Data.ByteString.Char8 as C8 import Data.ByteString.Conversion import qualified Data.ByteString.Lazy as Lazy import Data.Id +import Data.Qualified import Data.Text.Encoding (decodeLatin1) import Data.Time.Clock import Data.Time.Format @@ -135,12 +136,12 @@ testSimpleTokens c = do get (c . path loc . zUser uid2 . queryItem "asset_token" "acb123" . noRedirect) !!! const 404 === statusCode -- Token renewal fails if not done by owner - post (c . paths ["assets", "v3", toByteString' key, "token"] . zUser uid2) !!! do + post (c . paths ["assets", "v3", toByteString' (qUnqualified key), "token"] . zUser uid2) !!! do const 403 === statusCode const (Just "unauthorised") === fmap label . responseJsonMaybe -- Token renewal succeeds if done by owner r2 <- - post (c . paths ["assets", "v3", toByteString' key, "token"] . zUser uid) + post (c . paths ["assets", "v3", toByteString' (qUnqualified key), "token"] . zUser uid) responseJsonMaybe r2 liftIO $ assertBool "token unchanged" (tok /= tok') @@ -166,11 +167,11 @@ testSimpleTokens c = do get (c . path loc . queryItem "asset_token" (toByteString' tok') . zUser uid2 . noRedirect) !!! const 302 === statusCode -- Delete Token fails if not done by owner - delete (c . paths ["assets", "v3", toByteString' key, "token"] . zUser uid2) !!! do + delete (c . paths ["assets", "v3", toByteString' (qUnqualified key), "token"] . zUser uid2) !!! do const 403 === statusCode const (Just "unauthorised") === fmap label . responseJsonMaybe -- Delete Token succeeds by owner - delete (c . paths ["assets", "v3", toByteString' key, "token"] . zUser uid) !!! do + delete (c . paths ["assets", "v3", toByteString' (qUnqualified key), "token"] . zUser uid) !!! do const 200 === statusCode const Nothing === responseBody -- Access without token from different user (asset is now "public") @@ -263,8 +264,8 @@ uploadRaw c usr bs = . content "multipart/mixed" . lbytes bs -deleteAsset :: CargoHold -> UserId -> V3.AssetKey -> Http (Response (Maybe Lazy.ByteString)) -deleteAsset c u k = delete $ c . zUser u . paths ["assets", "v3", toByteString' k] +deleteAsset :: CargoHold -> UserId -> Qualified V3.AssetKey -> Http (Response (Maybe Lazy.ByteString)) +deleteAsset c u k = delete $ c . zUser u . paths ["assets", "v3", toByteString' (qUnqualified k)] -- Utilities ------------------------------------------------------------------ From eb43734f788a12c21d86d54c35b6e80a92c4ff6b Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Thu, 16 Dec 2021 18:05:32 +0100 Subject: [PATCH 31/74] Fix Asset golden tests --- .../src/Wire/API/Routes/Public/Cargohold.hs | 2 - .../Wire/API/Golden/Generated/Asset_asset.hs | 61 ++-- .../test/golden/testObject_Asset_asset_1.json | 1 + .../golden/testObject_Asset_asset_10.json | 1 + .../golden/testObject_Asset_asset_11.json | 1 + .../golden/testObject_Asset_asset_12.json | 1 + .../golden/testObject_Asset_asset_13.json | 1 + .../golden/testObject_Asset_asset_14.json | 1 + .../golden/testObject_Asset_asset_15.json | 1 + .../golden/testObject_Asset_asset_16.json | 1 + .../golden/testObject_Asset_asset_17.json | 1 + .../golden/testObject_Asset_asset_18.json | 1 + .../golden/testObject_Asset_asset_19.json | 1 + .../test/golden/testObject_Asset_asset_2.json | 1 + .../golden/testObject_Asset_asset_20.json | 1 + .../test/golden/testObject_Asset_asset_3.json | 1 + .../test/golden/testObject_Asset_asset_4.json | 1 + .../test/golden/testObject_Asset_asset_5.json | 1 + .../test/golden/testObject_Asset_asset_6.json | 1 + .../test/golden/testObject_Asset_asset_7.json | 1 + .../test/golden/testObject_Asset_asset_8.json | 1 + .../test/golden/testObject_Asset_asset_9.json | 1 + .../testObject_UserIdentity_user_16.json | 8 - .../testObject_UserIdentity_user_5.json | 8 - .../testObject_UserIdentity_user_8.json | 8 - .../golden/testObject_UserSSOId_user_1.json | 4 - .../golden/testObject_UserSSOId_user_10.json | 4 - .../golden/testObject_UserSSOId_user_11.json | 3 - .../golden/testObject_UserSSOId_user_12.json | 4 - .../golden/testObject_UserSSOId_user_14.json | 4 - .../golden/testObject_UserSSOId_user_15.json | 3 - .../golden/testObject_UserSSOId_user_16.json | 3 - .../golden/testObject_UserSSOId_user_17.json | 3 - .../golden/testObject_UserSSOId_user_18.json | 3 - .../golden/testObject_UserSSOId_user_19.json | 3 - .../golden/testObject_UserSSOId_user_20.json | 4 - .../golden/testObject_UserSSOId_user_3.json | 4 - .../golden/testObject_UserSSOId_user_4.json | 4 - .../golden/testObject_UserSSOId_user_5.json | 4 - .../golden/testObject_UserSSOId_user_6.json | 4 - .../golden/testObject_UserSSOId_user_7.json | 4 - .../golden/testObject_UserSSOId_user_8.json | 4 - .../Test/Wire/API/Roundtrip/ByteString.hs | 8 +- services/cargohold/cargohold.cabal | 4 +- services/cargohold/package.yaml | 1 + services/cargohold/test/integration/API/V4.hs | 282 ++++++++++++++++++ .../lib/src/Network/Wire/Simulations.hs | 2 +- 47 files changed, 335 insertions(+), 131 deletions(-) delete mode 100644 libs/wire-api/test/golden/testObject_UserIdentity_user_16.json delete mode 100644 libs/wire-api/test/golden/testObject_UserIdentity_user_5.json delete mode 100644 libs/wire-api/test/golden/testObject_UserIdentity_user_8.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_1.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_10.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_11.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_12.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_14.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_15.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_16.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_17.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_18.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_19.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_20.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_3.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_4.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_5.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_6.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_7.json delete mode 100644 libs/wire-api/test/golden/testObject_UserSSOId_user_8.json create mode 100644 services/cargohold/test/integration/API/V4.hs diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs b/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs index 8d14b8a57c6..92df6173a9c 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs @@ -1,5 +1,3 @@ -{-# LANGUAGE GeneralizedNewtypeDeriving #-} - -- This file is part of the Wire Server implementation. -- -- Copyright (C) 2021 Wire Swiss GmbH diff --git a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/Asset_asset.hs b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/Asset_asset.hs index 9e74080103a..98da5bf2c18 100644 --- a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/Asset_asset.hs +++ b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/Asset_asset.hs @@ -19,44 +19,31 @@ module Test.Wire.API.Golden.Generated.Asset_asset where import Control.Lens ((.~)) +import Data.Domain import Data.Id (Id (Id)) +import Data.Qualified import Data.Text.Ascii (AsciiChars (validate)) import qualified Data.UUID as UUID (fromString) import Imports (Functor (fmap), Maybe (Just, Nothing), fromJust, fromRight, read, undefined, (&)) import Wire.API.Asset - ( Asset, - AssetKey (AssetKeyV3), - AssetRetention - ( AssetEternal, - AssetEternalInfrequentAccess, - AssetExpiring, - AssetPersistent, - AssetVolatile - ), - AssetToken (AssetToken, assetTokenAscii), - assetExpires, - assetToken, - mkAsset, - ) testObject_Asset_asset_1 :: Asset testObject_Asset_asset_1 = - ( mkAsset (AssetKeyV3 (Id (fromJust (UUID.fromString "0000004b-0000-0017-0000-003e00000033"))) AssetExpiring) - & assetExpires .~ (fmap read (Just "1864-04-30 15:58:55.452 UTC")) - & assetToken .~ Just (AssetToken {assetTokenAscii = (fromRight undefined (validate ("Kun4JaxR6QuASXywDhzx")))}) - ) + mkAsset (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "0000004b-0000-0017-0000-003e00000033"))) AssetExpiring) (Domain "example.com")) + & assetExpires .~ (fmap read (Just "1864-04-30 15:58:55.452 UTC")) + & assetToken .~ Just (AssetToken {assetTokenAscii = (fromRight undefined (validate ("Kun4JaxR6QuASXywDhzx")))}) testObject_Asset_asset_2 :: Asset testObject_Asset_asset_2 = ( mkAsset - (AssetKeyV3 (Id (fromJust (UUID.fromString "00000008-0000-006c-0000-001900000036"))) AssetEternalInfrequentAccess) + (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "00000008-0000-006c-0000-001900000036"))) AssetEternalInfrequentAccess) (Domain "example.com")) & assetExpires .~ (fmap read (Just "1864-06-04 17:39:43.924 UTC")) & assetToken .~ Just (AssetToken {assetTokenAscii = (fromRight undefined (validate ("mPuul678vuJVZ_u9lQ==")))}) ) testObject_Asset_asset_3 :: Asset testObject_Asset_asset_3 = - ( mkAsset (AssetKeyV3 (Id (fromJust (UUID.fromString "00000055-0000-0071-0000-002e00000020"))) AssetEternal) + ( mkAsset (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "00000055-0000-0071-0000-002e00000020"))) AssetEternal) (Domain "example.com")) & assetExpires .~ (fmap read (Just "1864-05-18 20:18:13.438 UTC")) & assetToken .~ Nothing ) @@ -64,49 +51,49 @@ testObject_Asset_asset_3 = testObject_Asset_asset_4 :: Asset testObject_Asset_asset_4 = ( mkAsset - (AssetKeyV3 (Id (fromJust (UUID.fromString "00000063-0000-0044-0000-003000000059"))) AssetEternalInfrequentAccess) + (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "00000063-0000-0044-0000-003000000059"))) AssetEternalInfrequentAccess) (Domain "example.com")) & assetExpires .~ (fmap read (Nothing)) & assetToken .~ Just (AssetToken {assetTokenAscii = (fromRight undefined (validate ("IRKruiPSiANiX1fL")))}) ) testObject_Asset_asset_5 :: Asset testObject_Asset_asset_5 = - ( mkAsset (AssetKeyV3 (Id (fromJust (UUID.fromString "00000019-0000-005b-0000-001d00000056"))) AssetVolatile) + ( mkAsset (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "00000019-0000-005b-0000-001d00000056"))) AssetVolatile) (Domain "example.com")) & assetExpires .~ (fmap read (Just "1864-05-11 14:38:25.874 UTC")) & assetToken .~ Just (AssetToken {assetTokenAscii = (fromRight undefined (validate ("BrbiaM1RxJlqjlqq7quuPSc=")))}) ) testObject_Asset_asset_6 :: Asset testObject_Asset_asset_6 = - ( mkAsset (AssetKeyV3 (Id (fromJust (UUID.fromString "0000000e-0000-0046-0000-00560000005e"))) AssetPersistent) + ( mkAsset (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "0000000e-0000-0046-0000-00560000005e"))) AssetPersistent) (Domain "example.com")) & assetExpires .~ (fmap read (Just "1864-05-25 01:19:16.676 UTC")) & assetToken .~ Nothing ) testObject_Asset_asset_7 :: Asset testObject_Asset_asset_7 = - ( mkAsset (AssetKeyV3 (Id (fromJust (UUID.fromString "00000013-0000-002e-0000-003000000042"))) AssetEternal) + ( mkAsset (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "00000013-0000-002e-0000-003000000042"))) AssetEternal) (Domain "example.com")) & assetExpires .~ (fmap read (Just "1864-04-14 08:45:43.05 UTC")) & assetToken .~ Just (AssetToken {assetTokenAscii = (fromRight undefined (validate ("_N9ERJGmbZtd6XlW_6O12bxuNe4=")))}) ) testObject_Asset_asset_8 :: Asset testObject_Asset_asset_8 = - ( mkAsset (AssetKeyV3 (Id (fromJust (UUID.fromString "00000073-0000-003e-0000-00120000000c"))) AssetEternal) + ( mkAsset (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "00000073-0000-003e-0000-00120000000c"))) AssetEternal) (Domain "example.com")) & assetExpires .~ (fmap read (Nothing)) & assetToken .~ Nothing ) testObject_Asset_asset_9 :: Asset testObject_Asset_asset_9 = - ( mkAsset (AssetKeyV3 (Id (fromJust (UUID.fromString "00000006-0000-004b-0000-004f00000025"))) AssetPersistent) + ( mkAsset (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "00000006-0000-004b-0000-004f00000025"))) AssetPersistent) (Domain "example.com")) & assetExpires .~ (fmap read (Just "1864-05-21 01:34:09.726 UTC")) & assetToken .~ Nothing ) testObject_Asset_asset_10 :: Asset testObject_Asset_asset_10 = - ( mkAsset (AssetKeyV3 (Id (fromJust (UUID.fromString "00000065-0000-0080-0000-003400000061"))) AssetEternal) + ( mkAsset (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "00000065-0000-0080-0000-003400000061"))) AssetEternal) (Domain "example.com")) & assetExpires .~ (fmap read (Nothing)) & assetToken .~ Nothing ) @@ -114,14 +101,14 @@ testObject_Asset_asset_10 = testObject_Asset_asset_11 :: Asset testObject_Asset_asset_11 = ( mkAsset - (AssetKeyV3 (Id (fromJust (UUID.fromString "00000014-0000-0077-0000-001e00000076"))) AssetEternalInfrequentAccess) + (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "00000014-0000-0077-0000-001e00000076"))) AssetEternalInfrequentAccess) (Domain "example.com")) & assetExpires .~ (fmap read (Just "1864-05-11 16:58:59.746 UTC")) & assetToken .~ Just (AssetToken {assetTokenAscii = (fromRight undefined (validate ("DnlRW9Q=")))}) ) testObject_Asset_asset_12 :: Asset testObject_Asset_asset_12 = - ( mkAsset (AssetKeyV3 (Id (fromJust (UUID.fromString "0000001d-0000-0076-0000-003800000021"))) AssetPersistent) + ( mkAsset (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "0000001d-0000-0076-0000-003800000021"))) AssetPersistent) (Domain "example.com")) & assetExpires .~ (fmap read (Nothing)) & assetToken .~ Nothing ) @@ -129,7 +116,7 @@ testObject_Asset_asset_12 = testObject_Asset_asset_13 :: Asset testObject_Asset_asset_13 = ( mkAsset - (AssetKeyV3 (Id (fromJust (UUID.fromString "00000030-0000-0036-0000-003c0000000a"))) AssetEternalInfrequentAccess) + (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "00000030-0000-0036-0000-003c0000000a"))) AssetEternalInfrequentAccess) (Domain "example.com")) & assetExpires .~ (fmap read (Just "1864-04-30 19:37:57.302 UTC")) & assetToken .~ Just (AssetToken {assetTokenAscii = (fromRight undefined (validate ("n7CJBcdOSKznRmOypWXsGfEE0g==")))}) ) @@ -137,42 +124,42 @@ testObject_Asset_asset_13 = testObject_Asset_asset_14 :: Asset testObject_Asset_asset_14 = ( mkAsset - (AssetKeyV3 (Id (fromJust (UUID.fromString "00000047-0000-0012-0000-005500000062"))) AssetEternalInfrequentAccess) + (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "00000047-0000-0012-0000-005500000062"))) AssetEternalInfrequentAccess) (Domain "example.com")) & assetExpires .~ (fmap read (Just "1864-05-06 09:09:55.146 UTC")) & assetToken .~ Just (AssetToken {assetTokenAscii = (fromRight undefined (validate ("LYfUg4qlMjw=")))}) ) testObject_Asset_asset_15 :: Asset testObject_Asset_asset_15 = - ( mkAsset (AssetKeyV3 (Id (fromJust (UUID.fromString "00000030-0000-0074-0000-00660000004c"))) AssetPersistent) + ( mkAsset (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "00000030-0000-0074-0000-00660000004c"))) AssetPersistent) (Domain "example.com")) & assetExpires .~ (fmap read (Nothing)) & assetToken .~ Nothing ) testObject_Asset_asset_16 :: Asset testObject_Asset_asset_16 = - ( mkAsset (AssetKeyV3 (Id (fromJust (UUID.fromString "00000048-0000-0051-0000-005d00000070"))) AssetVolatile) + ( mkAsset (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "00000048-0000-0051-0000-005d00000070"))) AssetVolatile) (Domain "example.com")) & assetExpires .~ (fmap read (Just "1864-05-04 02:19:12.52 UTC")) & assetToken .~ Nothing ) testObject_Asset_asset_17 :: Asset testObject_Asset_asset_17 = - ( mkAsset (AssetKeyV3 (Id (fromJust (UUID.fromString "00000017-0000-000d-0000-00680000003e"))) AssetPersistent) + ( mkAsset (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "00000017-0000-000d-0000-00680000003e"))) AssetPersistent) (Domain "example.com")) & assetExpires .~ (fmap read (Just "1864-04-09 17:00:39.763 UTC")) & assetToken .~ Nothing ) testObject_Asset_asset_18 :: Asset testObject_Asset_asset_18 = - ( mkAsset (AssetKeyV3 (Id (fromJust (UUID.fromString "0000003e-0000-0032-0000-004d00000070"))) AssetEternal) + ( mkAsset (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "0000003e-0000-0032-0000-004d00000070"))) AssetEternal) (Domain "example.com")) & assetExpires .~ (fmap read (Just "1864-04-12 20:53:21.25 UTC")) & assetToken .~ Nothing ) testObject_Asset_asset_19 :: Asset testObject_Asset_asset_19 = - ( mkAsset (AssetKeyV3 (Id (fromJust (UUID.fromString "00000021-0000-0062-0000-002a0000006b"))) AssetVolatile) + ( mkAsset (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "00000021-0000-0062-0000-002a0000006b"))) AssetVolatile) (Domain "example.com")) & assetExpires .~ (fmap read (Nothing)) & assetToken @@ -181,7 +168,7 @@ testObject_Asset_asset_19 = testObject_Asset_asset_20 :: Asset testObject_Asset_asset_20 = - ( mkAsset (AssetKeyV3 (Id (fromJust (UUID.fromString "00000053-0000-0072-0000-001700000047"))) AssetVolatile) + ( mkAsset (Qualified (AssetKeyV3 (Id (fromJust (UUID.fromString "00000053-0000-0072-0000-001700000047"))) AssetVolatile) (Domain "example.com")) & assetExpires .~ (fmap read (Just "1864-04-25 16:48:39.986 UTC")) & assetToken .~ Nothing ) diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_1.json b/libs/wire-api/test/golden/testObject_Asset_asset_1.json index 9bef9870da2..d4f078bbfb5 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_1.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_1.json @@ -1,4 +1,5 @@ { + "domain": "example.com", "expires": "1864-04-30T15:58:55.452Z", "key": "3-5-0000004b-0000-0017-0000-003e00000033", "token": "Kun4JaxR6QuASXywDhzx" diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_10.json b/libs/wire-api/test/golden/testObject_Asset_asset_10.json index 1d25e3b58cd..c495e0a86cb 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_10.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_10.json @@ -1,3 +1,4 @@ { + "domain": "example.com", "key": "3-1-00000065-0000-0080-0000-003400000061" } diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_11.json b/libs/wire-api/test/golden/testObject_Asset_asset_11.json index d6e74f8e6ee..2b6ee63fdd3 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_11.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_11.json @@ -1,4 +1,5 @@ { + "domain": "example.com", "expires": "1864-05-11T16:58:59.746Z", "key": "3-4-00000014-0000-0077-0000-001e00000076", "token": "DnlRW9Q=" diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_12.json b/libs/wire-api/test/golden/testObject_Asset_asset_12.json index eda1a4fdd6d..85bd8fd2660 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_12.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_12.json @@ -1,3 +1,4 @@ { + "domain": "example.com", "key": "3-2-0000001d-0000-0076-0000-003800000021" } diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_13.json b/libs/wire-api/test/golden/testObject_Asset_asset_13.json index 8e07a56197e..2ac23c24eb5 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_13.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_13.json @@ -1,4 +1,5 @@ { + "domain": "example.com", "expires": "1864-04-30T19:37:57.302Z", "key": "3-4-00000030-0000-0036-0000-003c0000000a", "token": "n7CJBcdOSKznRmOypWXsGfEE0g==" diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_14.json b/libs/wire-api/test/golden/testObject_Asset_asset_14.json index 442e5562469..a70a668bc8b 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_14.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_14.json @@ -1,4 +1,5 @@ { + "domain": "example.com", "expires": "1864-05-06T09:09:55.146Z", "key": "3-4-00000047-0000-0012-0000-005500000062", "token": "LYfUg4qlMjw=" diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_15.json b/libs/wire-api/test/golden/testObject_Asset_asset_15.json index f49cd13e46a..770067088cd 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_15.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_15.json @@ -1,3 +1,4 @@ { + "domain": "example.com", "key": "3-2-00000030-0000-0074-0000-00660000004c" } diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_16.json b/libs/wire-api/test/golden/testObject_Asset_asset_16.json index 69e6e5f1816..bf6597141cf 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_16.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_16.json @@ -1,4 +1,5 @@ { + "domain": "example.com", "expires": "1864-05-04T02:19:12.520Z", "key": "3-3-00000048-0000-0051-0000-005d00000070" } diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_17.json b/libs/wire-api/test/golden/testObject_Asset_asset_17.json index ccb77a2d1c5..5a4e7a4811f 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_17.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_17.json @@ -1,4 +1,5 @@ { + "domain": "example.com", "expires": "1864-04-09T17:00:39.763Z", "key": "3-2-00000017-0000-000d-0000-00680000003e" } diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_18.json b/libs/wire-api/test/golden/testObject_Asset_asset_18.json index 516f95363ff..8f02aeae56c 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_18.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_18.json @@ -1,4 +1,5 @@ { + "domain": "example.com", "expires": "1864-04-12T20:53:21.250Z", "key": "3-1-0000003e-0000-0032-0000-004d00000070" } diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_19.json b/libs/wire-api/test/golden/testObject_Asset_asset_19.json index 4b62e85e30c..c8ea25d227f 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_19.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_19.json @@ -1,4 +1,5 @@ { + "domain": "example.com", "key": "3-3-00000021-0000-0062-0000-002a0000006b", "token": "4wm3D03aqvZ_0oKFtwXCYnSTC7m_z1E=" } diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_2.json b/libs/wire-api/test/golden/testObject_Asset_asset_2.json index 3a8d556bd57..a4e0765c06c 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_2.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_2.json @@ -1,4 +1,5 @@ { + "domain": "example.com", "expires": "1864-06-04T17:39:43.924Z", "key": "3-4-00000008-0000-006c-0000-001900000036", "token": "mPuul678vuJVZ_u9lQ==" diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_20.json b/libs/wire-api/test/golden/testObject_Asset_asset_20.json index ee08bfe6d13..3cd958cedc2 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_20.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_20.json @@ -1,4 +1,5 @@ { + "domain": "example.com", "expires": "1864-04-25T16:48:39.986Z", "key": "3-3-00000053-0000-0072-0000-001700000047" } diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_3.json b/libs/wire-api/test/golden/testObject_Asset_asset_3.json index eb3537825f3..13b90b2b057 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_3.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_3.json @@ -1,4 +1,5 @@ { + "domain": "example.com", "expires": "1864-05-18T20:18:13.438Z", "key": "3-1-00000055-0000-0071-0000-002e00000020" } diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_4.json b/libs/wire-api/test/golden/testObject_Asset_asset_4.json index d43de9f1101..fc65d82c3c0 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_4.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_4.json @@ -1,4 +1,5 @@ { + "domain": "example.com", "key": "3-4-00000063-0000-0044-0000-003000000059", "token": "IRKruiPSiANiX1fL" } diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_5.json b/libs/wire-api/test/golden/testObject_Asset_asset_5.json index 0bd2857635e..37a8a6a8dc7 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_5.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_5.json @@ -1,4 +1,5 @@ { + "domain": "example.com", "expires": "1864-05-11T14:38:25.874Z", "key": "3-3-00000019-0000-005b-0000-001d00000056", "token": "BrbiaM1RxJlqjlqq7quuPSc=" diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_6.json b/libs/wire-api/test/golden/testObject_Asset_asset_6.json index 8d9571f2dc1..506b8af9ecc 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_6.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_6.json @@ -1,4 +1,5 @@ { + "domain": "example.com", "expires": "1864-05-25T01:19:16.676Z", "key": "3-2-0000000e-0000-0046-0000-00560000005e" } diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_7.json b/libs/wire-api/test/golden/testObject_Asset_asset_7.json index b97f270a807..5d9fd890b0b 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_7.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_7.json @@ -1,4 +1,5 @@ { + "domain": "example.com", "expires": "1864-04-14T08:45:43.050Z", "key": "3-1-00000013-0000-002e-0000-003000000042", "token": "_N9ERJGmbZtd6XlW_6O12bxuNe4=" diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_8.json b/libs/wire-api/test/golden/testObject_Asset_asset_8.json index 434d93f714c..e23c34c5439 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_8.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_8.json @@ -1,3 +1,4 @@ { + "domain": "example.com", "key": "3-1-00000073-0000-003e-0000-00120000000c" } diff --git a/libs/wire-api/test/golden/testObject_Asset_asset_9.json b/libs/wire-api/test/golden/testObject_Asset_asset_9.json index 5e7097dcd98..5c33c2d979f 100644 --- a/libs/wire-api/test/golden/testObject_Asset_asset_9.json +++ b/libs/wire-api/test/golden/testObject_Asset_asset_9.json @@ -1,4 +1,5 @@ { + "domain": "example.com", "expires": "1864-05-21T01:34:09.726Z", "key": "3-2-00000006-0000-004b-0000-004f00000025" } diff --git a/libs/wire-api/test/golden/testObject_UserIdentity_user_16.json b/libs/wire-api/test/golden/testObject_UserIdentity_user_16.json deleted file mode 100644 index 156ade504d6..00000000000 --- a/libs/wire-api/test/golden/testObject_UserIdentity_user_16.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "email": "%x\u0013􀔑\u0004.@G빯t.6", - "phone": "+298116118047", - "sso_id": { - "subject": "me@example.com", - "tenant": "http://example.com" - } -} diff --git a/libs/wire-api/test/golden/testObject_UserIdentity_user_5.json b/libs/wire-api/test/golden/testObject_UserIdentity_user_5.json deleted file mode 100644 index 902e47fbe87..00000000000 --- a/libs/wire-api/test/golden/testObject_UserIdentity_user_5.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "email": null, - "phone": "+49198172826", - "sso_id": { - "subject": "me@example.com", - "tenant": "http://example.com" - } -} diff --git a/libs/wire-api/test/golden/testObject_UserIdentity_user_8.json b/libs/wire-api/test/golden/testObject_UserIdentity_user_8.json deleted file mode 100644 index f9a46004b6f..00000000000 --- a/libs/wire-api/test/golden/testObject_UserIdentity_user_8.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "email": null, - "phone": "+149548802116267", - "sso_id": { - "subject": "me@example.com", - "tenant": "http://example.com" - } -} diff --git a/libs/wire-api/test/golden/testObject_UserSSOId_user_1.json b/libs/wire-api/test/golden/testObject_UserSSOId_user_1.json deleted file mode 100644 index 520bcfc7dad..00000000000 --- a/libs/wire-api/test/golden/testObject_UserSSOId_user_1.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "subject": "𝢱􁱝S\u0006\\\u0017\\", - "tenant": "#ph􀽌" -} diff --git a/libs/wire-api/test/golden/testObject_UserSSOId_user_10.json b/libs/wire-api/test/golden/testObject_UserSSOId_user_10.json deleted file mode 100644 index 269300657d1..00000000000 --- a/libs/wire-api/test/golden/testObject_UserSSOId_user_10.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "subject": "􀞢^}Y7A\u0014󰐺\u001bF", - "tenant": "oo\"u/]5" -} diff --git a/libs/wire-api/test/golden/testObject_UserSSOId_user_11.json b/libs/wire-api/test/golden/testObject_UserSSOId_user_11.json deleted file mode 100644 index 46b703e246b..00000000000 --- a/libs/wire-api/test/golden/testObject_UserSSOId_user_11.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "scim_external_id": "U㞠\u00129[𮥂z􆔇ⵍ􎹘#~􀐽D\u0003[􏈫u𦷊h똶㕠2 c4􄯇\u000e" -} diff --git a/libs/wire-api/test/golden/testObject_UserSSOId_user_12.json b/libs/wire-api/test/golden/testObject_UserSSOId_user_12.json deleted file mode 100644 index db68edf1a29..00000000000 --- a/libs/wire-api/test/golden/testObject_UserSSOId_user_12.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "subject": "􏺁\u001bg𑄉", - "tenant": "\na," -} diff --git a/libs/wire-api/test/golden/testObject_UserSSOId_user_14.json b/libs/wire-api/test/golden/testObject_UserSSOId_user_14.json deleted file mode 100644 index 4d74fb56c90..00000000000 --- a/libs/wire-api/test/golden/testObject_UserSSOId_user_14.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "subject": "g\ta\u001d󳹝[a\u0013𢝝oA", - "tenant": "g􉙇)By𡑗h.\u000c\u00179@" -} diff --git a/libs/wire-api/test/golden/testObject_UserSSOId_user_15.json b/libs/wire-api/test/golden/testObject_UserSSOId_user_15.json deleted file mode 100644 index 69528cc5164..00000000000 --- a/libs/wire-api/test/golden/testObject_UserSSOId_user_15.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "scim_external_id": "a9qᩤ󶴏nM]vM\u0012t풣_'\u0010t1MJb{󼥁\u001dZC\u0006" -} diff --git a/libs/wire-api/test/golden/testObject_UserSSOId_user_16.json b/libs/wire-api/test/golden/testObject_UserSSOId_user_16.json deleted file mode 100644 index 9b9641de712..00000000000 --- a/libs/wire-api/test/golden/testObject_UserSSOId_user_16.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "scim_external_id": "Ltepz\u0006\u001c\u001c\u0000􇀶󽍉}𡃭N뫴7GJ" -} diff --git a/libs/wire-api/test/golden/testObject_UserSSOId_user_17.json b/libs/wire-api/test/golden/testObject_UserSSOId_user_17.json deleted file mode 100644 index 830c5048c44..00000000000 --- a/libs/wire-api/test/golden/testObject_UserSSOId_user_17.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "scim_external_id": "qj𤂎.^" -} diff --git a/libs/wire-api/test/golden/testObject_UserSSOId_user_18.json b/libs/wire-api/test/golden/testObject_UserSSOId_user_18.json deleted file mode 100644 index 764dfe765c5..00000000000 --- a/libs/wire-api/test/golden/testObject_UserSSOId_user_18.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "scim_external_id": "𒍧" -} diff --git a/libs/wire-api/test/golden/testObject_UserSSOId_user_19.json b/libs/wire-api/test/golden/testObject_UserSSOId_user_19.json deleted file mode 100644 index f1874f30cf6..00000000000 --- a/libs/wire-api/test/golden/testObject_UserSSOId_user_19.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "scim_external_id": "!𛉋mᅛ\u0018\u001dA\u0010󿃯𤧇x[h\n~􋁝" -} diff --git a/libs/wire-api/test/golden/testObject_UserSSOId_user_20.json b/libs/wire-api/test/golden/testObject_UserSSOId_user_20.json deleted file mode 100644 index 6476075a9dd..00000000000 --- a/libs/wire-api/test/golden/testObject_UserSSOId_user_20.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "subject": "\u0002b\u000e􇆽\u001b\u001d3,􅲈𠩀8𑿋", - "tenant": "X#\u0004 " -} diff --git a/libs/wire-api/test/golden/testObject_UserSSOId_user_3.json b/libs/wire-api/test/golden/testObject_UserSSOId_user_3.json deleted file mode 100644 index 1db46e5e7f4..00000000000 --- a/libs/wire-api/test/golden/testObject_UserSSOId_user_3.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "subject": "𨊌4X\u0019", - "tenant": "i\\\u0004\r𘑍\u0015󲛚줴Vi" -} diff --git a/libs/wire-api/test/golden/testObject_UserSSOId_user_4.json b/libs/wire-api/test/golden/testObject_UserSSOId_user_4.json deleted file mode 100644 index eb3dcf271d5..00000000000 --- a/libs/wire-api/test/golden/testObject_UserSSOId_user_4.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "subject": "􉹡0\u001b𬴯=0.7 , uuid >=1.3 , wai-utilities >=0.12 + , wire-api , yaml >=0.8 default-language: Haskell2010 diff --git a/services/cargohold/package.yaml b/services/cargohold/package.yaml index d4bd3dfbcdc..09cd81ee873 100644 --- a/services/cargohold/package.yaml +++ b/services/cargohold/package.yaml @@ -81,6 +81,7 @@ executables: - types-common >=0.7 - uuid >=1.3 - wai-utilities >=0.12 + - wire-api cargohold: main: src/Main.hs ghc-options: diff --git a/services/cargohold/test/integration/API/V4.hs b/services/cargohold/test/integration/API/V4.hs new file mode 100644 index 00000000000..b9b8e556705 --- /dev/null +++ b/services/cargohold/test/integration/API/V4.hs @@ -0,0 +1,282 @@ +{-# OPTIONS_GHC -Wno-incomplete-uni-patterns #-} + +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2020 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module API.V4 (tests) where + +import Bilge hiding (body) +import Bilge.Assert +import qualified Codec.MIME.Parse as MIME +import qualified Codec.MIME.Type as MIME +import Control.Lens hiding (sets) +import Data.ByteString.Builder +import qualified Data.ByteString.Char8 as C8 +import Data.ByteString.Conversion +import qualified Data.ByteString.Lazy as Lazy +import Data.Id +import Data.Qualified +import Data.Text.Encoding (decodeLatin1) +import Data.Time.Clock +import Data.Time.Format +import qualified Data.UUID as UUID +import Data.UUID.V4 +import Imports hiding (head) +import Network.HTTP.Client (parseUrlThrow) +import Network.HTTP.Types.Header +import Network.HTTP.Types.Method +import Network.HTTP.Types.Status (status200) +import Network.Wai.Utilities (Error (label)) +import Test.Tasty +import Test.Tasty.HUnit +import TestSetup +import Wire.API.Asset + +tests :: IO TestSetup -> TestTree +tests s = + testGroup + "API Integration v4" + [ testGroup + "simple" + [ test s "roundtrip" testSimpleRoundtrip, + test s "tokens" testSimpleTokens, + test s "s3-upstream-closed" testSimpleS3ClosedConnectionReuse, + test s "client-compatibility" testUploadCompatibility + ] + ] + +-------------------------------------------------------------------------------- +-- Simple (single-step) uploads + +testSimpleRoundtrip :: TestSignature () +testSimpleRoundtrip c = do + let def = defAssetSettings + let rets = [minBound ..] + let sets = def : map (\r -> def & setAssetRetention ?~ r) rets + mapM_ simpleRoundtrip sets + where + simpleRoundtrip sets = do + uid <- liftIO $ Id <$> nextRandom + uid2 <- liftIO $ Id <$> nextRandom + -- Initial upload + let bdy = (applicationText, "Hello World") + r1 <- + uploadSimple (c . path "/assets/v3") uid sets bdy + lookup "Date" (responseHeaders r1) + let utc = parseTimeOrError False defaultTimeLocale rfc822DateFormat date :: UTCTime + -- Potentially check for the expires header + when (isJust $ join (assetRetentionSeconds <$> (sets ^. setAssetRetention))) $ do + liftIO $ assertBool "invalid expiration" (Just utc < view assetExpires ast) + -- Lookup with token and download via redirect. + r2 <- + get (c . path loc . zUser uid . header "Asset-Token" (toByteString' tok) . noRedirect) lookup "Date" (responseHeaders r4) + let utc' = parseTimeOrError False defaultTimeLocale rfc822DateFormat date' :: UTCTime + liftIO $ assertBool "bad date" (utc' >= utc) + +testSimpleTokens :: TestSignature () +testSimpleTokens c = do + uid <- liftIO $ Id <$> nextRandom + uid2 <- liftIO $ Id <$> nextRandom + -- Initial upload + let sets = defAssetSettings & set setAssetRetention (Just AssetVolatile) + let bdy = (applicationText, "Hello World") + r1 <- + uploadSimple (c . path "/assets/v3") uid sets bdy + responseJsonMaybe r2 + liftIO $ assertBool "token unchanged" (tok /= tok') + -- Download by owner with new token. + r3 <- + get (c . path loc . zUser uid . header "Asset-Token" (toByteString' tok') . noRedirect) > wait >> go + where + wait = liftIO $ putStrLn "Waiting for S3 idle timeout ..." >> threadDelay 7000000 + go = do + uid <- liftIO $ Id <$> nextRandom + let sets = defAssetSettings & set setAssetRetention (Just AssetVolatile) + let part2 = (MIME.Type (MIME.Text "plain") [], C8.replicate 100000 'c') + uploadSimple (c . path "/assets/v3") uid sets part2 + !!! const 201 === statusCode + +-------------------------------------------------------------------------------- +-- Client compatibility tests + +-- Since the other tests use functions from the server code, it can happen that +-- an API change also changes the requests made here in the tests. +-- This test tries to prevent us from breaking the API without noticing. +-- +-- The body is taken directly from a request made by the web app +-- (just replaced the content with a shorter one and updated the MD5 header). +testUploadCompatibility :: TestSignature () +testUploadCompatibility c = do + uid <- liftIO $ Id <$> nextRandom + -- Initial upload + r1 <- + uploadRaw (c . path "/assets/v3") uid exampleMultipart + + UserId -> + AssetSettings -> + (MIME.Type, ByteString) -> + Http (Response (Maybe Lazy.ByteString)) +uploadSimple c usr sets (ct, bs) = + let mp = buildMultipartBody sets ct (Lazy.fromStrict bs) + in uploadRaw c usr (toLazyByteString mp) + +uploadRaw :: + CargoHold -> + UserId -> + Lazy.ByteString -> + Http (Response (Maybe Lazy.ByteString)) +uploadRaw c usr bs = + post $ + c + . method POST + . zUser usr + . zConn "conn" + . content "multipart/mixed" + . lbytes bs + +deleteAsset :: CargoHold -> UserId -> Qualified AssetKey -> Http (Response (Maybe Lazy.ByteString)) +deleteAsset c u k = delete $ c . zUser u . paths ["assets", "v3", toByteString' (qUnqualified k)] + +-- Utilities ------------------------------------------------------------------ + +decodeHeader :: FromByteString a => HeaderName -> Response b -> a +decodeHeader h = + fromMaybe (error $ "decodeHeader: missing or invalid header: " ++ show h) + . fromByteString + . getHeader' h + +getContentType :: Response a -> Maybe MIME.Type +getContentType = MIME.parseContentType . decodeLatin1 . getHeader' "Content-Type" + +applicationText :: MIME.Type +applicationText = MIME.Type (MIME.Application "text") [] + +applicationOctetStream :: MIME.Type +applicationOctetStream = MIME.Type (MIME.Application "octet-stream") [] + +zUser :: UserId -> Request -> Request +zUser = header "Z-User" . UUID.toASCIIBytes . toUUID + +zConn :: ByteString -> Request -> Request +zConn = header "Z-Connection" diff --git a/tools/api-simulations/lib/src/Network/Wire/Simulations.hs b/tools/api-simulations/lib/src/Network/Wire/Simulations.hs index 7a34d8280b9..f933ab94149 100644 --- a/tools/api-simulations/lib/src/Network/Wire/Simulations.hs +++ b/tools/api-simulations/lib/src/Network/Wire/Simulations.hs @@ -176,7 +176,7 @@ instance Serialize AssetInfo where AssetInfo k t <$> get mkAssetMsg :: Asset -> SymmetricKeys -> BotMessage -mkAssetMsg a = BotAssetMessage . AssetInfo (a ^. assetKey) (a ^. assetToken) +mkAssetMsg a = BotAssetMessage . AssetInfo (qUnqualified (a ^. assetKey)) (a ^. assetToken) mkTextMsg :: Text -> BotMessage mkTextMsg = BotTextMessage From 34819f01d09257e8295e3b8280113b160b4d761a Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Fri, 17 Dec 2021 11:07:17 +0100 Subject: [PATCH 32/74] Fix location url on upload Also make `AssetLocation` type more high level. --- libs/wire-api/src/Wire/API/Asset.hs | 49 ++++++++--- .../src/Wire/API/Routes/Public/Cargohold.hs | 27 ++++-- .../cargohold/src/CargoHold/API/Public.hs | 85 +++++++++++++------ 3 files changed, 120 insertions(+), 41 deletions(-) diff --git a/libs/wire-api/src/Wire/API/Asset.hs b/libs/wire-api/src/Wire/API/Asset.hs index 1e142ffe86c..49044a4e150 100644 --- a/libs/wire-api/src/Wire/API/Asset.hs +++ b/libs/wire-api/src/Wire/API/Asset.hs @@ -30,6 +30,7 @@ module Wire.API.Asset -- * AssetKey AssetKey (..), + assetKeyToText, -- * AssetToken AssetToken (..), @@ -77,15 +78,16 @@ import Data.Qualified import Data.SOP import Data.Schema import qualified Data.Swagger as S -import qualified Data.Swagger as Swagger import qualified Data.Text as T import Data.Text.Ascii (AsciiBase64Url) import qualified Data.Text.Encoding as T +import qualified Data.Text.Encoding.Error as T import Data.Time.Clock import qualified Data.UUID as UUID import GHC.TypeLits import Imports import Servant +import URI.ByteString import Wire.API.Arbitrary (Arbitrary (..), GenericUniform (..)) import Wire.API.ErrorDescription import Wire.API.Routes.MultiVerb @@ -169,6 +171,9 @@ instance ToByteString AssetKey where <> builder '-' <> builder (UUID.toASCIIBytes (toUUID i)) +assetKeyToText :: AssetKey -> Text +assetKeyToText = T.decodeUtf8 . toByteString' + instance ToSchema AssetKey where schema = (T.decodeUtf8 . toByteString') @@ -359,26 +364,48 @@ instance ToSchema AssetRetention where (\value -> element (retentionToTextRep value) value) [minBound .. maxBound] -newtype AssetLocation = AssetLocation {getAssetLocation :: Text} - deriving newtype - ( ToHttpApiData, - FromHttpApiData, - Swagger.ToParamSchema - ) - -instance AsHeaders '[AssetLocation] Asset (Asset, AssetLocation) where +-- FUTUREWORK: switch to a better URI library (e.g. modern-uri) +-- +-- This URI type is error-prone, since its internal representation is based on +-- ByteString, whereas URLs are defined in terms of characters, not octets (RFC +-- 3986). +newtype AssetLocation r = AssetLocation {getAssetLocation :: URIRef r} + +instance ToHttpApiData (AssetLocation r) where + toUrlPiece = T.decodeUtf8With T.lenientDecode . toHeader + toHeader = serializeURIRef' . getAssetLocation + +instance FromHttpApiData (AssetLocation Relative) where + parseUrlPiece = parseHeader . T.encodeUtf8 + parseHeader = + bimap (T.pack . show) AssetLocation + . parseRelativeRef strictURIParserOptions + +instance FromHttpApiData (AssetLocation Absolute) where + parseUrlPiece = parseHeader . T.encodeUtf8 + parseHeader = + bimap (T.pack . show) AssetLocation + . parseURI strictURIParserOptions + +instance S.ToParamSchema (AssetLocation r) where + toParamSchema _ = + mempty + & S.type_ ?~ S.SwaggerString + & S.format ?~ "url" + +instance AsHeaders '[AssetLocation r] Asset (Asset, AssetLocation r) where toHeaders (asset, loc) = (I loc :* Nil, asset) fromHeaders (I loc :* Nil, asset) = (asset, loc) -- | An asset as returned by the download API: if the asset is local, only a -- URL is returned, and if it is remote the content of the asset is streamed. data LocalOrRemoteAsset - = LocalAsset AssetLocation + = LocalAsset (AssetLocation Absolute) | RemoteAsset (SourceIO ByteString) instance ( ResponseType r0 ~ ErrorDescription code label desc, - ResponseType r1 ~ AssetLocation, + ResponseType r1 ~ AssetLocation Absolute, ResponseType r2 ~ SourceIO ByteString, KnownSymbol desc ) => diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs b/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs index 92df6173a9c..61206b38cc2 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs @@ -26,6 +26,7 @@ import Imports import Servant import Servant.Swagger.Internal import Servant.Swagger.Internal.Orphans () +import URI.ByteString import Wire.API.Asset import Wire.API.ErrorDescription import Wire.API.Routes.AssetBody @@ -60,10 +61,13 @@ instance HasServer (ApplyPrincipalPath tag api) ctx => HasServer (tag :> api) ct instance RoutesToPaths (ApplyPrincipalPath tag api) => RoutesToPaths (tag :> api) where getRoutes = getRoutes @(ApplyPrincipalPath tag api) +type AssetLocationHeader r = + '[DescHeader "Location" "Asset location" (AssetLocation r)] + type AssetRedirect = WithHeaders - '[DescHeader "Location" "Asset location" AssetLocation] - AssetLocation + (AssetLocationHeader Absolute) + (AssetLocation Absolute) (RespondEmpty 302 "Asset found") type AssetStreaming = @@ -78,7 +82,7 @@ type GetAsset = 'GET '[JSON] '[AssetNotFound, AssetRedirect] - (Maybe AssetLocation) + (Maybe (AssetLocation Absolute)) type ServantAPI = ( Summary "Renew an asset token" @@ -121,11 +125,11 @@ type BaseAPIv3 (tag :: PrincipalTag) = 'POST '[JSON] '[ WithHeaders - '[DescHeader "Location" "Asset location" AssetLocation] - (Asset, AssetLocation) + (AssetLocationHeader Relative) + (Asset, AssetLocation Relative) (Respond 201 "Asset posted" Asset) ] - (Asset, AssetLocation) + (Asset, AssetLocation Relative) ) :<|> ( Summary "Download an asset" :> tag @@ -146,6 +150,17 @@ type BaseAPIv3 (tag :: PrincipalTag) = () ) +type TestAPI = + ( MultiVerb + 'GET + '[JSON] + '[ AssetNotFound, + AssetRedirect, + AssetStreaming + ] + (Maybe LocalOrRemoteAsset) + ) + type QualifiedAPI = ( Summary "Download an asset" :> Description diff --git a/services/cargohold/src/CargoHold/API/Public.hs b/services/cargohold/src/CargoHold/API/Public.hs index 50b80f595ca..49d8df6f26c 100644 --- a/services/cargohold/src/CargoHold/API/Public.hs +++ b/services/cargohold/src/CargoHold/API/Public.hs @@ -23,12 +23,13 @@ import qualified CargoHold.API.V3 as V3 import CargoHold.App import qualified CargoHold.Types.V3 as V3 import Control.Lens -import Data.ByteString.Conversion +import Data.ByteString.Builder +import qualified Data.ByteString.Lazy as LBS +import Data.Domain import Data.Id import Data.Qualified -import qualified Data.Text.Encoding as Text -import qualified Data.Text.Encoding.Error as Text import Imports hiding (head) +import qualified Network.HTTP.Types as HTTP import Servant ((:<|>) (..)) import Servant.Server hiding (Handler) import URI.ByteString @@ -56,7 +57,32 @@ servantSitemap = qualifiedAPI = downloadAssetV4 :<|> deleteAssetV4 internalAPI = pure () -class MakePrincipal (tag :: PrincipalTag) (id :: *) | id -> tag, tag -> id where +class HasLocation (tag :: PrincipalTag) where + assetLocation :: Local AssetKey -> [Text] + +instance HasLocation 'UserPrincipalTag where + assetLocation key = + [ "assets", + "v4", + domainText (tDomain key), + assetKeyToText (tUnqualified key) + ] + +instance HasLocation 'BotPrincipalTag where + assetLocation key = + [ "bot", + "assets", + assetKeyToText (tUnqualified key) + ] + +instance HasLocation 'ProviderPrincipalTag where + assetLocation key = + [ "provider", + "assets", + assetKeyToText (tUnqualified key) + ] + +class HasLocation tag => MakePrincipal (tag :: PrincipalTag) (id :: *) | id -> tag, tag -> id where mkPrincipal :: id -> V3.Principal instance MakePrincipal 'UserPrincipalTag (Local UserId) where @@ -68,22 +94,36 @@ instance MakePrincipal 'BotPrincipalTag BotId where instance MakePrincipal 'ProviderPrincipalTag ProviderId where mkPrincipal = V3.ProviderPrincipal +mkAssetLocation :: + forall (tag :: PrincipalTag). + HasLocation tag => + Local AssetKey -> + AssetLocation Relative +mkAssetLocation key = + AssetLocation + RelativeRef + { rrAuthority = Nothing, + rrPath = path, + rrQuery = mempty, + rrFragment = Nothing + } + where + path = + LBS.toStrict + . toLazyByteString + . HTTP.encodePathSegmentsRelative + $ assetLocation @tag key + uploadAssetV3 :: + forall tag id. MakePrincipal tag id => id -> AssetSource -> - Handler (Asset, AssetLocation) + Handler (Asset, AssetLocation Relative) uploadAssetV3 pid req = do let principal = mkPrincipal pid asset <- V3.upload principal (getAssetSource req) - let key = - Text.decodeUtf8With Text.lenientDecode $ - toByteString' (tUnqualified (asset ^. assetKey)) - let loc = case principal of - V3.UserPrincipal {} -> "/assets/v3/" <> key - V3.BotPrincipal {} -> "/bot/assets/" <> key - V3.ProviderPrincipal {} -> "/provider/assets/" <> key - pure (fmap qUntagged asset, AssetLocation loc) + pure (fmap qUntagged asset, mkAssetLocation @tag (asset ^. assetKey)) downloadAssetV3 :: MakePrincipal tag id => @@ -91,10 +131,9 @@ downloadAssetV3 :: AssetKey -> Maybe AssetToken -> Maybe AssetToken -> - Handler (Maybe AssetLocation) + Handler (Maybe (AssetLocation Absolute)) downloadAssetV3 usr key tok1 tok2 = do - url <- V3.download (mkPrincipal usr) key (tok1 <|> tok2) - pure $ fmap (AssetLocation . Text.decodeUtf8With Text.lenientDecode . serializeURIRef') url + AssetLocation <$$> V3.download (mkPrincipal usr) key (tok1 <|> tok2) downloadAssetV4 :: Local UserId -> @@ -120,12 +159,10 @@ renewTokenV3 (tUnqualified -> usr) key = deleteTokenV3 :: Local UserId -> AssetKey -> Handler () deleteTokenV3 (tUnqualified -> usr) key = V3.deleteToken (V3.UserPrincipal usr) key -legacyDownloadPlain :: Local UserId -> ConvId -> AssetId -> Handler (Maybe AssetLocation) -legacyDownloadPlain (tUnqualified -> usr) cnv ast = do - url <- LegacyAPI.download usr cnv ast - pure $ fmap (AssetLocation . Text.decodeUtf8With Text.lenientDecode . serializeURIRef') url +legacyDownloadPlain :: Local UserId -> ConvId -> AssetId -> Handler (Maybe (AssetLocation Absolute)) +legacyDownloadPlain (tUnqualified -> usr) cnv ast = + AssetLocation <$$> LegacyAPI.download usr cnv ast -legacyDownloadOtr :: Local UserId -> ConvId -> AssetId -> Handler (Maybe AssetLocation) -legacyDownloadOtr (tUnqualified -> usr) cnv ast = do - url <- LegacyAPI.downloadOtr usr cnv ast - pure $ fmap (AssetLocation . Text.decodeUtf8With Text.lenientDecode . serializeURIRef') url +legacyDownloadOtr :: Local UserId -> ConvId -> AssetId -> Handler (Maybe (AssetLocation Absolute)) +legacyDownloadOtr (tUnqualified -> usr) cnv ast = + AssetLocation <$$> LegacyAPI.downloadOtr usr cnv ast From 15c2939c587ceac608bbea5cffe52338bf29c96e Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Fri, 17 Dec 2021 11:34:58 +0100 Subject: [PATCH 33/74] Test main API and v3 version separately --- .../src/Wire/API/Routes/Public/Cargohold.hs | 1 + services/cargohold/cargohold.cabal | 5 +- .../cargohold/src/CargoHold/API/Public.hs | 5 +- .../test/integration/{API/V4.hs => API.hs} | 105 +++------ .../cargohold/test/integration/API/Util.hs | 93 ++++++++ services/cargohold/test/integration/API/V3.hs | 218 ++---------------- services/cargohold/test/integration/Main.hs | 4 +- 7 files changed, 144 insertions(+), 287 deletions(-) rename services/cargohold/test/integration/{API/V4.hs => API.hs} (76%) create mode 100644 services/cargohold/test/integration/API/Util.hs diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs b/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs index 61206b38cc2..26795c1f46d 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs @@ -171,6 +171,7 @@ type QualifiedAPI = :> "v4" :> QualifiedCapture "key" AssetKey :> Header "Asset-Token" AssetToken + :> QueryParam "asset_token" AssetToken :> MultiVerb 'GET '[JSON] diff --git a/services/cargohold/cargohold.cabal b/services/cargohold/cargohold.cabal index 057edb3ebff..26330a6c3c2 100644 --- a/services/cargohold/cargohold.cabal +++ b/services/cargohold/cargohold.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: 253f7aaf3d9981e1ce5a3654d10d5aa926f6347fbb86c510d4eb2997e869e499 +-- hash: 3633e0db5e928a51056ef8474859ed8f6edf06bc84b9ade6db39f62c48bfce07 name: cargohold version: 1.5.0 @@ -134,8 +134,9 @@ executable cargohold executable cargohold-integration main-is: Main.hs other-modules: + API + API.Util API.V3 - API.V4 Metrics TestSetup Paths_cargohold diff --git a/services/cargohold/src/CargoHold/API/Public.hs b/services/cargohold/src/CargoHold/API/Public.hs index 49d8df6f26c..81e3d249109 100644 --- a/services/cargohold/src/CargoHold/API/Public.hs +++ b/services/cargohold/src/CargoHold/API/Public.hs @@ -139,10 +139,11 @@ downloadAssetV4 :: Local UserId -> Qualified AssetKey -> Maybe AssetToken -> + Maybe AssetToken -> Handler (Maybe LocalOrRemoteAsset) -downloadAssetV4 usr qkey tok = do +downloadAssetV4 usr qkey tok1 tok2 = do key <- tUnqualified <$> ensureLocal qkey - LocalAsset <$$> downloadAssetV3 usr key tok + LocalAsset <$$> downloadAssetV3 usr key tok1 tok2 deleteAssetV3 :: MakePrincipal tag id => id -> AssetKey -> Handler () deleteAssetV3 usr key = V3.delete (mkPrincipal usr) key diff --git a/services/cargohold/test/integration/API/V4.hs b/services/cargohold/test/integration/API.hs similarity index 76% rename from services/cargohold/test/integration/API/V4.hs rename to services/cargohold/test/integration/API.hs index b9b8e556705..0ad220ce526 100644 --- a/services/cargohold/test/integration/API/V4.hs +++ b/services/cargohold/test/integration/API.hs @@ -2,7 +2,7 @@ -- This file is part of the Wire Server implementation. -- --- Copyright (C) 2020 Wire Swiss GmbH +-- Copyright (C) 2021 Wire Swiss GmbH -- -- This program is free software: you can redistribute it and/or modify it under -- the terms of the GNU Affero General Public License as published by the Free @@ -17,39 +17,33 @@ -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . -module API.V4 (tests) where +module API (tests) where +import API.Util import Bilge hiding (body) import Bilge.Assert -import qualified Codec.MIME.Parse as MIME +import qualified CargoHold.Types.V3 as V3 import qualified Codec.MIME.Type as MIME import Control.Lens hiding (sets) -import Data.ByteString.Builder import qualified Data.ByteString.Char8 as C8 import Data.ByteString.Conversion -import qualified Data.ByteString.Lazy as Lazy import Data.Id import Data.Qualified -import Data.Text.Encoding (decodeLatin1) import Data.Time.Clock import Data.Time.Format -import qualified Data.UUID as UUID import Data.UUID.V4 import Imports hiding (head) import Network.HTTP.Client (parseUrlThrow) -import Network.HTTP.Types.Header -import Network.HTTP.Types.Method import Network.HTTP.Types.Status (status200) import Network.Wai.Utilities (Error (label)) import Test.Tasty import Test.Tasty.HUnit import TestSetup -import Wire.API.Asset tests :: IO TestSetup -> TestTree tests s = testGroup - "API Integration v4" + "API Integration" [ testGroup "simple" [ test s "roundtrip" testSimpleRoundtrip, @@ -64,9 +58,9 @@ tests s = testSimpleRoundtrip :: TestSignature () testSimpleRoundtrip c = do - let def = defAssetSettings + let def = V3.defAssetSettings let rets = [minBound ..] - let sets = def : map (\r -> def & setAssetRetention ?~ r) rets + let sets = def : map (\r -> def & V3.setAssetRetention ?~ r) rets mapM_ simpleRoundtrip sets where simpleRoundtrip sets = do @@ -78,14 +72,14 @@ testSimpleRoundtrip c = do uploadSimple (c . path "/assets/v3") uid sets bdy lookup "Date" (responseHeaders r1) let utc = parseTimeOrError False defaultTimeLocale rfc822DateFormat date :: UTCTime -- Potentially check for the expires header - when (isJust $ join (assetRetentionSeconds <$> (sets ^. setAssetRetention))) $ do - liftIO $ assertBool "invalid expiration" (Just utc < view assetExpires ast) + when (isJust $ join (V3.assetRetentionSeconds <$> (sets ^. V3.setAssetRetention))) $ do + liftIO $ assertBool "invalid expiration" (Just utc < view V3.assetExpires ast) -- Lookup with token and download via redirect. r2 <- get (c . path loc . zUser uid . header "Asset-Token" (toByteString' tok) . noRedirect) nextRandom uid2 <- liftIO $ Id <$> nextRandom -- Initial upload - let sets = defAssetSettings & set setAssetRetention (Just AssetVolatile) + let sets = V3.defAssetSettings & set V3.setAssetRetention (Just V3.AssetVolatile) let bdy = (applicationText, "Hello World") r1 <- uploadSimple (c . path "/assets/v3") uid sets bdy responseJsonMaybe r2 + let Just tok' = V3.newAssetToken <$> responseJsonMaybe r2 liftIO $ assertBool "token unchanged" (tok /= tok') -- Download by owner with new token. r3 <- @@ -157,6 +157,9 @@ testSimpleTokens c = do -- Verify access with new token from a different user. get (c . path loc . header "Asset-Token" (toByteString' tok') . zUser uid2 . noRedirect) !!! const 302 === statusCode + -- Verify access with new token as query parameter from a different user + get (c . path loc . queryItem "asset_token" (toByteString' tok') . zUser uid2 . noRedirect) + !!! const 302 === statusCode -- Delete Token fails if not done by owner delete (c . paths ["assets", "v3", toByteString' (qUnqualified key), "token"] . zUser uid2) !!! do const 403 === statusCode @@ -179,7 +182,7 @@ testSimpleS3ClosedConnectionReuse c = go >> wait >> go wait = liftIO $ putStrLn "Waiting for S3 idle timeout ..." >> threadDelay 7000000 go = do uid <- liftIO $ Id <$> nextRandom - let sets = defAssetSettings & set setAssetRetention (Just AssetVolatile) + let sets = V3.defAssetSettings & set V3.setAssetRetention (Just V3.AssetVolatile) let part2 = (MIME.Type (MIME.Text "plain") [], C8.replicate 100000 'c') uploadSimple (c . path "/assets/v3") uid sets part2 !!! const 201 === statusCode @@ -228,55 +231,3 @@ testUploadCompatibility c = do \test\r\n\ \--FrontierIyj6RcVrqMcxNtMEWPsNpuPm325QsvWQ--\r\n\ \\r\n" - --- API Calls ------------------------------------------------------------------ - -uploadSimple :: - CargoHold -> - UserId -> - AssetSettings -> - (MIME.Type, ByteString) -> - Http (Response (Maybe Lazy.ByteString)) -uploadSimple c usr sets (ct, bs) = - let mp = buildMultipartBody sets ct (Lazy.fromStrict bs) - in uploadRaw c usr (toLazyByteString mp) - -uploadRaw :: - CargoHold -> - UserId -> - Lazy.ByteString -> - Http (Response (Maybe Lazy.ByteString)) -uploadRaw c usr bs = - post $ - c - . method POST - . zUser usr - . zConn "conn" - . content "multipart/mixed" - . lbytes bs - -deleteAsset :: CargoHold -> UserId -> Qualified AssetKey -> Http (Response (Maybe Lazy.ByteString)) -deleteAsset c u k = delete $ c . zUser u . paths ["assets", "v3", toByteString' (qUnqualified k)] - --- Utilities ------------------------------------------------------------------ - -decodeHeader :: FromByteString a => HeaderName -> Response b -> a -decodeHeader h = - fromMaybe (error $ "decodeHeader: missing or invalid header: " ++ show h) - . fromByteString - . getHeader' h - -getContentType :: Response a -> Maybe MIME.Type -getContentType = MIME.parseContentType . decodeLatin1 . getHeader' "Content-Type" - -applicationText :: MIME.Type -applicationText = MIME.Type (MIME.Application "text") [] - -applicationOctetStream :: MIME.Type -applicationOctetStream = MIME.Type (MIME.Application "octet-stream") [] - -zUser :: UserId -> Request -> Request -zUser = header "Z-User" . UUID.toASCIIBytes . toUUID - -zConn :: ByteString -> Request -> Request -zConn = header "Z-Connection" diff --git a/services/cargohold/test/integration/API/Util.hs b/services/cargohold/test/integration/API/Util.hs new file mode 100644 index 00000000000..df2e708ecff --- /dev/null +++ b/services/cargohold/test/integration/API/Util.hs @@ -0,0 +1,93 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2020 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module API.Util where + +import Bilge hiding (body) +import qualified CargoHold.Types.V3 as V3 +import qualified Codec.MIME.Parse as MIME +import qualified Codec.MIME.Type as MIME +import Data.ByteString.Builder +import Data.ByteString.Conversion +import qualified Data.ByteString.Lazy as Lazy +import Data.Id +import Data.Qualified +import Data.Text.Encoding (decodeLatin1) +import qualified Data.UUID as UUID +import Imports hiding (head) +import Network.HTTP.Types.Header +import Network.HTTP.Types.Method +import TestSetup + +uploadSimple :: + CargoHold -> + UserId -> + V3.AssetSettings -> + (MIME.Type, ByteString) -> + Http (Response (Maybe Lazy.ByteString)) +uploadSimple c usr sets (ct, bs) = + let mp = V3.buildMultipartBody sets ct (Lazy.fromStrict bs) + in uploadRaw c usr (toLazyByteString mp) + +decodeHeader :: FromByteString a => HeaderName -> Response b -> a +decodeHeader h = + fromMaybe (error $ "decodeHeader: missing or invalid header: " ++ show h) + . fromByteString + . getHeader' h + +uploadRaw :: + CargoHold -> + UserId -> + Lazy.ByteString -> + Http (Response (Maybe Lazy.ByteString)) +uploadRaw c usr bs = + post $ + c + . method POST + . zUser usr + . zConn "conn" + . content "multipart/mixed" + . lbytes bs + +getContentType :: Response a -> Maybe MIME.Type +getContentType = MIME.parseContentType . decodeLatin1 . getHeader' "Content-Type" + +applicationText :: MIME.Type +applicationText = MIME.Type (MIME.Application "text") [] + +applicationOctetStream :: MIME.Type +applicationOctetStream = MIME.Type (MIME.Application "octet-stream") [] + +zUser :: UserId -> Request -> Request +zUser = header "Z-User" . UUID.toASCIIBytes . toUUID + +zConn :: ByteString -> Request -> Request +zConn = header "Z-Connection" + +deleteAssetV3 :: CargoHold -> UserId -> Qualified V3.AssetKey -> Http (Response (Maybe Lazy.ByteString)) +deleteAssetV3 c u k = delete $ c . zUser u . paths ["assets", "v3", toByteString' (qUnqualified k)] + +deleteAsset :: CargoHold -> UserId -> Qualified V3.AssetKey -> Http (Response (Maybe Lazy.ByteString)) +deleteAsset c u k = + delete $ + c . zUser u + . paths + [ "assets", + "v4", + toByteString' (qDomain k), + toByteString' (qUnqualified k) + ] diff --git a/services/cargohold/test/integration/API/V3.hs b/services/cargohold/test/integration/API/V3.hs index 750cb2d0264..323be274205 100644 --- a/services/cargohold/test/integration/API/V3.hs +++ b/services/cargohold/test/integration/API/V3.hs @@ -19,32 +19,24 @@ module API.V3 (tests) where +import API.Util import Bilge hiding (body) import Bilge.Assert -import qualified CargoHold.Types.V3 as V3 -import qualified Codec.MIME.Parse as MIME -import qualified Codec.MIME.Type as MIME import Control.Lens hiding (sets) -import Data.ByteString.Builder import qualified Data.ByteString.Char8 as C8 import Data.ByteString.Conversion -import qualified Data.ByteString.Lazy as Lazy import Data.Id import Data.Qualified -import Data.Text.Encoding (decodeLatin1) import Data.Time.Clock import Data.Time.Format -import qualified Data.UUID as UUID import Data.UUID.V4 import Imports hiding (head) import Network.HTTP.Client (parseUrlThrow) -import Network.HTTP.Types.Header -import Network.HTTP.Types.Method import Network.HTTP.Types.Status (status200) -import Network.Wai.Utilities (Error (label)) import Test.Tasty import Test.Tasty.HUnit import TestSetup +import Wire.API.Asset tests :: IO TestSetup -> TestTree tests s = @@ -52,11 +44,7 @@ tests s = "API Integration v3" [ testGroup "simple" - [ test s "roundtrip" testSimpleRoundtrip, - test s "tokens" testSimpleTokens, - test s "s3-upstream-closed" testSimpleS3ClosedConnectionReuse, - test s "client-compatibility" testUploadCompatibility - ] + [test s "roundtrip using v3 API" testSimpleRoundtrip] ] -------------------------------------------------------------------------------- @@ -64,9 +52,9 @@ tests s = testSimpleRoundtrip :: TestSignature () testSimpleRoundtrip c = do - let def = V3.defAssetSettings + let def = defAssetSettings let rets = [minBound ..] - let sets = def : map (\r -> def & V3.setAssetRetention ?~ r) rets + let sets = def : map (\r -> def & setAssetRetention ?~ r) rets mapM_ simpleRoundtrip sets where simpleRoundtrip sets = do @@ -77,15 +65,16 @@ testSimpleRoundtrip c = do r1 <- uploadSimple (c . path "/assets/v3") uid sets bdy toByteString' (qUnqualified (ast ^. assetKey)) + let Just tok = view assetToken ast -- Check mandatory Date header let Just date = C8.unpack <$> lookup "Date" (responseHeaders r1) let utc = parseTimeOrError False defaultTimeLocale rfc822DateFormat date :: UTCTime -- Potentially check for the expires header - when (isJust $ join (V3.assetRetentionSeconds <$> (sets ^. V3.setAssetRetention))) $ do - liftIO $ assertBool "invalid expiration" (Just utc < view V3.assetExpires ast) + when (isJust $ join (assetRetentionSeconds <$> (sets ^. setAssetRetention))) $ do + liftIO $ assertBool "invalid expiration" (Just utc < view assetExpires ast) -- Lookup with token and download via redirect. r2 <- get (c . path loc . zUser uid . header "Asset-Token" (toByteString' tok) . noRedirect) lookup "Date" (responseHeaders r4) let utc' = parseTimeOrError False defaultTimeLocale rfc822DateFormat date' :: UTCTime liftIO $ assertBool "bad date" (utc' >= utc) - -testSimpleTokens :: TestSignature () -testSimpleTokens c = do - uid <- liftIO $ Id <$> nextRandom - uid2 <- liftIO $ Id <$> nextRandom - -- Initial upload - let sets = V3.defAssetSettings & set V3.setAssetRetention (Just V3.AssetVolatile) - let bdy = (applicationText, "Hello World") - r1 <- - uploadSimple (c . path "/assets/v3") uid sets bdy - responseJsonMaybe r2 - liftIO $ assertBool "token unchanged" (tok /= tok') - -- Download by owner with new token. - r3 <- - get (c . path loc . zUser uid . header "Asset-Token" (toByteString' tok') . noRedirect) > wait >> go - where - wait = liftIO $ putStrLn "Waiting for S3 idle timeout ..." >> threadDelay 7000000 - go = do - uid <- liftIO $ Id <$> nextRandom - let sets = V3.defAssetSettings & set V3.setAssetRetention (Just V3.AssetVolatile) - let part2 = (MIME.Type (MIME.Text "plain") [], C8.replicate 100000 'c') - uploadSimple (c . path "/assets/v3") uid sets part2 - !!! const 201 === statusCode - --------------------------------------------------------------------------------- --- Client compatibility tests - --- Since the other tests use functions from the server code, it can happen that --- an API change also changes the requests made here in the tests. --- This test tries to prevent us from breaking the API without noticing. --- --- The body is taken directly from a request made by the web app --- (just replaced the content with a shorter one and updated the MD5 header). -testUploadCompatibility :: TestSignature () -testUploadCompatibility c = do - uid <- liftIO $ Id <$> nextRandom - -- Initial upload - r1 <- - uploadRaw (c . path "/assets/v3") uid exampleMultipart - - UserId -> - V3.AssetSettings -> - (MIME.Type, ByteString) -> - Http (Response (Maybe Lazy.ByteString)) -uploadSimple c usr sets (ct, bs) = - let mp = V3.buildMultipartBody sets ct (Lazy.fromStrict bs) - in uploadRaw c usr (toLazyByteString mp) - -uploadRaw :: - CargoHold -> - UserId -> - Lazy.ByteString -> - Http (Response (Maybe Lazy.ByteString)) -uploadRaw c usr bs = - post $ - c - . method POST - . zUser usr - . zConn "conn" - . content "multipart/mixed" - . lbytes bs - -deleteAsset :: CargoHold -> UserId -> Qualified V3.AssetKey -> Http (Response (Maybe Lazy.ByteString)) -deleteAsset c u k = delete $ c . zUser u . paths ["assets", "v3", toByteString' (qUnqualified k)] - --- Utilities ------------------------------------------------------------------ - -decodeHeader :: FromByteString a => HeaderName -> Response b -> a -decodeHeader h = - fromMaybe (error $ "decodeHeader: missing or invalid header: " ++ show h) - . fromByteString - . getHeader' h - -getContentType :: Response a -> Maybe MIME.Type -getContentType = MIME.parseContentType . decodeLatin1 . getHeader' "Content-Type" - -applicationText :: MIME.Type -applicationText = MIME.Type (MIME.Application "text") [] - -applicationOctetStream :: MIME.Type -applicationOctetStream = MIME.Type (MIME.Application "octet-stream") [] - -zUser :: UserId -> Request -> Request -zUser = header "Z-User" . UUID.toASCIIBytes . toUUID - -zConn :: ByteString -> Request -> Request -zConn = header "Z-Connection" diff --git a/services/cargohold/test/integration/Main.hs b/services/cargohold/test/integration/Main.hs index 21e1e336102..9301e222c58 100644 --- a/services/cargohold/test/integration/Main.hs +++ b/services/cargohold/test/integration/Main.hs @@ -20,6 +20,7 @@ module Main ) where +import qualified API import qualified API.V3 import Bilge hiding (body, header) import Data.Proxy @@ -81,7 +82,8 @@ main = runTests go go c i = withResource (getOpts c i) releaseOpts $ \opts -> testGroup "Cargohold" - [ API.V3.tests opts, + [ API.tests opts, + API.V3.tests opts, Metrics.tests opts ] getOpts _ i = do From 6c900994cd2a46726805056dfad2d6a92b05ec52 Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Fri, 17 Dec 2021 12:19:13 +0100 Subject: [PATCH 34/74] Add CHANGELOG entry --- changelog.d/1-api-changes/qualified-assets | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/1-api-changes/qualified-assets diff --git a/changelog.d/1-api-changes/qualified-assets b/changelog.d/1-api-changes/qualified-assets new file mode 100644 index 00000000000..62e988dde3e --- /dev/null +++ b/changelog.d/1-api-changes/qualified-assets @@ -0,0 +1 @@ +Add qualified v4 endpoints for downloading and deleting assets. The upload API is still on the same path, but the asset object it returns now contains a `domain` field. Note that federated behaviour is still not implemented. From 31f2de42d906ff47b19a0f1667809f3b3f8bbc6e Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Mon, 20 Dec 2021 10:25:53 +0100 Subject: [PATCH 35/74] Remove leftover debug definition --- libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs b/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs index 26795c1f46d..f08fdf1c3c5 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs @@ -150,17 +150,6 @@ type BaseAPIv3 (tag :: PrincipalTag) = () ) -type TestAPI = - ( MultiVerb - 'GET - '[JSON] - '[ AssetNotFound, - AssetRedirect, - AssetStreaming - ] - (Maybe LocalOrRemoteAsset) - ) - type QualifiedAPI = ( Summary "Download an asset" :> Description From 6c3fbf95d0f83199ac5d58b9dec107f97a1a44ec Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Thu, 23 Dec 2021 19:17:38 +0100 Subject: [PATCH 36/74] Fix doubtless HLint issues in Federator --- changelog.d/5-internal/hlint-federator | 1 + .../federator/src/Federator/ExternalServer.hs | 4 ++-- .../federator/src/Federator/InternalServer.hs | 2 +- services/federator/src/Federator/MockServer.hs | 4 ++-- .../federator/src/Federator/Monitor/Internal.hs | 2 +- .../integration/Test/Federator/IngressSpec.hs | 9 ++++++--- .../federator/test/unit/Test/Federator/Client.hs | 15 +++++++-------- .../test/unit/Test/Federator/InternalServer.hs | 3 ++- .../federator/test/unit/Test/Federator/Util.hs | 7 +++---- .../test/unit/Test/Federator/Validation.hs | 9 ++++++--- 10 files changed, 31 insertions(+), 25 deletions(-) create mode 100644 changelog.d/5-internal/hlint-federator diff --git a/changelog.d/5-internal/hlint-federator b/changelog.d/5-internal/hlint-federator new file mode 100644 index 00000000000..96c1fac4dc7 --- /dev/null +++ b/changelog.d/5-internal/hlint-federator @@ -0,0 +1 @@ +Fix non-controversial HLint issues in federator to improve code quality diff --git a/services/federator/src/Federator/ExternalServer.hs b/services/federator/src/Federator/ExternalServer.hs index b81c7b18691..092c1a31aff 100644 --- a/services/federator/src/Federator/ExternalServer.hs +++ b/services/federator/src/Federator/ExternalServer.hs @@ -107,14 +107,14 @@ parseRequestData req = do when (Wai.requestMethod req /= HTTP.methodPost) $ throw InvalidRoute -- No query parameters are allowed - when (not . BS.null . Wai.rawQueryString $ req) $ + unless (BS.null . Wai.rawQueryString $ req) $ throw InvalidRoute -- check that the path has the expected form (componentSeg, rpcPath) <- case Wai.pathInfo req of ["federation", comp, rpc] -> pure (comp, rpc) _ -> throw InvalidRoute - when (not (Text.all isAllowedRPCChar rpcPath)) $ + unless (Text.all isAllowedRPCChar rpcPath) $ throw InvalidRoute when (Text.null rpcPath) $ diff --git a/services/federator/src/Federator/InternalServer.hs b/services/federator/src/Federator/InternalServer.hs index 5493448f32b..c9aeace7b55 100644 --- a/services/federator/src/Federator/InternalServer.hs +++ b/services/federator/src/Federator/InternalServer.hs @@ -90,7 +90,7 @@ parseRequestData req = do when (Wai.requestMethod req /= HTTP.methodPost) $ throw InvalidRoute -- No query parameters are allowed - when (not . BS.null . Wai.rawQueryString $ req) $ + unless (BS.null . Wai.rawQueryString $ req) $ throw InvalidRoute -- check that the path has the expected form (domain, componentSeg, rpcPath) <- case Wai.pathInfo req of diff --git a/services/federator/src/Federator/MockServer.hs b/services/federator/src/Federator/MockServer.hs index ec28270ca9f..1284e2e2817 100644 --- a/services/federator/src/Federator/MockServer.hs +++ b/services/federator/src/Federator/MockServer.hs @@ -53,7 +53,7 @@ import Wire.API.Federation.Domain -- | Thrown in IO by mock federator if the server could not be started after 10 -- seconds. -data MockTimeout = MockTimeout Warp.Port +newtype MockTimeout = MockTimeout Warp.Port deriving (Eq, Show, Typeable) instance Exception MockTimeout @@ -159,7 +159,7 @@ withTempMockFederator headers resp action = do frBody = rdBody } ) - embed @IO $ modifyIORef remoteCalls $ (<> [fedRequest]) + embed @IO $ modifyIORef remoteCalls (<> [fedRequest]) body <- fromException @MockException . handle (throw . handleException) diff --git a/services/federator/src/Federator/Monitor/Internal.hs b/services/federator/src/Federator/Monitor/Internal.hs index 28351d384db..9833077cdfe 100644 --- a/services/federator/src/Federator/Monitor/Internal.hs +++ b/services/federator/src/Federator/Monitor/Internal.hs @@ -72,7 +72,7 @@ data WatchedPath deriving stock (Eq, Ord, Show, Generic) deriving (Arbitrary) via (GenericUniform WatchedPath) -mergePaths :: [WatchedPath] -> (Set WatchedPath) +mergePaths :: [WatchedPath] -> Set WatchedPath mergePaths = Set.fromList . merge . sort where merge [] = [] diff --git a/services/federator/test/integration/Test/Federator/IngressSpec.hs b/services/federator/test/integration/Test/Federator/IngressSpec.hs index 34b6af1d327..f1b2b891bf3 100644 --- a/services/federator/test/integration/Test/Federator/IngressSpec.hs +++ b/services/federator/test/integration/Test/Federator/IngressSpec.hs @@ -62,13 +62,14 @@ spec env = do resp <- runTestSem . assertNoError @RemoteError - $ inwardBrigCallViaIngress "get-user-by-handle" $ + $ inwardBrigCallViaIngress + "get-user-by-handle" (Aeson.fromEncoding (Aeson.toEncoding hdl)) liftIO $ do bdy <- streamingResponseStrictBody resp let actualProfile = Aeson.decode (toLazyByteString bdy) responseStatusCode resp `shouldBe` HTTP.status200 - actualProfile `shouldBe` (Just expectedProfile) + actualProfile `shouldBe` Just expectedProfile -- @SF.Federation @TSFI.RESTfulAPI @S2 @S3 @S7 -- @@ -96,7 +97,9 @@ spec env = do r <- runTestSem . runError @RemoteError - $ inwardBrigCallViaIngressWithSettings tlsSettings "get-user-by-handle" $ + $ inwardBrigCallViaIngressWithSettings + tlsSettings + "get-user-by-handle" (Aeson.fromEncoding (Aeson.toEncoding hdl)) liftIO $ case r of Right _ -> expectationFailure "Expected client certificate error, got response" diff --git a/services/federator/test/unit/Test/Federator/Client.hs b/services/federator/test/unit/Test/Federator/Client.hs index 854ac570874..13d92f5cd33 100644 --- a/services/federator/test/unit/Test/Federator/Client.hs +++ b/services/federator/test/unit/Test/Federator/Client.hs @@ -131,13 +131,12 @@ testClientStreaming = withInfiniteMockServer $ \port -> do ceFederator = Endpoint "127.0.0.1" (fromIntegral port) } let c = clientIn (Proxy @StreamingAPI) (Proxy @(FederatorClient 'Brig)) - runCodensity (runFederatorClientToCodensity env c) $ \eout -> - case eout of - Left err -> assertFailure $ "Unexpected error: " <> displayException err - Right out -> do - let expected = mconcat (replicate 500 "Hello") - actual <- takeSourceT (fromIntegral (LBS.length expected)) (fmap Text.encodeUtf8 out) - actual @?= expected + runCodensity (runFederatorClientToCodensity env c) $ \case + Left err -> assertFailure $ "Unexpected error: " <> displayException err + Right out -> do + let expected = mconcat (replicate 500 "Hello") + actual <- takeSourceT (fromIntegral (LBS.length expected)) (fmap Text.encodeUtf8 out) + actual @?= expected testClientFailure :: IO () testClientFailure = do @@ -232,7 +231,7 @@ withInfiniteMockServer k = bracket (startMockServer Nothing app) fst (k . snd) app _ respond = respond $ Wai.responseStream HTTP.ok200 mempty $ \write flush -> let go n = do - when (n == 0) $ flush + when (n == 0) flush write (byteString "Hello\n") *> go (if n == 0 then 100 else n - 1) in go (1000 :: Int) diff --git a/services/federator/test/unit/Test/Federator/InternalServer.hs b/services/federator/test/unit/Test/Federator/InternalServer.hs index b384d0b88fc..55452911f4b 100644 --- a/services/federator/test/unit/Test/Federator/InternalServer.hs +++ b/services/federator/test/unit/Test/Federator/InternalServer.hs @@ -49,7 +49,8 @@ tests :: TestTree tests = testGroup "Federate" - [ testGroup "with remote" $ + [ testGroup + "with remote" [ federatedRequestSuccess, federatedRequestFailureAllowList ] diff --git a/services/federator/test/unit/Test/Federator/Util.hs b/services/federator/test/unit/Test/Federator/Util.hs index a299276c396..ff8acc1be24 100644 --- a/services/federator/test/unit/Test/Federator/Util.hs +++ b/services/federator/test/unit/Test/Federator/Util.hs @@ -62,10 +62,9 @@ testRequest tr = do pure . flip Wai.setPath (trPath tr) $ Wai.defaultRequest { Wai.requestMethod = trMethod tr, - Wai.requestBody = atomicModifyIORef refChunks $ \bss -> - case bss of - [] -> ([], mempty) - x : y -> (y, x), + Wai.requestBody = atomicModifyIORef refChunks $ \case + [] -> ([], mempty) + x : y -> (y, x), Wai.requestHeaders = [("X-SSL-Certificate", HTTP.urlEncode True h) | h <- toList (trCertificateHeader tr)] <> [(originDomainHeaderName, h) | h <- toList (trDomainHeader tr)] diff --git a/services/federator/test/unit/Test/Federator/Validation.hs b/services/federator/test/unit/Test/Federator/Validation.hs index 785757927fb..012ab593083 100644 --- a/services/federator/test/unit/Test/Federator/Validation.hs +++ b/services/federator/test/unit/Test/Federator/Validation.hs @@ -60,12 +60,15 @@ mockDiscoveryFailure = Polysemy.interpret $ \case tests :: TestTree tests = - testGroup "Validation" $ - [ testGroup "federateWith" $ + testGroup + "Validation" + [ testGroup + "federateWith" [ federateWithAllowListSuccess, federateWithAllowListFail ], - testGroup "validateDomain" $ + testGroup + "validateDomain" [ validateDomainAllowListFailSemantic, validateDomainAllowListFail, validateDomainAllowListSuccess, From b358b0b7135707d421f54b2dc1e65b2f597a73df Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 27 Dec 2021 14:28:28 +0100 Subject: [PATCH 37/74] Rename decodeHeader to decodeHeaderOrFail and introduce HasCallStack constraint The HasCallStack constraint leads to more descriptive stacktraces. The new name reflects better, that this function may fail/throw. --- services/cargohold/test/integration/API.hs | 16 ++++++++-------- services/cargohold/test/integration/API/Util.hs | 6 +++--- services/cargohold/test/integration/API/V3.hs | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/services/cargohold/test/integration/API.hs b/services/cargohold/test/integration/API.hs index 0ad220ce526..51f60a757ba 100644 --- a/services/cargohold/test/integration/API.hs +++ b/services/cargohold/test/integration/API.hs @@ -71,7 +71,7 @@ testSimpleRoundtrip c = do r1 <- uploadSimple (c . path "/assets/v3") uid sets bdy HeaderName -> Response b -> a -decodeHeader h = - fromMaybe (error $ "decodeHeader: missing or invalid header: " ++ show h) +decodeHeaderOrFail :: (HasCallStack, FromByteString a) => HeaderName -> Response b -> a +decodeHeaderOrFail h = + fromMaybe (error $ "decodeHeaderOrFail: missing or invalid header: " ++ show h) . fromByteString . getHeader' h diff --git a/services/cargohold/test/integration/API/V3.hs b/services/cargohold/test/integration/API/V3.hs index 323be274205..1b4faaabd54 100644 --- a/services/cargohold/test/integration/API/V3.hs +++ b/services/cargohold/test/integration/API/V3.hs @@ -84,8 +84,8 @@ testSimpleRoundtrip c = do liftIO $ do assertEqual "status" status200 (responseStatus r3) assertEqual "content-type should always be application/octet-stream" (Just applicationOctetStream) (getContentType r3) - assertEqual "token mismatch" tok (decodeHeader "x-amz-meta-token" r3) - assertEqual "user mismatch" uid (decodeHeader "x-amz-meta-user" r3) + assertEqual "token mismatch" tok (decodeHeaderOrFail "x-amz-meta-token" r3) + assertEqual "user mismatch" uid (decodeHeaderOrFail "x-amz-meta-user" r3) assertEqual "data mismatch" (Just "Hello World") (responseBody r3) -- Delete (forbidden for other users) deleteAssetV3 c uid2 (view assetKey ast) !!! const 403 === statusCode From de7aa833f8a9d4b83cc5d0ad5169835006124e28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Dimja=C5=A1evi=C4=87?= Date: Tue, 28 Dec 2021 14:58:26 +0100 Subject: [PATCH 38/74] FS-266: Add sft_servers_all to calls/config/v2 (#2012) * Add two SFT parameters to a Brig template configmap.yaml * Introduce a new DNS action * Extend the DNS interpreters * Propagate new SFT parameters from options to environments * Perform lookupA actions * Extend the return type of newConfig to include all SFT servers * Test fixing * Fix an assertion message in a Brig calling unit test (This is not related to the PR, but it is a single line fix) * Add a couple of unit tests * Expand an integration test * Add a change log --- changelog.d/6-federation/sft-servers-all | 1 + charts/brig/templates/configmap.yaml | 2 + libs/dns-util/dns-util.cabal | 5 +- libs/dns-util/package.yaml | 1 + libs/dns-util/src/Wire/Network/DNS/A.hs | 32 ++++++++ libs/dns-util/src/Wire/Network/DNS/Effect.hs | 23 ++++-- libs/wire-api/src/Wire/API/Call/Config.hs | 28 ++++++- .../Golden/Generated/RTCConfiguration_user.hs | 6 ++ services/brig/brig.cabal | 4 +- services/brig/package.yaml | 2 + services/brig/src/Brig/Calling.hs | 18 ++++- services/brig/src/Brig/Calling/API.hs | 56 +++++++++++-- services/brig/src/Brig/Options.hs | 8 +- services/brig/test/integration/API/Calling.hs | 10 ++- services/brig/test/unit/Test/Brig/Calling.hs | 79 ++++++++++++++----- 15 files changed, 228 insertions(+), 47 deletions(-) create mode 100644 changelog.d/6-federation/sft-servers-all create mode 100644 libs/dns-util/src/Wire/Network/DNS/A.hs diff --git a/changelog.d/6-federation/sft-servers-all b/changelog.d/6-federation/sft-servers-all new file mode 100644 index 00000000000..ec9339bdca7 --- /dev/null +++ b/changelog.d/6-federation/sft-servers-all @@ -0,0 +1 @@ +Extend GET /calls/config/v2 to include all SFT servers in federation diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml index e01d29bd9a7..967217453d1 100644 --- a/charts/brig/templates/configmap.yaml +++ b/charts/brig/templates/configmap.yaml @@ -175,6 +175,8 @@ data: {{- if .sftDiscoveryIntervalSeconds }} sftDiscoveryIntervalSeconds: {{ .sftDiscoveryIntervalSeconds }} {{- end }} + sftLookupDomain: {{ required "Missing value: .sft.sftLookupDomain" .sftLookupDomain }} + sftLookupPort: {{ required "Missing value: .sft.sftLookupPort" .sftLookupPort }} {{- end }} {{- end }} diff --git a/libs/dns-util/dns-util.cabal b/libs/dns-util/dns-util.cabal index 023ad5af62f..63bc4a8e470 100644 --- a/libs/dns-util/dns-util.cabal +++ b/libs/dns-util/dns-util.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: eb1c3d83585fec582c135dbe676c68498f2546468581882f07fdba8f0d16aec3 +-- hash: d71d0f07f44620f73670692eb75544d71d97efd5ddfd25d72388d073f8cf2494 name: dns-util version: 0.1.0 @@ -20,6 +20,7 @@ build-type: Simple library exposed-modules: + Wire.Network.DNS.A Wire.Network.DNS.Effect Wire.Network.DNS.Helper Wire.Network.DNS.SRV @@ -33,6 +34,7 @@ library base >=4.6 && <5.0 , dns , imports + , iproute , polysemy , random default-language: Haskell2010 @@ -55,6 +57,7 @@ test-suite spec , dns-util , hspec , imports + , iproute , polysemy , random default-language: Haskell2010 diff --git a/libs/dns-util/package.yaml b/libs/dns-util/package.yaml index 2a5cb7b9cc2..b0f2b30d5a6 100644 --- a/libs/dns-util/package.yaml +++ b/libs/dns-util/package.yaml @@ -14,6 +14,7 @@ dependencies: - dns - random - imports +- iproute - polysemy library: source-dirs: src diff --git a/libs/dns-util/src/Wire/Network/DNS/A.hs b/libs/dns-util/src/Wire/Network/DNS/A.hs new file mode 100644 index 00000000000..ecac28be2cf --- /dev/null +++ b/libs/dns-util/src/Wire/Network/DNS/A.hs @@ -0,0 +1,32 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2021 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Wire.Network.DNS.A where + +import Data.IP +import Imports +import Network.DNS + +-- | A response to lookupA, i.e., a reponse to a lookup of IPv4 addresses of a +-- domain. +data AResponse + = AIPv4s [IPv4] + | AResponseError DNSError + deriving (Eq) + +interpretResponse :: Either DNSError [IPv4] -> AResponse +interpretResponse = either AResponseError AIPv4s diff --git a/libs/dns-util/src/Wire/Network/DNS/Effect.hs b/libs/dns-util/src/Wire/Network/DNS/Effect.hs index 5484d971d12..b7e3bafd23c 100644 --- a/libs/dns-util/src/Wire/Network/DNS/Effect.hs +++ b/libs/dns-util/src/Wire/Network/DNS/Effect.hs @@ -21,20 +21,29 @@ import Imports import Network.DNS (Domain, Resolver) import qualified Network.DNS as DNS import Polysemy -import Wire.Network.DNS.SRV +import qualified Wire.Network.DNS.A as A +import qualified Wire.Network.DNS.SRV as SRV data DNSLookup m a where - LookupSRV :: Domain -> DNSLookup m SrvResponse + LookupSRV :: Domain -> DNSLookup m SRV.SrvResponse + LookupA :: Domain -> DNSLookup m A.AResponse makeSem ''DNSLookup runDNSLookupDefault :: Member (Embed IO) r => Sem (DNSLookup ': r) a -> Sem r a runDNSLookupDefault = - interpret $ \(LookupSRV domain) -> embed $ do - rs <- DNS.makeResolvSeed DNS.defaultResolvConf - DNS.withResolver rs $ \resolver -> - interpretResponse <$> DNS.lookupSRV resolver domain + interpret $ \case + LookupSRV domain -> embed $ do + rs <- DNS.makeResolvSeed DNS.defaultResolvConf + DNS.withResolver rs $ \resolver -> + SRV.interpretResponse <$> DNS.lookupSRV resolver domain + LookupA domain -> embed $ do + rs <- DNS.makeResolvSeed DNS.defaultResolvConf + DNS.withResolver rs $ \resolver -> + A.interpretResponse <$> DNS.lookupA resolver domain runDNSLookupWithResolver :: Member (Embed IO) r => Resolver -> Sem (DNSLookup ': r) a -> Sem r a runDNSLookupWithResolver resolver = - interpret $ \(LookupSRV domain) -> embed (interpretResponse <$> DNS.lookupSRV resolver domain) + interpret $ \case + LookupSRV domain -> embed (SRV.interpretResponse <$> DNS.lookupSRV resolver domain) + LookupA domain -> embed (A.interpretResponse <$> DNS.lookupA resolver domain) diff --git a/libs/wire-api/src/Wire/API/Call/Config.hs b/libs/wire-api/src/Wire/API/Call/Config.hs index c239de17344..85c315254be 100644 --- a/libs/wire-api/src/Wire/API/Call/Config.hs +++ b/libs/wire-api/src/Wire/API/Call/Config.hs @@ -25,6 +25,7 @@ module Wire.API.Call.Config rtcConfiguration, rtcConfIceServers, rtcConfSftServers, + rtcConfSftServersAll, rtcConfTTL, -- * RTCIceServer @@ -104,12 +105,18 @@ import Wire.API.Arbitrary (Arbitrary (arbitrary), GenericUniform (..)) data RTCConfiguration = RTCConfiguration { _rtcConfIceServers :: NonEmpty RTCIceServer, _rtcConfSftServers :: Maybe (NonEmpty SFTServer), - _rtcConfTTL :: Word32 + _rtcConfTTL :: Word32, + _rtcConfSftServersAll :: Maybe [SFTServer] } deriving stock (Eq, Show, Generic) deriving (Arbitrary) via (GenericUniform RTCConfiguration) -rtcConfiguration :: NonEmpty RTCIceServer -> Maybe (NonEmpty SFTServer) -> Word32 -> RTCConfiguration +rtcConfiguration :: + NonEmpty RTCIceServer -> + Maybe (NonEmpty SFTServer) -> + Word32 -> + Maybe [SFTServer] -> + RTCConfiguration rtcConfiguration = RTCConfiguration modelRtcConfiguration :: Doc.Model @@ -121,19 +128,26 @@ modelRtcConfiguration = Doc.defineModel "RTCConfiguration" $ do Doc.description "Array of 'SFTServer' objects (optional)" Doc.property "ttl" Doc.int32' $ Doc.description "Number of seconds after which the configuration should be refreshed (advisory)" + Doc.property "sft_servers_all" (Doc.array (Doc.ref modelRtcSftServerUrl)) $ + Doc.description "Array of 'SFTServer' URLs (optional)" instance ToJSON RTCConfiguration where - toJSON (RTCConfiguration srvs sfts ttl) = + toJSON (RTCConfiguration srvs sfts ttl all_servers) = object ( [ "ice_servers" .= srvs, "ttl" .= ttl ] <> ["sft_servers" .= sfts | isJust sfts] + <> ["sft_servers_all" .= all_servers | isJust all_servers] ) instance FromJSON RTCConfiguration where parseJSON = withObject "RTCConfiguration" $ \o -> - RTCConfiguration <$> o .: "ice_servers" <*> o .:? "sft_servers" <*> o .: "ttl" + RTCConfiguration + <$> o .: "ice_servers" + <*> o .:? "sft_servers" + <*> o .: "ttl" + <*> o .:? "sft_servers_all" -------------------------------------------------------------------------------- -- SFTServer @@ -165,6 +179,12 @@ modelRtcSftServer = Doc.defineModel "RTC SFT Server" $ do Doc.property "urls" (Doc.array Doc.string') $ Doc.description "Array containing exactly one SFT server address of the form 'https://:'" +modelRtcSftServerUrl :: Doc.Model +modelRtcSftServerUrl = Doc.defineModel "RTC SFT Server URL" $ do + Doc.description "Inspired by WebRTC 'RTCIceServer' object, contains details of SFT servers" + Doc.property "urls" (Doc.array Doc.string') $ + Doc.description "Array containing exactly one SFT server URL" + -------------------------------------------------------------------------------- -- RTCIceServer diff --git a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/RTCConfiguration_user.hs b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/RTCConfiguration_user.hs index cb52eaec7a7..5ef3d79325d 100644 --- a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/RTCConfiguration_user.hs +++ b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/RTCConfiguration_user.hs @@ -152,6 +152,7 @@ testObject_RTCConfiguration_user_1 = ) (Nothing) (2) + Nothing ) testObject_RTCConfiguration_user_2 :: RTCConfiguration @@ -334,6 +335,7 @@ testObject_RTCConfiguration_user_2 = ) ) (4) + Nothing ) testObject_RTCConfiguration_user_3 :: RTCConfiguration @@ -480,6 +482,7 @@ testObject_RTCConfiguration_user_3 = ) ) (9) + Nothing ) testObject_RTCConfiguration_user_4 :: RTCConfiguration @@ -685,6 +688,7 @@ testObject_RTCConfiguration_user_4 = ) ) (2) + Nothing ) testObject_RTCConfiguration_user_5 :: RTCConfiguration @@ -728,6 +732,7 @@ testObject_RTCConfiguration_user_5 = ) ) (2) + Nothing ) testObject_RTCConfiguration_user_6 :: RTCConfiguration @@ -750,4 +755,5 @@ testObject_RTCConfiguration_user_6 = ) Nothing (2) + Nothing ) diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal index 94a4b541681..17a6c70c7b8 100644 --- a/services/brig/brig.cabal +++ b/services/brig/brig.cabal @@ -4,7 +4,7 @@ cabal-version: 2.0 -- -- see: https://github.com/sol/hpack -- --- hash: 513c0f5104342fb14b0246f7c44733a84ec36fae97633fc55cb209e0e0bcd087 +-- hash: 244ca5e0c12867ef47ffcfbc056775381028fbc85e5c57eb280afe842e3499d9 name: brig version: 2.0 @@ -167,6 +167,7 @@ library , errors >=1.4 , exceptions >=0.5 , extended + , extra , filepath >=1.3 , fsnotify >=0.2 , galley-types >=0.75.3 @@ -496,6 +497,7 @@ test-suite brig-tests , dns-util , http-types , imports + , iproute , polysemy , polysemy-wire-zoo , retry diff --git a/services/brig/package.yaml b/services/brig/package.yaml index 05329979f73..02cd7dbf32b 100644 --- a/services/brig/package.yaml +++ b/services/brig/package.yaml @@ -57,6 +57,7 @@ library: - errors >=1.4 - exceptions >=0.5 - extended + - extra - filepath >=1.3 - fsnotify >=0.2 - galley-types >=0.75.3 @@ -160,6 +161,7 @@ tests: - dns-util - http-types - imports + - iproute - polysemy - polysemy-wire-zoo - retry diff --git a/services/brig/src/Brig/Calling.hs b/services/brig/src/Brig/Calling.hs index 73e981873a5..9c102058288 100644 --- a/services/brig/src/Brig/Calling.hs +++ b/services/brig/src/Brig/Calling.hs @@ -29,6 +29,7 @@ module Brig.Calling newEnv, sftDiscoveryLoop, discoverSFTServers, + discoverSFTServersAll, discoveryToMaybe, randomize, startSFTServiceDiscovery, @@ -46,9 +47,11 @@ import qualified Brig.Options as Opts import Brig.Types (TurnURI) import Control.Lens import Control.Monad.Random.Class (MonadRandom) +import Data.IP import Data.List.NonEmpty (NonEmpty (..)) import qualified Data.List.NonEmpty as NonEmpty import Data.List1 +import Data.Misc (Port (..)) import Data.Range import Data.Time.Clock (DiffTime, diffTimeToPicoseconds) import Imports @@ -59,6 +62,7 @@ import Polysemy.TinyLog import qualified System.Logger as Log import System.Random.MWC (GenIO, createSystemRandom) import System.Random.Shuffle +import Wire.Network.DNS.A import Wire.Network.DNS.Effect import Wire.Network.DNS.SRV @@ -109,7 +113,9 @@ data SFTEnv = SFTEnv sftDiscoveryInterval :: Int, -- | maximum amount of servers to give out, -- even if more are in the SRV record - sftListLength :: Range 1 100 Int + sftListLength :: Range 1 100 Int, + sftLookupDomain :: DNS.Domain, + sftLookupPort :: Port } data Discovery a @@ -133,6 +139,14 @@ discoverSFTServers domain = err (Log.msg ("DNS Lookup failed for SFT Discovery" :: ByteString) . Log.field "Error" (show e)) pure Nothing +discoverSFTServersAll :: Members [DNSLookup, TinyLog] r => DNS.Domain -> Sem r (Maybe [IPv4]) +discoverSFTServersAll domain = + lookupA domain >>= \case + AIPv4s ips -> pure . Just $ ips + AResponseError e -> do + err (Log.msg ("DNS Lookup failed for SFT Discovery" :: ByteString) . Log.field "Error" (show e)) + pure Nothing + mkSFTDomain :: SFTOptions -> DNS.Domain mkSFTDomain SFTOptions {..} = DNS.normalize $ maybe defSftServiceName ("_" <>) sftSRVServiceName <> "._tcp." <> sftBaseDomain @@ -153,6 +167,8 @@ mkSFTEnv opts = <*> pure (mkSFTDomain opts) <*> pure (diffTimeToMicroseconds (fromMaybe defSftDiscoveryIntervalSeconds (Opts.sftDiscoveryIntervalSeconds opts))) <*> pure (fromMaybe defSftListLength (Opts.sftListLength opts)) + <*> pure (Opts.sftLookupDomain opts) + <*> pure (Opts.sftLookupPort opts) startSFTServiceDiscovery :: Log.Logger -> SFTEnv -> IO () startSFTServiceDiscovery logger = diff --git a/services/brig/src/Brig/Calling/API.hs b/services/brig/src/Brig/Calling/API.hs index e2654a488bc..4e500c9f417 100644 --- a/services/brig/src/Brig/Calling/API.hs +++ b/services/brig/src/Brig/Calling/API.hs @@ -29,8 +29,9 @@ import qualified Brig.Calling as Calling import Brig.Calling.Internal import qualified Brig.Options as Opt import Control.Lens -import Data.ByteString.Conversion (toByteString') +import Data.ByteString.Conversion import Data.ByteString.Lens +import Data.Either.Extra (eitherToMaybe) import Data.Id import Data.List.NonEmpty (NonEmpty (..)) import qualified Data.List.NonEmpty as NonEmpty @@ -42,14 +43,20 @@ import Data.Text.Ascii (AsciiBase64, encodeBase64) import Data.Text.Strict.Lens import Data.Time.Clock.POSIX (getPOSIXTime) import Imports hiding (head) +import Network.HTTP.Client hiding (Response) import Network.Wai (Response) import Network.Wai.Predicate hiding (and, result, setStatus, (#)) import Network.Wai.Routing hiding (toList) import Network.Wai.Utilities hiding (code, message) import Network.Wai.Utilities.Swagger (document) import OpenSSL.EVP.Digest (Digest, hmacBS) +import Polysemy (runM) +import Polysemy.TinyLog +import System.Logger (Logger) import qualified System.Random.MWC as MWC +import Wire.API.Call.Config (SFTServer) import qualified Wire.API.Call.Config as Public +import Wire.Network.DNS.Effect import Wire.Network.DNS.SRV (srvTarget) routesPublic :: Routes Doc.ApiBuilder Handler () @@ -91,10 +98,12 @@ getCallsConfigV2H (_ ::: uid ::: connid ::: limit) = -- | ('UserId', 'ConnId' are required as args here to make sure this is an authenticated end-point.) getCallsConfigV2 :: UserId -> ConnId -> Maybe (Range 1 10 Int) -> Handler Public.RTCConfiguration getCallsConfigV2 _ _ limit = do - env <- liftIO =<< readIORef <$> view turnEnvV2 + env <- liftIO . readIORef =<< view turnEnvV2 staticUrl <- view $ settings . Opt.sftStaticUrl sftEnv' <- view sftEnv - newConfig env staticUrl sftEnv' limit + httpMan <- view httpManager + logger <- view applog + newConfig env staticUrl sftEnv' limit httpMan logger getCallsConfigH :: JSON ::: UserId ::: ConnId -> Handler Response getCallsConfigH (_ ::: uid ::: connid) = @@ -102,8 +111,10 @@ getCallsConfigH (_ ::: uid ::: connid) = getCallsConfig :: UserId -> ConnId -> Handler Public.RTCConfiguration getCallsConfig _ _ = do - env <- liftIO =<< readIORef <$> view turnEnv - dropTransport <$> newConfig env Nothing Nothing Nothing + env <- liftIO . readIORef =<< view turnEnv + httpMan <- view httpManager + logger <- view applog + dropTransport <$> newConfig env Nothing Nothing Nothing httpMan logger where -- In order to avoid being backwards incompatible, remove the `transport` query param from the URIs dropTransport :: Public.RTCConfiguration -> Public.RTCConfiguration @@ -112,8 +123,16 @@ getCallsConfig _ _ = do (Public.rtcConfIceServers . traverse . Public.iceURLs . traverse . Public.turiTransport) Nothing -newConfig :: MonadIO m => Calling.Env -> Maybe HttpsUrl -> Maybe SFTEnv -> Maybe (Range 1 10 Int) -> m Public.RTCConfiguration -newConfig env sftStaticUrl mSftEnv limit = do +newConfig :: + MonadIO m => + Calling.Env -> + Maybe HttpsUrl -> + Maybe SFTEnv -> + Maybe (Range 1 10 Int) -> + Manager -> + Logger -> + m Public.RTCConfiguration +newConfig env sftStaticUrl mSftEnv limit httpMan logger = do let (sha, secret, tTTL, cTTL, prng) = (env ^. turnSHA512, env ^. turnSecret, env ^. turnTokenTTL, env ^. turnConfigTTL, env ^. turnPrng) -- randomize list of servers (before limiting the list, to ensure not always the same servers are chosen if limit is set) randomizedUris <- liftIO $ randomize (List1.toNonEmpty $ env ^. turnServers) @@ -134,7 +153,28 @@ newConfig env sftStaticUrl mSftEnv limit = do let subsetLength = Calling.sftListLength actualSftEnv liftIO $ mapM (getRandomSFTServers subsetLength) sftSrvEntries - pure $ Public.rtcConfiguration srvs (staticSft <|> sftServerFromSrvTarget . srvTarget <$$> sftEntries) cTTL + let mSftServers = staticSft <|> sftServerFromSrvTarget . srvTarget <$$> sftEntries + mSftServersAll :: Maybe (Maybe [SFTServer]) <- for mSftEnv $ \e -> liftIO $ do + response <- runM . runTinyLog logger . runDNSLookupDefault . discoverSFTServersAll . sftLookupDomain $ e + case response of + Nothing -> pure $ Nothing @[SFTServer] + Just ips -> fmap (eitherToMaybe @String @[SFTServer] . sequence) $ + for ips $ \ip -> do + let req = + parseRequest_ $ + mconcat + [ "GET ", + show ip, + ":", + show . sftLookupPort $ e, + "/sft/url" + ] + -- TODO: introduce an effect for talking to SFT. Perhaps this could be a + -- part of an existing effect External. + sftUrlResponse <- liftIO (responseBody <$> httpLbs req httpMan) + pure @IO . fmap Public.sftServer . runParser' (parser @HttpsUrl) $ sftUrlResponse + + pure $ Public.rtcConfiguration srvs mSftServers cTTL (join mSftServersAll) where limitedList :: NonEmpty Public.TurnURI -> Range 1 10 Int -> NonEmpty Public.TurnURI limitedList uris lim = diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs index ad631e49325..5d73b27e89e 100644 --- a/services/brig/src/Brig/Options.hs +++ b/services/brig/src/Brig/Options.hs @@ -33,7 +33,7 @@ import Data.Aeson.Types (typeMismatch) import qualified Data.Char as Char import Data.Domain (Domain (..)) import Data.Id -import Data.Misc (HttpsUrl) +import Data.Misc (HttpsUrl, Port (..)) import Data.Range import Data.Scientific (toBoundedInteger) import qualified Data.Text as Text @@ -612,7 +612,9 @@ data SFTOptions = SFTOptions { sftBaseDomain :: !DNS.Domain, sftSRVServiceName :: !(Maybe ByteString), -- defaults to defSftServiceName if unset sftDiscoveryIntervalSeconds :: !(Maybe DiffTime), -- defaults to defSftDiscoveryIntervalSeconds - sftListLength :: !(Maybe (Range 1 100 Int)) -- defaults to defSftListLength + sftListLength :: !(Maybe (Range 1 100 Int)), -- defaults to defSftListLength + sftLookupDomain :: !DNS.Domain, + sftLookupPort :: !Port } deriving (Show, Generic) @@ -623,6 +625,8 @@ instance FromJSON SFTOptions where <*> (mapM asciiOnly =<< o .:? "sftSRVServiceName") <*> (secondsToDiffTime <$$> o .:? "sftDiscoveryIntervalSeconds") <*> (o .:? "sftListLength") + <*> (asciiOnly =<< o .: "sftLookupDomain") + <*> o .: "sftLookupPort" where asciiOnly :: Text -> Y.Parser ByteString asciiOnly t = diff --git a/services/brig/test/integration/API/Calling.hs b/services/brig/test/integration/API/Calling.hs index be0185af2d0..6515f2272ff 100644 --- a/services/brig/test/integration/API/Calling.hs +++ b/services/brig/test/integration/API/Calling.hs @@ -32,7 +32,7 @@ import Data.Id import qualified Data.List.NonEmpty as NonEmpty import Data.List1 (List1) import qualified Data.List1 as List1 -import Data.Misc (Port, mkHttpsUrl) +import Data.Misc (Port (..), mkHttpsUrl) import qualified Data.Set as Set import Imports import System.FilePath (()) @@ -103,12 +103,16 @@ testSFT :: Brig -> Opts.Opts -> Http () testSFT b opts = do uid <- userId <$> randomUser b cfg <- getTurnConfigurationV2 uid b - liftIO $ + liftIO $ do assertEqual "when SFT discovery is not enabled, sft_servers shouldn't be returned" Nothing (cfg ^. rtcConfSftServers) - withSettingsOverrides (opts & Opts.sftL ?~ Opts.SFTOptions "integration-tests.zinfra.io" Nothing (Just 0.001) Nothing) $ do + assertEqual + "when SFT discovery is not enabled, sft_servers_all shouldn't be returned" + Nothing + (cfg ^. rtcConfSftServersAll) + withSettingsOverrides (opts & Opts.sftL ?~ Opts.SFTOptions "integration-tests.zinfra.io" Nothing (Just 0.001) Nothing "integration-tests.zinfra.io" (Port 8585)) $ do cfg1 <- retryWhileN 10 (isNothing . view rtcConfSftServers) (getTurnConfigurationV2 uid b) -- These values are controlled by https://github.com/zinfra/cailleach/tree/77ca2d23cf2959aa183dd945d0a0b13537a8950d/environments/dns-integration-tests let Right server1 = mkHttpsUrl =<< first show (parseURI laxURIParserOptions "https://sft01.integration-tests.zinfra.io:443") diff --git a/services/brig/test/unit/Test/Brig/Calling.hs b/services/brig/test/unit/Test/Brig/Calling.hs index 8fab0a2731f..64af8c99b84 100644 --- a/services/brig/test/unit/Test/Brig/Calling.hs +++ b/services/brig/test/unit/Test/Brig/Calling.hs @@ -22,8 +22,10 @@ module Test.Brig.Calling where import Brig.Calling import Brig.Options import Control.Retry +import Data.IP import Data.List.NonEmpty (NonEmpty (..)) import qualified Data.List.NonEmpty as NonEmpty +import Data.Misc (Port (..)) import Data.Range import qualified Data.Set as Set import Imports @@ -34,23 +36,28 @@ import qualified System.Logger as Log import Test.Tasty import Test.Tasty.HUnit import qualified UnliftIO.Async as Async +import qualified Wire.Network.DNS.A as A import Wire.Network.DNS.Effect import Wire.Network.DNS.SRV data FakeDNSEnv = FakeDNSEnv - { fakeLookupFn :: Domain -> SrvResponse, + { fakeLookupSrv :: Domain -> SrvResponse, + fakeLookupA :: Domain -> A.AResponse, fakeLookupCalls :: IORef [Domain] } -newFakeDNSEnv :: (Domain -> SrvResponse) -> IO FakeDNSEnv -newFakeDNSEnv lookupFn = - FakeDNSEnv lookupFn <$> newIORef [] +newFakeDNSEnv :: (Domain -> SrvResponse) -> (Domain -> A.AResponse) -> IO FakeDNSEnv +newFakeDNSEnv lookupSrvFn lookupAFn = + FakeDNSEnv lookupSrvFn lookupAFn <$> newIORef [] runFakeDNSLookup :: Member (Embed IO) r => FakeDNSEnv -> Sem (DNSLookup ': r) a -> Sem r a runFakeDNSLookup FakeDNSEnv {..} = interpret $ \case LookupSRV domain -> do modifyIORef' fakeLookupCalls (++ [domain]) - pure $ fakeLookupFn domain + pure $ fakeLookupSrv domain + LookupA domain -> do + modifyIORef' fakeLookupCalls (++ [domain]) + pure $ fakeLookupA domain newtype LogRecorder = LogRecorder {recordedLogs :: IORef [(Log.Level, LByteString)]} @@ -67,18 +74,19 @@ ignoreLogs = interpret $ \(Polylog _ _) -> pure () {-# ANN tests ("HLint: ignore" :: String) #-} tests :: TestTree tests = - testGroup "Calling" $ + testGroup + "Calling" [ testGroup "mkSFTDomain" $ [ testCase "when service name is provided" $ assertEqual "should use the service name to form domain" "_foo._tcp.example.com." - (mkSFTDomain (SFTOptions "example.com" (Just "foo") Nothing Nothing)), + (mkSFTDomain (SFTOptions "example.com" (Just "foo") Nothing Nothing "example.com" (Port 8585))), testCase "when service name is not provided" $ assertEqual "should assume service name to be 'sft'" "_sft._tcp.example.com." - (mkSFTDomain (SFTOptions "example.com" Nothing Nothing Nothing)) + (mkSFTDomain (SFTOptions "example.com" Nothing Nothing Nothing "example.com" (Port 8585))) ], testGroup "sftDiscoveryLoop" $ [ testCase "when service can be discovered" $ void testDiscoveryLoopWhenSuccessful, @@ -95,6 +103,10 @@ tests = [ testCase "more servers in SRV than limit" testSFTManyServers, testCase "fewer servers in SRV than limit" testSFTFewerServers -- the randomization part is not (yet?) tested here. + ], + testGroup "discoverSFTServersAll" $ + [ testCase "when service is available" testSFTDiscoverAWhenAvailable, + testCase "when dns lookup fails" testSFTDiscoverAWhenDNSFails ] ] @@ -104,8 +116,8 @@ testDiscoveryLoopWhenSuccessful = do entry2 = SrvEntry 0 0 (SrvTarget "sft2.foo.example.com." 443) entry3 = SrvEntry 0 0 (SrvTarget "sft3.foo.example.com." 443) returnedEntries = entry1 :| [entry2, entry3] - fakeDNSEnv <- newFakeDNSEnv (\_ -> SrvAvailable returnedEntries) - sftEnv <- mkSFTEnv (SFTOptions "foo.example.com" Nothing (Just 0.001) Nothing) + fakeDNSEnv <- newFakeDNSEnv (\_ -> SrvAvailable returnedEntries) undefined + sftEnv <- mkSFTEnv (SFTOptions "foo.example.com" Nothing (Just 0.001) Nothing "foo.example.com" (Port 8585)) discoveryLoop <- Async.async $ runM . ignoreLogs . runFakeDNSLookup fakeDNSEnv $ sftDiscoveryLoop sftEnv void $ retryEvery10MicrosWhileN 2000 (== 0) (length <$> readIORef (fakeLookupCalls fakeDNSEnv)) @@ -119,8 +131,8 @@ testDiscoveryLoopWhenSuccessful = do testDiscoveryLoopWhenUnsuccessful :: IO () testDiscoveryLoopWhenUnsuccessful = do - fakeDNSEnv <- newFakeDNSEnv (\_ -> SrvNotAvailable) - sftEnv <- mkSFTEnv (SFTOptions "foo.example.com" Nothing (Just 0.001) Nothing) + fakeDNSEnv <- newFakeDNSEnv (const SrvNotAvailable) undefined + sftEnv <- mkSFTEnv (SFTOptions "foo.example.com" Nothing (Just 0.001) Nothing "foo.example.com" (Port 8585)) discoveryLoop <- Async.async $ runM . ignoreLogs . runFakeDNSLookup fakeDNSEnv $ sftDiscoveryLoop sftEnv -- We wait for at least two lookups to be sure that the lookup loop looped at @@ -138,7 +150,7 @@ testDiscoveryLoopWhenUnsuccessfulAfterSuccess = do -- In the following lines we re-use the 'sftEnv' from a successful lookup to -- replicate what will happen when a dns lookup fails after success - failingFakeDNSEnv <- newFakeDNSEnv (\_ -> SrvNotAvailable) + failingFakeDNSEnv <- newFakeDNSEnv (const SrvNotAvailable) undefined discoveryLoop <- Async.async $ runM . ignoreLogs . runFakeDNSLookup failingFakeDNSEnv $ sftDiscoveryLoop sftEnv -- We wait for at least two lookups to be sure that the lookup loop looped at -- least once @@ -156,9 +168,9 @@ testDiscoveryLoopWhenURLsChange = do -- replicate what will happen when a dns lookup returns new URLs let entry1 = SrvEntry 0 0 (SrvTarget "sft4.foo.example.com." 443) entry2 = SrvEntry 0 0 (SrvTarget "sft5.foo.example.com." 443) - newEntries = (entry1 :| [entry2]) + newEntries = entry1 :| [entry2] - fakeDNSEnv <- newFakeDNSEnv (\_ -> SrvAvailable newEntries) + fakeDNSEnv <- newFakeDNSEnv (const $ SrvAvailable newEntries) undefined discoveryLoop <- Async.async $ runM . ignoreLogs . runFakeDNSLookup fakeDNSEnv $ sftDiscoveryLoop sftEnv void $ retryEvery10MicrosWhileN 2000 (== 0) (length <$> readIORef (fakeLookupCalls fakeDNSEnv)) -- We don't want to stop the loop before it has written to the sftServers IORef @@ -173,8 +185,8 @@ testSFTDiscoverWhenAvailable = do logRecorder <- newLogRecorder let entry1 = SrvEntry 0 0 (SrvTarget "sft7.foo.example.com." 443) entry2 = SrvEntry 0 0 (SrvTarget "sft8.foo.example.com." 8843) - returnedEntries = (entry1 :| [entry2]) - fakeDNSEnv <- newFakeDNSEnv (\_ -> SrvAvailable returnedEntries) + returnedEntries = entry1 :| [entry2] + fakeDNSEnv <- newFakeDNSEnv (\_ -> SrvAvailable returnedEntries) undefined assertEqual "discovered servers should be returned" (Just returnedEntries) =<< ( runM . recordLogs logRecorder . runFakeDNSLookup fakeDNSEnv $ @@ -186,7 +198,7 @@ testSFTDiscoverWhenAvailable = do testSFTDiscoverWhenNotAvailable :: IO () testSFTDiscoverWhenNotAvailable = do logRecorder <- newLogRecorder - fakeDNSEnv <- newFakeDNSEnv (\_ -> SrvNotAvailable) + fakeDNSEnv <- newFakeDNSEnv (const SrvNotAvailable) undefined assertEqual "discovered servers should be returned" Nothing =<< ( runM . recordLogs logRecorder . runFakeDNSLookup fakeDNSEnv $ @@ -198,9 +210,9 @@ testSFTDiscoverWhenNotAvailable = do testSFTDiscoverWhenDNSFails :: IO () testSFTDiscoverWhenDNSFails = do logRecorder <- newLogRecorder - fakeDNSEnv <- newFakeDNSEnv (\_ -> SrvResponseError IllegalDomain) + fakeDNSEnv <- newFakeDNSEnv (const $ SrvResponseError IllegalDomain) undefined - assertEqual "discovered servers should be returned" Nothing + assertEqual "no servers should be returned" Nothing =<< ( runM . recordLogs logRecorder . runFakeDNSLookup fakeDNSEnv $ discoverSFTServers "_sft._tcp.foo.example.com" ) @@ -239,3 +251,30 @@ retryEvery10MicrosWhileN n f m = (constantDelay 10 <> limitRetries n) (const (return . f)) (const m) + +testSFTDiscoverAWhenAvailable :: IO () +testSFTDiscoverAWhenAvailable = do + logRecorder <- newLogRecorder + let entry1 = read "192.0.2.1" :: IPv4 + entry2 = read "192.0.2.2" :: IPv4 + returnedEntries = [entry1, entry2] + fakeDNSEnv <- newFakeDNSEnv undefined (const $ A.AIPv4s returnedEntries) + + assertEqual "discovered servers should be returned" (Just returnedEntries) + =<< ( runM . recordLogs logRecorder . runFakeDNSLookup fakeDNSEnv $ + discoverSFTServersAll "foo.example.com" + ) + assertEqual "nothing should be logged" [] + =<< readIORef (recordedLogs logRecorder) + +testSFTDiscoverAWhenDNSFails :: IO () +testSFTDiscoverAWhenDNSFails = do + logRecorder <- newLogRecorder + fakeDNSEnv <- newFakeDNSEnv undefined (const $ A.AResponseError IllegalDomain) + + assertEqual "no servers should be returned" Nothing + =<< ( runM . recordLogs logRecorder . runFakeDNSLookup fakeDNSEnv $ + discoverSFTServersAll "foo.example.com" + ) + assertEqual "should warn about it in the logs" [(Log.Error, "DNS Lookup failed for SFT Discovery, Error=IllegalDomain\n")] + =<< readIORef (recordedLogs logRecorder) From dd6bdcda76d3f81b3e3d4b17f7947a8ae48b655c Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Wed, 29 Dec 2021 11:27:12 +0100 Subject: [PATCH 39/74] Add documentation about linting --- .gitignore | 5 +++- docs/developer/linting.md | 55 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 docs/developer/linting.md diff --git a/.gitignore b/.gitignore index d1c8a56e869..a906ed007c6 100644 --- a/.gitignore +++ b/.gitignore @@ -85,6 +85,9 @@ hie.yaml hie.orig.yaml stack-dev.yaml +# HIE db files (e.g. generated for stan) +*.hie + # generated files under .local .local @@ -102,4 +105,4 @@ telepresence.log # local config .envrc.local -cabal.project.local \ No newline at end of file +cabal.project.local diff --git a/docs/developer/linting.md b/docs/developer/linting.md new file mode 100644 index 00000000000..e0d787433ae --- /dev/null +++ b/docs/developer/linting.md @@ -0,0 +1,55 @@ +# Linting + +# HLint + +To run [HLint](https://github.com/ndmitchell/hlint) you need it's binary, e.g. +by executing: + +```sh +nix-shell -p hlint +``` + +To run it on the whole project (Warning: This takes long!): + +```sh +hlint . +``` + +To run it on a sub-project: + +```sh +hlint services/federator +``` + +# Stan + +To run [Stan](https://github.com/kowainik/stan), you need it's binary compiled +by the same GHC version as used in the project. + +```sh +nix-shell -p haskell.packages.ghc884.stan +``` + +Stan depends on [*hie*](https://www.haskell.org/ghc/blog/20190626-HIEFiles.html) +database files that are created during compilation. To generate them for all +packages add this to your `cabal.project.local` file: + +``` +package * + ghc-options: -fwrite-ide-info -hiedir=.hie +``` + +Of course, you can append the `ghc-options` to the respective entry of a package or +add a new one: + +```sh +package cargohold + ghc-options: -fwrite-ide-info -hiedir=.hie +``` + +To analyze a sub-project with stan: + +```sh +cd services/cargohold +stan +``` From 0ec9c66237d8403fe7239289e690e78cf5dd0b59 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Wed, 29 Dec 2021 13:52:37 +0100 Subject: [PATCH 40/74] Update docs/developer/linting.md Co-authored-by: Julia Longtin --- docs/developer/linting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer/linting.md b/docs/developer/linting.md index e0d787433ae..5dcbf75ab0f 100644 --- a/docs/developer/linting.md +++ b/docs/developer/linting.md @@ -9,7 +9,7 @@ by executing: nix-shell -p hlint ``` -To run it on the whole project (Warning: This takes long!): +To run it on the whole project (Warning: This takes a long time!): ```sh hlint . From 8613748a62b138e331b7848b50811f09bda1f26c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Dimja=C5=A1evi=C4=87?= Date: Thu, 30 Dec 2021 15:42:12 +0100 Subject: [PATCH 41/74] Enable Insecure Requests for SFT Lookups in a Test Environment (#2014) * Move SFT lookup parameters to another configuration section * Fix a request to an SFT's `GET /sft/url` * Group the SFT lookup domain and port in Brig configuration * Fix how the SFT environment is computed. Before, it would rely on the SFT lookup parameters to be set to keep on working. This change makes SFT lookup parameters truly optional. * Add a Brig configuration flag for insecure connections in test environments * Introduce an effect for SFT --- .../6-federation/sft-servers-all-fixup | 1 + charts/brig/templates/configmap.yaml | 10 ++- services/brig/brig.cabal | 5 +- services/brig/brig.integration.yaml | 6 ++ services/brig/package.yaml | 2 + services/brig/src/Brig/App.hs | 2 +- services/brig/src/Brig/Calling.hs | 11 ++- services/brig/src/Brig/Calling/API.hs | 49 +++++++------ services/brig/src/Brig/Effects/SFT.hs | 68 +++++++++++++++++++ services/brig/src/Brig/Options.hs | 47 +++++++++++-- services/brig/test/integration/API/Calling.hs | 10 ++- services/brig/test/unit/Test/Brig/Calling.hs | 26 +++---- 12 files changed, 179 insertions(+), 58 deletions(-) create mode 100644 changelog.d/6-federation/sft-servers-all-fixup create mode 100644 services/brig/src/Brig/Effects/SFT.hs diff --git a/changelog.d/6-federation/sft-servers-all-fixup b/changelog.d/6-federation/sft-servers-all-fixup new file mode 100644 index 00000000000..25c1a8fb06d --- /dev/null +++ b/changelog.d/6-federation/sft-servers-all-fixup @@ -0,0 +1 @@ +Improve Brig's configuration for SFTs and fix a call to SFT servers diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml index 967217453d1..88be747050b 100644 --- a/charts/brig/templates/configmap.yaml +++ b/charts/brig/templates/configmap.yaml @@ -175,8 +175,6 @@ data: {{- if .sftDiscoveryIntervalSeconds }} sftDiscoveryIntervalSeconds: {{ .sftDiscoveryIntervalSeconds }} {{- end }} - sftLookupDomain: {{ required "Missing value: .sft.sftLookupDomain" .sftLookupDomain }} - sftLookupPort: {{ required "Missing value: .sft.sftLookupPort" .sftLookupPort }} {{- end }} {{- end }} @@ -230,5 +228,13 @@ data: {{- if .setSftStaticUrl }} setSftStaticUrl: {{ .setSftStaticUrl }} {{- end }} + {{- if .setSftLookup }} + {{- with .setSftLookup }} + setSftLookup: + domain: {{ required "Missing value: .setSftLookup.domain" .setSftLookup.domain }} + port: {{ required "Missing value: .setSftLookup.port" .setSftLookup.port }} + isTestingEnvironment: {{ .setSftLookup.isTestingEnvironment }} + {{- end }} + {{- end }} {{- end }} {{- end }} diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal index 17a6c70c7b8..fe097e533ae 100644 --- a/services/brig/brig.cabal +++ b/services/brig/brig.cabal @@ -4,7 +4,7 @@ cabal-version: 2.0 -- -- see: https://github.com/sol/hpack -- --- hash: 244ca5e0c12867ef47ffcfbc056775381028fbc85e5c57eb280afe842e3499d9 +-- hash: addf4b080e564149f44c6cdb33c824734aa68cc9ff021f41268938902e2958b0 name: brig version: 2.0 @@ -64,6 +64,7 @@ library Brig.Data.User Brig.Data.UserKey Brig.Data.UserPendingActivation + Brig.Effects.SFT Brig.Email Brig.Federation.Client Brig.Index.Eval @@ -154,6 +155,7 @@ library , cassandra-util >=0.16.2 , comonad , conduit >=1.2.8 + , connection , containers >=0.5 , cookie >=0.4 , cryptobox-haskell >=0.1.1 @@ -177,6 +179,7 @@ library , html-entities >=1.1 , http-client >=0.5 , http-client-openssl >=0.2 + , http-client-tls , http-media , http-types >=0.8 , imports diff --git a/services/brig/brig.integration.yaml b/services/brig/brig.integration.yaml index b5ca314dfb9..07175fdd7be 100644 --- a/services/brig/brig.integration.yaml +++ b/services/brig/brig.integration.yaml @@ -178,6 +178,12 @@ optSettings: # Remember to keep it the same in Galley. setFederationDomain: example.com setFeatureFlags: # see #RefConfigOptions in `/docs/reference` + # FUTUREWORK: Replace the following SFT DNS lookup parameters with something + # to be added to the CI environment + setSftLookup: + domain: sftd.integration-tests.zinfra.io + port: 443 + isTestingEnvironment: true logLevel: Warn # ^ NOTE: We log too much in brig, if we set this to Info like other services, running tests diff --git a/services/brig/package.yaml b/services/brig/package.yaml index 02cd7dbf32b..0f10e5f4b24 100644 --- a/services/brig/package.yaml +++ b/services/brig/package.yaml @@ -44,6 +44,7 @@ library: - cassandra-util >=0.16.2 - comonad - conduit >=1.2.8 + - connection - containers >=0.5 - cookie >=0.4 - cryptobox-haskell >=0.1.1 @@ -71,6 +72,7 @@ library: - html-entities >=1.1 - http-client >=0.5 - http-client-openssl >=0.2 + - http-client-tls - http-media - http-types >=0.8 - imports diff --git a/services/brig/src/Brig/App.hs b/services/brig/src/Brig/App.hs index 326ca0b0a24..6f68e03adf2 100644 --- a/services/brig/src/Brig/App.hs +++ b/services/brig/src/Brig/App.hs @@ -214,7 +214,7 @@ newEnv o = do eventsQueue <- case Opt.internalEventsQueue (Opt.internalEvents o) of StompQueue q -> pure (StompQueue q) SqsQueue q -> SqsQueue <$> AWS.getQueueUrl (aws ^. AWS.amazonkaEnv) q - mSFTEnv <- mapM Calling.mkSFTEnv $ Opt.sft o + mSFTEnv <- mapM Calling.mkSFTEnv $ (,Opt.setSftLookup . Opt.optSettings $ o) <$> Opt.sft o prekeyLocalLock <- case Opt.randomPrekeys o of Just True -> Just <$> newMVar () _ -> pure Nothing diff --git a/services/brig/src/Brig/Calling.hs b/services/brig/src/Brig/Calling.hs index 9c102058288..97537d82cbe 100644 --- a/services/brig/src/Brig/Calling.hs +++ b/services/brig/src/Brig/Calling.hs @@ -51,7 +51,6 @@ import Data.IP import Data.List.NonEmpty (NonEmpty (..)) import qualified Data.List.NonEmpty as NonEmpty import Data.List1 -import Data.Misc (Port (..)) import Data.Range import Data.Time.Clock (DiffTime, diffTimeToPicoseconds) import Imports @@ -114,8 +113,7 @@ data SFTEnv = SFTEnv -- | maximum amount of servers to give out, -- even if more are in the SRV record sftListLength :: Range 1 100 Int, - sftLookupDomain :: DNS.Domain, - sftLookupPort :: Port + sftLookup :: Maybe Opts.SFTLookup } data Discovery a @@ -160,15 +158,14 @@ sftDiscoveryLoop SFTEnv {..} = forever $ do Just es -> atomicWriteIORef sftServers (Discovered (SFTServers es)) threadDelay sftDiscoveryInterval -mkSFTEnv :: SFTOptions -> IO SFTEnv -mkSFTEnv opts = +mkSFTEnv :: (SFTOptions, Maybe Opts.SFTLookup) -> IO SFTEnv +mkSFTEnv (opts, msftLookup) = SFTEnv <$> newIORef NotDiscoveredYet <*> pure (mkSFTDomain opts) <*> pure (diffTimeToMicroseconds (fromMaybe defSftDiscoveryIntervalSeconds (Opts.sftDiscoveryIntervalSeconds opts))) <*> pure (fromMaybe defSftListLength (Opts.sftListLength opts)) - <*> pure (Opts.sftLookupDomain opts) - <*> pure (Opts.sftLookupPort opts) + <*> pure msftLookup startSFTServiceDiscovery :: Log.Logger -> SFTEnv -> IO () startSFTServiceDiscovery logger = diff --git a/services/brig/src/Brig/Calling/API.hs b/services/brig/src/Brig/Calling/API.hs index 4e500c9f417..44c1d2b7d2a 100644 --- a/services/brig/src/Brig/Calling/API.hs +++ b/services/brig/src/Brig/Calling/API.hs @@ -27,6 +27,7 @@ import Brig.App import Brig.Calling import qualified Brig.Calling as Calling import Brig.Calling.Internal +import Brig.Effects.SFT import qualified Brig.Options as Opt import Control.Lens import Data.ByteString.Conversion @@ -43,7 +44,8 @@ import Data.Text.Ascii (AsciiBase64, encodeBase64) import Data.Text.Strict.Lens import Data.Time.Clock.POSIX (getPOSIXTime) import Imports hiding (head) -import Network.HTTP.Client hiding (Response) +import Network.Connection +import Network.HTTP.Client.TLS (mkManagerSettings, newTlsManager, newTlsManagerWith) import Network.Wai (Response) import Network.Wai.Predicate hiding (and, result, setStatus, (#)) import Network.Wai.Routing hiding (toList) @@ -101,9 +103,8 @@ getCallsConfigV2 _ _ limit = do env <- liftIO . readIORef =<< view turnEnvV2 staticUrl <- view $ settings . Opt.sftStaticUrl sftEnv' <- view sftEnv - httpMan <- view httpManager logger <- view applog - newConfig env staticUrl sftEnv' limit httpMan logger + newConfig env staticUrl sftEnv' limit logger getCallsConfigH :: JSON ::: UserId ::: ConnId -> Handler Response getCallsConfigH (_ ::: uid ::: connid) = @@ -112,9 +113,8 @@ getCallsConfigH (_ ::: uid ::: connid) = getCallsConfig :: UserId -> ConnId -> Handler Public.RTCConfiguration getCallsConfig _ _ = do env <- liftIO . readIORef =<< view turnEnv - httpMan <- view httpManager logger <- view applog - dropTransport <$> newConfig env Nothing Nothing Nothing httpMan logger + dropTransport <$> newConfig env Nothing Nothing Nothing logger where -- In order to avoid being backwards incompatible, remove the `transport` query param from the URIs dropTransport :: Public.RTCConfiguration -> Public.RTCConfiguration @@ -129,10 +129,9 @@ newConfig :: Maybe HttpsUrl -> Maybe SFTEnv -> Maybe (Range 1 10 Int) -> - Manager -> Logger -> m Public.RTCConfiguration -newConfig env sftStaticUrl mSftEnv limit httpMan logger = do +newConfig env sftStaticUrl mSftEnv limit logger = do let (sha, secret, tTTL, cTTL, prng) = (env ^. turnSHA512, env ^. turnSecret, env ^. turnTokenTTL, env ^. turnConfigTTL, env ^. turnPrng) -- randomize list of servers (before limiting the list, to ensure not always the same servers are chosen if limit is set) randomizedUris <- liftIO $ randomize (List1.toNonEmpty $ env ^. turnServers) @@ -153,27 +152,27 @@ newConfig env sftStaticUrl mSftEnv limit httpMan logger = do let subsetLength = Calling.sftListLength actualSftEnv liftIO $ mapM (getRandomSFTServers subsetLength) sftSrvEntries - let mSftServers = staticSft <|> sftServerFromSrvTarget . srvTarget <$$> sftEntries - mSftServersAll :: Maybe (Maybe [SFTServer]) <- for mSftEnv $ \e -> liftIO $ do - response <- runM . runTinyLog logger . runDNSLookupDefault . discoverSFTServersAll . sftLookupDomain $ e + mSftServersAll :: Maybe (Maybe [SFTServer]) <- for (mSftEnv >>= sftLookup) $ \sftl -> liftIO $ do + httpMan <- + if Opt.sftlIsTestEnv sftl + then + let s = TLSSettingsSimple True False True + in newTlsManagerWith $ mkManagerSettings s Nothing + else newTlsManager + response <- + runM + . runTinyLog logger + . runDNSLookupDefault + . discoverSFTServersAll + . Opt.unLookupDomain + . Opt.sftlDomain + $ sftl case response of Nothing -> pure $ Nothing @[SFTServer] - Just ips -> fmap (eitherToMaybe @String @[SFTServer] . sequence) $ - for ips $ \ip -> do - let req = - parseRequest_ $ - mconcat - [ "GET ", - show ip, - ":", - show . sftLookupPort $ e, - "/sft/url" - ] - -- TODO: introduce an effect for talking to SFT. Perhaps this could be a - -- part of an existing effect External. - sftUrlResponse <- liftIO (responseBody <$> httpLbs req httpMan) - pure @IO . fmap Public.sftServer . runParser' (parser @HttpsUrl) $ sftUrlResponse + Just ips -> fmap (eitherToMaybe @SFTError @[SFTServer] . sequence) $ + for ips $ \ip -> runM . interpretSFT httpMan $ sftGetClientUrl ip (Opt.sftlPort sftl) + let mSftServers = staticSft <|> sftServerFromSrvTarget . srvTarget <$$> sftEntries pure $ Public.rtcConfiguration srvs mSftServers cTTL (join mSftServersAll) where limitedList :: NonEmpty Public.TurnURI -> Range 1 10 Int -> NonEmpty Public.TurnURI diff --git a/services/brig/src/Brig/Effects/SFT.hs b/services/brig/src/Brig/Effects/SFT.hs new file mode 100644 index 00000000000..2297a7a1df1 --- /dev/null +++ b/services/brig/src/Brig/Effects/SFT.hs @@ -0,0 +1,68 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2021 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Brig.Effects.SFT where + +import Data.ByteString (unsnoc) +import qualified Data.ByteString.Char8 as BS8 +import Data.ByteString.Conversion.From +import Data.ByteString.Internal (ByteString (PS), w2c) +import Data.ByteString.Unsafe (unsafeTake) +import Data.IP +import Data.Misc +import Data.String.Conversions (cs) +import Imports +import Network.HTTP.Client +import Polysemy +import Polysemy.Internal +import Wire.API.Call.Config + +newtype SFTError = SFTError {unSFTError :: String} + +data SFT m a where + SFTGetClientUrl :: IPv4 -> Port -> SFT m (Either SFTError SFTServer) + +sftGetClientUrl :: Member SFT r => IPv4 -> Port -> Sem r (Either SFTError SFTServer) +sftGetClientUrl ipAddr port = send $ SFTGetClientUrl ipAddr port + +interpretSFT :: Member (Embed IO) r => Manager -> Sem (SFT ': r) a -> Sem r a +interpretSFT httpManager = interpret $ \(SFTGetClientUrl ipAddr port) -> do + let req = + parseRequest_ $ + mconcat + [ "GET https://", + show ipAddr, + ":", + show . portNumber $ port, + "/sft/url" + ] + sftUrlResponse <- liftIO (responseBody <$> httpLbs req httpManager) + pure . bimap SFTError sftServer . runParser' (parser @HttpsUrl) . cs . strip . cs $ sftUrlResponse + where + -- FUTUREWORK: remove this adopted code once upgraded to bytestring >= 0.10.12.0 + strip :: BS8.ByteString -> BS8.ByteString + strip = BS8.dropWhile isSpace . dropWhileEnd' isSpace + where + dropWhileEnd' :: (Char -> Bool) -> BS8.ByteString -> BS8.ByteString + dropWhileEnd' f ps = unsafeTake (findFromEndUntil (not . f . w2c) ps) ps + findFromEndUntil :: (Word8 -> Bool) -> BS8.ByteString -> Int + findFromEndUntil f ps@(PS _ _ l) = case unsnoc ps of + Nothing -> 0 + Just (b, c) -> + if f c + then l + else findFromEndUntil f b diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs index 5d73b27e89e..ce19a26eded 100644 --- a/services/brig/src/Brig/Options.hs +++ b/services/brig/src/Brig/Options.hs @@ -495,10 +495,46 @@ data Settings = Settings -- config will always return this entry. This is useful in Kubernetes -- where SFTs are deployed behind a load-balancer. In the long-run the SRV -- fetching logic can go away completely - setSftStaticUrl :: !(Maybe HttpsUrl) + setSftStaticUrl :: !(Maybe HttpsUrl), + setSftLookup :: !(Maybe SFTLookup) } deriving (Show, Generic) +data SFTLookup = SFTLookup + { sftlDomain :: !LookupDomain, + sftlPort :: !Port, + -- FUTUREWORK: Get rid of the test environment flag below. This is to be + -- done by not looking up A records and consequently making GET requests via + -- HTTPS based on IP addresses, instead of domain names. + + -- | Set to True if running in a test environment. This will avoid + -- performing SSL checks in a request to an SFT server. The default value is + -- False. + sftlIsTestEnv :: Bool + } + deriving (Show, Generic) + +instance FromJSON SFTLookup where + parseJSON = Aeson.withObject "SFTLookup" $ \o -> do + d <- o Aeson..: "domain" + p <- o Aeson..: "port" + t <- o Aeson..:? "isTestingEnvironment" Aeson..!= False + pure $ SFTLookup d p t + +newtype LookupDomain = LookupDomain {unLookupDomain :: DNS.Domain} + deriving stock (Show, Generic) + +instance FromJSON LookupDomain where + parseJSON (Y.String s) = + LookupDomain <$> Y.withText "LookupDomain" asciiOnly (Y.String s) + where + asciiOnly :: Text -> Y.Parser ByteString + asciiOnly t = + if Text.all Char.isAscii t + then pure $ Text.encodeUtf8 t + else fail $ "Expected ascii string only, found: " <> Text.unpack t + parseJSON _ = fail "Expected a String" + -- | The analog to `GT.FeatureFlags`. This type tracks only the things that we need to -- express our current cloud business logic. -- @@ -612,9 +648,7 @@ data SFTOptions = SFTOptions { sftBaseDomain :: !DNS.Domain, sftSRVServiceName :: !(Maybe ByteString), -- defaults to defSftServiceName if unset sftDiscoveryIntervalSeconds :: !(Maybe DiffTime), -- defaults to defSftDiscoveryIntervalSeconds - sftListLength :: !(Maybe (Range 1 100 Int)), -- defaults to defSftListLength - sftLookupDomain :: !DNS.Domain, - sftLookupPort :: !Port + sftListLength :: !(Maybe (Range 1 100 Int)) -- defaults to defSftListLength } deriving (Show, Generic) @@ -625,8 +659,6 @@ instance FromJSON SFTOptions where <*> (mapM asciiOnly =<< o .:? "sftSRVServiceName") <*> (secondsToDiffTime <$$> o .:? "sftDiscoveryIntervalSeconds") <*> (o .:? "sftListLength") - <*> (asciiOnly =<< o .: "sftLookupDomain") - <*> o .: "sftLookupPort" where asciiOnly :: Text -> Y.Parser ByteString asciiOnly t = @@ -688,7 +720,8 @@ Lens.makeLensesFor ("setUserMaxPermClients", "userMaxPermClients"), ("setFederationDomain", "federationDomain"), ("setSqsThrottleMillis", "sqsThrottleMillis"), - ("setSftStaticUrl", "sftStaticUrl") + ("setSftStaticUrl", "sftStaticUrl"), + ("setSftLookup", "sftLookup") ] ''Settings diff --git a/services/brig/test/integration/API/Calling.hs b/services/brig/test/integration/API/Calling.hs index 6515f2272ff..7b06963fdab 100644 --- a/services/brig/test/integration/API/Calling.hs +++ b/services/brig/test/integration/API/Calling.hs @@ -112,16 +112,22 @@ testSFT b opts = do "when SFT discovery is not enabled, sft_servers_all shouldn't be returned" Nothing (cfg ^. rtcConfSftServersAll) - withSettingsOverrides (opts & Opts.sftL ?~ Opts.SFTOptions "integration-tests.zinfra.io" Nothing (Just 0.001) Nothing "integration-tests.zinfra.io" (Port 8585)) $ do + withSettingsOverrides (opts & Opts.sftL ?~ Opts.SFTOptions "integration-tests.zinfra.io" Nothing (Just 0.001) Nothing) $ do cfg1 <- retryWhileN 10 (isNothing . view rtcConfSftServers) (getTurnConfigurationV2 uid b) -- These values are controlled by https://github.com/zinfra/cailleach/tree/77ca2d23cf2959aa183dd945d0a0b13537a8950d/environments/dns-integration-tests let Right server1 = mkHttpsUrl =<< first show (parseURI laxURIParserOptions "https://sft01.integration-tests.zinfra.io:443") let Right server2 = mkHttpsUrl =<< first show (parseURI laxURIParserOptions "https://sft02.integration-tests.zinfra.io:8443") - liftIO $ + liftIO $ do assertEqual "when SFT discovery is enabled, sft_servers should be returned" (Set.fromList [sftServer server1, sftServer server2]) (Set.fromList $ maybe [] NonEmpty.toList $ cfg1 ^. rtcConfSftServers) + void . for (cfg1 ^. rtcConfSftServersAll) $ \allServers -> do + let Right clientUrl = mkHttpsUrl =<< first show (parseURI laxURIParserOptions "https://sft01.avs.zinfra.io") + assertEqual + "when SFT discovery is enabled and SFT lookup configured, sft_servers_all should be returned" + (Set.singleton . sftServer $ clientUrl) + (Set.fromList allServers) modifyAndAssert :: Brig -> diff --git a/services/brig/test/unit/Test/Brig/Calling.hs b/services/brig/test/unit/Test/Brig/Calling.hs index 64af8c99b84..192262f7cd4 100644 --- a/services/brig/test/unit/Test/Brig/Calling.hs +++ b/services/brig/test/unit/Test/Brig/Calling.hs @@ -25,7 +25,6 @@ import Control.Retry import Data.IP import Data.List.NonEmpty (NonEmpty (..)) import qualified Data.List.NonEmpty as NonEmpty -import Data.Misc (Port (..)) import Data.Range import qualified Data.Set as Set import Imports @@ -43,20 +42,21 @@ import Wire.Network.DNS.SRV data FakeDNSEnv = FakeDNSEnv { fakeLookupSrv :: Domain -> SrvResponse, fakeLookupA :: Domain -> A.AResponse, - fakeLookupCalls :: IORef [Domain] + fakeLookupSrvCalls :: IORef [Domain], + fakeLookupACalls :: IORef [Domain] } newFakeDNSEnv :: (Domain -> SrvResponse) -> (Domain -> A.AResponse) -> IO FakeDNSEnv newFakeDNSEnv lookupSrvFn lookupAFn = - FakeDNSEnv lookupSrvFn lookupAFn <$> newIORef [] + FakeDNSEnv lookupSrvFn lookupAFn <$> newIORef [] <*> newIORef [] runFakeDNSLookup :: Member (Embed IO) r => FakeDNSEnv -> Sem (DNSLookup ': r) a -> Sem r a runFakeDNSLookup FakeDNSEnv {..} = interpret $ \case LookupSRV domain -> do - modifyIORef' fakeLookupCalls (++ [domain]) + modifyIORef' fakeLookupSrvCalls (++ [domain]) pure $ fakeLookupSrv domain LookupA domain -> do - modifyIORef' fakeLookupCalls (++ [domain]) + modifyIORef' fakeLookupACalls (++ [domain]) pure $ fakeLookupA domain newtype LogRecorder = LogRecorder {recordedLogs :: IORef [(Log.Level, LByteString)]} @@ -81,12 +81,12 @@ tests = assertEqual "should use the service name to form domain" "_foo._tcp.example.com." - (mkSFTDomain (SFTOptions "example.com" (Just "foo") Nothing Nothing "example.com" (Port 8585))), + (mkSFTDomain (SFTOptions "example.com" (Just "foo") Nothing Nothing)), testCase "when service name is not provided" $ assertEqual "should assume service name to be 'sft'" "_sft._tcp.example.com." - (mkSFTDomain (SFTOptions "example.com" Nothing Nothing Nothing "example.com" (Port 8585))) + (mkSFTDomain (SFTOptions "example.com" Nothing Nothing Nothing)) ], testGroup "sftDiscoveryLoop" $ [ testCase "when service can be discovered" $ void testDiscoveryLoopWhenSuccessful, @@ -117,10 +117,10 @@ testDiscoveryLoopWhenSuccessful = do entry3 = SrvEntry 0 0 (SrvTarget "sft3.foo.example.com." 443) returnedEntries = entry1 :| [entry2, entry3] fakeDNSEnv <- newFakeDNSEnv (\_ -> SrvAvailable returnedEntries) undefined - sftEnv <- mkSFTEnv (SFTOptions "foo.example.com" Nothing (Just 0.001) Nothing "foo.example.com" (Port 8585)) + sftEnv <- mkSFTEnv (SFTOptions "foo.example.com" Nothing (Just 0.001) Nothing, Nothing) discoveryLoop <- Async.async $ runM . ignoreLogs . runFakeDNSLookup fakeDNSEnv $ sftDiscoveryLoop sftEnv - void $ retryEvery10MicrosWhileN 2000 (== 0) (length <$> readIORef (fakeLookupCalls fakeDNSEnv)) + void $ retryEvery10MicrosWhileN 2000 (== 0) (length <$> readIORef (fakeLookupSrvCalls fakeDNSEnv)) -- We don't want to stop the loop before it has written to the sftServers IORef void $ retryEvery10MicrosWhileN 2000 (== NotDiscoveredYet) (readIORef (sftServers sftEnv)) Async.cancel discoveryLoop @@ -132,12 +132,12 @@ testDiscoveryLoopWhenSuccessful = do testDiscoveryLoopWhenUnsuccessful :: IO () testDiscoveryLoopWhenUnsuccessful = do fakeDNSEnv <- newFakeDNSEnv (const SrvNotAvailable) undefined - sftEnv <- mkSFTEnv (SFTOptions "foo.example.com" Nothing (Just 0.001) Nothing "foo.example.com" (Port 8585)) + sftEnv <- mkSFTEnv (SFTOptions "foo.example.com" Nothing (Just 0.001) Nothing, Nothing) discoveryLoop <- Async.async $ runM . ignoreLogs . runFakeDNSLookup fakeDNSEnv $ sftDiscoveryLoop sftEnv -- We wait for at least two lookups to be sure that the lookup loop looped at -- least once - void $ retryEvery10MicrosWhileN 2000 (<= 1) (length <$> readIORef (fakeLookupCalls fakeDNSEnv)) + void $ retryEvery10MicrosWhileN 2000 (<= 1) (length <$> readIORef (fakeLookupSrvCalls fakeDNSEnv)) Async.cancel discoveryLoop actualServers <- readIORef (sftServers sftEnv) @@ -154,7 +154,7 @@ testDiscoveryLoopWhenUnsuccessfulAfterSuccess = do discoveryLoop <- Async.async $ runM . ignoreLogs . runFakeDNSLookup failingFakeDNSEnv $ sftDiscoveryLoop sftEnv -- We wait for at least two lookups to be sure that the lookup loop looped at -- least once - void $ retryEvery10MicrosWhileN 2000 (<= 1) (length <$> readIORef (fakeLookupCalls failingFakeDNSEnv)) + void $ retryEvery10MicrosWhileN 2000 (<= 1) (length <$> readIORef (fakeLookupSrvCalls failingFakeDNSEnv)) Async.cancel discoveryLoop actualServers <- readIORef (sftServers sftEnv) @@ -172,7 +172,7 @@ testDiscoveryLoopWhenURLsChange = do fakeDNSEnv <- newFakeDNSEnv (const $ SrvAvailable newEntries) undefined discoveryLoop <- Async.async $ runM . ignoreLogs . runFakeDNSLookup fakeDNSEnv $ sftDiscoveryLoop sftEnv - void $ retryEvery10MicrosWhileN 2000 (== 0) (length <$> readIORef (fakeLookupCalls fakeDNSEnv)) + void $ retryEvery10MicrosWhileN 2000 (== 0) (length <$> readIORef (fakeLookupSrvCalls fakeDNSEnv)) -- We don't want to stop the loop before it has written to the sftServers IORef void $ retryEvery10MicrosWhileN 2000 (== Discovered (mkSFTServers newEntries)) (readIORef (sftServers sftEnv)) Async.cancel discoveryLoop From d540f9bcb154f880219dfc0635b991140a5ab525 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Dimja=C5=A1evi=C4=87?= Date: Fri, 31 Dec 2021 15:26:16 +0100 Subject: [PATCH 42/74] Fix Brig's configmap for SFT Lookup Templating (#2015) --- changelog.d/6-federation/sft-brig-configmap-fix | 1 + charts/brig/templates/configmap.yaml | 8 +++++--- services/brig/src/Brig/Options.hs | 3 +-- 3 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 changelog.d/6-federation/sft-brig-configmap-fix diff --git a/changelog.d/6-federation/sft-brig-configmap-fix b/changelog.d/6-federation/sft-brig-configmap-fix new file mode 100644 index 00000000000..d271b241cf3 --- /dev/null +++ b/changelog.d/6-federation/sft-brig-configmap-fix @@ -0,0 +1 @@ +Fix Brig's configmap for SFT lookups diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml index 88be747050b..2f7a0be6735 100644 --- a/charts/brig/templates/configmap.yaml +++ b/charts/brig/templates/configmap.yaml @@ -231,9 +231,11 @@ data: {{- if .setSftLookup }} {{- with .setSftLookup }} setSftLookup: - domain: {{ required "Missing value: .setSftLookup.domain" .setSftLookup.domain }} - port: {{ required "Missing value: .setSftLookup.port" .setSftLookup.port }} - isTestingEnvironment: {{ .setSftLookup.isTestingEnvironment }} + domain: {{ required "Missing value: .setSftLookup.domain" .domain }} + port: {{ required "Missing value: .setSftLookup.port" .port }} + {{- if .isTestingEnvironment }} + isTestingEnvironment: {{ .isTestingEnvironment }} + {{- end }} {{- end }} {{- end }} {{- end }} diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs index ce19a26eded..798b5c1f988 100644 --- a/services/brig/src/Brig/Options.hs +++ b/services/brig/src/Brig/Options.hs @@ -720,8 +720,7 @@ Lens.makeLensesFor ("setUserMaxPermClients", "userMaxPermClients"), ("setFederationDomain", "federationDomain"), ("setSqsThrottleMillis", "sqsThrottleMillis"), - ("setSftStaticUrl", "sftStaticUrl"), - ("setSftLookup", "sftLookup") + ("setSftStaticUrl", "sftStaticUrl") ] ''Settings From f6fd70d6d378f6db8b5c57fe9bc4d842f6e8d949 Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Mon, 3 Jan 2022 09:28:24 +0100 Subject: [PATCH 43/74] SQSERVICES 1011 Servantify Galley/Team API (#2008) * ToSchema instances for types in Team.Member * Servantify `PUT /teams/:tid` endpoint Co-authored-by: Paolo Capriotti --- changelog.d/5-internal/pr-2008 | 1 + libs/galley-types/src/Galley/Types/Teams.hs | 7 +- libs/schema-profunctor/src/Data/Schema.hs | 6 + .../src/Wire/API/Routes/Public/Galley.hs | 22 +- libs/wire-api/src/Wire/API/Team.hs | 175 +- libs/wire-api/src/Wire/API/Team/Member.hs | 362 +-- libs/wire-api/src/Wire/API/Team/Permission.hs | 33 +- libs/wire-api/src/Wire/API/User.hs | 11 +- .../Golden/Generated/NewTeamMember_team.hs | 42 +- .../Golden/Generated/TeamMemberList_team.hs | 2011 ++++++++--------- .../API/Golden/Generated/TeamMember_team.hs | 538 +++-- services/brig/src/Brig/IO/Intra.hs | 3 +- .../brig/test/integration/API/Team/Util.hs | 3 +- services/galley/src/Galley/API/Public.hs | 15 +- services/galley/src/Galley/API/Teams.hs | 56 +- services/galley/src/Galley/Cassandra/Team.hs | 4 +- .../test/integration/API/MessageTimer.hs | 2 +- services/galley/test/integration/API/Teams.hs | 30 +- services/galley/test/integration/API/Util.hs | 7 +- services/spar/test-integration/Util/Core.hs | 7 +- tools/stern/src/Stern/Types.hs | 1 + 21 files changed, 1633 insertions(+), 1703 deletions(-) create mode 100644 changelog.d/5-internal/pr-2008 diff --git a/changelog.d/5-internal/pr-2008 b/changelog.d/5-internal/pr-2008 new file mode 100644 index 00000000000..a5c36a513b0 --- /dev/null +++ b/changelog.d/5-internal/pr-2008 @@ -0,0 +1 @@ +Servantify Galley Teams API. (#2008) diff --git a/libs/galley-types/src/Galley/Types/Teams.hs b/libs/galley-types/src/Galley/Types/Teams.hs index f1251eff0a5..a37bbdc466d 100644 --- a/libs/galley-types/src/Galley/Types/Teams.hs +++ b/libs/galley-types/src/Galley/Types/Teams.hs @@ -65,16 +65,17 @@ module Galley.Types.Teams teamListHasMore, TeamMember, userId, + nUserId, permissions, + nPermissions, invitation, + nInvitation, legalHoldStatus, - teamMemberJson, TeamMemberList, ListType (..), newTeamMemberList, teamMembers, teamMemberListType, - teamMemberListJson, TeamConversation, newTeamConversation, conversationId, @@ -105,8 +106,6 @@ module Galley.Types.Teams newTeamIconKey, newTeamMembers, NewTeamMember, - newNewTeamMember, - ntmNewTeamMember, Event, newEvent, eventType, diff --git a/libs/schema-profunctor/src/Data/Schema.hs b/libs/schema-profunctor/src/Data/Schema.hs index 398d4caaabd..0e8c5e8cacb 100644 --- a/libs/schema-profunctor/src/Data/Schema.hs +++ b/libs/schema-profunctor/src/Data/Schema.hs @@ -500,6 +500,9 @@ instance With Text where instance With Integer where with _ = (A.parseJSON >=>) +instance With Bool where + with = A.withBool + -- | A schema for a single value of an enumeration. element :: forall a b. @@ -766,6 +769,9 @@ instance HasEnum Text NamedSwaggerDoc where instance HasEnum Integer NamedSwaggerDoc where mkEnum = mkSwaggerEnum S.SwaggerInteger +instance HasEnum Bool NamedSwaggerDoc where + mkEnum = mkSwaggerEnum S.SwaggerBoolean + mkSwaggerEnum :: S.SwaggerType 'S.SwaggerKindSchema -> Text -> diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs b/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs index 63949b1d841..fe3ec2ec6e3 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs @@ -1,4 +1,5 @@ {-# LANGUAGE DerivingVia #-} +{-# LANGUAGE GeneralizedNewtypeDeriving #-} {-# OPTIONS_GHC -Wno-orphans #-} -- This file is part of the Wire Server implementation. @@ -43,6 +44,7 @@ import Wire.API.Routes.Public import Wire.API.Routes.Public.Util import Wire.API.Routes.QualifiedCapture import Wire.API.ServantProto (Proto, RawProto) +import Wire.API.Team import Wire.API.Team.Conversation import Wire.API.Team.Feature @@ -705,7 +707,25 @@ data Api routes = Api :- FeatureConfigGet 'WithLockStatus 'TeamFeatureSelfDeletingMessages, featureConfigGuestLinksGet :: routes - :- FeatureConfigGet 'WithLockStatus 'TeamFeatureGuestLinks + :- FeatureConfigGet 'WithLockStatus 'TeamFeatureGuestLinks, + -- teams + createNonBindingTeam :: + routes + :- Summary "Create a new non binding team" + :> ZUser + :> ZConn + :> CanThrow NotConnected + :> "teams" + :> ReqBody '[Servant.JSON] NonBindingNewTeam + :> MultiVerb + 'POST + '[JSON] + '[ WithHeaders + '[DescHeader "Location" "Team ID" TeamId] + TeamId + (RespondEmpty 201 "Team ID as `Location` header value") + ] + TeamId } deriving (Generic) diff --git a/libs/wire-api/src/Wire/API/Team.hs b/libs/wire-api/src/Wire/API/Team.hs index b1307bea120..d639b314d06 100644 --- a/libs/wire-api/src/Wire/API/Team.hs +++ b/libs/wire-api/src/Wire/API/Team.hs @@ -46,7 +46,6 @@ module Wire.API.Team newTeamIcon, newTeamIconKey, newTeamMembers, - newTeamJson, -- * TeamUpdateData TeamUpdateData (..), @@ -71,12 +70,14 @@ module Wire.API.Team where import Control.Lens (makeLenses) -import Data.Aeson -import Data.Aeson.Types (Pair) +import Data.Aeson (FromJSON, ToJSON, Value (..)) +import Data.Aeson.Types (Parser) import Data.Id (TeamId, UserId) -import Data.Json.Util import Data.Misc (PlainTextPassword (..)) import Data.Range +import Data.Schema +import Data.Singletons (sing) +import qualified Data.Swagger as S import qualified Data.Swagger.Build.Api as Doc import Imports import Test.QuickCheck.Gen (suchThat) @@ -96,9 +97,10 @@ data Team = Team } deriving stock (Eq, Show, Generic) deriving (Arbitrary) via (GenericUniform Team) + deriving (ToJSON, FromJSON, S.ToSchema) via (Schema Team) newTeam :: TeamId -> UserId -> Text -> Text -> TeamBinding -> Team -newTeam tid uid nme ico bnd = Team tid uid nme ico Nothing bnd +newTeam tid uid nme ico = Team tid uid nme ico Nothing modelTeam :: Doc.Model modelTeam = Doc.defineModel "Team" $ do @@ -117,41 +119,28 @@ modelTeam = Doc.defineModel "Team" $ do Doc.property "binding" Doc.bool' $ Doc.description "user binding team" -instance ToJSON Team where - toJSON t = - object $ - "id" .= _teamId t - # "creator" .= _teamCreator t - # "name" .= _teamName t - # "icon" .= _teamIcon t - # "icon_key" .= _teamIconKey t - # "binding" .= _teamBinding t - # [] - -instance FromJSON Team where - parseJSON = withObject "team" $ \o -> do - Team - <$> o .: "id" - <*> o .: "creator" - <*> o .: "name" - <*> o .: "icon" - <*> o .:? "icon_key" - <*> o .:? "binding" .!= NonBinding +instance ToSchema Team where + schema = + object "Team" $ + Team + <$> _teamId .= field "id" schema + <*> _teamCreator .= field "creator" schema + <*> _teamName .= field "name" schema + <*> _teamIcon .= field "icon" schema + <*> _teamIconKey .= maybe_ (optField "icon_key" schema) + <*> _teamBinding .= (fromMaybe Binding <$> optField "binding" schema) data TeamBinding = Binding | NonBinding deriving stock (Eq, Show, Generic) deriving (Arbitrary) via (GenericUniform TeamBinding) + deriving (ToJSON, FromJSON, S.ToSchema) via (Schema TeamBinding) -instance ToJSON TeamBinding where - toJSON Binding = Bool True - toJSON NonBinding = Bool False - -instance FromJSON TeamBinding where - parseJSON (Bool True) = pure Binding - parseJSON (Bool False) = pure NonBinding - parseJSON other = fail $ "Unknown binding type: " <> show other +instance ToSchema TeamBinding where + schema = + enum @Bool "TeamBinding" $ + mconcat [element True Binding, element False NonBinding] -------------------------------------------------------------------------------- -- TeamList @@ -162,6 +151,7 @@ data TeamList = TeamList } deriving stock (Eq, Show, Generic) deriving (Arbitrary) via (GenericUniform TeamList) + deriving (ToJSON, FromJSON, S.ToSchema) via (Schema TeamList) newTeamList :: [Team] -> Bool -> TeamList newTeamList = TeamList @@ -174,24 +164,18 @@ modelTeamList = Doc.defineModel "TeamList" $ do Doc.property "has_more" Doc.bool' $ Doc.description "if more teams are available" -instance ToJSON TeamList where - toJSON t = - object $ - "teams" .= _teamListTeams t - # "has_more" .= _teamListHasMore t - # [] - -instance FromJSON TeamList where - parseJSON = withObject "teamlist" $ \o -> do - TeamList - <$> o .: "teams" - <*> o .: "has_more" +instance ToSchema TeamList where + schema = + object "TeamList" $ + TeamList <$> _teamListTeams .= field "teams" (array schema) + <*> _teamListHasMore .= field "has_more" schema -------------------------------------------------------------------------------- -- NewTeam newtype BindingNewTeam = BindingNewTeam (NewTeam ()) deriving stock (Eq, Show, Generic) + deriving (ToJSON, FromJSON, S.ToSchema) via (Schema BindingNewTeam) modelNewBindingTeam :: Doc.Model modelNewBindingTeam = Doc.defineModel "NewBindingTeam" $ do @@ -204,17 +188,13 @@ modelNewBindingTeam = Doc.defineModel "NewBindingTeam" $ do Doc.description "team icon asset key" Doc.optional -instance ToJSON BindingNewTeam where - toJSON (BindingNewTeam t) = object $ newTeamJson t - -newTeamJson :: NewTeam a -> [Pair] -newTeamJson (NewTeam n i ik _) = - "name" .= fromRange n - # "icon" .= fromRange i - # "icon_key" .= (fromRange <$> ik) - # [] +instance ToSchema BindingNewTeam where + schema = BindingNewTeam <$> unwrap .= newTeamSchema "BindingNewTeam" sch + where + unwrap (BindingNewTeam nt) = nt -deriving newtype instance FromJSON BindingNewTeam + sch :: ValueSchema SwaggerDoc () + sch = null_ -- FUTUREWORK: since new team members do not get serialized, we zero them here. -- it may be worth looking into how this can be solved in the types. @@ -227,6 +207,15 @@ instance Arbitrary BindingNewTeam where -- | FUTUREWORK: this is dead code! remove! newtype NonBindingNewTeam = NonBindingNewTeam (NewTeam (Range 1 127 [TeamMember])) deriving stock (Eq, Show, Generic) + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema NonBindingNewTeam) + +instance ToSchema NonBindingNewTeam where + schema = NonBindingNewTeam <$> unwrap .= newTeamSchema "NonBindingNewTeam" sch + where + unwrap (NonBindingNewTeam nt) = nt + + sch :: ValueSchema SwaggerDoc (Range 1 127 [TeamMember]) + sch = fromRange .= rangedSchema sing sing (array schema) modelNewNonBindingTeam :: Doc.Model modelNewNonBindingTeam = Doc.defineModel "newNonBindingTeam" $ do @@ -242,14 +231,6 @@ modelNewNonBindingTeam = Doc.defineModel "newNonBindingTeam" $ do Doc.description "initial team member ids (between 1 and 127)" Doc.optional -instance ToJSON NonBindingNewTeam where - toJSON (NonBindingNewTeam t) = - object $ - "members" .= (fromRange <$> _newTeamMembers t) - # newTeamJson t - -deriving newtype instance FromJSON NonBindingNewTeam - data NewTeam a = NewTeam { _newTeamName :: Range 1 256 Text, _newTeamIcon :: Range 1 256 Text, @@ -262,17 +243,14 @@ data NewTeam a = NewTeam newNewTeam :: Range 1 256 Text -> Range 1 256 Text -> NewTeam a newNewTeam nme ico = NewTeam nme ico Nothing Nothing -instance (FromJSON a) => FromJSON (NewTeam a) where - parseJSON = withObject "new-team" $ \o -> do - name <- o .: "name" - icon <- o .: "icon" - key <- o .:? "icon_key" - mems <- o .:? "members" - either fail pure $ - NewTeam <$> checkedEitherMsg "name" name - <*> checkedEitherMsg "icon" icon - <*> maybe (pure Nothing) (fmap Just . checkedEitherMsg "icon_key") key - <*> pure mems +newTeamSchema :: HasSchemaRef d => Text -> ValueSchema d a -> ValueSchema NamedSwaggerDoc (NewTeam a) +newTeamSchema name sch = + object name $ + NewTeam + <$> _newTeamName .= field "name" schema + <*> _newTeamIcon .= field "icon" schema + <*> _newTeamIconKey .= maybe_ (optField "icon_key" schema) + <*> _newTeamMembers .= maybe_ (optField "members" sch) -------------------------------------------------------------------------------- -- TeamUpdateData @@ -283,6 +261,7 @@ data TeamUpdateData = TeamUpdateData _iconKeyUpdate :: Maybe (Range 1 256 Text) } deriving stock (Eq, Show, Generic) + deriving (ToJSON, FromJSON, S.ToSchema) via (Schema TeamUpdateData) instance Arbitrary TeamUpdateData where arbitrary = arb `suchThat` valid @@ -307,25 +286,21 @@ modelUpdateData = Doc.defineModel "TeamUpdateData" $ do newTeamUpdateData :: TeamUpdateData newTeamUpdateData = TeamUpdateData Nothing Nothing Nothing -instance ToJSON TeamUpdateData where - toJSON u = - object $ - "name" .= _nameUpdate u - # "icon" .= _iconUpdate u - # "icon_key" .= _iconKeyUpdate u - # [] - -instance FromJSON TeamUpdateData where - parseJSON = withObject "team update data" $ \o -> do - name <- o .:? "name" - icon <- o .:? "icon" - icon_key <- o .:? "icon_key" - when (isNothing name && isNothing icon && isNothing icon_key) $ - fail "TeamUpdateData: no update data specified" - either fail pure $ - TeamUpdateData <$> maybe (pure Nothing) (fmap Just . checkedEitherMsg "name") name - <*> maybe (pure Nothing) (fmap Just . checkedEitherMsg "icon") icon - <*> maybe (pure Nothing) (fmap Just . checkedEitherMsg "icon_key") icon_key +validateTeamUpdateData :: TeamUpdateData -> Parser TeamUpdateData +validateTeamUpdateData u = + when + (isNothing (_nameUpdate u) && isNothing (_iconUpdate u) && isNothing (_iconKeyUpdate u)) + (fail "TeamUpdateData: no update data specified") + $> u + +instance ToSchema TeamUpdateData where + schema = + (`withParser` validateTeamUpdateData) + . object "TeamUpdateData" + $ TeamUpdateData + <$> _nameUpdate .= maybe_ (optField "name" schema) + <*> _iconUpdate .= maybe_ (optField "icon" schema) + <*> _iconKeyUpdate .= maybe_ (optField "icon_key" schema) -------------------------------------------------------------------------------- -- TeamDeleteData @@ -335,6 +310,7 @@ newtype TeamDeleteData = TeamDeleteData } deriving stock (Eq, Show) deriving newtype (Arbitrary) + deriving (ToJSON, FromJSON, S.ToSchema) via (Schema TeamDeleteData) newTeamDeleteData :: Maybe PlainTextPassword -> TeamDeleteData newTeamDeleteData = TeamDeleteData @@ -346,15 +322,10 @@ modelTeamDelete = Doc.defineModel "teamDeleteData" $ do Doc.property "password" Doc.string' $ Doc.description "The account password to authorise the deletion." -instance FromJSON TeamDeleteData where - parseJSON = withObject "team-delete-data" $ \o -> - TeamDeleteData <$> o .: "password" - -instance ToJSON TeamDeleteData where - toJSON tdd = - object - [ "password" .= _tdAuthPassword tdd - ] +instance ToSchema TeamDeleteData where + schema = + object "TeamDeleteData" $ + TeamDeleteData <$> _tdAuthPassword .= optField "password" (maybeWithDefault Null schema) makeLenses ''Team makeLenses ''TeamList diff --git a/libs/wire-api/src/Wire/API/Team/Member.hs b/libs/wire-api/src/Wire/API/Team/Member.hs index 905bd2f9718..9a7cfda570e 100644 --- a/libs/wire-api/src/Wire/API/Team/Member.hs +++ b/libs/wire-api/src/Wire/API/Team/Member.hs @@ -21,12 +21,17 @@ module Wire.API.Team.Member ( -- * TeamMember - TeamMember (..), + TeamMember, + mkTeamMember, userId, permissions, invitation, legalHoldStatus, + ntmNewTeamMember, + + -- * TODO: remove after servantification teamMemberJson, + teamMemberListJson, -- * TeamMemberList TeamMemberList, @@ -38,12 +43,13 @@ module Wire.API.Team.Member NewListType (..), toNewListType, ListType (..), - teamMemberListJson, -- * NewTeamMember NewTeamMember, - newNewTeamMember, - ntmNewTeamMember, + mkNewTeamMember, + nUserId, + nPermissions, + nInvitation, -- * TeamMemberDeleteData TeamMemberDeleteData, @@ -58,35 +64,95 @@ module Wire.API.Team.Member ) where -import Control.Lens (makeLenses) -import Data.Aeson -import Data.Aeson.Types (Parser) -import qualified Data.HashMap.Strict as HM +import Control.Lens (Lens, Lens', makeLenses, (%~)) +import Data.Aeson (FromJSON (..), ToJSON (..), Value (..)) import Data.Id (UserId) import Data.Json.Util import Data.LegalHold (UserLegalHoldStatus (..), defUserLegalHoldStatus, typeUserLegalHoldStatus) import Data.Misc (PlainTextPassword (..)) import Data.Proxy -import Data.String.Conversions (cs) +import Data.Schema import qualified Data.Swagger.Build.Api as Doc -import Data.Swagger.Schema (ToSchema) -import Deriving.Swagger (CamelToSnake, ConstructorTagModifier, CustomSwagger, StripPrefix) +import qualified Data.Swagger.Schema as S import GHC.TypeLits import Imports -import Wire.API.Arbitrary (Arbitrary, GenericUniform (..), arbitrary, shrink) +import Wire.API.Arbitrary (Arbitrary, GenericUniform (..)) import Wire.API.Team.Permission (Permissions, modelPermissions) +data PermissionTag = Required | Optional + +type family PermissionType (tag :: PermissionTag) = (t :: *) | t -> tag where + PermissionType 'Required = Permissions + PermissionType 'Optional = Maybe Permissions + -------------------------------------------------------------------------------- -- TeamMember -data TeamMember = TeamMember - { _userId :: UserId, - _permissions :: Permissions, - _invitation :: Maybe (UserId, UTCTimeMillis), +type TeamMember = TeamMember' 'Required + +data TeamMember' (tag :: PermissionTag) = TeamMember + { _newTeamMember :: NewTeamMember' tag, _legalHoldStatus :: UserLegalHoldStatus } - deriving stock (Eq, Ord, Show, Generic) - deriving (Arbitrary) via (GenericUniform TeamMember) + deriving stock (Generic) + +ntmNewTeamMember :: NewTeamMember' tag -> TeamMember' tag +ntmNewTeamMember ntm = TeamMember ntm defUserLegalHoldStatus + +deriving instance Eq (PermissionType tag) => Eq (TeamMember' tag) + +deriving instance Ord (PermissionType tag) => Ord (TeamMember' tag) + +deriving instance Show (PermissionType tag) => Show (TeamMember' tag) + +deriving via (GenericUniform TeamMember) instance Arbitrary TeamMember + +deriving via (GenericUniform (TeamMember' 'Optional)) instance Arbitrary (TeamMember' 'Optional) + +deriving via + (Schema (TeamMember' tag)) + instance + (ToSchema (TeamMember' tag)) => + ToJSON (TeamMember' tag) + +deriving via + (Schema (TeamMember' tag)) + instance + (ToSchema (TeamMember' tag)) => + FromJSON (TeamMember' tag) + +deriving via + (Schema (TeamMember' tag)) + instance + (ToSchema (TeamMember' tag)) => + S.ToSchema (TeamMember' tag) + +mkTeamMember :: + UserId -> + PermissionType tag -> + Maybe (UserId, UTCTimeMillis) -> + UserLegalHoldStatus -> + TeamMember' tag +mkTeamMember uid perms inv = TeamMember (NewTeamMember uid perms inv) + +instance ToSchema TeamMember where + schema = + object "TeamMember" $ + TeamMember + <$> _newTeamMember .= newTeamMemberSchema + <*> _legalHoldStatus .= (fromMaybe defUserLegalHoldStatus <$> optField "legalhold_status" schema) + +instance ToSchema (TeamMember' 'Optional) where + schema = + object "TeamMember" $ + TeamMember + <$> _newTeamMember + .= ( NewTeamMember + <$> _nUserId .= field "user" schema + <*> _nPermissions .= maybe_ (optField "permissions" schema) + <*> _nInvitation .= invitedSchema' + ) + <*> _legalHoldStatus .= (fromMaybe defUserLegalHoldStatus <$> optField "legalhold_status" schema) modelTeamMember :: Doc.Model modelTeamMember = Doc.defineModel "TeamMember" $ do @@ -111,54 +177,46 @@ modelTeamMember = Doc.defineModel "TeamMember" $ do Doc.description "The state of Legal Hold compliance for the member" Doc.optional -instance ToJSON TeamMember where - toJSON = teamMemberJson (const True) - -instance FromJSON TeamMember where - parseJSON = parseTeamMember - --- | Show 'Permissions' conditionally. The condition takes the member that will receive the result --- into account. See 'canSeePermsOf'. --- --- FUTUREWORK: --- There must be a cleaner way to do this, with a separate type --- instead of logic in the JSON instance. -teamMemberJson :: (TeamMember -> Bool) -> TeamMember -> Value -teamMemberJson withPerms m = - object $ - ["user" .= _userId m] - <> ["permissions" .= _permissions m | withPerms m] - <> ["created_by" .= (fst <$> _invitation m)] - <> ["created_at" .= (snd <$> _invitation m)] - <> ["legalhold_status" .= _legalHoldStatus m] - -parseTeamMember :: Value -> Parser TeamMember -parseTeamMember = withObject "team-member" $ \o -> - TeamMember - <$> o .: "user" - <*> o .: "permissions" - <*> parseInvited o - -- Default to disabled if missing - <*> o .:? "legalhold_status" .!= defUserLegalHoldStatus - where - parseInvited :: Object -> Parser (Maybe (UserId, UTCTimeMillis)) - parseInvited o = do - invby <- o .:? "created_by" - invat <- o .:? "created_at" - case (invby, invat) of - (Just b, Just a) -> pure $ Just (b, a) - (Nothing, Nothing) -> pure $ Nothing - _ -> fail "created_by, created_at" +setPerm :: Bool -> Permissions -> Maybe Permissions +setPerm True = Just +setPerm False = const Nothing -------------------------------------------------------------------------------- -- TeamMemberList -data TeamMemberList = TeamMemberList - { _teamMembers :: [TeamMember], +type TeamMemberList = TeamMemberList' 'Required + +data TeamMemberList' (tag :: PermissionTag) = TeamMemberList + { _teamMembers :: [TeamMember' tag], _teamMemberListType :: ListType } - deriving stock (Eq, Show, Generic) - deriving (Arbitrary) via (GenericUniform TeamMemberList) + deriving stock (Generic) + +deriving instance Eq (PermissionType tag) => Eq (TeamMemberList' tag) + +deriving instance Show (PermissionType tag) => Show (TeamMemberList' tag) + +deriving via (GenericUniform (TeamMemberList' 'Optional)) instance Arbitrary (TeamMemberList' 'Optional) + +deriving via (GenericUniform TeamMemberList) instance Arbitrary TeamMemberList + +deriving via + (Schema (TeamMemberList' tag)) + instance + ToSchema (TeamMemberList' tag) => + FromJSON (TeamMemberList' tag) + +deriving via + (Schema (TeamMemberList' tag)) + instance + ToSchema (TeamMemberList' tag) => + ToJSON (TeamMemberList' tag) + +deriving via + (Schema (TeamMemberList' tag)) + instance + ToSchema (TeamMemberList' tag) => + S.ToSchema (TeamMemberList' tag) newTeamMemberList :: [TeamMember] -> ListType -> TeamMemberList newTeamMemberList = TeamMemberList @@ -171,20 +229,12 @@ modelTeamMemberList = Doc.defineModel "TeamMemberList" $ do Doc.property "hasMore" Doc.bool' $ Doc.description "true if 'members' doesn't contain all team members" -instance ToJSON TeamMemberList where - toJSON = teamMemberListJson (const True) - --- | Show a list of team members using 'teamMemberJson'. -teamMemberListJson :: (TeamMember -> Bool) -> TeamMemberList -> Value -teamMemberListJson withPerms l = - object - [ "members" .= map (teamMemberJson withPerms) (_teamMembers l), - "hasMore" .= _teamMemberListType l - ] - -instance FromJSON TeamMemberList where - parseJSON = withObject "team member list" $ \o -> - TeamMemberList <$> o .: "members" <*> o .: "hasMore" +instance ToSchema (TeamMember' tag) => ToSchema (TeamMemberList' tag) where + schema = + object "TeamMemberList" $ + TeamMemberList + <$> _teamMembers .= field "members" (array schema) + <*> _teamMemberListType .= field "hasMore" schema type HardTruncationLimit = (2000 :: Nat) @@ -197,18 +247,15 @@ data NewListType | NewListTruncated deriving stock (Eq, Ord, Show, Generic) deriving (Arbitrary) via (GenericUniform NewListType) - deriving (ToSchema) via (CustomSwagger '[ConstructorTagModifier (StripPrefix "New", CamelToSnake)] NewListType) - --- This replaces the previous `hasMore` but has no boolean blindness. At the API level --- though we do want this to remain true/false -instance ToJSON NewListType where - toJSON NewListComplete = String "list_complete" - toJSON NewListTruncated = String "list_truncated" + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema NewListType) -instance FromJSON NewListType where - parseJSON (String "list_complete") = pure NewListComplete - parseJSON (String "list_truncated") = pure NewListTruncated - parseJSON bad = fail $ "NewListType: " <> cs (encode bad) +instance ToSchema NewListType where + schema = + enum @Text "NewListType" $ + mconcat + [ element "list_complete" NewListComplete, + element "list_truncated" NewListTruncated + ] toNewListType :: ListType -> NewListType toNewListType ListComplete = NewListComplete @@ -219,37 +266,83 @@ data ListType | ListTruncated deriving stock (Eq, Ord, Show, Generic) deriving (Arbitrary) via (GenericUniform ListType) + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema ListType) -- This replaces the previous `hasMore` but has no boolean blindness. At the API level -- though we do want this to remain true/false -instance ToJSON ListType where - toJSON ListComplete = Bool False - toJSON ListTruncated = Bool True - -instance FromJSON ListType where - parseJSON (Bool False) = pure ListComplete - parseJSON (Bool True) = pure ListTruncated - parseJSON bad = fail $ "ListType: " <> cs (encode bad) +instance ToSchema ListType where + schema = + enum @Bool "ListType" $ + mconcat [element True ListTruncated, element False ListComplete] -------------------------------------------------------------------------------- -- NewTeamMember +type NewTeamMember = NewTeamMember' 'Required + +mkNewTeamMember :: UserId -> PermissionType 'Required -> Maybe (UserId, UTCTimeMillis) -> NewTeamMember +mkNewTeamMember = NewTeamMember + -- | Like 'TeamMember', but we can receive this from the clients. Clients are not allowed to --- set 'UserLegalHoldStatus', so both 'newNewTeamMember and {To,From}JSON make sure that is --- always the default. I decided to keep the 'TeamMember' inside (rather than making an --- entirely new type because (1) it's a smaller change and I'm in a hurry; (2) it encodes the --- identity relationship between the fields that *do* occur in both more explicit. -newtype NewTeamMember = NewTeamMember - { _ntmNewTeamMember :: TeamMember +-- set 'UserLegalHoldStatus'. +data NewTeamMember' (tag :: PermissionTag) = NewTeamMember + { _nUserId :: UserId, + _nPermissions :: PermissionType tag, + _nInvitation :: Maybe (UserId, UTCTimeMillis) } - deriving stock (Eq, Show) + deriving stock (Generic) + +deriving instance (Eq (PermissionType tag)) => Eq (NewTeamMember' tag) + +deriving instance (Ord (PermissionType tag)) => Ord (NewTeamMember' tag) + +deriving instance (Show (PermissionType tag)) => Show (NewTeamMember' tag) + +deriving via + (Schema (NewTeamMember' tag)) + instance + (ToSchema (NewTeamMember' tag)) => + ToJSON (NewTeamMember' tag) + +deriving via + (Schema (NewTeamMember' tag)) + instance + (ToSchema (NewTeamMember' tag)) => + FromJSON (NewTeamMember' tag) + +deriving via + (Schema (NewTeamMember' tag)) + instance + (ToSchema (NewTeamMember' tag)) => + S.ToSchema (NewTeamMember' tag) + +deriving via (GenericUniform NewTeamMember) instance Arbitrary NewTeamMember + +deriving via (GenericUniform (NewTeamMember' 'Optional)) instance Arbitrary (NewTeamMember' 'Optional) + +newTeamMemberSchema :: ObjectSchema SwaggerDoc NewTeamMember +newTeamMemberSchema = + NewTeamMember + <$> _nUserId .= field "user" schema + <*> _nPermissions .= field "permissions" schema + <*> _nInvitation .= invitedSchema' -instance Arbitrary NewTeamMember where - arbitrary = newNewTeamMember <$> arbitrary <*> arbitrary <*> arbitrary - shrink (NewTeamMember (TeamMember uid perms _mbinv _)) = [newNewTeamMember uid perms Nothing] +invitedSchema :: ObjectSchemaP SwaggerDoc (Maybe (UserId, UTCTimeMillis)) (Maybe UserId, Maybe UTCTimeMillis) +invitedSchema = + (,) <$> fmap fst .= optField "created_by" (maybeWithDefault Null schema) + <*> fmap snd .= optField "created_at" (maybeWithDefault Null schema) -newNewTeamMember :: UserId -> Permissions -> Maybe (UserId, UTCTimeMillis) -> NewTeamMember -newNewTeamMember uid perms mbinv = NewTeamMember $ TeamMember uid perms mbinv defUserLegalHoldStatus +invitedSchema' :: ObjectSchema SwaggerDoc (Maybe (UserId, UTCTimeMillis)) +invitedSchema' = withParser invitedSchema $ \(invby, invat) -> + case (invby, invat) of + (Just b, Just a) -> pure $ Just (b, a) + (Nothing, Nothing) -> pure $ Nothing + _ -> fail "created_by, created_at" + +instance ToSchema NewTeamMember where + schema = + object "NewTeamMember" $ + field "member" $ unnamed (object "Unnamed" newTeamMemberSchema) modelNewTeamMember :: Doc.Model modelNewTeamMember = Doc.defineModel "NewTeamMember" $ do @@ -257,22 +350,6 @@ modelNewTeamMember = Doc.defineModel "NewTeamMember" $ do Doc.property "member" (Doc.ref modelTeamMember) $ Doc.description "the team member to add (the legalhold_status field must be null or missing!)" -instance ToJSON NewTeamMember where - toJSON t = object ["member" .= mem] - where - mem = Object . HM.fromList . fltr . HM.toList $ o - o = case toJSON (_ntmNewTeamMember t) of - Object o_ -> o_ - _ -> error "impossible" - fltr = filter ((`elem` ["user", "permissions", "created_by", "created_at"]) . fst) - -instance FromJSON NewTeamMember where - parseJSON = withObject "add team member" $ \o -> do - mem <- o .: "member" - if (_legalHoldStatus mem == defUserLegalHoldStatus) - then pure $ NewTeamMember mem - else fail "legalhold_status field cannot be set in NewTeamMember" - -------------------------------------------------------------------------------- -- TeamMemberDeleteData @@ -281,6 +358,12 @@ newtype TeamMemberDeleteData = TeamMemberDeleteData } deriving stock (Eq, Show) deriving newtype (Arbitrary) + deriving (ToJSON, FromJSON, S.ToSchema) via (Schema TeamMemberDeleteData) + +instance ToSchema TeamMemberDeleteData where + schema = + object "TeamMemberDeleteData" $ + TeamMemberDeleteData <$> _tmdAuthPassword .= optField "password" (maybeWithDefault Null schema) newTeamMemberDeleteData :: Maybe PlainTextPassword -> TeamMemberDeleteData newTeamMemberDeleteData = TeamMemberDeleteData @@ -292,17 +375,30 @@ modelTeamMemberDelete = Doc.defineModel "teamDeleteData" $ do Doc.property "password" Doc.string' $ Doc.description "The account password to authorise the deletion." -instance FromJSON TeamMemberDeleteData where - parseJSON = withObject "team-member-delete-data" $ \o -> - TeamMemberDeleteData <$> (o .:? "password") +makeLenses ''TeamMember' +makeLenses ''TeamMemberList' +makeLenses ''NewTeamMember' +makeLenses ''TeamMemberDeleteData -instance ToJSON TeamMemberDeleteData where - toJSON tmd = - object - [ "password" .= _tmdAuthPassword tmd - ] +userId :: Lens' TeamMember UserId +userId = newTeamMember . nUserId -makeLenses ''TeamMember -makeLenses ''TeamMemberList -makeLenses ''NewTeamMember -makeLenses ''TeamMemberDeleteData +permissions :: Lens (TeamMember' tag1) (TeamMember' tag2) (PermissionType tag1) (PermissionType tag2) +permissions = newTeamMember . nPermissions + +invitation :: Lens' TeamMember (Maybe (UserId, UTCTimeMillis)) +invitation = newTeamMember . nInvitation + +-- JSON serialisation utilities (FUTUREWORK(leif): remove after servantification) + +teamMemberJson :: (TeamMember -> Bool) -> TeamMember -> Value +teamMemberJson withPerms = toJSON . setOptionalPerms withPerms + +setOptionalPerms :: (TeamMember -> Bool) -> TeamMember -> TeamMember' 'Optional +setOptionalPerms withPerms m = m & permissions %~ setPerm (withPerms m) + +-- | Show a list of team members using 'teamMemberJson'. +teamMemberListJson :: (TeamMember -> Bool) -> TeamMemberList -> Value +teamMemberListJson withPerms l = + toJSON $ + l {_teamMembers = map (setOptionalPerms withPerms) (_teamMembers l)} diff --git a/libs/wire-api/src/Wire/API/Team/Permission.hs b/libs/wire-api/src/Wire/API/Team/Permission.hs index 00ef0ed71c0..272d7281b26 100644 --- a/libs/wire-api/src/Wire/API/Team/Permission.hs +++ b/libs/wire-api/src/Wire/API/Team/Permission.hs @@ -44,10 +44,11 @@ where import qualified Cassandra as Cql import qualified Control.Error.Util as Err import Control.Lens (makeLenses, (^.)) -import Data.Aeson +import Data.Aeson (FromJSON (..), ToJSON (..)) import Data.Bits (testBit, (.|.)) -import Data.Json.Util +import Data.Schema import qualified Data.Set as Set +import qualified Data.Swagger as S import qualified Data.Swagger.Build.Api as Doc import Imports import Wire.API.Arbitrary (Arbitrary (arbitrary), GenericUniform (..)) @@ -60,6 +61,19 @@ data Permissions = Permissions _copy :: Set Perm } deriving stock (Eq, Ord, Show, Generic) + deriving (FromJSON, ToJSON, S.ToSchema) via (Schema Permissions) + +permissionsSchema :: ValueSchemaP NamedSwaggerDoc Permissions (Set Perm, Set Perm) +permissionsSchema = + object "Permissions" $ + (,) <$> (permsToInt . _self) .= field "self" (intToPerms <$> schema) + <*> (permsToInt . _copy) .= field "copy" (intToPerms <$> schema) + +instance ToSchema Permissions where + schema = withParser permissionsSchema $ \(s, d) -> + case newPermissions s d of + Nothing -> fail "invalid permissions" + Just ps -> pure ps modelPermissions :: Doc.Model modelPermissions = Doc.defineModel "Permissions" $ do @@ -72,21 +86,6 @@ modelPermissions = Doc.defineModel "Permissions" $ do Doc.property "copy" (Doc.int64 $ Doc.min 0 . Doc.max 0x7FFFFFFFFFFFFFFF) $ Doc.description "The permissions bitmask which this user can assign to others" -instance ToJSON Permissions where - toJSON p = - object $ - "self" .= permsToInt (_self p) - # "copy" .= permsToInt (_copy p) - # [] - -instance FromJSON Permissions where - parseJSON = withObject "permissions" $ \o -> do - s <- intToPerms <$> o .: "self" - d <- intToPerms <$> o .: "copy" - case newPermissions s d of - Nothing -> fail "invalid permissions" - Just ps -> pure ps - instance Arbitrary Permissions where arbitrary = maybe (error "instance Arbitrary Permissions") pure =<< do diff --git a/libs/wire-api/src/Wire/API/User.hs b/libs/wire-api/src/Wire/API/User.hs index 18464c0b185..40780bd55fe 100644 --- a/libs/wire-api/src/Wire/API/User.hs +++ b/libs/wire-api/src/Wire/API/User.hs @@ -129,7 +129,7 @@ import qualified SAML2.WebSSO as SAML import qualified Test.QuickCheck as QC import Wire.API.Arbitrary (Arbitrary (arbitrary), GenericUniform (..)) import Wire.API.Provider.Service (ServiceRef, modelServiceRef) -import Wire.API.Team (BindingNewTeam (BindingNewTeam), modelNewBindingTeam, newTeamJson) +import Wire.API.Team (BindingNewTeam (BindingNewTeam), NewTeam (..), modelNewBindingTeam) import Wire.API.User.Activation (ActivationCode) import Wire.API.User.Auth (CookieLabel) import Wire.API.User.Identity @@ -802,6 +802,15 @@ instance ToJSON BindingNewTeamUser where A.object $ "currency" A..= c # newTeamJson t + where + -- FUTUREWORK(leif): this was originally defined in libs/wire-api/src/Wire/API/Team.hs and I moved it here + -- during the process of servantifying, it should go away when servantification is complete + newTeamJson :: NewTeam a -> [A.Pair] + newTeamJson (NewTeam n i ik _) = + "name" A..= fromRange n + # "icon" A..= fromRange i + # "icon_key" A..= (fromRange <$> ik) + # [] instance FromJSON BindingNewTeamUser where parseJSON j@(A.Object o) = do diff --git a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/NewTeamMember_team.hs b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/NewTeamMember_team.hs index ebbffc91d2f..4092949a185 100644 --- a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/NewTeamMember_team.hs +++ b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/NewTeamMember_team.hs @@ -23,7 +23,7 @@ import Data.Json.Util (readUTCTimeMillis) import qualified Data.UUID as UUID (fromString) import GHC.Exts (IsList (fromList)) import Imports (Maybe (Just, Nothing), fromJust) -import Wire.API.Team.Member (NewTeamMember, newNewTeamMember) +import Wire.API.Team.Member (NewTeamMember, mkNewTeamMember) import Wire.API.Team.Permission ( Perm ( AddTeamMember, @@ -45,7 +45,7 @@ import Wire.API.Team.Permission testObject_NewTeamMember_team_1 :: NewTeamMember testObject_NewTeamMember_team_1 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000002-0000-0007-0000-000200000002"))) (Permissions {_self = fromList [], _copy = fromList []}) ( Just @@ -57,7 +57,7 @@ testObject_NewTeamMember_team_1 = testObject_NewTeamMember_team_2 :: NewTeamMember testObject_NewTeamMember_team_2 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000004-0000-0000-0000-000200000003"))) ( Permissions { _self = @@ -81,7 +81,7 @@ testObject_NewTeamMember_team_2 = testObject_NewTeamMember_team_3 :: NewTeamMember testObject_NewTeamMember_team_3 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000008-0000-0008-0000-000700000005"))) ( Permissions { _self = @@ -99,7 +99,7 @@ testObject_NewTeamMember_team_3 = testObject_NewTeamMember_team_4 :: NewTeamMember testObject_NewTeamMember_team_4 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000700000005"))) ( Permissions { _self = fromList [CreateConversation, AddTeamMember, SetTeamData], @@ -111,7 +111,7 @@ testObject_NewTeamMember_team_4 = testObject_NewTeamMember_team_5 :: NewTeamMember testObject_NewTeamMember_team_5 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000002"))) (Permissions {_self = fromList [AddTeamMember, SetBilling, GetTeamConversations], _copy = fromList [AddTeamMember]}) ( Just @@ -123,7 +123,7 @@ testObject_NewTeamMember_team_5 = testObject_NewTeamMember_team_6 :: NewTeamMember testObject_NewTeamMember_team_6 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000002-0000-0006-0000-000400000003"))) ( Permissions { _self = @@ -141,7 +141,7 @@ testObject_NewTeamMember_team_6 = testObject_NewTeamMember_team_7 :: NewTeamMember testObject_NewTeamMember_team_7 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000007-0000-0004-0000-000500000005"))) ( Permissions { _self = @@ -159,7 +159,7 @@ testObject_NewTeamMember_team_7 = testObject_NewTeamMember_team_8 :: NewTeamMember testObject_NewTeamMember_team_8 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000008-0000-0003-0000-000200000003"))) ( Permissions { _self = fromList [DoNotUseDeprecatedModifyConvName], @@ -175,7 +175,7 @@ testObject_NewTeamMember_team_8 = testObject_NewTeamMember_team_9 :: NewTeamMember testObject_NewTeamMember_team_9 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000001-0000-0008-0000-000300000004"))) (Permissions {_self = fromList [SetBilling], _copy = fromList []}) ( Just @@ -187,7 +187,7 @@ testObject_NewTeamMember_team_9 = testObject_NewTeamMember_team_10 :: NewTeamMember testObject_NewTeamMember_team_10 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000008-0000-0003-0000-000600000003"))) (Permissions {_self = fromList [GetBilling], _copy = fromList []}) ( Just @@ -199,7 +199,7 @@ testObject_NewTeamMember_team_10 = testObject_NewTeamMember_team_11 :: NewTeamMember testObject_NewTeamMember_team_11 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000006-0000-0005-0000-000000000002"))) ( Permissions { _self = fromList [CreateConversation, DoNotUseDeprecatedModifyConvName, SetTeamData], @@ -215,7 +215,7 @@ testObject_NewTeamMember_team_11 = testObject_NewTeamMember_team_12 :: NewTeamMember testObject_NewTeamMember_team_12 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000001-0000-0004-0000-000000000007"))) (Permissions {_self = fromList [SetBilling, SetTeamData, GetTeamConversations], _copy = fromList []}) (Nothing) @@ -223,7 +223,7 @@ testObject_NewTeamMember_team_12 = testObject_NewTeamMember_team_13 :: NewTeamMember testObject_NewTeamMember_team_13 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000002-0000-0004-0000-000600000001"))) ( Permissions { _self = fromList [AddTeamMember, DoNotUseDeprecatedAddRemoveConvMember, SetTeamData, GetTeamConversations], @@ -235,7 +235,7 @@ testObject_NewTeamMember_team_13 = testObject_NewTeamMember_team_14 :: NewTeamMember testObject_NewTeamMember_team_14 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000500000004"))) ( Permissions { _self = @@ -253,7 +253,7 @@ testObject_NewTeamMember_team_14 = testObject_NewTeamMember_team_15 :: NewTeamMember testObject_NewTeamMember_team_15 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000000-0000-0008-0000-000800000007"))) ( Permissions { _self = fromList [RemoveTeamMember, GetMemberPermissions, DeleteTeam], @@ -269,7 +269,7 @@ testObject_NewTeamMember_team_15 = testObject_NewTeamMember_team_16 :: NewTeamMember testObject_NewTeamMember_team_16 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000000-0000-0006-0000-000300000005"))) ( Permissions { _self = fromList [CreateConversation, RemoveTeamMember, GetBilling, GetTeamConversations, DeleteTeam], @@ -281,7 +281,7 @@ testObject_NewTeamMember_team_16 = testObject_NewTeamMember_team_17 :: NewTeamMember testObject_NewTeamMember_team_17 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000000-0000-0008-0000-000400000005"))) (Permissions {_self = fromList [], _copy = fromList []}) ( Just @@ -293,7 +293,7 @@ testObject_NewTeamMember_team_17 = testObject_NewTeamMember_team_18 :: NewTeamMember testObject_NewTeamMember_team_18 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000006-0000-0003-0000-000000000001"))) (Permissions {_self = fromList [], _copy = fromList []}) ( Just @@ -305,7 +305,7 @@ testObject_NewTeamMember_team_18 = testObject_NewTeamMember_team_19 :: NewTeamMember testObject_NewTeamMember_team_19 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000004-0000-0005-0000-000100000008"))) ( Permissions { _self = fromList [DoNotUseDeprecatedDeleteConversation, RemoveTeamMember, SetBilling, SetMemberPermissions], @@ -317,7 +317,7 @@ testObject_NewTeamMember_team_19 = testObject_NewTeamMember_team_20 :: NewTeamMember testObject_NewTeamMember_team_20 = - ( newNewTeamMember + ( mkNewTeamMember (Id (fromJust (UUID.fromString "00000008-0000-0000-0000-000000000004"))) ( Permissions { _self = diff --git a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/TeamMemberList_team.hs b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/TeamMemberList_team.hs index 9cb5bc130eb..5b6a14c481b 100644 --- a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/TeamMemberList_team.hs +++ b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/TeamMemberList_team.hs @@ -32,14 +32,8 @@ import GHC.Exts (IsList (fromList)) import Imports (Maybe (Just, Nothing), fromJust) import Wire.API.Team.Member ( ListType (ListComplete, ListTruncated), - TeamMember - ( TeamMember, - _invitation, - _legalHoldStatus, - _permissions, - _userId - ), TeamMemberList, + mkTeamMember, newTeamMemberList, ) import Wire.API.Team.Permission @@ -56,95 +50,85 @@ import Wire.API.Team.Permission ) testObject_TeamMemberList_team_1 :: TeamMemberList -testObject_TeamMemberList_team_1 = (newTeamMemberList ([]) (ListComplete)) +testObject_TeamMemberList_team_1 = newTeamMemberList ([]) (ListComplete) testObject_TeamMemberList_team_2 :: TeamMemberList testObject_TeamMemberList_team_2 = - ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000002-0000-0000-0000-000000000002"))), - _permissions = Permissions {_self = fromList [GetBilling, SetMemberPermissions], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000002"))), - (fromJust (readUTCTimeMillis "1864-05-10T10:05:44.332Z")) - ), - _legalHoldStatus = UserLegalHoldPending - } - ] - ) - (ListComplete) - ) + newTeamMemberList + [ mkTeamMember + (Id (fromJust (UUID.fromString "00000002-0000-0000-0000-000000000002"))) + (Permissions {_self = fromList [GetBilling, SetMemberPermissions], _copy = fromList []}) + ( Just + ( Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000002")), + fromJust (readUTCTimeMillis "1864-05-10T10:05:44.332Z") + ) + ) + UserLegalHoldPending + ] + ListComplete testObject_TeamMemberList_team_3 :: TeamMemberList testObject_TeamMemberList_team_3 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T06:07:36.175Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T14:28:10.448Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T16:05:37.642Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T13:06:20.504Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T16:37:10.774Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T04:36:55.388Z")) - ), - _legalHoldStatus = UserLegalHoldPending - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T06:07:36.175Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T14:28:10.448Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T16:05:37.642Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T13:06:20.504Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T16:37:10.774Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T04:36:55.388Z")) + ) + ) + UserLegalHoldPending ] ) (ListComplete) @@ -153,26 +137,24 @@ testObject_TeamMemberList_team_3 = testObject_TeamMemberList_team_4 :: TeamMemberList testObject_TeamMemberList_team_4 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), - _permissions = Permissions {_self = fromList [GetTeamConversations], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-08T16:05:11.696Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-08T07:09:26.753Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))) + (Permissions {_self = fromList [GetTeamConversations], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-08T16:05:11.696Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-08T07:09:26.753Z")) + ) + ) + UserLegalHoldDisabled ] ) (ListTruncated) @@ -181,46 +163,42 @@ testObject_TeamMemberList_team_4 = testObject_TeamMemberList_team_5 :: TeamMemberList testObject_TeamMemberList_team_5 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T23:10:04.963Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T15:40:17.119Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T00:40:38.004Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T07:30:49.028Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T23:10:04.963Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T15:40:17.119Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T00:40:38.004Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T07:30:49.028Z")) + ) + ) + UserLegalHoldEnabled ] ) (ListComplete) @@ -229,106 +207,96 @@ testObject_TeamMemberList_team_5 = testObject_TeamMemberList_team_6 :: TeamMemberList testObject_TeamMemberList_team_6 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T17:07:48.156Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T00:04:10.559Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T10:39:19.860Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T13:40:56.648Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T12:13:40.273Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T13:28:04.561Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T02:59:55.584Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T22:57:33.947Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T01:02:39.691Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T13:39:38.488Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T17:07:48.156Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T00:04:10.559Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T10:39:19.860Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T13:40:56.648Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T12:13:40.273Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T13:28:04.561Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T02:59:55.584Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T22:57:33.947Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T01:02:39.691Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T13:39:38.488Z")) + ) + ) + UserLegalHoldEnabled ] ) (ListComplete) @@ -337,28 +305,25 @@ testObject_TeamMemberList_team_6 = testObject_TeamMemberList_team_7 :: TeamMemberList testObject_TeamMemberList_team_7 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), - _permissions = Permissions {_self = fromList [SetTeamData], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-10T03:11:36.961Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldEnabled - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))) + (Permissions {_self = fromList [SetTeamData], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-10T03:11:36.961Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldEnabled ] ) (ListTruncated) @@ -367,108 +332,97 @@ testObject_TeamMemberList_team_7 = testObject_TeamMemberList_team_8 :: TeamMemberList testObject_TeamMemberList_team_8 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T07:35:03.629Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T00:48:38.818Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T06:12:10.151Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T03:45:53.520Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T17:14:59.798Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T17:51:55.340Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T01:38:35.880Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T18:06:10.660Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T07:30:46.880Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldPending - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T07:35:03.629Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T00:48:38.818Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T06:12:10.151Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T03:45:53.520Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T17:14:59.798Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T17:51:55.340Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T01:38:35.880Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T18:06:10.660Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T07:30:46.880Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldPending ] ) (ListTruncated) @@ -477,26 +431,24 @@ testObject_TeamMemberList_team_8 = testObject_TeamMemberList_team_9 :: TeamMemberList testObject_TeamMemberList_team_9 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))), - _permissions = Permissions {_self = fromList [AddTeamMember], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-08T22:16:59.050Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), - _permissions = Permissions {_self = fromList [CreateConversation], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-08T21:43:37.550Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))) + (Permissions {_self = fromList [AddTeamMember], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-08T22:16:59.050Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))) + (Permissions {_self = fromList [CreateConversation], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-08T21:43:37.550Z")) + ) + ) + UserLegalHoldEnabled ] ) (ListTruncated) @@ -505,214 +457,192 @@ testObject_TeamMemberList_team_9 = testObject_TeamMemberList_team_10 :: TeamMemberList testObject_TeamMemberList_team_10 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T04:44:28.366Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T06:22:04.036Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T12:10:11.701Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T21:54:05.305Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T00:26:06.221Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T20:12:04.856Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T23:35:44.986Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T07:36:17.730Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T19:36:57.529Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T19:45:56.914Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T13:42:17.107Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T03:42:46.106Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T09:41:44.679Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T09:26:44.717Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T00:40:00.056Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T07:47:20.635Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T15:58:21.895Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T19:25:51.873Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T03:19:55.569Z")) - ), - _legalHoldStatus = UserLegalHoldPending - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T04:44:28.366Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T06:22:04.036Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T12:10:11.701Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T21:54:05.305Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T00:26:06.221Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T20:12:04.856Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T23:35:44.986Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T07:36:17.730Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T19:36:57.529Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T19:45:56.914Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T13:42:17.107Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T03:42:46.106Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T09:41:44.679Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T09:26:44.717Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T00:40:00.056Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T07:47:20.635Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T15:58:21.895Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T19:25:51.873Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T03:19:55.569Z")) + ) + ) + UserLegalHoldPending ] ) (ListComplete) @@ -721,120 +651,107 @@ testObject_TeamMemberList_team_10 = testObject_TeamMemberList_team_11 :: TeamMemberList testObject_TeamMemberList_team_11 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T06:08:50.626Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T08:23:53.653Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T16:28:42.815Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T11:47:57.498Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T17:22:07.538Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T19:14:48.836Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T14:53:49.059Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T10:44:04.209Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T23:34:24.831Z")) - ), - _legalHoldStatus = UserLegalHoldPending - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T06:08:50.626Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T08:23:53.653Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T16:28:42.815Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T11:47:57.498Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T17:22:07.538Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T19:14:48.836Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T14:53:49.059Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T10:44:04.209Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T23:34:24.831Z")) + ) + ) + UserLegalHoldPending ] ) (ListTruncated) @@ -843,38 +760,34 @@ testObject_TeamMemberList_team_11 = testObject_TeamMemberList_team_12 :: TeamMemberList testObject_TeamMemberList_team_12 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T15:59:09.462Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T00:27:17.631Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T15:59:09.462Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T00:27:17.631Z")) + ) + ) + UserLegalHoldEnabled ] ) (ListTruncated) @@ -883,32 +796,29 @@ testObject_TeamMemberList_team_12 = testObject_TeamMemberList_team_13 :: TeamMemberList testObject_TeamMemberList_team_13 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))), - _permissions = Permissions {_self = fromList [GetMemberPermissions], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-10T04:37:19.686Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T13:22:20.368Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))) + (Permissions {_self = fromList [GetMemberPermissions], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-10T04:37:19.686Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T13:22:20.368Z")) + ) + ) + UserLegalHoldEnabled ] ) (ListTruncated) @@ -917,136 +827,121 @@ testObject_TeamMemberList_team_13 = testObject_TeamMemberList_team_14 :: TeamMemberList testObject_TeamMemberList_team_14 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T07:01:56.077Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T09:34:46.900Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T10:40:24.034Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T10:17:53.056Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T18:37:38.894Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T06:25:10.534Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T02:42:16.433Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T07:25:18.248Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T15:31:36.237Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T15:23:38.616Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldEnabled - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T07:01:56.077Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T09:34:46.900Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T10:40:24.034Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T10:17:53.056Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T18:37:38.894Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T06:25:10.534Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T02:42:16.433Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T07:25:18.248Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T15:31:36.237Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T15:23:38.616Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldEnabled ] ) (ListTruncated) @@ -1055,44 +950,39 @@ testObject_TeamMemberList_team_14 = testObject_TeamMemberList_team_15 :: TeamMemberList testObject_TeamMemberList_team_15 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T20:33:17.912Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T09:03:59.579Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldDisabled - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T20:33:17.912Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T09:03:59.579Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldDisabled ] ) (ListTruncated) @@ -1104,52 +994,47 @@ testObject_TeamMemberList_team_16 = (newTeamMemberList ([]) (ListComplete)) testObject_TeamMemberList_team_17 :: TeamMemberList testObject_TeamMemberList_team_17 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T10:04:36.715Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T03:02:37.641Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T23:21:44.944Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T08:47:48.774Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T10:04:36.715Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T03:02:37.641Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0000-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T23:21:44.944Z")) + ) + ) + UserLegalHoldDisabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T08:47:48.774Z")) + ) + ) + UserLegalHoldDisabled ] ) (ListTruncated) @@ -1158,56 +1043,51 @@ testObject_TeamMemberList_team_17 = testObject_TeamMemberList_team_18 :: TeamMemberList testObject_TeamMemberList_team_18 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T17:44:12.611Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T05:14:06.040Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), - (fromJust (readUTCTimeMillis "1864-05-09T05:24:40.864Z")) - ), - _legalHoldStatus = UserLegalHoldPending - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T20:09:48.156Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T20:09:31.059Z")) - ), - _legalHoldStatus = UserLegalHoldPending - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T17:44:12.611Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T05:14:06.040Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000001"))), + (fromJust (readUTCTimeMillis "1864-05-09T05:24:40.864Z")) + ) + ) + UserLegalHoldPending, + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0001-0000-000000000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T20:09:48.156Z")) + ) + ) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T20:09:31.059Z")) + ) + ) + UserLegalHoldPending ] ) (ListTruncated) @@ -1216,20 +1096,19 @@ testObject_TeamMemberList_team_18 = testObject_TeamMemberList_team_19 :: TeamMemberList testObject_TeamMemberList_team_19 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0002-0000-000200000000"))), - _permissions = - Permissions - { _self = fromList [CreateConversation, SetTeamData, SetMemberPermissions], - _copy = fromList [] - }, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0002-0000-000200000002"))), - (fromJust (readUTCTimeMillis "1864-05-09T19:12:15.962Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0002-0000-000200000000"))) + ( Permissions + { _self = fromList [CreateConversation, SetTeamData, SetMemberPermissions], + _copy = fromList [] + } + ) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0002-0000-000200000002"))), + (fromJust (readUTCTimeMillis "1864-05-09T19:12:15.962Z")) + ) + ) + UserLegalHoldDisabled ] ) (ListTruncated) @@ -1238,22 +1117,20 @@ testObject_TeamMemberList_team_19 = testObject_TeamMemberList_team_20 :: TeamMemberList testObject_TeamMemberList_team_20 = ( newTeamMemberList - ( [ TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldEnabled - }, - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))), - _permissions = Permissions {_self = fromList [], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), - (fromJust (readUTCTimeMillis "1864-05-08T15:41:51.601Z")) - ), - _legalHoldStatus = UserLegalHoldPending - } + ( [ mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000100000001"))) + (Permissions {_self = fromList [], _copy = fromList []}) + (Nothing) + UserLegalHoldEnabled, + mkTeamMember + (Id (fromJust (UUID.fromString "00000001-0000-0001-0000-000100000000"))) + (Permissions {_self = fromList [], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000000"))), + (fromJust (readUTCTimeMillis "1864-05-08T15:41:51.601Z")) + ) + ) + UserLegalHoldPending ] ) (ListComplete) diff --git a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/TeamMember_team.hs b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/TeamMember_team.hs index 432dba47f5a..395af66b78f 100644 --- a/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/TeamMember_team.hs +++ b/libs/wire-api/test/golden/Test/Wire/API/Golden/Generated/TeamMember_team.hs @@ -30,7 +30,7 @@ import Data.LegalHold import qualified Data.UUID as UUID (fromString) import GHC.Exts (IsList (fromList)) import Imports (Maybe (Just, Nothing), fromJust) -import Wire.API.Team.Member (TeamMember (..)) +import Wire.API.Team.Member (TeamMember, mkTeamMember) import Wire.API.Team.Permission ( Perm ( AddTeamMember, @@ -52,340 +52,316 @@ import Wire.API.Team.Permission testObject_TeamMember_team_1 :: TeamMember testObject_TeamMember_team_1 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000007-0000-0005-0000-000500000002"))), - _permissions = - Permissions - { _self = fromList [GetBilling, GetMemberPermissions, SetMemberPermissions, DeleteTeam], - _copy = fromList [GetBilling] - }, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000001-0000-0003-0000-000300000004"))), - (fromJust (readUTCTimeMillis "1864-05-12T22:05:34.634Z")) - ), - _legalHoldStatus = UserLegalHoldPending - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000007-0000-0005-0000-000500000002"))) + ( Permissions + { _self = fromList [GetBilling, GetMemberPermissions, SetMemberPermissions, DeleteTeam], + _copy = fromList [GetBilling] + } + ) + ( Just + ( (Id (fromJust (UUID.fromString "00000001-0000-0003-0000-000300000004"))), + (fromJust (readUTCTimeMillis "1864-05-12T22:05:34.634Z")) + ) + ) + UserLegalHoldPending testObject_TeamMember_team_2 :: TeamMember testObject_TeamMember_team_2 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000003-0000-0000-0000-000500000005"))), - _permissions = - Permissions {_self = fromList [DoNotUseDeprecatedModifyConvName, SetMemberPermissions], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000004"))), - (fromJust (readUTCTimeMillis "1864-05-03T14:56:52.508Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000003-0000-0000-0000-000500000005"))) + (Permissions {_self = fromList [DoNotUseDeprecatedModifyConvName, SetMemberPermissions], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0000-0000-000000000004"))), + (fromJust (readUTCTimeMillis "1864-05-03T14:56:52.508Z")) + ) + ) + (UserLegalHoldDisabled) testObject_TeamMember_team_3 :: TeamMember testObject_TeamMember_team_3 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000005-0000-0003-0000-000400000003"))), - _permissions = - Permissions - { _self = - fromList - [DoNotUseDeprecatedDeleteConversation, AddTeamMember, DoNotUseDeprecatedAddRemoveConvMember, GetBilling], - _copy = fromList [GetBilling] - }, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0005-0000-000200000007"))), - (fromJust (readUTCTimeMillis "1864-05-06T14:02:04.371Z")) - ), - _legalHoldStatus = UserLegalHoldPending - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000005-0000-0003-0000-000400000003"))) + ( Permissions + { _self = + fromList + [DoNotUseDeprecatedDeleteConversation, AddTeamMember, DoNotUseDeprecatedAddRemoveConvMember, GetBilling], + _copy = fromList [GetBilling] + } + ) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0005-0000-000200000007"))), + (fromJust (readUTCTimeMillis "1864-05-06T14:02:04.371Z")) + ) + ) + (UserLegalHoldPending) testObject_TeamMember_team_4 :: TeamMember testObject_TeamMember_team_4 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000008-0000-0005-0000-000100000006"))), - _permissions = - Permissions - { _self = fromList [DoNotUseDeprecatedModifyConvName, SetMemberPermissions], - _copy = fromList [SetMemberPermissions] - }, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000006-0000-0002-0000-000500000001"))), - (fromJust (readUTCTimeMillis "1864-05-12T15:36:56.285Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000008-0000-0005-0000-000100000006"))) + ( Permissions + { _self = fromList [DoNotUseDeprecatedModifyConvName, SetMemberPermissions], + _copy = fromList [SetMemberPermissions] + } + ) + ( Just + ( (Id (fromJust (UUID.fromString "00000006-0000-0002-0000-000500000001"))), + (fromJust (readUTCTimeMillis "1864-05-12T15:36:56.285Z")) + ) + ) + (UserLegalHoldEnabled) testObject_TeamMember_team_5 :: TeamMember testObject_TeamMember_team_5 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000007-0000-0000-0000-000200000001"))), - _permissions = - Permissions - { _self = fromList [DoNotUseDeprecatedDeleteConversation, GetBilling, SetBilling, GetMemberPermissions], - _copy = fromList [DoNotUseDeprecatedDeleteConversation, GetMemberPermissions] - }, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000004-0000-0002-0000-000300000007"))), - (fromJust (readUTCTimeMillis "1864-05-07T21:02:57.104Z")) - ), - _legalHoldStatus = UserLegalHoldPending - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000007-0000-0000-0000-000200000001"))) + ( Permissions + { _self = fromList [DoNotUseDeprecatedDeleteConversation, GetBilling, SetBilling, GetMemberPermissions], + _copy = fromList [DoNotUseDeprecatedDeleteConversation, GetMemberPermissions] + } + ) + ( Just + ( (Id (fromJust (UUID.fromString "00000004-0000-0002-0000-000300000007"))), + (fromJust (readUTCTimeMillis "1864-05-07T21:02:57.104Z")) + ) + ) + (UserLegalHoldPending) testObject_TeamMember_team_6 :: TeamMember testObject_TeamMember_team_6 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000006-0000-0007-0000-000800000005"))), - _permissions = - Permissions - { _self = - fromList - [CreateConversation, AddTeamMember, DoNotUseDeprecatedAddRemoveConvMember, SetBilling, SetTeamData], - _copy = fromList [] - }, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000005-0000-0007-0000-000800000000"))), - (fromJust (readUTCTimeMillis "1864-05-09T03:11:26.909Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000006-0000-0007-0000-000800000005"))) + ( Permissions + { _self = + fromList + [CreateConversation, AddTeamMember, DoNotUseDeprecatedAddRemoveConvMember, SetBilling, SetTeamData], + _copy = fromList [] + } + ) + ( Just + ( (Id (fromJust (UUID.fromString "00000005-0000-0007-0000-000800000000"))), + (fromJust (readUTCTimeMillis "1864-05-09T03:11:26.909Z")) + ) + ) + (UserLegalHoldEnabled) testObject_TeamMember_team_7 :: TeamMember testObject_TeamMember_team_7 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000007-0000-0000-0000-000200000001"))), - _permissions = - Permissions - { _self = - fromList - [ DoNotUseDeprecatedDeleteConversation, - DoNotUseDeprecatedAddRemoveConvMember, - SetBilling, - SetMemberPermissions, - GetTeamConversations - ], - _copy = fromList [] - }, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldPending - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000007-0000-0000-0000-000200000001"))) + ( Permissions + { _self = + fromList + [ DoNotUseDeprecatedDeleteConversation, + DoNotUseDeprecatedAddRemoveConvMember, + SetBilling, + SetMemberPermissions, + GetTeamConversations + ], + _copy = fromList [] + } + ) + (Nothing) + (UserLegalHoldPending) testObject_TeamMember_team_8 :: TeamMember testObject_TeamMember_team_8 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000005-0000-0007-0000-000300000000"))), - _permissions = - Permissions - { _self = - fromList - [ DoNotUseDeprecatedAddRemoveConvMember, - DoNotUseDeprecatedModifyConvName, - SetTeamData, - SetMemberPermissions, - DeleteTeam - ], - _copy = fromList [] - }, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000007-0000-0003-0000-000400000003"))), - (fromJust (readUTCTimeMillis "1864-05-05T18:40:11.956Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000005-0000-0007-0000-000300000000"))) + ( Permissions + { _self = + fromList + [ DoNotUseDeprecatedAddRemoveConvMember, + DoNotUseDeprecatedModifyConvName, + SetTeamData, + SetMemberPermissions, + DeleteTeam + ], + _copy = fromList [] + } + ) + ( Just + ( (Id (fromJust (UUID.fromString "00000007-0000-0003-0000-000400000003"))), + (fromJust (readUTCTimeMillis "1864-05-05T18:40:11.956Z")) + ) + ) + (UserLegalHoldDisabled) testObject_TeamMember_team_9 :: TeamMember testObject_TeamMember_team_9 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000008-0000-0006-0000-000300000003"))), - _permissions = - Permissions - { _self = fromList [AddTeamMember, DoNotUseDeprecatedModifyConvName], - _copy = fromList [DoNotUseDeprecatedModifyConvName] - }, - _invitation = Nothing, - _legalHoldStatus = UserLegalHoldPending - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000008-0000-0006-0000-000300000003"))) + ( Permissions + { _self = fromList [AddTeamMember, DoNotUseDeprecatedModifyConvName], + _copy = fromList [DoNotUseDeprecatedModifyConvName] + } + ) + (Nothing) + (UserLegalHoldPending) testObject_TeamMember_team_10 :: TeamMember testObject_TeamMember_team_10 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000002-0000-0000-0000-000100000006"))), - _permissions = - Permissions {_self = fromList [DoNotUseDeprecatedDeleteConversation, AddTeamMember], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000008-0000-0005-0000-000000000002"))), - (fromJust (readUTCTimeMillis "1864-05-03T19:02:13.669Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000002-0000-0000-0000-000100000006"))) + (Permissions {_self = fromList [DoNotUseDeprecatedDeleteConversation, AddTeamMember], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000008-0000-0005-0000-000000000002"))), + (fromJust (readUTCTimeMillis "1864-05-03T19:02:13.669Z")) + ) + ) + (UserLegalHoldDisabled) testObject_TeamMember_team_11 :: TeamMember testObject_TeamMember_team_11 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000004-0000-0001-0000-000400000007"))), - _permissions = - Permissions - { _self = - fromList [CreateConversation, DoNotUseDeprecatedDeleteConversation, SetTeamData, SetMemberPermissions], - _copy = fromList [] - }, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000003-0000-0001-0000-000100000005"))), - (fromJust (readUTCTimeMillis "1864-05-04T18:20:29.420Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000004-0000-0001-0000-000400000007"))) + ( Permissions + { _self = + fromList [CreateConversation, DoNotUseDeprecatedDeleteConversation, SetTeamData, SetMemberPermissions], + _copy = fromList [] + } + ) + ( Just + ( (Id (fromJust (UUID.fromString "00000003-0000-0001-0000-000100000005"))), + (fromJust (readUTCTimeMillis "1864-05-04T18:20:29.420Z")) + ) + ) + (UserLegalHoldEnabled) testObject_TeamMember_team_12 :: TeamMember testObject_TeamMember_team_12 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000002-0000-0006-0000-000200000005"))), - _permissions = Permissions {_self = fromList [GetTeamConversations], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000005-0000-0001-0000-000300000003"))), - (fromJust (readUTCTimeMillis "1864-05-10T22:34:18.259Z")) - ), - _legalHoldStatus = UserLegalHoldPending - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000002-0000-0006-0000-000200000005"))) + (Permissions {_self = fromList [GetTeamConversations], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000005-0000-0001-0000-000300000003"))), + (fromJust (readUTCTimeMillis "1864-05-10T22:34:18.259Z")) + ) + ) + (UserLegalHoldPending) testObject_TeamMember_team_13 :: TeamMember testObject_TeamMember_team_13 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000006-0000-0001-0000-000800000006"))), - _permissions = - Permissions {_self = fromList [CreateConversation, GetMemberPermissions], _copy = fromList [CreateConversation]}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000000-0000-0003-0000-000400000007"))), - (fromJust (readUTCTimeMillis "1864-05-06T08:18:27.514Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000006-0000-0001-0000-000800000006"))) + (Permissions {_self = fromList [CreateConversation, GetMemberPermissions], _copy = fromList [CreateConversation]}) + ( Just + ( (Id (fromJust (UUID.fromString "00000000-0000-0003-0000-000400000007"))), + (fromJust (readUTCTimeMillis "1864-05-06T08:18:27.514Z")) + ) + ) + (UserLegalHoldDisabled) testObject_TeamMember_team_14 :: TeamMember testObject_TeamMember_team_14 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000004-0000-0000-0000-000300000007"))), - _permissions = - Permissions - { _self = fromList [DoNotUseDeprecatedDeleteConversation, AddTeamMember, GetBilling, GetMemberPermissions], - _copy = fromList [GetBilling, GetMemberPermissions] - }, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000008-0000-0000-0000-000200000002"))), - (fromJust (readUTCTimeMillis "1864-05-12T15:53:41.144Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000004-0000-0000-0000-000300000007"))) + ( Permissions + { _self = fromList [DoNotUseDeprecatedDeleteConversation, AddTeamMember, GetBilling, GetMemberPermissions], + _copy = fromList [GetBilling, GetMemberPermissions] + } + ) + ( Just + ( (Id (fromJust (UUID.fromString "00000008-0000-0000-0000-000200000002"))), + (fromJust (readUTCTimeMillis "1864-05-12T15:53:41.144Z")) + ) + ) + (UserLegalHoldDisabled) testObject_TeamMember_team_15 :: TeamMember testObject_TeamMember_team_15 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000005-0000-0006-0000-000800000006"))), - _permissions = Permissions {_self = fromList [DeleteTeam], _copy = fromList [DeleteTeam]}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000008-0000-0000-0000-000500000003"))), - (fromJust (readUTCTimeMillis "1864-05-04T06:15:13.870Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000005-0000-0006-0000-000800000006"))) + (Permissions {_self = fromList [DeleteTeam], _copy = fromList [DeleteTeam]}) + ( Just + ( (Id (fromJust (UUID.fromString "00000008-0000-0000-0000-000500000003"))), + (fromJust (readUTCTimeMillis "1864-05-04T06:15:13.870Z")) + ) + ) + (UserLegalHoldEnabled) testObject_TeamMember_team_16 :: TeamMember testObject_TeamMember_team_16 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000000-0000-0008-0000-000200000008"))), - _permissions = - Permissions {_self = fromList [DoNotUseDeprecatedDeleteConversation, GetTeamConversations], _copy = fromList []}, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000006-0000-0000-0000-000400000002"))), - (fromJust (readUTCTimeMillis "1864-05-10T04:27:37.101Z")) - ), - _legalHoldStatus = UserLegalHoldPending - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000000-0000-0008-0000-000200000008"))) + (Permissions {_self = fromList [DoNotUseDeprecatedDeleteConversation, GetTeamConversations], _copy = fromList []}) + ( Just + ( (Id (fromJust (UUID.fromString "00000006-0000-0000-0000-000400000002"))), + (fromJust (readUTCTimeMillis "1864-05-10T04:27:37.101Z")) + ) + ) + (UserLegalHoldPending) testObject_TeamMember_team_17 :: TeamMember testObject_TeamMember_team_17 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000006-0000-0006-0000-000500000007"))), - _permissions = - Permissions - { _self = - fromList - [ DoNotUseDeprecatedAddRemoveConvMember, - DoNotUseDeprecatedModifyConvName, - GetBilling, - SetTeamData, - GetTeamConversations - ], - _copy = fromList [DoNotUseDeprecatedAddRemoveConvMember] - }, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000006-0000-0003-0000-000700000004"))), - (fromJust (readUTCTimeMillis "1864-05-07T23:22:37.991Z")) - ), - _legalHoldStatus = UserLegalHoldDisabled - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000006-0000-0006-0000-000500000007"))) + ( Permissions + { _self = + fromList + [ DoNotUseDeprecatedAddRemoveConvMember, + DoNotUseDeprecatedModifyConvName, + GetBilling, + SetTeamData, + GetTeamConversations + ], + _copy = fromList [DoNotUseDeprecatedAddRemoveConvMember] + } + ) + ( Just + ( (Id (fromJust (UUID.fromString "00000006-0000-0003-0000-000700000004"))), + (fromJust (readUTCTimeMillis "1864-05-07T23:22:37.991Z")) + ) + ) + (UserLegalHoldDisabled) testObject_TeamMember_team_18 :: TeamMember testObject_TeamMember_team_18 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000005-0000-0005-0000-000200000008"))), - _permissions = - Permissions - { _self = - fromList [RemoveTeamMember, DoNotUseDeprecatedModifyConvName, GetMemberPermissions, SetMemberPermissions], - _copy = fromList [SetMemberPermissions] - }, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000007-0000-0008-0000-000500000006"))), - (fromJust (readUTCTimeMillis "1864-05-15T14:48:55.847Z")) - ), - _legalHoldStatus = UserLegalHoldPending - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000005-0000-0005-0000-000200000008"))) + ( Permissions + { _self = + fromList [RemoveTeamMember, DoNotUseDeprecatedModifyConvName, GetMemberPermissions, SetMemberPermissions], + _copy = fromList [SetMemberPermissions] + } + ) + ( Just + ( (Id (fromJust (UUID.fromString "00000007-0000-0008-0000-000500000006"))), + (fromJust (readUTCTimeMillis "1864-05-15T14:48:55.847Z")) + ) + ) + (UserLegalHoldPending) testObject_TeamMember_team_19 :: TeamMember testObject_TeamMember_team_19 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000003-0000-0002-0000-000200000008"))), - _permissions = - Permissions - { _self = - fromList [AddTeamMember, DoNotUseDeprecatedModifyConvName, GetBilling, SetBilling, SetMemberPermissions], - _copy = fromList [SetMemberPermissions] - }, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000006-0000-0001-0000-000200000008"))), - (fromJust (readUTCTimeMillis "1864-05-12T01:37:35.003Z")) - ), - _legalHoldStatus = UserLegalHoldPending - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000003-0000-0002-0000-000200000008"))) + ( Permissions + { _self = + fromList [AddTeamMember, DoNotUseDeprecatedModifyConvName, GetBilling, SetBilling, SetMemberPermissions], + _copy = fromList [SetMemberPermissions] + } + ) + ( Just + ( (Id (fromJust (UUID.fromString "00000006-0000-0001-0000-000200000008"))), + (fromJust (readUTCTimeMillis "1864-05-12T01:37:35.003Z")) + ) + ) + (UserLegalHoldPending) testObject_TeamMember_team_20 :: TeamMember testObject_TeamMember_team_20 = - TeamMember - { _userId = (Id (fromJust (UUID.fromString "00000005-0000-0007-0000-000100000005"))), - _permissions = - Permissions - { _self = fromList [CreateConversation, AddTeamMember, DoNotUseDeprecatedModifyConvName, GetBilling], - _copy = fromList [] - }, - _invitation = - Just - ( (Id (fromJust (UUID.fromString "00000005-0000-0001-0000-000800000007"))), - (fromJust (readUTCTimeMillis "1864-05-04T22:12:50.096Z")) - ), - _legalHoldStatus = UserLegalHoldEnabled - } + mkTeamMember + (Id (fromJust (UUID.fromString "00000005-0000-0007-0000-000100000005"))) + ( Permissions + { _self = fromList [CreateConversation, AddTeamMember, DoNotUseDeprecatedModifyConvName, GetBilling], + _copy = fromList [] + } + ) + ( Just + ( (Id (fromJust (UUID.fromString "00000005-0000-0001-0000-000800000007"))), + (fromJust (readUTCTimeMillis "1864-05-04T22:12:50.096Z")) + ) + ) + (UserLegalHoldEnabled) diff --git a/services/brig/src/Brig/IO/Intra.hs b/services/brig/src/Brig/IO/Intra.hs index 55fbb8f54bd..a7ed7bc5068 100644 --- a/services/brig/src/Brig/IO/Intra.hs +++ b/services/brig/src/Brig/IO/Intra.hs @@ -122,6 +122,7 @@ import Wire.API.Federation.Error import Wire.API.Message (UserClients) import Wire.API.Team.Feature (IncludeLockStatus (..), TeamFeatureName (..), TeamFeatureStatus) import Wire.API.Team.LegalHold (LegalholdProtectee) +import qualified Wire.API.Team.Member as Member ----------------------------------------------------------------------------- -- Event Handlers @@ -853,7 +854,7 @@ addTeamMember u tid (minvmeta, role) = do _ -> False where prm = Team.rolePermissions role - bdy = Team.newNewTeamMember u prm minvmeta + bdy = Member.mkNewTeamMember u prm minvmeta req = paths ["i", "teams", toByteString' tid, "members"] . header "Content-Type" "application/json" diff --git a/services/brig/test/integration/API/Team/Util.hs b/services/brig/test/integration/API/Team/Util.hs index aa44fc75895..36811ec25d3 100644 --- a/services/brig/test/integration/API/Team/Util.hs +++ b/services/brig/test/integration/API/Team/Util.hs @@ -48,6 +48,7 @@ import Util import Web.Cookie (parseSetCookie, setCookieName) import Wire.API.Team.Feature (TeamFeatureStatusValue (..)) import qualified Wire.API.Team.Feature as Public +import qualified Wire.API.Team.Member as Member import qualified Wire.API.User as Public -- | FUTUREWORK: Remove 'createPopulatedBindingTeam', 'createPopulatedBindingTeamWithNames', @@ -204,7 +205,7 @@ updatePermissions from tid (to, perm) galley = !!! const 200 === statusCode where - changeMember = Team.newNewTeamMember to perm Nothing + changeMember = Member.mkNewTeamMember to perm Nothing createTeamConv :: HasCallStack => Galley -> TeamId -> UserId -> [UserId] -> Maybe Milliseconds -> Http ConvId createTeamConv g tid u us mtimer = do diff --git a/services/galley/src/Galley/API/Public.hs b/services/galley/src/Galley/API/Public.hs index 31d41ae0531..42db3104f91 100644 --- a/services/galley/src/Galley/API/Public.hs +++ b/services/galley/src/Galley/API/Public.hs @@ -187,25 +187,14 @@ servantSitemap = GalleyAPI.featureConfigClassifiedDomainsGet = Features.getFeatureConfig @'Public.WithoutLockStatus @'Public.TeamFeatureClassifiedDomains Features.getClassifiedDomainsInternal, GalleyAPI.featureConfigConferenceCallingGet = Features.getFeatureConfig @'Public.WithoutLockStatus @'Public.TeamFeatureConferenceCalling Features.getConferenceCallingInternal, GalleyAPI.featureConfigSelfDeletingMessagesGet = Features.getFeatureConfig @'Public.WithLockStatus @'Public.TeamFeatureSelfDeletingMessages Features.getSelfDeletingMessagesInternal, - GalleyAPI.featureConfigGuestLinksGet = Features.getFeatureConfig @'Public.WithLockStatus @'Public.TeamFeatureGuestLinks Features.getGuestLinkInternal + GalleyAPI.featureConfigGuestLinksGet = Features.getFeatureConfig @'Public.WithLockStatus @'Public.TeamFeatureGuestLinks Features.getGuestLinkInternal, + GalleyAPI.createNonBindingTeam = Teams.createNonBindingTeamH } sitemap :: Routes ApiBuilder (Sem GalleyEffects) () sitemap = do -- Team API ----------------------------------------------------------- - post "/teams" (continue Teams.createNonBindingTeamH) $ - zauthUserId - .&. zauthConnId - .&. jsonRequest @Public.NonBindingNewTeam - .&. accept "application" "json" - document "POST" "createNonBindingTeam" $ do - summary "Create a new non binding team" - body (ref Public.modelNewNonBindingTeam) $ - description "JSON body" - response 201 "Team ID as `Location` header value" end - errorResponse (Error.errorDescriptionTypeToWai @Error.NotConnected) - put "/teams/:tid" (continue Teams.updateTeamH) $ zauthUserId .&. zauthConnId diff --git a/services/galley/src/Galley/API/Teams.hs b/services/galley/src/Galley/API/Teams.hs index 9d46c2fcd7a..34e27ae7fc1 100644 --- a/services/galley/src/Galley/API/Teams.hs +++ b/services/galley/src/Galley/API/Teams.hs @@ -135,6 +135,7 @@ import qualified Wire.API.Team as Public import qualified Wire.API.Team.Conversation as Public import Wire.API.Team.Export (TeamExportUser (..)) import qualified Wire.API.Team.Feature as Public +import Wire.API.Team.Member (ntmNewTeamMember, teamMemberJson, teamMemberListJson) import qualified Wire.API.Team.Member as Public import qualified Wire.API.Team.SearchVisibility as Public import Wire.API.User (User, UserSSOId (UserScimExternalId), userSCIMExternalId, userSSOId) @@ -203,41 +204,22 @@ lookupTeam zusr tid = do else pure Nothing createNonBindingTeamH :: - Members - '[ BrigAccess, - Error ActionError, - Error TeamError, - GundeckAccess, - Input UTCTime, - P.TinyLog, - TeamStore, - WaiRoutes - ] - r => - UserId ::: ConnId ::: JsonRequest Public.NonBindingNewTeam ::: JSON -> - Sem r Response -createNonBindingTeamH (zusr ::: zcon ::: req ::: _) = do - newTeam <- fromJsonBody req - newTeamId <- createNonBindingTeam zusr zcon newTeam - pure (empty & setStatus status201 . location newTeamId) - -createNonBindingTeam :: - Members - '[ BrigAccess, - Error ActionError, - Error TeamError, - GundeckAccess, - Input UTCTime, - TeamStore, - P.TinyLog - ] - r => + forall r. + ( Member BrigAccess r, + Member (Error ActionError) r, + Member (Error TeamError) r, + Member GundeckAccess r, + Member (Input UTCTime) r, + Member P.TinyLog r, + Member TeamStore r, + Member WaiRoutes r + ) => UserId -> ConnId -> Public.NonBindingNewTeam -> Sem r TeamId -createNonBindingTeam zusr zcon (Public.NonBindingNewTeam body) = do - let owner = Public.TeamMember zusr fullPermissions Nothing LH.defUserLegalHoldStatus +createNonBindingTeamH zusr zcon (Public.NonBindingNewTeam body) = do + let owner = Public.mkTeamMember zusr fullPermissions Nothing LH.defUserLegalHoldStatus let others = filter ((zusr /=) . view userId) . maybe [] fromRange @@ -275,7 +257,7 @@ createBindingTeam :: BindingNewTeam -> Sem r TeamId createBindingTeam zusr tid (BindingNewTeam body) = do - let owner = Public.TeamMember zusr fullPermissions Nothing LH.defUserLegalHoldStatus + let owner = Public.mkTeamMember zusr fullPermissions Nothing LH.defUserLegalHoldStatus team <- E.createTeam (Just tid) zusr (body ^. newTeamName) (body ^. newTeamIcon) (body ^. newTeamIconKey) Binding finishCreateTeam team owner [] Nothing @@ -835,7 +817,7 @@ addTeamMember :: Public.NewTeamMember -> Sem r () addTeamMember zusr zcon tid nmem = do - let uid = nmem ^. ntmNewTeamMember . userId + let uid = nmem ^. nUserId P.debug $ Log.field "targets" (toByteString uid) . Log.field "action" (Log.val "Teams.addTeamMember") @@ -843,7 +825,7 @@ addTeamMember zusr zcon tid nmem = do zusrMembership <- E.getTeamMember tid zusr >>= permissionCheck AddTeamMember - let targetPermissions = nmem ^. ntmNewTeamMember . permissions + let targetPermissions = nmem ^. nPermissions targetPermissions `ensureNotElevated` zusrMembership ensureNonBindingTeam tid ensureUnboundUsers [uid] @@ -904,7 +886,7 @@ uncheckedAddTeamMember tid nmem = do (TeamSize sizeBeforeJoin) <- E.getSize tid ensureNotTooLargeForLegalHold tid (fromIntegral sizeBeforeJoin + 1) (TeamSize sizeBeforeAdd) <- addTeamMemberInternal tid Nothing Nothing nmem mems - billingUserIds <- Journal.getBillingUserIds tid $ Just $ newTeamMemberList ((nmem ^. ntmNewTeamMember) : mems ^. teamMembers) (mems ^. teamMemberListType) + billingUserIds <- Journal.getBillingUserIds tid $ Just $ newTeamMemberList (ntmNewTeamMember nmem : mems ^. teamMembers) (mems ^. teamMemberListType) Journal.teamUpdate tid (sizeBeforeAdd + 1) billingUserIds updateTeamMemberH :: @@ -925,7 +907,7 @@ updateTeamMemberH :: Sem r Response updateTeamMemberH (zusr ::: zcon ::: tid ::: req ::: _) = do -- the team member to be updated - targetMember <- view ntmNewTeamMember <$> (fromJsonBody req) + targetMember <- ntmNewTeamMember <$> fromJsonBody req updateTeamMember zusr zcon tid targetMember pure empty @@ -1385,7 +1367,7 @@ addTeamMemberInternal :: NewTeamMember -> TeamMemberList -> Sem r TeamSize -addTeamMemberInternal tid origin originConn (view ntmNewTeamMember -> new) memList = do +addTeamMemberInternal tid origin originConn (ntmNewTeamMember -> new) memList = do P.debug $ Log.field "targets" (toByteString (new ^. userId)) . Log.field "action" (Log.val "Teams.addTeamMemberInternal") diff --git a/services/galley/src/Galley/Cassandra/Team.hs b/services/galley/src/Galley/Cassandra/Team.hs index 7d0f373a290..a89af42d6d4 100644 --- a/services/galley/src/Galley/Cassandra/Team.hs +++ b/services/galley/src/Galley/Cassandra/Team.hs @@ -410,8 +410,8 @@ newTeamMember' lh tid (uid, perms, minvu, minvt, fromMaybe defUserLegalHoldStatu UserLegalHoldPending -> UserLegalHoldPending UserLegalHoldEnabled -> UserLegalHoldEnabled - mk (Just invu) (Just invt) = pure $ TeamMember uid perms (Just (invu, invt)) lhStatus - mk Nothing Nothing = pure $ TeamMember uid perms Nothing lhStatus + mk (Just invu) (Just invt) = pure $ mkTeamMember uid perms (Just (invu, invt)) lhStatus + mk Nothing Nothing = pure $ mkTeamMember uid perms Nothing lhStatus mk _ _ = throwM $ ErrorCall "TeamMember with incomplete metadata." teamConversationsForPagination :: TeamId -> Maybe ConvId -> Range 1 HardTruncationLimit Int32 -> Client (Page TeamConversation) diff --git a/services/galley/test/integration/API/MessageTimer.hs b/services/galley/test/integration/API/MessageTimer.hs index f0e1d91fb21..e773e3a5491 100644 --- a/services/galley/test/integration/API/MessageTimer.hs +++ b/services/galley/test/integration/API/MessageTimer.hs @@ -189,7 +189,7 @@ messageTimerChangeWithoutAllowedAction = do -- Create a team and a guest user [owner, member, guest] <- randomUsers 3 connectUsers owner (list1 member [guest]) - tid <- createNonBindingTeam "team" owner [Member.TeamMember member Teams.fullPermissions Nothing LH.defUserLegalHoldStatus] + tid <- createNonBindingTeam "team" owner [Member.mkTeamMember member Teams.fullPermissions Nothing LH.defUserLegalHoldStatus] -- Create a conversation cid <- createTeamConvWithRole owner tid [member, guest] Nothing Nothing Nothing roleNameWireMember -- Try to change the timer (as a non admin, guest user) and observe failure diff --git a/services/galley/test/integration/API/Teams.hs b/services/galley/test/integration/API/Teams.hs index 837f499864b..2e0890f7c4a 100644 --- a/services/galley/test/integration/API/Teams.hs +++ b/services/galley/test/integration/API/Teams.hs @@ -480,7 +480,7 @@ testAddTeamMember = do Util.connectUsers (mem1 ^. userId) (list1 (mem2 ^. userId) []) tid <- Util.createNonBindingTeam "foo" owner [mem1, mem2] mem3 <- newTeamMember' p1 <$> Util.randomUser - let payload = json (newNewTeamMember (mem3 ^. userId) (mem3 ^. permissions) (mem3 ^. invitation)) + let payload = json (Member.mkNewTeamMember (mem3 ^. userId) (mem3 ^. permissions) (mem3 ^. invitation)) Util.connectUsers (mem1 ^. userId) (list1 (mem3 ^. userId) []) Util.connectUsers (mem2 ^. userId) (list1 (mem3 ^. userId) []) -- `mem1` lacks permission to add new team members @@ -566,7 +566,7 @@ testAddTeamMemberCheckBound = do ( g . paths ["teams", toByteString' tidBound, "members"] . zUser ownerBound . zConn "conn" - . json (newNewTeamMember (rndMem ^. userId) (rndMem ^. permissions) (rndMem ^. invitation)) + . json (Member.mkNewTeamMember (rndMem ^. userId) (rndMem ^. permissions) (rndMem ^. invitation)) ) !!! const 403 === statusCode owner <- Util.randomUser @@ -576,7 +576,7 @@ testAddTeamMemberCheckBound = do post ( g . paths ["teams", toByteString' tid, "members"] . zUser owner . zConn "conn" - . json (newNewTeamMember (boundMem ^. userId) (boundMem ^. permissions) (boundMem ^. invitation)) + . json (Member.mkNewTeamMember (boundMem ^. userId) (boundMem ^. permissions) (boundMem ^. invitation)) ) !!! const 403 === statusCode @@ -970,7 +970,7 @@ testUpdateTeamConv (rolePermissions -> perms) convRole = do owner <- Util.randomUser member <- Util.randomUser Util.connectUsers owner (list1 member []) - tid <- Util.createNonBindingTeam "foo" owner [Member.TeamMember member perms Nothing LH.defUserLegalHoldStatus] + tid <- Util.createNonBindingTeam "foo" owner [Member.mkTeamMember member perms Nothing LH.defUserLegalHoldStatus] cid <- Util.createTeamConvWithRole owner tid [member] (Just "gossip") Nothing Nothing convRole resp <- updateTeamConv member cid (ConversationRename "not gossip") -- FUTUREWORK: Ensure that the team role _really_ does not matter @@ -1459,7 +1459,7 @@ testBillingInLargeTeamWithoutIndexedBillingTeamMembers = do foldM ( \billingMembers n -> do newBillingMemberId <- randomUser - let mem = json $ newNewTeamMember newBillingMemberId (rolePermissions RoleOwner) Nothing + let mem = json $ Member.mkNewTeamMember newBillingMemberId (rolePermissions RoleOwner) Nothing -- We cannot properly add the new owner with an invite as we don't have a way to -- override galley settings while making a call to brig withoutIndexedBillingTeamMembers $ @@ -1477,7 +1477,7 @@ testBillingInLargeTeamWithoutIndexedBillingTeamMembers = do -- If we add another owner, one of them won't get notified ownerFanoutPlusTwo <- randomUser - let memFanoutPlusTwo = json $ newNewTeamMember ownerFanoutPlusTwo (rolePermissions RoleOwner) Nothing + let memFanoutPlusTwo = json $ Member.mkNewTeamMember ownerFanoutPlusTwo (rolePermissions RoleOwner) Nothing -- We cannot properly add the new owner with an invite as we don't have a way to -- override galley settings while making a call to brig withoutIndexedBillingTeamMembers $ @@ -1513,7 +1513,7 @@ testBillingInLargeTeamWithoutIndexedBillingTeamMembers = do refreshIndex -- Promotions and demotion should also be kept track of regardless of feature being enabled - let demoteFanoutPlusThree = newNewTeamMember ownerFanoutPlusThree (rolePermissions RoleAdmin) Nothing + let demoteFanoutPlusThree = Member.mkNewTeamMember ownerFanoutPlusThree (rolePermissions RoleAdmin) Nothing withoutIndexedBillingTeamMembers $ updateTeamMember galley team firstOwner demoteFanoutPlusThree !!! const 200 === statusCode ensureQueueEmpty @@ -1522,7 +1522,7 @@ testBillingInLargeTeamWithoutIndexedBillingTeamMembers = do tUpdateUncertainCount [4, 5] (allOwnersBeforeFanoutLimit <> [ownerFanoutPlusFour, ownerFanoutPlusFive]) refreshIndex - let promoteFanoutPlusThree = newNewTeamMember ownerFanoutPlusThree (rolePermissions RoleOwner) Nothing + let promoteFanoutPlusThree = Member.mkNewTeamMember ownerFanoutPlusThree (rolePermissions RoleOwner) Nothing withoutIndexedBillingTeamMembers $ updateTeamMember galley team firstOwner promoteFanoutPlusThree !!! const 200 === statusCode ensureQueueEmpty @@ -1548,28 +1548,28 @@ testUpdateTeamMember = do assertQueue "add member" $ tUpdate 2 [owner] refreshIndex -- non-owner can **NOT** demote owner - let demoteOwner = newNewTeamMember owner (rolePermissions RoleAdmin) Nothing + let demoteOwner = Member.mkNewTeamMember owner (rolePermissions RoleAdmin) Nothing updateTeamMember g tid (member ^. userId) demoteOwner !!! do const 403 === statusCode const "access-denied" === (Error.label . responseJsonUnsafeWithMsg "error label") -- owner can demote non-owner - let demoteMember = newNewTeamMember (member ^. userId) noPermissions (member ^. invitation) + let demoteMember = Member.mkNewTeamMember (member ^. userId) noPermissions (member ^. invitation) WS.bracketR2 c owner (member ^. userId) $ \(wsOwner, wsMember) -> do updateTeamMember g tid owner demoteMember !!! do const 200 === statusCode member' <- Util.getTeamMember owner tid (member ^. userId) - liftIO $ assertEqual "permissions" (member' ^. permissions) (demoteMember ^. ntmNewTeamMember . permissions) + liftIO $ assertEqual "permissions" (member' ^. permissions) (demoteMember ^. nPermissions) checkTeamMemberUpdateEvent tid (member ^. userId) wsOwner (pure noPermissions) checkTeamMemberUpdateEvent tid (member ^. userId) wsMember (pure noPermissions) WS.assertNoEvent timeout [wsOwner, wsMember] assertQueue "Member demoted" $ tUpdate 2 [owner] -- owner can promote non-owner - let promoteMember = newNewTeamMember (member ^. userId) fullPermissions (member ^. invitation) + let promoteMember = Member.mkNewTeamMember (member ^. userId) fullPermissions (member ^. invitation) WS.bracketR2 c owner (member ^. userId) $ \(wsOwner, wsMember) -> do updateTeamMember g tid owner promoteMember !!! do const 200 === statusCode member' <- Util.getTeamMember owner tid (member ^. userId) - liftIO $ assertEqual "permissions" (member' ^. permissions) (promoteMember ^. ntmNewTeamMember . permissions) + liftIO $ assertEqual "permissions" (member' ^. permissions) (promoteMember ^. nPermissions) checkTeamMemberUpdateEvent tid (member ^. userId) wsOwner (pure fullPermissions) checkTeamMemberUpdateEvent tid (member ^. userId) wsMember (pure fullPermissions) WS.assertNoEvent timeout [wsOwner, wsMember] @@ -1582,7 +1582,7 @@ testUpdateTeamMember = do updateTeamMember g tid (member ^. userId) demoteOwner !!! do const 200 === statusCode owner' <- Util.getTeamMember (member ^. userId) tid owner - liftIO $ assertEqual "permissions" (owner' ^. permissions) (demoteOwner ^. ntmNewTeamMember . permissions) + liftIO $ assertEqual "permissions" (owner' ^. permissions) (demoteOwner ^. nPermissions) -- owner no longer has GetPermissions, but she can still see the update because it's about her! checkTeamMemberUpdateEvent tid owner wsOwner (pure (rolePermissions RoleAdmin)) checkTeamMemberUpdateEvent tid owner wsMember (pure (rolePermissions RoleAdmin)) @@ -1881,7 +1881,7 @@ postCryptoBroadcastMessage100OrMaxConns = do (xxx, yyy, _, _) -> error ("Unexpected while connecting users: " ++ show xxx ++ " and " ++ show yyy) newTeamMember' :: Permissions -> UserId -> TeamMember -newTeamMember' perms uid = Member.TeamMember uid perms Nothing LH.defUserLegalHoldStatus +newTeamMember' perms uid = Member.mkTeamMember uid perms Nothing LH.defUserLegalHoldStatus -- NOTE: all client functions calling @{/i,}/teams/*/features/*@ can be replaced by -- hypothetical functions 'getTeamFeatureFlag', 'getTeamFeatureFlagInternal', diff --git a/services/galley/test/integration/API/Util.hs b/services/galley/test/integration/API/Util.hs index 31af5c651a5..3a767e13e9a 100644 --- a/services/galley/test/integration/API/Util.hs +++ b/services/galley/test/integration/API/Util.hs @@ -122,6 +122,7 @@ import Wire.API.Message import qualified Wire.API.Message.Proto as Proto import Wire.API.Routes.Internal.Brig.Connection import Wire.API.Routes.MultiTablePaging +import Wire.API.Team.Member (mkNewTeamMember) import Wire.API.User.Client (ClientCapability (..), UserClientsFull (UserClientsFull)) import qualified Wire.API.User.Client as Client import Wire.API.User.Identity (mkSimpleSampleUref) @@ -358,7 +359,7 @@ getTeamMemberInternal tid mid = do addTeamMember :: HasCallStack => UserId -> TeamId -> UserId -> Permissions -> Maybe (UserId, UTCTimeMillis) -> TestM () addTeamMember usr tid muid mperms mmbinv = do g <- view tsGalley - let payload = json (newNewTeamMember muid mperms mmbinv) + let payload = json (mkNewTeamMember muid mperms mmbinv) post (g . paths ["teams", toByteString' tid, "members"] . zUser usr . zConn "conn" . payload) !!! const 200 === statusCode @@ -370,7 +371,7 @@ addTeamMemberInternal tid muid mperms mmbinv = addTeamMemberInternal' tid muid m addTeamMemberInternal' :: HasCallStack => TeamId -> UserId -> Permissions -> Maybe (UserId, UTCTimeMillis) -> TestM ResponseLBS addTeamMemberInternal' tid muid mperms mmbinv = do g <- view tsGalley - let payload = json (newNewTeamMember muid mperms mmbinv) + let payload = json (mkNewTeamMember muid mperms mmbinv) post (g . paths ["i", "teams", toByteString' tid, "members"] . payload) addUserToTeam :: HasCallStack => UserId -> TeamId -> TestM TeamMember @@ -417,7 +418,7 @@ addUserToTeamWithSSO hasEmail tid = do makeOwner :: HasCallStack => UserId -> TeamMember -> TeamId -> TestM () makeOwner owner mem tid = do galley <- view tsGalley - let changeMember = newNewTeamMember (mem ^. Team.userId) fullPermissions (mem ^. Team.invitation) + let changeMember = mkNewTeamMember (mem ^. Team.userId) fullPermissions (mem ^. Team.invitation) put ( galley . paths ["teams", toByteString' tid, "members"] diff --git a/services/spar/test-integration/Util/Core.hs b/services/spar/test-integration/Util/Core.hs index f51f8789748..a621f285f1d 100644 --- a/services/spar/test-integration/Util/Core.hs +++ b/services/spar/test-integration/Util/Core.hs @@ -199,6 +199,7 @@ import Wire.API.Routes.Public.Spar import Wire.API.Team.Feature (TeamFeatureStatusValue (..)) import qualified Wire.API.Team.Feature as Public import qualified Wire.API.Team.Invitation as TeamInvitation +import qualified Wire.API.Team.Member as Member import Wire.API.User (HandleUpdate (HandleUpdate), UserUpdate) import qualified Wire.API.User as User import Wire.API.User.Identity (mkSampleUref) @@ -472,7 +473,7 @@ createTeamMember brigreq galleyreq teamid perms = do postUser name False (Just ssoid) (Just teamid) brigreq UserId -> TeamId -> UserId -> TestSpar () promoteTeamMember usr tid memid = do gly <- view teGalley let bdy :: Galley.NewTeamMember - bdy = Galley.newNewTeamMember memid Galley.fullPermissions Nothing + bdy = Member.mkNewTeamMember memid Galley.fullPermissions Nothing call $ put (gly . paths ["teams", toByteString' tid, "members"] . zAuthAccess usr "conn" . json bdy) !!! const 200 === statusCode @@ -1209,7 +1210,7 @@ ssoToUidSpar tid ssoid = do veid <- either (error . ("could not parse brig sso_id: " <>)) pure $ Intra.veidFromUserSSOId ssoid runSpar $ runValidExternalId - (SAMLUserStore.get) + SAMLUserStore.get (ScimExternalIdStore.lookup tid) veid diff --git a/tools/stern/src/Stern/Types.hs b/tools/stern/src/Stern/Types.hs index 6f2e43a2729..3fc157ab118 100644 --- a/tools/stern/src/Stern/Types.hs +++ b/tools/stern/src/Stern/Types.hs @@ -35,6 +35,7 @@ import Data.Range import Galley.Types.Teams import Galley.Types.Teams.Intra import Imports +import Wire.API.Team.Member (teamMemberJson) newtype TeamMemberInfo = TeamMemberInfo {tm :: TeamMember} From fb40c898e41711dc07c3a4c3e6054636fa7e4e28 Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Mon, 3 Jan 2022 17:05:40 +0100 Subject: [PATCH 44/74] Do not regenerate Swagger doc module if unchanged (#2018) --- changelog.d/5-internal/brig-faster-doc | 1 + services/brig/Setup.hs | 26 +++++++++++++++++++++----- services/brig/brig.cabal | 1 + services/brig/package.yaml | 1 + 4 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 changelog.d/5-internal/brig-faster-doc diff --git a/changelog.d/5-internal/brig-faster-doc b/changelog.d/5-internal/brig-faster-doc new file mode 100644 index 00000000000..edd39960037 --- /dev/null +++ b/changelog.d/5-internal/brig-faster-doc @@ -0,0 +1 @@ +The Swagger documentation module is not regenerated anymore if its content is unchanged diff --git a/services/brig/Setup.hs b/services/brig/Setup.hs index b7f084b8ca1..ce0b496ab7c 100644 --- a/services/brig/Setup.hs +++ b/services/brig/Setup.hs @@ -1,3 +1,6 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + -- This file is part of the Wire Server implementation. -- -- Copyright (C) 2021 Wire Swiss GmbH @@ -15,10 +18,15 @@ -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . +import Control.Exception +import Control.Monad import Data.Char import Data.Foldable import qualified Data.Map as Map import Data.Maybe +import Data.Text (Text) +import qualified Data.Text as Text +import qualified Data.Text.IO as Text import Distribution.Simple import Distribution.Simple.BuildPaths import Distribution.Simple.LocalBuildInfo @@ -49,17 +57,25 @@ generateDocs base src = do let name = moduleName src dest = base (moduleName src <> ".hs") createDirectoryIfMissing True base - putStrLn ("Generating " <> dest <> " ...") let out = - unlines - [ "module Brig.Docs." <> name <> " where", + Text.unlines + [ "module Brig.Docs." <> Text.pack name <> " where", "", "import Imports", "", "contents :: Text", - "contents = " ++ show contents + "contents = " <> Text.pack (show contents) ] - writeFile dest out + writeFileIfChanged dest out + +writeFileIfChanged :: FilePath -> Text -> IO () +writeFileIfChanged fp c' = do + changed <- handle @IOException (const (pure True)) $ do + c <- Text.readFile fp + pure $ c /= c' + when changed $ do + putStrLn ("Generating " <> fp <> " ...") + Text.writeFile fp c' moduleName :: String -> String moduleName = go . dropExtension diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal index fe097e533ae..dcd60278065 100644 --- a/services/brig/brig.cabal +++ b/services/brig/brig.cabal @@ -26,6 +26,7 @@ custom-setup , containers , directory , filepath + , text library exposed-modules: diff --git a/services/brig/package.yaml b/services/brig/package.yaml index 0f10e5f4b24..b67cac85e5e 100644 --- a/services/brig/package.yaml +++ b/services/brig/package.yaml @@ -17,6 +17,7 @@ custom-setup: - containers - directory - filepath + - text extra-source-files: - docs/* library: From 918f988c1bdfc4c0c6b35aec67a342e968f0d0ae Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Tue, 4 Jan 2022 06:49:52 +0100 Subject: [PATCH 45/74] Simplify `rangedSchema` function (#2017) --- changelog.d/5-internal/simplify-ranged-schema | 1 + libs/types-common/src/Data/Range.hs | 10 ++++------ libs/wire-api/src/Wire/API/Conversation.hs | 3 +-- libs/wire-api/src/Wire/API/Team.hs | 3 +-- 4 files changed, 7 insertions(+), 10 deletions(-) create mode 100644 changelog.d/5-internal/simplify-ranged-schema diff --git a/changelog.d/5-internal/simplify-ranged-schema b/changelog.d/5-internal/simplify-ranged-schema new file mode 100644 index 00000000000..c6a3e875811 --- /dev/null +++ b/changelog.d/5-internal/simplify-ranged-schema @@ -0,0 +1 @@ +`rangedSchema` does not need to be passed singletons explicitly anymore diff --git a/libs/types-common/src/Data/Range.hs b/libs/types-common/src/Data/Range.hs index d244a6fa656..6eb80f5de71 100644 --- a/libs/types-common/src/Data/Range.hs +++ b/libs/types-common/src/Data/Range.hs @@ -122,14 +122,12 @@ instance (Within a n m, FromJSON a) => FromJSON (Range n m a) where msg sn sm = fail (errorMsg (fromSing sn) (fromSing sm) "") rangedSchema :: + forall n m d v w a b. (Within a n m, HasRangedSchemaDocModifier d b) => - SNat n -> - SNat m -> SchemaP d v w a b -> SchemaP d v w a (Range n m b) -rangedSchema sn sm sch = Range <$> untypedRangedSchema (get sn) (get sm) sch - where - get = toInteger . fromSing +rangedSchema sch = + Range <$> untypedRangedSchema (toInteger (demote @n)) (toInteger (demote @m)) sch untypedRangedSchema :: forall d v w a b. @@ -181,7 +179,7 @@ instance S.HasSchema d S.Schema => HasRangedSchemaDocModifier d Word32 where ran instance S.HasSchema d S.Schema => HasRangedSchemaDocModifier d Word64 where rangedSchemaDocModifier _ = numRangedSchemaDocModifier instance (Within a n m, ToSchema a, HasRangedSchemaDocModifier NamedSwaggerDoc a) => ToSchema (Range n m a) where - schema = fromRange .= rangedSchema sing sing schema + schema = fromRange .= rangedSchema schema instance (Within a n m, Cql a) => Cql (Range n m a) where ctype = retag (ctype :: Tagged a ColumnType) diff --git a/libs/wire-api/src/Wire/API/Conversation.hs b/libs/wire-api/src/Wire/API/Conversation.hs index 896ac033bf2..36f6cc93dcc 100644 --- a/libs/wire-api/src/Wire/API/Conversation.hs +++ b/libs/wire-api/src/Wire/API/Conversation.hs @@ -100,7 +100,6 @@ import Data.Qualified (Qualified (qUnqualified), deprecatedSchema) import Data.Range (Range, fromRange, rangedSchema) import Data.Schema import qualified Data.Set as Set -import Data.Singletons (sing) import Data.String.Conversions (cs) import qualified Data.Swagger as S import qualified Data.Swagger.Build.Api as Doc @@ -366,7 +365,7 @@ instance ToSchema ListConversations where "ListConversations" (description ?~ "A request to list some of a user's conversations, including remote ones. Maximum 1000 qualified conversation IDs") $ ListConversations - <$> (fromRange . lcQualifiedIds) .= field "qualified_ids" (rangedSchema sing sing (array schema)) + <$> (fromRange . lcQualifiedIds) .= field "qualified_ids" (rangedSchema (array schema)) data ConversationsResponse = ConversationsResponse { crFound :: [Conversation], diff --git a/libs/wire-api/src/Wire/API/Team.hs b/libs/wire-api/src/Wire/API/Team.hs index d639b314d06..fda2311e6e1 100644 --- a/libs/wire-api/src/Wire/API/Team.hs +++ b/libs/wire-api/src/Wire/API/Team.hs @@ -76,7 +76,6 @@ import Data.Id (TeamId, UserId) import Data.Misc (PlainTextPassword (..)) import Data.Range import Data.Schema -import Data.Singletons (sing) import qualified Data.Swagger as S import qualified Data.Swagger.Build.Api as Doc import Imports @@ -215,7 +214,7 @@ instance ToSchema NonBindingNewTeam where unwrap (NonBindingNewTeam nt) = nt sch :: ValueSchema SwaggerDoc (Range 1 127 [TeamMember]) - sch = fromRange .= rangedSchema sing sing (array schema) + sch = fromRange .= rangedSchema (array schema) modelNewNonBindingTeam :: Doc.Model modelNewNonBindingTeam = Doc.defineModel "newNonBindingTeam" $ do From da3800a5ec125337f4647a858db10d3143691ce8 Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Tue, 4 Jan 2022 11:07:48 +0100 Subject: [PATCH 46/74] SQSERVICES 1011 servantify gally team api (PUT /teams/:tid) (#2010) * ToSchema instances * change API impl * update changelog * error description for the route --- changelog.d/5-internal/pr-2008 | 2 +- .../wire-api/src/Wire/API/ErrorDescription.hs | 6 ++++++ .../src/Wire/API/Routes/Public/Galley.hs | 18 +++++++++++++++++- libs/wire-api/src/Wire/API/Team/Member.hs | 2 +- services/galley/src/Galley/API/Public.hs | 18 ++---------------- services/galley/src/Galley/API/Teams.hs | 19 +------------------ 6 files changed, 28 insertions(+), 37 deletions(-) diff --git a/changelog.d/5-internal/pr-2008 b/changelog.d/5-internal/pr-2008 index a5c36a513b0..a378dfd9c78 100644 --- a/changelog.d/5-internal/pr-2008 +++ b/changelog.d/5-internal/pr-2008 @@ -1 +1 @@ -Servantify Galley Teams API. (#2008) +Servantify Galley Teams API. (#2008, #2010) diff --git a/libs/wire-api/src/Wire/API/ErrorDescription.hs b/libs/wire-api/src/Wire/API/ErrorDescription.hs index 30f7ebf71ba..f6312d19f98 100644 --- a/libs/wire-api/src/Wire/API/ErrorDescription.hs +++ b/libs/wire-api/src/Wire/API/ErrorDescription.hs @@ -20,6 +20,7 @@ import Servant.API.Status (KnownStatus, statusVal) import Servant.Client.Core import Servant.Swagger.Internal import Wire.API.Routes.MultiVerb +import Wire.API.Team.Permission -- This can be added to an endpoint to document a possible failure -- case outside its return type (usually through an exception). @@ -234,6 +235,11 @@ noIdentity n = ErrorDescription (Text.pack (symbolVal (Proxy @desc)) <> " (code type OperationDenied = ErrorDescription 403 "operation-denied" "Insufficient permissions" +-- FUTUREWORK(leif): We need this to document possible (operation denied) errors in the servant routes. +-- Be aware that this is redundant and should be replaced by a more type safe solution in the future. +type family OperationDeniedError (a :: Perm) :: * where + OperationDeniedError 'SetTeamData = ErrorDescription 403 "operation-denied" "Insufficient permissions (missing SetTeamData)" + operationDeniedSpecialized :: String -> OperationDenied operationDeniedSpecialized p = ErrorDescription $ diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs b/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs index fe3ec2ec6e3..22ed3a43c27 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Galley.hs @@ -47,6 +47,7 @@ import Wire.API.ServantProto (Proto, RawProto) import Wire.API.Team import Wire.API.Team.Conversation import Wire.API.Team.Feature +import Wire.API.Team.Permission (Perm (..)) instance AsHeaders '[ConvId] Conversation Conversation where toHeaders c = (I (qUnqualified (cnvQualifiedId c)) :* Nil, c) @@ -725,7 +726,22 @@ data Api routes = Api TeamId (RespondEmpty 201 "Team ID as `Location` header value") ] - TeamId + TeamId, + updateTeam :: + routes + :- Summary "Update team properties" + :> ZUser + :> ZConn + :> CanThrow NotATeamMember + :> CanThrow (OperationDeniedError 'SetTeamData) + :> "teams" + :> Capture "tid" TeamId + :> ReqBody '[JSON] TeamUpdateData + :> MultiVerb + 'PUT + '[JSON] + '[RespondEmpty 200 "Team updated"] + () } deriving (Generic) diff --git a/libs/wire-api/src/Wire/API/Team/Member.hs b/libs/wire-api/src/Wire/API/Team/Member.hs index 9a7cfda570e..600978c6b12 100644 --- a/libs/wire-api/src/Wire/API/Team/Member.hs +++ b/libs/wire-api/src/Wire/API/Team/Member.hs @@ -336,7 +336,7 @@ invitedSchema' :: ObjectSchema SwaggerDoc (Maybe (UserId, UTCTimeMillis)) invitedSchema' = withParser invitedSchema $ \(invby, invat) -> case (invby, invat) of (Just b, Just a) -> pure $ Just (b, a) - (Nothing, Nothing) -> pure $ Nothing + (Nothing, Nothing) -> pure Nothing _ -> fail "created_by, created_at" instance ToSchema NewTeamMember where diff --git a/services/galley/src/Galley/API/Public.hs b/services/galley/src/Galley/API/Public.hs index 42db3104f91..4ee1c27b97d 100644 --- a/services/galley/src/Galley/API/Public.hs +++ b/services/galley/src/Galley/API/Public.hs @@ -188,28 +188,14 @@ servantSitemap = GalleyAPI.featureConfigConferenceCallingGet = Features.getFeatureConfig @'Public.WithoutLockStatus @'Public.TeamFeatureConferenceCalling Features.getConferenceCallingInternal, GalleyAPI.featureConfigSelfDeletingMessagesGet = Features.getFeatureConfig @'Public.WithLockStatus @'Public.TeamFeatureSelfDeletingMessages Features.getSelfDeletingMessagesInternal, GalleyAPI.featureConfigGuestLinksGet = Features.getFeatureConfig @'Public.WithLockStatus @'Public.TeamFeatureGuestLinks Features.getGuestLinkInternal, - GalleyAPI.createNonBindingTeam = Teams.createNonBindingTeamH + GalleyAPI.createNonBindingTeam = Teams.createNonBindingTeamH, + GalleyAPI.updateTeam = Teams.updateTeamH } sitemap :: Routes ApiBuilder (Sem GalleyEffects) () sitemap = do -- Team API ----------------------------------------------------------- - put "/teams/:tid" (continue Teams.updateTeamH) $ - zauthUserId - .&. zauthConnId - .&. capture "tid" - .&. jsonRequest @Public.TeamUpdateData - .&. accept "application" "json" - document "PUT" "updateTeam" $ do - summary "Update team properties" - parameter Path "tid" bytes' $ - description "Team ID" - body (ref Public.modelUpdateData) $ - description "JSON body" - errorResponse (Error.errorDescriptionTypeToWai @Error.NotATeamMember) - errorResponse (Error.errorDescriptionToWai (Error.operationDenied Public.SetTeamData)) - get "/teams" (continue Teams.getManyTeamsH) $ zauthUserId .&. opt (query "ids" ||| query "start") diff --git a/services/galley/src/Galley/API/Teams.hs b/services/galley/src/Galley/API/Teams.hs index 34e27ae7fc1..5883ebdb88c 100644 --- a/services/galley/src/Galley/API/Teams.hs +++ b/services/galley/src/Galley/API/Teams.hs @@ -326,23 +326,6 @@ updateTeamStatus tid (TeamStatusUpdate newStatus cur) = do (_, _) -> throw InvalidTeamStatusUpdate updateTeamH :: - Members - '[ Error ActionError, - Error NotATeamMember, - GundeckAccess, - Input UTCTime, - TeamStore, - WaiRoutes - ] - r => - UserId ::: ConnId ::: TeamId ::: JsonRequest Public.TeamUpdateData ::: JSON -> - Sem r Response -updateTeamH (zusr ::: zcon ::: tid ::: req ::: _) = do - updateData <- fromJsonBody req - updateTeam zusr zcon tid updateData - pure empty - -updateTeam :: Members '[ Error ActionError, Error NotATeamMember, @@ -356,7 +339,7 @@ updateTeam :: TeamId -> Public.TeamUpdateData -> Sem r () -updateTeam zusr zcon tid updateData = do +updateTeamH zusr zcon tid updateData = do zusrMembership <- E.getTeamMember tid zusr -- let zothers = map (view userId) membs -- Log.debug $ From 81cb9405ee58c79997dfbc830b581c3c353314d4 Mon Sep 17 00:00:00 2001 From: Leif Battermann Date: Tue, 4 Jan 2022 16:46:03 +0100 Subject: [PATCH 47/74] Return specific error when a domain is blocked (#2023) --- changelog.d/5-internal/pr-2023 | 1 + services/brig/src/Brig/API/Public.hs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/5-internal/pr-2023 diff --git a/changelog.d/5-internal/pr-2023 b/changelog.d/5-internal/pr-2023 new file mode 100644 index 00000000000..826977a5870 --- /dev/null +++ b/changelog.d/5-internal/pr-2023 @@ -0,0 +1 @@ +When sending an activation code, the blocked domains are checked before the whitelist. This only affects the wire SaaS staging environment (there is no whitelist configuration in prod, and blocked domains are not applicable to on-prem installations). diff --git a/services/brig/src/Brig/API/Public.hs b/services/brig/src/Brig/API/Public.hs index 7bf438b4186..a4685355fa4 100644 --- a/services/brig/src/Brig/API/Public.hs +++ b/services/brig/src/Brig/API/Public.hs @@ -999,8 +999,8 @@ sendActivationCodeH req = -- docs/reference/user/registration.md {#RefRegistration} sendActivationCode :: Public.SendActivationCode -> Handler () sendActivationCode Public.SendActivationCode {..} = do + either customerExtensionCheckBlockedDomains (const $ pure ()) saUserKey checkWhitelist saUserKey - either customerExtensionCheckBlockedDomains (\_ -> pure ()) saUserKey API.sendActivationCode saUserKey saLocale saCall !>> sendActCodeError -- | If the user presents an email address from a blocked domain, throw an error. From 7cf3b24d89a7a4b64bd7ad8cbfb498a0443df6de Mon Sep 17 00:00:00 2001 From: jschaul Date: Tue, 4 Jan 2022 20:38:34 +0100 Subject: [PATCH 48/74] Sftd helm chart: provide a list of all sftd urls over http(s) using sftd_disco sidecar process. (#2019) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Related to https://wearezeta.atlassian.net/browse/FS-266. Implements querying the list of all SFT servers from the sftd's sidecar nginx pod. This is a different take on the above issue and if this list provided by sft servers is made use of; then that makes some of the work introduced in #2012 #2014 #2015 obsolete (sorry 😟 ). From brig, using the url configured in `setSftStaticUrl`, and calling `/sft_servers_all.json` on that URL, this PR provides a list of all SFT servers available. The list is computed on the SFTD chart itself (using a sftd_disco sidecar container). This has the advantage of allowing brig to be hosted on a separate kubernetes cluster to sftd, and moves the business logic of knowing sftd servers to sftd itself. From a brig pod on a test cluster: ``` # cat /etc/wire/brig/conf/brig.yaml | grep sft setSftStaticUrl: https://sftd.a.adhoc-testing.wire.link:443 / # curl -ks https://sftd.a.adhoc-testing.wire.link:443/sft_servers_all.json { "sft_servers_all": [ "https://sftd.a.adhoc-testing.wire.link/sfts/wire-server-a-sftd-0" ] } ``` Co-authored-by: Marko Dimjašević --- .dockerignore | 1 + Makefile | 2 +- changelog.d/6-federation/sftd_disco | 1 + .../sftd/templates/configmap-join-call.yaml | 4 + .../sftd/templates/deployment-join-call.yaml | 16 ++++ charts/sftd/templates/ingress.yaml | 4 + tools/nginz_disco/Dockerfile | 2 +- tools/nginz_disco/README.md | 4 +- tools/sftd_disco/Dockerfile | 7 ++ tools/sftd_disco/Makefile | 6 ++ tools/sftd_disco/README.md | 5 ++ tools/sftd_disco/sftd_disco.sh | 77 +++++++++++++++++++ 12 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 changelog.d/6-federation/sftd_disco create mode 100644 tools/sftd_disco/Dockerfile create mode 100644 tools/sftd_disco/Makefile create mode 100644 tools/sftd_disco/README.md create mode 100755 tools/sftd_disco/sftd_disco.sh diff --git a/.dockerignore b/.dockerignore index e3606f86bea..f23c963a415 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ **/.* **/dist +**/dist-buildah **/target **/*.aci **/*.tgz diff --git a/Makefile b/Makefile index 1401d4d4c49..6d54583ecf8 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ DOCKER_TAG ?= $(USER) # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything) HELM_SEMVER ?= 0.0.42 # The list of helm charts needed for integration tests on kubernetes -CHARTS_INTEGRATION := wire-server databases-ephemeral fake-aws nginx-ingress-controller nginx-ingress-services wire-server-metrics fluent-bit kibana +CHARTS_INTEGRATION := wire-server databases-ephemeral fake-aws nginx-ingress-controller nginx-ingress-services wire-server-metrics fluent-bit kibana sftd # The list of helm charts to publish on S3 # FUTUREWORK: after we "inline local subcharts", # (e.g. move charts/brig to charts/wire-server/brig) diff --git a/changelog.d/6-federation/sftd_disco b/changelog.d/6-federation/sftd_disco new file mode 100644 index 00000000000..f7754d070a9 --- /dev/null +++ b/changelog.d/6-federation/sftd_disco @@ -0,0 +1 @@ +SFTD chart: provide a /sft_servers_all.json url that can be used by brig to populate /calls/config/v2 diff --git a/charts/sftd/templates/configmap-join-call.yaml b/charts/sftd/templates/configmap-join-call.yaml index fd4ec86717a..523d741a2b5 100644 --- a/charts/sftd/templates/configmap-join-call.yaml +++ b/charts/sftd/templates/configmap-join-call.yaml @@ -17,4 +17,8 @@ data: proxy_pass http://$1.{{ include "sftd.fullname" . }}.${POD_NAMESPACE}.svc.cluster.local:8585/$2; } + location ~ ^/sft_servers_all.json$ { + root /etc/wire/sftd-disco/; + } + } diff --git a/charts/sftd/templates/deployment-join-call.yaml b/charts/sftd/templates/deployment-join-call.yaml index 0c247f7e6f7..3574bf04815 100644 --- a/charts/sftd/templates/deployment-join-call.yaml +++ b/charts/sftd/templates/deployment-join-call.yaml @@ -26,7 +26,20 @@ spec: - name: nginx-config configMap: name: {{ include "sftd.fullname" . }}-join-call + - name: sftd-disco + emptyDir: {} containers: + - name: sftd-disco + image: quay.io/wire/sftd_disco:wip-2 # TODO configure + version + volumeMounts: + - name: sftd-disco + mountPath: /etc/wire/sftd-disco + readOnly: false + command: + - "/bin/sh" + - "-c" + - | + /usr/bin/sftd_disco.sh _sft._tcp.{{ include "sftd.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local - name: nginx securityContext: {{- toYaml .Values.securityContext | nindent 12 }} @@ -50,6 +63,9 @@ spec: - mountPath: /etc/nginx/conf.d/default.conf.template name: nginx-config subPath: default.conf.template + - name: sftd-disco + mountPath: /etc/wire/sftd-disco + readOnly: true env: - name: POD_NAMESPACE valueFrom: diff --git a/charts/sftd/templates/ingress.yaml b/charts/sftd/templates/ingress.yaml index a14c53981a6..9bf7958fa0b 100644 --- a/charts/sftd/templates/ingress.yaml +++ b/charts/sftd/templates/ingress.yaml @@ -25,3 +25,7 @@ spec: backend: serviceName: "{{ include "sftd.fullname" . }}-join-call" servicePort: http + - path: /sft_servers_all.json + backend: + serviceName: "{{ include "sftd.fullname" . }}-join-call" + servicePort: http diff --git a/tools/nginz_disco/Dockerfile b/tools/nginz_disco/Dockerfile index 9d15426f594..67c5a5b1ee4 100644 --- a/tools/nginz_disco/Dockerfile +++ b/tools/nginz_disco/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.12.3 +FROM alpine:3.15.0 RUN apk add --no-cache curl bash openssl bind-tools diff --git a/tools/nginz_disco/README.md b/tools/nginz_disco/README.md index 5571c7a1a38..33fa7f108db 100644 --- a/tools/nginz_disco/README.md +++ b/tools/nginz_disco/README.md @@ -1,6 +1,6 @@ # nginz-disco -Due to nginx not supporting DNS names for its list of upstream servers (unless you pay extra), the nginz-disco container is a simple bash script to do DNS lookups and write the resulting IPs to a file. Nginz reloads on changes to this file. +Due to nginx not supporting DNS names for its list of upstream servers (unless you pay extra), the nginz-disco container is a simple bash script to do DNS lookups and write the resulting IPs to a file. Nginz reloads on changes to this file. -This is useful as a sidecar container to nginz in kubernetes. See also [wire-server-deploy/nginz](https://github.com/wireapp/wire-server-deploy/charts/nginz/) +# [2022-01-18] + +## Release notes + +* This release introduces a mandatory `federationDomain` configuration setting to cargohold. Please update your `values/wire-server/values.yaml` to set `cargohold.settings.federationDomain` to the same value as the corresponding option in galley (and brig). (#1990) +* The brig server config option `setDefaultLocale` has been replaced by `setDefaultUserLocale` and `setDefaultTemplateLocale` (see docs/reference/config-options.md for details) (#2028) +* From this release onwards, the images for haskell components (brig, galley, + cargohold, etc.) will be using Ubuntu 20.04 as the base. The images are about + 30-35 MB larger than the previous alpine based images. (#1852) +* Wire cloud operators: Make sure [#35](https://github.com/wireapp/ansible-sft/pull/35) is applied to all SFT servers before deploying. (#2030) + +## API changes + +* The deprecated endpoint `GET /teams` now ignores query parameters `ids`, `start` (#2027) +* Add qualified v4 endpoints for downloading and deleting assets. The upload API is still on the same path, but the asset object it returns now contains a `domain` field. Note that federated behaviour is still not implemented. (#2002) +* Enable downloading assets from a remote (federated) cargohold instance via the v4 API. + The content of remote assets is returned as stream with content type + `application/octet-stream`. + Please refer to the Swagger API documentation for more details. (#2004) +* Remove resumable upload API (#1998) + +## Features + +* Allow configuring setDefaultLocale in brig using helm chart (#2025) +* If the guest links team feature is disabled guest links will be revoked. (#1976) +* Revoke guest links if feature is disabled. If the guest links team feature is disabled `get /conversations/join`, `post /conversations/:cnv/code`, and `get /conversations/:cnv/code` will return an error. (#1980) +* Specialize `setDefaultLocale` to distinguish between default user locale and default template locale if the user's locale is n/a. (#2028) + +## Bug fixes and other updates + +* Fix an issue with remote asset streaming (#2037, #2038) + +## Documentation + +* Annotate a first batch of integration and unit tests to map them to externally-facing documentation (#1869) +* Add the description to several test cases (#1991) +* Improve documentation for stern tool and helm chart (#2032) + +## Internal changes + +* Replace servant-generic in Galley with a custom `Named` combinator (#2022) +* The Swagger documentation module is not regenerated anymore if its content is unchanged (#2018) +* cabal-run-integration.sh - remove Makefile indirection (#2044) +* Fix test runner for global cabal make target (#1987) +* The `cabal-install-artefacts.sh` script now creates the `dist` directory if it does not exist (#2007) +* Set `purge: false` in fake-s3 chart (#1981) +* Add missing backendTwo.carghold in integration.yaml (#2039) +* Use GHC 8.10.7 and stack 2.7.3 for builds (#1852) +* Fix non-controversial HLint issues in federator to improve code quality (#2011) +* Added laws for DefaultSsoCode, Now, IdP and ScimExternalIdStore (#1940) +* Moved specifications for Spar effects out of the test suite and into the library (#2005) +* Tag integration tests for security audit. (#2000) +* Upgrade nixpkgs pin used to provision developement dependencies (#1852) +* Servantify Galley Teams API. (#2008, #2010, #2027) +* When sending an activation code, the blocked domains are checked before the whitelist. This only affects the wire SaaS staging environment (there is no whitelist configuration in prod, and blocked domains are not applicable to on-prem installations). (#2023) +* Add a helm chart that deploys [restund](https://docs.wire.com/understand/restund.html) (#2003) +* Publish restund helm chart (#2036) +* Improve optional field API in schema-profunctor (#1988) +* Migrate the public API of Cannon to Servant. (There is an internal API that is not yet migrated.) (#2024) +* sftd chart: Add multiSFT option, remove additionalArgs option (#1992) +* sftd chart: Fix quoted args for multiSFT option (#1999) +* `rangedSchema` does not need to be passed singletons explicitly anymore (#2017) +* Split cannon benchmarks and tests (#1986) +* Tag integration tests for certification. (#1985) +* Tag integration tests for certification. (#2001) +* New internal endpoint to configure the guest links team feature. (#1993) + +## Federation changes + +* Make federator capable of streaming responses (#1966) +* Use `Named` routes for the federation API (#2033) +* Fix Brig's configmap for SFT lookups (#2015) +* SFTD chart: provide a /sft_servers_all.json url that can be used by brig to populate /calls/config/v2 (#2019) +* Allow making HTTP-only requests to SFTs via an IPv4 address (#2026) +* Replace IPv4-HTTP-only Approach to SFT Server Lookup with /sft_servers_all.json (#2030) +* Extend GET /calls/config/v2 to include all SFT servers in federation (#2012) +* Improve Brig's configuration for SFTs and fix a call to SFT servers (#2014) + # [2021-12-10] ## Release notes diff --git a/changelog.d/0-release-notes/cargohold-domain b/changelog.d/0-release-notes/cargohold-domain deleted file mode 100644 index 33e8403cf87..00000000000 --- a/changelog.d/0-release-notes/cargohold-domain +++ /dev/null @@ -1 +0,0 @@ -This release introduces a mandatory `federationDomain` configuration setting to cargohold. Please update your `values/wire-server/values.yaml` to set `cargohold.settings.federationDomain` to the same value as the corresponding option in galley (and brig). diff --git a/changelog.d/0-release-notes/pr-2028 b/changelog.d/0-release-notes/pr-2028 deleted file mode 100644 index cc4cf7d5a47..00000000000 --- a/changelog.d/0-release-notes/pr-2028 +++ /dev/null @@ -1 +0,0 @@ -The brig server config option `setDefaultLocale` has been replaced by `setDefaultUserLocale` and `setDefaultTemplateLocale` (see docs/reference/config-options.md for details) diff --git a/changelog.d/0-release-notes/ubuntu-images b/changelog.d/0-release-notes/ubuntu-images deleted file mode 100644 index 16c9c4dd5ba..00000000000 --- a/changelog.d/0-release-notes/ubuntu-images +++ /dev/null @@ -1,3 +0,0 @@ -From this release onwards, the images for haskell components (brig, galley, -cargohold, etc.) will be using Ubuntu 20.04 as the base. The images are about -30-35 MB larger than the previous alpine based images. \ No newline at end of file diff --git a/changelog.d/0-release-notes/update-sft-urls b/changelog.d/0-release-notes/update-sft-urls deleted file mode 100644 index baafc1bae80..00000000000 --- a/changelog.d/0-release-notes/update-sft-urls +++ /dev/null @@ -1 +0,0 @@ -Wire cloud operators: Make sure [#35](https://github.com/wireapp/ansible-sft/pull/35) is applied to all SFT servers before deploying. (#2030) diff --git a/changelog.d/1-api-changes/pr-2027 b/changelog.d/1-api-changes/pr-2027 deleted file mode 100644 index 4c7f70a85f5..00000000000 --- a/changelog.d/1-api-changes/pr-2027 +++ /dev/null @@ -1 +0,0 @@ -The deprecated endpoint `GET /teams` now ignores query parameters `ids`, `start` diff --git a/changelog.d/1-api-changes/qualified-assets b/changelog.d/1-api-changes/qualified-assets deleted file mode 100644 index 62e988dde3e..00000000000 --- a/changelog.d/1-api-changes/qualified-assets +++ /dev/null @@ -1 +0,0 @@ -Add qualified v4 endpoints for downloading and deleting assets. The upload API is still on the same path, but the asset object it returns now contains a `domain` field. Note that federated behaviour is still not implemented. diff --git a/changelog.d/1-api-changes/remote-assets b/changelog.d/1-api-changes/remote-assets deleted file mode 100644 index a80a382c27a..00000000000 --- a/changelog.d/1-api-changes/remote-assets +++ /dev/null @@ -1,4 +0,0 @@ -Enable downloading assets from a remote (federated) cargohold instance via the v4 API. -The content of remote assets is returned as stream with content type -`application/octet-stream`. -Please refer to the Swagger API documentation for more details. diff --git a/changelog.d/1-api-changes/remove-resumable-uploads b/changelog.d/1-api-changes/remove-resumable-uploads deleted file mode 100644 index a73bc51b5bc..00000000000 --- a/changelog.d/1-api-changes/remove-resumable-uploads +++ /dev/null @@ -1 +0,0 @@ -Remove resumable upload API diff --git a/changelog.d/2-features/helm-brig-setDefaultLocale b/changelog.d/2-features/helm-brig-setDefaultLocale deleted file mode 100644 index 7fcdf370267..00000000000 --- a/changelog.d/2-features/helm-brig-setDefaultLocale +++ /dev/null @@ -1 +0,0 @@ -Allow configuring setDefaultLocale in brig using helm chart \ No newline at end of file diff --git a/changelog.d/2-features/pr-1976 b/changelog.d/2-features/pr-1976 deleted file mode 100644 index e67dd816a08..00000000000 --- a/changelog.d/2-features/pr-1976 +++ /dev/null @@ -1 +0,0 @@ -If the guest links team feature is disabled guest links will be revoked. diff --git a/changelog.d/2-features/pr-1980 b/changelog.d/2-features/pr-1980 deleted file mode 100644 index 1eafba32ca6..00000000000 --- a/changelog.d/2-features/pr-1980 +++ /dev/null @@ -1 +0,0 @@ -Revoke guest links if feature is disabled. If the guest links team feature is disabled `get /conversations/join`, `post /conversations/:cnv/code`, and `get /conversations/:cnv/code` will return an error. diff --git a/changelog.d/2-features/pr-2028 b/changelog.d/2-features/pr-2028 deleted file mode 100644 index 03d788286ee..00000000000 --- a/changelog.d/2-features/pr-2028 +++ /dev/null @@ -1 +0,0 @@ -Specialize `setDefaultLocale` to distinguish between default user locale and default template locale if the user's locale is n/a. diff --git a/changelog.d/3-bug-fixes/fix-cargohold-streaming b/changelog.d/3-bug-fixes/fix-cargohold-streaming deleted file mode 100644 index 38d8b2fe84d..00000000000 --- a/changelog.d/3-bug-fixes/fix-cargohold-streaming +++ /dev/null @@ -1 +0,0 @@ -Fix an issue with remote asset streaming (#2037, #2038) diff --git a/changelog.d/4-docs/pr-1869 b/changelog.d/4-docs/pr-1869 deleted file mode 100644 index 5b3e780a9e3..00000000000 --- a/changelog.d/4-docs/pr-1869 +++ /dev/null @@ -1 +0,0 @@ -Annotate a first batch of integration and unit tests to map them to externally-facing documentation diff --git a/changelog.d/4-docs/sqservices-1127 b/changelog.d/4-docs/sqservices-1127 deleted file mode 100644 index bb5629509ae..00000000000 --- a/changelog.d/4-docs/sqservices-1127 +++ /dev/null @@ -1 +0,0 @@ -Add the description to several test cases diff --git a/changelog.d/4-docs/stern-docs b/changelog.d/4-docs/stern-docs deleted file mode 100644 index 6b94b16d142..00000000000 --- a/changelog.d/4-docs/stern-docs +++ /dev/null @@ -1 +0,0 @@ -Improve documentation for stern tool and helm chart \ No newline at end of file diff --git a/changelog.d/5-internal/avoid-servant-generics b/changelog.d/5-internal/avoid-servant-generics deleted file mode 100644 index 0ab6f411a3c..00000000000 --- a/changelog.d/5-internal/avoid-servant-generics +++ /dev/null @@ -1 +0,0 @@ -Replace servant-generic in Galley with a custom `Named` combinator diff --git a/changelog.d/5-internal/brig-faster-doc b/changelog.d/5-internal/brig-faster-doc deleted file mode 100644 index edd39960037..00000000000 --- a/changelog.d/5-internal/brig-faster-doc +++ /dev/null @@ -1 +0,0 @@ -The Swagger documentation module is not regenerated anymore if its content is unchanged diff --git a/changelog.d/5-internal/cabal-integration-test-improvement b/changelog.d/5-internal/cabal-integration-test-improvement deleted file mode 100644 index e46afe7e4a2..00000000000 --- a/changelog.d/5-internal/cabal-integration-test-improvement +++ /dev/null @@ -1 +0,0 @@ -cabal-run-integration.sh - remove Makefile indirection diff --git a/changelog.d/5-internal/cabal-make-c-test-all b/changelog.d/5-internal/cabal-make-c-test-all deleted file mode 100644 index 305c0c8d6ce..00000000000 --- a/changelog.d/5-internal/cabal-make-c-test-all +++ /dev/null @@ -1 +0,0 @@ -Fix test runner for global cabal make target diff --git a/changelog.d/5-internal/fix-cabal-install b/changelog.d/5-internal/fix-cabal-install deleted file mode 100644 index f09f1ea0b38..00000000000 --- a/changelog.d/5-internal/fix-cabal-install +++ /dev/null @@ -1 +0,0 @@ -The `cabal-install-artefacts.sh` script now creates the `dist` directory if it does not exist diff --git a/changelog.d/5-internal/fix-cargohold-flake b/changelog.d/5-internal/fix-cargohold-flake deleted file mode 100644 index 3281ab93589..00000000000 --- a/changelog.d/5-internal/fix-cargohold-flake +++ /dev/null @@ -1 +0,0 @@ -Set `purge: false` in fake-s3 chart diff --git a/changelog.d/5-internal/fix-integration-yaml b/changelog.d/5-internal/fix-integration-yaml deleted file mode 100644 index 789f497c98f..00000000000 --- a/changelog.d/5-internal/fix-integration-yaml +++ /dev/null @@ -1 +0,0 @@ -Add missing backendTwo.carghold in integration.yaml diff --git a/changelog.d/5-internal/ghc-8107 b/changelog.d/5-internal/ghc-8107 deleted file mode 100644 index b1581527d97..00000000000 --- a/changelog.d/5-internal/ghc-8107 +++ /dev/null @@ -1 +0,0 @@ -Use GHC 8.10.7 and stack 2.7.3 for builds \ No newline at end of file diff --git a/changelog.d/5-internal/hlint-federator b/changelog.d/5-internal/hlint-federator deleted file mode 100644 index 96c1fac4dc7..00000000000 --- a/changelog.d/5-internal/hlint-federator +++ /dev/null @@ -1 +0,0 @@ -Fix non-controversial HLint issues in federator to improve code quality diff --git a/changelog.d/5-internal/in-mem-specs b/changelog.d/5-internal/in-mem-specs deleted file mode 100644 index 57fd8d6b57a..00000000000 --- a/changelog.d/5-internal/in-mem-specs +++ /dev/null @@ -1 +0,0 @@ -Added laws for DefaultSsoCode, Now, IdP and ScimExternalIdStore diff --git a/changelog.d/5-internal/integration-spar-polysemy b/changelog.d/5-internal/integration-spar-polysemy deleted file mode 100644 index f12d69de1cf..00000000000 --- a/changelog.d/5-internal/integration-spar-polysemy +++ /dev/null @@ -1 +0,0 @@ -Moved specifications for Spar effects out of the test suite and into the library diff --git a/changelog.d/5-internal/map-federation-tests b/changelog.d/5-internal/map-federation-tests deleted file mode 100644 index b53c1449dde..00000000000 --- a/changelog.d/5-internal/map-federation-tests +++ /dev/null @@ -1 +0,0 @@ -Tag integration tests for security audit. \ No newline at end of file diff --git a/changelog.d/5-internal/nixpkgs b/changelog.d/5-internal/nixpkgs deleted file mode 100644 index 750845b483e..00000000000 --- a/changelog.d/5-internal/nixpkgs +++ /dev/null @@ -1 +0,0 @@ -Upgrade nixpkgs pin used to provision developement dependencies \ No newline at end of file diff --git a/changelog.d/5-internal/pr-2008 b/changelog.d/5-internal/pr-2008 deleted file mode 100644 index 49afa225615..00000000000 --- a/changelog.d/5-internal/pr-2008 +++ /dev/null @@ -1 +0,0 @@ -Servantify Galley Teams API. (#2008, #2010, #2027) diff --git a/changelog.d/5-internal/pr-2023 b/changelog.d/5-internal/pr-2023 deleted file mode 100644 index 826977a5870..00000000000 --- a/changelog.d/5-internal/pr-2023 +++ /dev/null @@ -1 +0,0 @@ -When sending an activation code, the blocked domains are checked before the whitelist. This only affects the wire SaaS staging environment (there is no whitelist configuration in prod, and blocked domains are not applicable to on-prem installations). diff --git a/changelog.d/5-internal/restund-chart b/changelog.d/5-internal/restund-chart deleted file mode 100644 index b0addd486b1..00000000000 --- a/changelog.d/5-internal/restund-chart +++ /dev/null @@ -1 +0,0 @@ -Add a helm chart that deploys [restund](https://docs.wire.com/understand/restund.html) diff --git a/changelog.d/5-internal/restund-chart-followup b/changelog.d/5-internal/restund-chart-followup deleted file mode 100644 index d702a17ab53..00000000000 --- a/changelog.d/5-internal/restund-chart-followup +++ /dev/null @@ -1 +0,0 @@ -Publish restund helm chart diff --git a/changelog.d/5-internal/schema-profunctor-optional b/changelog.d/5-internal/schema-profunctor-optional deleted file mode 100644 index 4e66ad936cc..00000000000 --- a/changelog.d/5-internal/schema-profunctor-optional +++ /dev/null @@ -1 +0,0 @@ -Improve optional field API in schema-profunctor diff --git a/changelog.d/5-internal/servantify-cannon-public-api b/changelog.d/5-internal/servantify-cannon-public-api deleted file mode 100644 index 1a60973e2b6..00000000000 --- a/changelog.d/5-internal/servantify-cannon-public-api +++ /dev/null @@ -1 +0,0 @@ -Migrate the public API of Cannon to Servant. (There is an internal API that is not yet migrated.) diff --git a/changelog.d/5-internal/sftd-multi-sft b/changelog.d/5-internal/sftd-multi-sft deleted file mode 100644 index a0324fe7496..00000000000 --- a/changelog.d/5-internal/sftd-multi-sft +++ /dev/null @@ -1 +0,0 @@ -sftd chart: Add multiSFT option, remove additionalArgs option diff --git a/changelog.d/5-internal/sftd-multi-sft-fixup b/changelog.d/5-internal/sftd-multi-sft-fixup deleted file mode 100644 index 78b95439e69..00000000000 --- a/changelog.d/5-internal/sftd-multi-sft-fixup +++ /dev/null @@ -1 +0,0 @@ -sftd chart: Fix quoted args for multiSFT option diff --git a/changelog.d/5-internal/simplify-ranged-schema b/changelog.d/5-internal/simplify-ranged-schema deleted file mode 100644 index c6a3e875811..00000000000 --- a/changelog.d/5-internal/simplify-ranged-schema +++ /dev/null @@ -1 +0,0 @@ -`rangedSchema` does not need to be passed singletons explicitly anymore diff --git a/changelog.d/5-internal/split-cannon-benchmarks b/changelog.d/5-internal/split-cannon-benchmarks deleted file mode 100644 index 2122a84d6b9..00000000000 --- a/changelog.d/5-internal/split-cannon-benchmarks +++ /dev/null @@ -1 +0,0 @@ -Split cannon benchmarks and tests diff --git a/changelog.d/5-internal/sqservices-1118 b/changelog.d/5-internal/sqservices-1118 deleted file mode 100644 index 0d240283035..00000000000 --- a/changelog.d/5-internal/sqservices-1118 +++ /dev/null @@ -1 +0,0 @@ -Tag integration tests for certification. diff --git a/changelog.d/5-internal/sqservices-1118-2 b/changelog.d/5-internal/sqservices-1118-2 deleted file mode 100644 index 0d240283035..00000000000 --- a/changelog.d/5-internal/sqservices-1118-2 +++ /dev/null @@ -1 +0,0 @@ -Tag integration tests for certification. diff --git a/changelog.d/5-internal/sqservices-1169 b/changelog.d/5-internal/sqservices-1169 deleted file mode 100644 index cc2869e0494..00000000000 --- a/changelog.d/5-internal/sqservices-1169 +++ /dev/null @@ -1 +0,0 @@ -New internal endpoint to configure the guest links team feature. diff --git a/changelog.d/6-federation/federator-streaming b/changelog.d/6-federation/federator-streaming deleted file mode 100644 index 7901572b90b..00000000000 --- a/changelog.d/6-federation/federator-streaming +++ /dev/null @@ -1 +0,0 @@ -Make federator capable of streaming responses diff --git a/changelog.d/6-federation/named-fed-api-routes b/changelog.d/6-federation/named-fed-api-routes deleted file mode 100644 index c41471a3636..00000000000 --- a/changelog.d/6-federation/named-fed-api-routes +++ /dev/null @@ -1 +0,0 @@ -Use `Named` routes for the federation API diff --git a/changelog.d/6-federation/sft-brig-configmap-fix b/changelog.d/6-federation/sft-brig-configmap-fix deleted file mode 100644 index d271b241cf3..00000000000 --- a/changelog.d/6-federation/sft-brig-configmap-fix +++ /dev/null @@ -1 +0,0 @@ -Fix Brig's configmap for SFT lookups diff --git a/changelog.d/6-federation/sft-fix-ipv4-http b/changelog.d/6-federation/sft-fix-ipv4-http deleted file mode 100644 index bc0be239d8f..00000000000 --- a/changelog.d/6-federation/sft-fix-ipv4-http +++ /dev/null @@ -1 +0,0 @@ -Allow making HTTP-only requests to SFTs via an IPv4 address diff --git a/changelog.d/6-federation/sft-replace-ipv4-w-disco b/changelog.d/6-federation/sft-replace-ipv4-w-disco deleted file mode 100644 index 569d5284c7c..00000000000 --- a/changelog.d/6-federation/sft-replace-ipv4-w-disco +++ /dev/null @@ -1 +0,0 @@ -Replace IPv4-HTTP-only Approach to SFT Server Lookup with /sft_servers_all.json diff --git a/changelog.d/6-federation/sft-servers-all b/changelog.d/6-federation/sft-servers-all deleted file mode 100644 index ec9339bdca7..00000000000 --- a/changelog.d/6-federation/sft-servers-all +++ /dev/null @@ -1 +0,0 @@ -Extend GET /calls/config/v2 to include all SFT servers in federation diff --git a/changelog.d/6-federation/sft-servers-all-fixup b/changelog.d/6-federation/sft-servers-all-fixup deleted file mode 100644 index 25c1a8fb06d..00000000000 --- a/changelog.d/6-federation/sft-servers-all-fixup +++ /dev/null @@ -1 +0,0 @@ -Improve Brig's configuration for SFTs and fix a call to SFT servers diff --git a/changelog.d/6-federation/sftd_disco b/changelog.d/6-federation/sftd_disco deleted file mode 100644 index f7754d070a9..00000000000 --- a/changelog.d/6-federation/sftd_disco +++ /dev/null @@ -1 +0,0 @@ -SFTD chart: provide a /sft_servers_all.json url that can be used by brig to populate /calls/config/v2 From 2fd80aaeca592fa0bd688c393dea6cf72e890ab3 Mon Sep 17 00:00:00 2001 From: Paolo Capriotti Date: Wed, 19 Jan 2022 09:20:16 +0100 Subject: [PATCH 74/74] Update CHANGELOG --- CHANGELOG.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a29b97fdf47..e6b66eb36bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,11 +14,7 @@ ## API changes * The deprecated endpoint `GET /teams` now ignores query parameters `ids`, `start` (#2027) -* Add qualified v4 endpoints for downloading and deleting assets. The upload API is still on the same path, but the asset object it returns now contains a `domain` field. Note that federated behaviour is still not implemented. (#2002) -* Enable downloading assets from a remote (federated) cargohold instance via the v4 API. - The content of remote assets is returned as stream with content type - `application/octet-stream`. - Please refer to the Swagger API documentation for more details. (#2004) +* Add qualified v4 endpoints for downloading and deleting assets. The upload API is still on the same path, but the asset object it returns now contains a `domain` field. (#2002) * Remove resumable upload API (#1998) ## Features @@ -77,6 +73,7 @@ * Replace IPv4-HTTP-only Approach to SFT Server Lookup with /sft_servers_all.json (#2030) * Extend GET /calls/config/v2 to include all SFT servers in federation (#2012) * Improve Brig's configuration for SFTs and fix a call to SFT servers (#2014) +* Enable downloading assets from a remote (federated) cargohold instance via the v4 API. The content of remote assets is returned as stream with content type `application/octet-stream`. Please refer to the Swagger API documentation for more details. (#2004) # [2021-12-10]