diff --git a/cabal.project b/cabal.project index 932f0f55399..5ebc608c29e 100644 --- a/cabal.project +++ b/cabal.project @@ -48,6 +48,7 @@ packages: , tools/db/inconsistencies/ , tools/db/migrate-sso-feature-flag/ , tools/db/move-team/ + , tools/db/phone-users/ , tools/db/repair-handles/ , tools/db/repair-brig-clients-table/ , tools/db/service-backfill/ @@ -127,6 +128,8 @@ package proxy ghc-options: -Werror package mlsstats ghc-options: -Werror +package phone-users + ghc-options: -Werror package rabbitmq-consumer ghc-options: -Werror package repair-handles @@ -179,6 +182,6 @@ package fedcalls -- - these packages have bounds that are justified with their current -- dependency set, however, we have updated their dependencies, such -- that they work with newer base and ghc (api) versions -allow-newer: +allow-newer: , proto-lens-protoc:base , proto-lens-protoc:ghc diff --git a/changelog.d/5-internal/WPB-8702 b/changelog.d/5-internal/WPB-8702 new file mode 100644 index 00000000000..442ee8a831a --- /dev/null +++ b/changelog.d/5-internal/WPB-8702 @@ -0,0 +1 @@ +Add tool to determine number of phone-only users diff --git a/nix/local-haskell-packages.nix b/nix/local-haskell-packages.nix index 289d38bdd7c..89527deeb19 100644 --- a/nix/local-haskell-packages.nix +++ b/nix/local-haskell-packages.nix @@ -49,6 +49,7 @@ inconsistencies = hself.callPackage ../tools/db/inconsistencies/default.nix { inherit gitignoreSource; }; migrate-sso-feature-flag = hself.callPackage ../tools/db/migrate-sso-feature-flag/default.nix { inherit gitignoreSource; }; move-team = hself.callPackage ../tools/db/move-team/default.nix { inherit gitignoreSource; }; + phone-users = hself.callPackage ../tools/db/phone-users/default.nix { inherit gitignoreSource; }; repair-brig-clients-table = hself.callPackage ../tools/db/repair-brig-clients-table/default.nix { inherit gitignoreSource; }; repair-handles = hself.callPackage ../tools/db/repair-handles/default.nix { inherit gitignoreSource; }; service-backfill = hself.callPackage ../tools/db/service-backfill/default.nix { inherit gitignoreSource; }; diff --git a/tools/db/phone-users/.ormolu b/tools/db/phone-users/.ormolu new file mode 120000 index 00000000000..ffc2ca9745e --- /dev/null +++ b/tools/db/phone-users/.ormolu @@ -0,0 +1 @@ +../../../.ormolu \ No newline at end of file diff --git a/tools/db/phone-users/README.md b/tools/db/phone-users/README.md new file mode 100644 index 00000000000..ab03b0b8fa1 --- /dev/null +++ b/tools/db/phone-users/README.md @@ -0,0 +1,44 @@ +# Phone users + +This program scans brig's users table and determines the number of users that can only login by phone/sms. + +Example usage: + +```shell +phone-users --brig-cassandra-keyspace brig --galley-cassandra-keyspace galley -l 100000 +``` + +Display usage: + +```shell +phone-users -h +``` + +```text +phone-users + +Usage: phone-users [--brig-cassandra-host HOST] [--brig-cassandra-port PORT] + [--brig-cassandra-keyspace STRING] + [--galley-cassandra-host HOST] [--galley-cassandra-port PORT] + [--galley-cassandra-keyspace STRING] [-l|--limit INT] + + This program scans brig's users table and determines the number of users that + can only login by phone/sms + +Available options: + -h,--help Show this help text + --brig-cassandra-host HOST + Cassandra Host for brig (default: "localhost") + --brig-cassandra-port PORT + Cassandra Port for brig (default: 9042) + --brig-cassandra-keyspace STRING + Cassandra Keyspace for brig (default: "brig_test") + --galley-cassandra-host HOST + Cassandra Host for galley (default: "localhost") + --galley-cassandra-port PORT + Cassandra Port for galley (default: 9043) + --galley-cassandra-keyspace STRING + Cassandra Keyspace for galley + (default: "galley_test") + -l,--limit INT Limit the number of users to process +``` diff --git a/tools/db/phone-users/app/Main.hs b/tools/db/phone-users/app/Main.hs new file mode 100644 index 00000000000..be8658b8005 --- /dev/null +++ b/tools/db/phone-users/app/Main.hs @@ -0,0 +1,23 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2024 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Main where + +import qualified PhoneUsers.Lib as Lib + +main :: IO () +main = Lib.main diff --git a/tools/db/phone-users/default.nix b/tools/db/phone-users/default.nix new file mode 100644 index 00000000000..2903ef57701 --- /dev/null +++ b/tools/db/phone-users/default.nix @@ -0,0 +1,48 @@ +# WARNING: GENERATED FILE, DO NOT EDIT. +# This file is generated by running hack/bin/generate-local-nix-packages.sh and +# must be regenerated whenever local packages are added or removed, or +# dependencies are added or removed. +{ mkDerivation +, aeson +, aeson-pretty +, base +, bytestring +, cassandra-util +, conduit +, cql +, gitignoreSource +, imports +, lens +, lib +, optparse-applicative +, time +, tinylog +, types-common +, wire-api +}: +mkDerivation { + pname = "phone-users"; + version = "1.0.0"; + src = gitignoreSource ./.; + isLibrary = true; + isExecutable = true; + libraryHaskellDepends = [ + aeson + aeson-pretty + bytestring + cassandra-util + conduit + cql + imports + lens + optparse-applicative + time + tinylog + types-common + wire-api + ]; + executableHaskellDepends = [ base ]; + description = "Check users that are only able to login via phone"; + license = lib.licenses.agpl3Only; + mainProgram = "phone-users"; +} diff --git a/tools/db/phone-users/phone-users.cabal b/tools/db/phone-users/phone-users.cabal new file mode 100644 index 00000000000..ab9c01f8284 --- /dev/null +++ b/tools/db/phone-users/phone-users.cabal @@ -0,0 +1,96 @@ +cabal-version: 3.0 +name: phone-users +version: 1.0.0 +synopsis: Check users that are only able to login via phone +category: Network +author: Wire Swiss GmbH +maintainer: Wire Swiss GmbH +copyright: (c) 2024 Wire Swiss GmbH +license: AGPL-3.0-only +build-type: Simple + +library + hs-source-dirs: src + exposed-modules: + PhoneUsers.Lib + PhoneUsers.Types + + ghc-options: + -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates + -Wpartial-fields -fwarn-tabs -optP-Wno-nonportable-include-path + -funbox-strict-fields -threaded -with-rtsopts=-N + -Wredundant-constraints -Wunused-packages + + build-depends: + , aeson + , aeson-pretty + , bytestring + , cassandra-util + , conduit + , cql + , imports + , lens + , optparse-applicative + , time + , tinylog + , types-common + , wire-api + + default-extensions: + AllowAmbiguousTypes + BangPatterns + ConstraintKinds + DataKinds + DefaultSignatures + DeriveFunctor + DeriveGeneric + DeriveLift + DeriveTraversable + DerivingStrategies + DerivingVia + DuplicateRecordFields + EmptyCase + FlexibleContexts + FlexibleInstances + FunctionalDependencies + GADTs + GeneralizedNewtypeDeriving + InstanceSigs + KindSignatures + LambdaCase + MultiParamTypeClasses + MultiWayIf + NamedFieldPuns + NoImplicitPrelude + OverloadedLabels + OverloadedRecordDot + OverloadedStrings + PackageImports + PatternSynonyms + PolyKinds + QuasiQuotes + RankNTypes + RecordWildCards + ScopedTypeVariables + StandaloneDeriving + TupleSections + TypeApplications + TypeFamilies + TypeFamilyDependencies + TypeOperators + UndecidableInstances + ViewPatterns + +executable phone-users + main-is: Main.hs + build-depends: + , base + , phone-users + + hs-source-dirs: app + default-language: Haskell2010 + ghc-options: + -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates + -Wpartial-fields -fwarn-tabs -optP-Wno-nonportable-include-path + -funbox-strict-fields -threaded -with-rtsopts=-N + -Wredundant-constraints -Wunused-packages diff --git a/tools/db/phone-users/src/PhoneUsers/Lib.hs b/tools/db/phone-users/src/PhoneUsers/Lib.hs new file mode 100644 index 00000000000..8c913b7a0bf --- /dev/null +++ b/tools/db/phone-users/src/PhoneUsers/Lib.hs @@ -0,0 +1,177 @@ +{-# LANGUAGE OverloadedStrings #-} + +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2024 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module PhoneUsers.Lib where + +import Cassandra as C +import Cassandra.Settings as C +import Data.Conduit +import qualified Data.Conduit.Combinators as Conduit +import qualified Data.Conduit.List as CL +import Data.Id (TeamId, UserId) +import Data.Time +import qualified Database.CQL.Protocol as CQL +import Imports +import Options.Applicative +import PhoneUsers.Types +-- import qualified System.IO as SIO +import qualified System.Logger as Log +import System.Logger.Message ((.=), (~~)) +import Wire.API.Team.Feature (FeatureStatus (FeatureStatusDisabled, FeatureStatusEnabled)) +import Wire.API.User (AccountStatus (Active)) + +lookupClientsLastActiveTimestamps :: ClientState -> UserId -> IO [Maybe UTCTime] +lookupClientsLastActiveTimestamps client u = do + runClient client $ runIdentity <$$> retry x1 (query selectClients (params One (Identity u))) + where + selectClients :: PrepQuery R (Identity UserId) (Identity (Maybe UTCTime)) + selectClients = "SELECT last_active from clients where user = ?" + +readUsers :: ClientState -> ConduitM () [UserRow] IO () +readUsers client = + transPipe (runClient client) (paginateC selectUsersAll (paramsP One () 1000) x5) + .| Conduit.map (fmap CQL.asRecord) + where + selectUsersAll :: C.PrepQuery C.R () (CQL.TupleType UserRow) + selectUsersAll = + "SELECT id, email, phone, activated, status, team FROM user" + +getConferenceCalling :: ClientState -> TeamId -> IO (Maybe FeatureStatus) +getConferenceCalling client tid = do + runClient client $ runIdentity <$$> retry x1 (query1 select (params One (Identity tid))) + where + select :: PrepQuery R (Identity TeamId) (Identity FeatureStatus) + select = + "select conference_calling from team_features where team_id = ?" + +process :: Log.Logger -> Maybe Int -> ClientState -> ClientState -> IO Result +process logger limit brigClient galleyClient = + runConduit $ + readUsers brigClient + -- .| Conduit.mapM (\chunk -> SIO.hPutStr stderr "." $> chunk) + .| Conduit.concat + .| (maybe (Conduit.filter (const True)) Conduit.take limit) + .| Conduit.mapM (getUserInfo logger brigClient galleyClient) + .| forever (CL.isolate 10000 .| (Conduit.foldMap infoToResult >>= yield)) + .| Conduit.takeWhile ((> 0) . usersSearched) + .| CL.scan (<>) mempty + `fuseUpstream` Conduit.mapM_ (\r -> Log.info logger $ "intermediate_result" .= show r) + +getUserInfo :: Log.Logger -> ClientState -> ClientState -> UserRow -> IO UserInfo +getUserInfo logger brigClient galleyClient ur = do + if not $ isCandidate + then pure NoPhoneUser + else do + -- should we give C* a little break here and add a small threadDelay? + -- threadDelay 200 + lastActiveTimeStamps <- lookupClientsLastActiveTimestamps brigClient ur.id + now <- getCurrentTime + -- activity: + -- inactive: they have no client or client's last_active is greater than 90 days ago + -- active: otherwise + -- last_active is null on client creation, but it will be set once notifications are fetched + -- therefore we can consider empty last_active as inactive + let activeLast90Days = any (clientWasActiveLast90Days now) $ catMaybes lastActiveTimeStamps + userInfo <- + if activeLast90Days + then do + apu <- case ur.team of + Nothing -> pure ActivePersonalUser + Just tid -> do + isPaying <- isPayingTeam galleyClient tid + pure $ + if isPaying + then ActiveTeamUser Free + else ActiveTeamUser Paid + Log.info logger $ + "active_phone_user" .= show apu + ~~ "user_record" .= show ur + ~~ "last_active_timestamps" .= show lastActiveTimeStamps + ~~ Log.msg (Log.val "active phone user found") + pure apu + else pure InactiveLast90Days + pure $ PhoneUser userInfo + where + -- to qualify as an active phone user candidate, their account must be active and they must have a phone number but no verified email + isCandidate :: Bool + isCandidate = + ur.activated && ur.status == Just Active && isJust ur.phone && isNothing ur.email + + clientWasActiveLast90Days :: UTCTime -> UTCTime -> Bool + clientWasActiveLast90Days now lastActive = diffUTCTime now lastActive < 90 * nominalDay + + -- if conference_calling is enabled for the team, then it's a paying team + isPayingTeam :: ClientState -> TeamId -> IO Bool + isPayingTeam client tid = do + status <- getConferenceCalling client tid + pure $ case status of + Just FeatureStatusEnabled -> True + Just FeatureStatusDisabled -> False + Nothing -> False + +infoToResult :: UserInfo -> Result +infoToResult = \case + NoPhoneUser -> mempty {usersSearched = 1} + PhoneUser InactiveLast90Days -> mempty {usersSearched = 1, phoneUsersTotal = 1, inactivePhoneUsers = 1} + PhoneUser ActivePersonalUser -> mempty {usersSearched = 1, phoneUsersTotal = 1, activePersonalPhoneUsers = 1} + PhoneUser (ActiveTeamUser Free) -> + Result + { usersSearched = 1, + phoneUsersTotal = 1, + inactivePhoneUsers = 0, + activePersonalPhoneUsers = 0, + activeFreeTeamPhoneUsers = 1, + activePaidTeamPhoneUsers = 0 + } + PhoneUser (ActiveTeamUser Paid) -> + Result + { usersSearched = 1, + phoneUsersTotal = 1, + inactivePhoneUsers = 0, + activePersonalPhoneUsers = 0, + activeFreeTeamPhoneUsers = 0, + activePaidTeamPhoneUsers = 1 + } + +main :: IO () +main = do + opts <- execParser (info (helper <*> optsParser) desc) + logger <- initLogger + brigClient <- initCas opts.brigDb logger + galleyClient <- initCas opts.galleyDb logger + putStrLn "scanning users table..." + res <- process logger opts.limit brigClient galleyClient + Log.info logger $ "result" .= show res + where + initLogger = + Log.new + . Log.setLogLevel Log.Info + . Log.setOutput Log.StdOut + . Log.setFormat Nothing + . Log.setBufSize 0 + $ Log.defSettings + initCas settings l = + C.init + . C.setLogger (C.mkLogger l) + . C.setContacts settings.host [] + . C.setPortNumber (fromIntegral settings.port) + . C.setKeyspace settings.keyspace + . C.setProtocolVersion C.V4 + $ C.defSettings + desc = header "phone-users" <> progDesc "This program scans brig's users table and determines the number of users that can only login by phone/sms" <> fullDesc diff --git a/tools/db/phone-users/src/PhoneUsers/Types.hs b/tools/db/phone-users/src/PhoneUsers/Types.hs new file mode 100644 index 00000000000..fc60a3ee038 --- /dev/null +++ b/tools/db/phone-users/src/PhoneUsers/Types.hs @@ -0,0 +1,184 @@ +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE TemplateHaskell #-} + +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2024 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module PhoneUsers.Types where + +import Cassandra as C +import Control.Lens +import qualified Data.Aeson as A +import qualified Data.Aeson.Encode.Pretty as A +import qualified Data.ByteString.Lazy.Char8 as LC8 +import Data.Id +import Data.Text.Strict.Lens +import Database.CQL.Protocol hiding (Result) +import Imports +import Options.Applicative +import Wire.API.User + +data CassandraSettings = CassandraSettings + { host :: String, + port :: Int, + keyspace :: C.Keyspace + } + +data Opts = Opts + { brigDb :: CassandraSettings, + galleyDb :: CassandraSettings, + limit :: Maybe Int + } + +optsParser :: Parser Opts +optsParser = + Opts + <$> brigCassandraParser + <*> galleyCassandraParser + <*> optional + ( option + auto + ( long "limit" + <> short 'l' + <> metavar "INT" + <> help "Limit the number of users to process" + ) + ) + +galleyCassandraParser :: Parser CassandraSettings +galleyCassandraParser = + CassandraSettings + <$> strOption + ( long "galley-cassandra-host" + <> metavar "HOST" + <> help "Cassandra Host for galley" + <> value "localhost" + <> showDefault + ) + <*> option + auto + ( long "galley-cassandra-port" + <> metavar "PORT" + <> help "Cassandra Port for galley" + <> value 9043 + <> showDefault + ) + <*> ( C.Keyspace . view packed + <$> strOption + ( long "galley-cassandra-keyspace" + <> metavar "STRING" + <> help "Cassandra Keyspace for galley" + <> value "galley_test" + <> showDefault + ) + ) + +brigCassandraParser :: Parser CassandraSettings +brigCassandraParser = + CassandraSettings + <$> strOption + ( long "brig-cassandra-host" + <> metavar "HOST" + <> help "Cassandra Host for brig" + <> value "localhost" + <> showDefault + ) + <*> option + auto + ( long "brig-cassandra-port" + <> metavar "PORT" + <> help "Cassandra Port for brig" + <> value 9042 + <> showDefault + ) + <*> ( C.Keyspace . view packed + <$> strOption + ( long "brig-cassandra-keyspace" + <> metavar "STRING" + <> help "Cassandra Keyspace for brig" + <> value "brig_test" + <> showDefault + ) + ) + +data Result = Result + { usersSearched :: Int, + phoneUsersTotal :: Int, + inactivePhoneUsers :: Int, + activePersonalPhoneUsers :: Int, + activeFreeTeamPhoneUsers :: Int, + activePaidTeamPhoneUsers :: Int + } + deriving (Generic) + +instance A.ToJSON Result + +instance Show Result where + show = LC8.unpack . A.encodePretty + +instance Semigroup Result where + r1 <> r2 = + Result + { usersSearched = r1.usersSearched + r2.usersSearched, + phoneUsersTotal = r1.phoneUsersTotal + r2.phoneUsersTotal, + inactivePhoneUsers = r1.inactivePhoneUsers + r2.inactivePhoneUsers, + activePersonalPhoneUsers = r1.activePersonalPhoneUsers + r2.activePersonalPhoneUsers, + activeFreeTeamPhoneUsers = r1.activeFreeTeamPhoneUsers + r2.activeFreeTeamPhoneUsers, + activePaidTeamPhoneUsers = r1.activePaidTeamPhoneUsers + r2.activePaidTeamPhoneUsers + } + +instance Monoid Result where + mempty = + Result + { usersSearched = 0, + phoneUsersTotal = 0, + inactivePhoneUsers = 0, + activePersonalPhoneUsers = 0, + activeFreeTeamPhoneUsers = 0, + activePaidTeamPhoneUsers = 0 + } + +type Activated = Bool + +data UserRow = UserRow + { id :: UserId, + email :: Maybe Email, + phone :: Maybe Phone, + activated :: Activated, + status :: Maybe AccountStatus, + team :: Maybe TeamId + } + deriving (Generic) + +instance A.ToJSON UserRow + +recordInstance ''UserRow + +instance Show UserRow where + show = LC8.unpack . A.encodePretty + +data TeamUser = Free | Paid + deriving (Show) + +data UserInfo = NoPhoneUser | PhoneUser PhoneUserInfo + deriving (Show) + +data PhoneUserInfo + = InactiveLast90Days + | ActivePersonalUser + | ActiveTeamUser TeamUser + deriving (Show)