Merge pull request #710 from wireapp/release/2019-04-09

Release/2019-04-09

CHANGELOG.md

@@ -1,3 +1,36 @@
+# 2019-04-09 #710
+
+## API changes
+
+- Do not allow provisioning saml users if SCIM is configured (#706)
+
+## Documentation changes
+
+- Docs for user deletion via SCIM. (#691)
+- Docs for jump-to-definition with Emacs (#693)
+- Add missing config options in demo (#694)
+- Move the connections doc, add haddocks (#695)
+
+## Bug fixes
+
+- Fix templating in outgoing SMSs. (#696)
+- Saml implicit user creation no longer chokes on odd but legal names. (#702)
+- Fix: user deletion via scim (#698)
+
+## Internal changes
+
+- Remove redundant cassandra write in renewCookie (#676)
+- Add Prometheus middleware for wire-services (#672)
+- Improve logging of spar errors (#654)
+- Upgrade cql-io-1.1.0 (#697)
+- Switch metrics-core to be backed by Prometheus (#704)
+- Refactorings:
+    - #665, #687, #685, #686
+
+## Changes (potentially) requiring action for self-hosters
+
+- Switch proxy to use YAML-only config (#684)
+
 # 2019-03-25 #674
 
 ## API changes

build/alpine/Dockerfile.builder

@@ -14,8 +14,11 @@ WORKDIR /
 #
 # Finally, we build docs for haskell-src-exts without hyperlinking enabled
 # to avoid a Haddock segfault. See https://github.com/haskell/haddock/issues/928
+#
+# Note: git, ncurses, sed are added here for historical reasons; since
+# roughly 2019-03-28, they are included in prebuilder as well.
 
-RUN apk add --no-cache git ncurses && \
+RUN apk add --no-cache git ncurses sed && \
     git clone -b develop https://github.com/wireapp/wire-server.git && \
     cd /wire-server && \
     stack update && \

build/alpine/Dockerfile.prebuilder

@@ -33,10 +33,23 @@ RUN apk add --no-cache \
         llvm-libunwind-dev \
         bash \
         xz \
-        libxml2-dev
+        libxml2-dev \
+        git \
+        ncurses \
+        sed
 
 # get static version of Haskell Stack and use system ghc by default
 ARG STACK_ALPINE_VERSION=1.9.1
 RUN curl -sSfL https://github.com/commercialhaskell/stack/releases/download/v${STACK_ALPINE_VERSION}/stack-${STACK_ALPINE_VERSION}-linux-x86_64-static.tar.gz \
     | tar --wildcards -C /usr/local/bin --strip-components=1 -xzvf - '*/stack' && chmod 755 /usr/local/bin/stack && \
     stack config set system-ghc --global true
+
+# upgrade stack to current master (2019-03-28).  this fixes an issue
+# with building internal libraries as used in cql-io-1.1.0.
+# details: https://github.com/commercialhaskell/stack/pull/4596
+RUN git clone https://github.com/commercialhaskell/stack && \
+    cd stack && \
+    git checkout f0b66a1ab60fb5be85f6cb60491915bf53d3cd3c && \
+    sed -i -e 's/lts-12.20/lts-12.14/' snapshot-lts-12.yaml && \
+    stack install --system-ghc --stack-yaml=stack-lts-12.yaml --system-ghc && \
+    cp /root/.local/bin/stack /usr/local/bin/stack

deploy/services-demo/conf/brig.demo.yaml

@@ -43,7 +43,17 @@ emailSMS:
     templateDir: resources/templates
     emailSender: [email protected]
     smsSender: "<insert-sender-number-here>"
-
+    templateBranding:
+      brand: Wire
+      brandUrl: https://wire.com
+      brandLabelUrl: wire.com # This is the text in the label for the above URL
+      brandLogoUrl: https://wire.com/p/img/email/logo-email-black.png
+      brandService: Wire Service Provider
+      copyright: © WIRE SWISS GmbH
+      misuse: [email protected]
+      legal: https://wire.com/legal/
+      forgot: https://wire.com/forgot/
+      support: https://support.wire.com/
   user:
     activationUrl: http://127.0.0.1:8080/activate?key=${key}&code=${code}
     smsActivationUrl: http://127.0.0.1:8080/v/${code}

deploy/services-demo/conf/proxy.demo.yaml

@@ -4,3 +4,6 @@ port: 8087
 httpPoolSize: 1000
 maxConns: 5000
 secretsConfig: resources/proxy.config
+
+logLevel: Info
+logNetStrings: false

deploy/services-demo/demo.sh

@@ -99,8 +99,6 @@ blueish=4
 function run_haskell_service() {
     service=$1
     colour=$2
-    # TODO can be removed once all services have been switched to YAML configs
-    [ $# -gt 2 ] && export LOG_LEVEL=$3
     (cd ${SCRIPT_DIR} && ${DIR}/../dist/${service} -c ${SCRIPT_DIR}/conf/${service}.demo.yaml || kill_all) \
         | sed -e "s/^/$(tput setaf ${colour})[${service}] /" -e "s/$/$(tput sgr0)/" &
 }
@@ -145,7 +143,7 @@ if [ "$docker_deployment" = "false" ]; then
     run_haskell_service gundeck ${blue}
     run_haskell_service cannon ${orange}
     run_haskell_service cargohold ${purpleish}
-    run_haskell_service proxy ${redish} Info
+    run_haskell_service proxy ${redish}
     run_haskell_service spar ${orange}
     run_nginz ${blueish}
 else

docs/README.md

@@ -2,12 +2,13 @@
 
 # Reference documentation
 
-What you need to know as a user of the Wire backend: concepts, features, and API.
+What you need to know as a user of the Wire backend: concepts, features, and API. We strive to keep these up to date.
 
 ## Users
 
 We support the following functionality related to users and user profiles:
 
+* [Connections between users](reference/user/connection.md) `{#RefConnection}`
 * [Rich info](reference/user/rich-info.md) `{#RefRichInfo}`
 * TODO
 
@@ -23,7 +24,10 @@ We have support for provisioning users via SCIM ([RFC 7664][], [RFC 7643][]). It
 
 # Developer documentation
 
-What you need to know as a Wire backend developer. All of these documents can and should be referenced in the code.
+Internal documentation detailing what you need to know as a Wire backend developer. All of these documents can and should be referenced in the code.
+
+If you're not a member of the Wire backend team, you might still find these documents useful, but keep in mind that they are a work in progress.
 
 * [Development setup](developer/dependencies.md) `{#DevDeps}`
+* [Editor setup](developer/editor-setup.md) `{#DevEditor}`
 * TODO

docs/developer/dependencies.md

@@ -51,6 +51,12 @@ extra-lib-dirs:
 - /usr/local/opt/icu4c/lib
 ```
 
+_Note_: if you're getting `fatal error: 'libxml/parser.h' file not found` and you're on macOS Mojave, try doing:
+
+```bash
+sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /
+```
+
 ## Haskell Stack
 
 When you're done, ensure `stack --version` is >= 1.6.5

docs/developer/editor-setup.md

@@ -0,0 +1,50 @@
+# Editor setup {#DevEditor}
+
+This page provides tips for setting up editors to work with the Wire codebase.
+
+## Emacs {#DevEmacs}
+
+### Jump-to-definition {#DevEmacsJump}
+
+Jump-to-definition is possible with [hasktags][]. First you need to install it and make sure it's on your PATH (if you don't want hasktags to be on your PATH, do `M-x customize-variable haskell-hasktags-path`):
+
+[hasktags]: https://hackage.haskell.org/package/hasktags
+
+```bash
+stack install hasktags    # or cabal install hasktags
+```
+
+To generate tags, do `M-x haskell-mode-generate-tags`. You can also add a Git hook to regenerate tags on checkout:
+
+```bash
+echo "hasktags -e -x ." > .git/hooks/post-checkout
+chmod +x .git/hooks/post-checkout
+```
+
+To jump to an identifier, press `M-.`. You can also do `C-u M-x xref-find-definitions` to get interactive search through identifiers. Press `M-,` to return to where you were before the jump.
+
+Jump-to-definition is case-insensitive by default, which is probably not what you want. To change that, do `M-x customize-variable tags-case-fold-search`.
+
+By default hasktags only generates tags for the current package. The Wire backend is composed of many packages, and it's useful to be able to jump to any identifier in wire-server. One way to do it is to setup Emacs to check if there's a Projectile project that the current directory belongs to, and if so, override the "current package" default.
+
+Install the [projectile][] package for Emacs and do `M-x projectile-add-known-project <path to wire-server>`. Then add the following snippet to your `init.el`:
+
+[projectile]: https://www.projectile.mx/en/latest/installation/
+
+```
+(require 'haskell)
+(require 'projectile)
+
+;; When inside a project, even if there is a cabal file in the current
+;; folder, use the project folder to generate tags. This is useful for
+;; projects with several services or subprojects.
+(defadvice haskell-cabal--find-tags-dir (around project-tags act)
+  (setq ad-return-value
+    (if (projectile-project-root)
+      (projectile-project-root)
+      ad-do-it)))
+```
+
+## Vim {#DevVim}
+
+TODO.

docs/reference/provisioning/scim-via-curl.md

@@ -169,7 +169,11 @@ curl -X PUT \
 
 ### delete user
 
-**Not implemented yet.**
+```
+curl -X DELETE \
+  --header "Authorization: Bearer $SCIM_TOKEN" \
+  $WIRE_BACKEND/scim/v2/Users/$STORED_USER_ID
+```
 
 ### groups
 

services/brig/doc/connections.md → docs/reference/user/connection.md

@@ -1,7 +1,4 @@
-
-## User Connections
-
-### Concepts
+# Connection {#RefConnection}
 
 Connections between users are a means for giving users some control over their
 privacy and a degree of spam protection. Connections control whether a user can:
@@ -25,52 +22,51 @@ group conversations where the blocked user is a member.
 In the implementation, a connection is a directed edge from one user to another that is
 attributed with a relation state and potentially other meta information.
 
-### Connection States
+## Connection states {#RefConnectionStates}
 
 The following are the existing states that connections can be in and their meaning.
 
-#### Sent
+### Sent {#RefConnectionSent}
 
 The creator of a new connection (i.e. the sender of the connection request) ends
 up in this state. From the point of view of the creator, it indicates that a
 connection request has been sent but not accepted (it might be blocked or ignored).
 
-#### Pending
+### Pending {#RefConnectionPending}
 
 The recipient of a connection request automatically ends up in this state.
 From his point of view, the state indicates that the connection is pending
 and awaiting further action (i.e. through accepting, ignoring or blocking it).
 
-#### Blocked
+### Blocked {#RefConnectionBlocked}
 
 When a connection is in this state it indicates that the user does not want
 to be bothered by the other user, e.g. by receiving messages, calls or being added
 to conversations.
 
-#### Ignored
+### Ignored {#RefConnectionIgnored}
 
-This is a temporary state for the recipient of a connection request. It indicates
-that the recipient neither wanted to accept nor block the connection (yet). In this state
-the sender can continue to send further connection attempts.
+The recipient of a connection request may decide to explicitly "ignore" the request
+In this state the sender can continue to send further connection attempts. The
+recipient can change their mind and accept the request later.
 
-#### Cancelled
+### Cancelled {#RefConnectionCancelled}
 
 This is a state that the sender can change to if the connection has not
 yet been accepted. The state will also change for the recipient, unless
 blocked.
 
-#### Accepted
+### Accepted {#RefConnectionAccepted}
 
 A connection in this state is fully accepted by a user. The user thus allows the
 user at the other end of the connection to add him to conversations, view his full
 profile information, etc.
 
-
-### Transitions
+## Transitions between connection states {#RefConnectionTransitions}
 
 The following diagram depicts the transitions between connection states.
 
-![Connection State Transitions](connections.png)
+![Connection State Transitions](connection-transitions.png)
 
-In order to edit this diagram, open the [connections.xml](connections.xml)
+In order to edit this diagram, open [connection-transitions.xml](connection-transitions.xml)
 with [draw.io](https://www.draw.io/).

libs/api-bot/src/Network/Wire/Bot/Monad.hs

@@ -778,9 +778,10 @@ timed p ma = do
     start <- liftIO getCurrentTime
     a <- ma
     stop <- liftIO getCurrentTime
-    let d = round . (* 1000) $ stop `diffUTCTime` start
+    let durationInMillis = realToFrac . (* 1000) $ stop `diffUTCTime` start
     m <- getMetrics
-    liftIO $ Metrics.bucketsIncr 30 12 d p m
+    let timeHisto = Metrics.deprecatedRequestDurationHistogram p
+    liftIO $ Metrics.histoSubmit durationInMillis timeHisto m
     return a
 
 incrAssertTotal :: MonadBotNet m => m ()
@@ -806,10 +807,10 @@ decrBotsAlive = getMetrics >>= liftIO . Metrics.gaugeDecr Metrics.botsAlive
 
 -- Note: Separate TVars to avoid contention.
 data BotMetrics = BotMetrics
-    { botEventsRcvd :: TVar (HashMap Metrics.Path Word)
-    , botEventsAckd :: TVar (HashMap Metrics.Path Word)
-    , botEventsIgnd :: TVar (HashMap Metrics.Path Word)
-    , botEventsMssd :: TVar (HashMap Metrics.Path Word)
+    { botEventsRcvd :: TVar (HashMap Metrics.Path Double)
+    , botEventsAckd :: TVar (HashMap Metrics.Path Double)
+    , botEventsIgnd :: TVar (HashMap Metrics.Path Double)
+    , botEventsMssd :: TVar (HashMap Metrics.Path Double)
     }
 
 newBotMetrics :: IO BotMetrics