From 0ec3c624cc3d5553d2294b07788ab03f8c71b84e Mon Sep 17 00:00:00 2001 From: Vladimir Dementyev Date: Wed, 1 Nov 2023 17:36:34 -0700 Subject: [PATCH] dev: add broker benchmarks --- etc/k6/broker.js | 165 ++++++++++++++++++++++++ etc/k6/results/2023-10-broker.md | 215 +++++++++++++++++++++++++++++++ etc/traefik/rules.toml | 13 ++ etc/traefik/traefik.yml | 9 ++ 4 files changed, 402 insertions(+) create mode 100644 etc/k6/broker.js create mode 100644 etc/k6/results/2023-10-broker.md create mode 100644 etc/traefik/rules.toml create mode 100644 etc/traefik/traefik.yml diff --git a/etc/k6/broker.js b/etc/k6/broker.js new file mode 100644 index 00000000..b7506363 --- /dev/null +++ b/etc/k6/broker.js @@ -0,0 +1,165 @@ +// Build k6 with xk6-cable like this: +// xk6 build v0.38.3 --with github.com/anycable/xk6-cable@v0.3.0 + +import { check, sleep, fail } from "k6"; +import cable from "k6/x/cable"; +import { randomIntBetween } from "https://jslib.k6.io/k6-utils/1.1.0/index.js"; + +const rampingOptions = { + scenarios: { + default: { + executor: "ramping-vus", + startVUs: 100, + stages: [ + { duration: "10s", target: 300 }, + { duration: "10s", target: 500 }, + { duration: "10s", target: 1000 }, + { duration: "10s", target: 1300 }, + { duration: "10s", target: 1500 }, + { duration: "30s", target: 1500 }, + { duration: "30s", target: 0 }, + ], + gracefulStop: "1m", + gracefulRampDown: "1m", + }, + }, +}; + +export const options = __ENV.SKIP_OPTIONS ? {} : rampingOptions; + +import { Trend, Counter } from "k6/metrics"; +let rttTrend = new Trend("rtt", true); +let subTrend = new Trend("suback", true); +let broadcastTrend = new Trend("broadcast_duration", true); +let historyTrend = new Trend("history_duration", true); +let historyRcvd = new Counter("history_rcvd"); +let broadcastsSent = new Counter("broadcasts_sent"); +let broadcastsRcvd = new Counter("broadcasts_rcvd"); +let acksRcvd = new Counter("acks_rcvd"); + +// Load ENV from .env +function loadDotEnv() { + try { + let dotenv = open("./.env"); + dotenv.split(/[\n\r]/m).forEach((line) => { + // Ignore comments + if (line[0] === "#") return; + + let parts = line.split("=", 2); + + __ENV[parts[0]] = parts[1]; + }); + } catch (_err) {} +} + +loadDotEnv(); + +let config = __ENV; + +config.URL = config.URL || "ws://localhost:8080/cable"; + +let url = config.URL; +let channelName = config.CHANNEL_ID || "BenchmarkChannel"; + +let numChannels = parseInt(config.NUM_CHANNELS || "5") || 5; +let channelStreamId = __VU % numChannels; + +let sendersRatio = parseFloat(config.SENDERS_RATIO || "0.3") || 1; +let sendersMod = (1 / sendersRatio) | 0; +let sender = __VU % sendersMod == 0; + +let sendingRate = parseFloat(config.SENDING_RATE || "0.3"); + +let iterations = (config.N || "20") | 0; + +export default function () { + let cableOptions = { + receiveTimeoutMs: 15000, + logLevel: config.DEBUG === "true" ? "debug" : "info", + }; + + // Prevent from creating a lot of connections at once + sleep(randomIntBetween(2, 10) / 5); + + let client = cable.connect(url, cableOptions); + + if ( + !check(client, { + "successful connection": (obj) => obj, + }) + ) { + // Cooldown + sleep(randomIntBetween(5, 10) / 5); + fail("connection failed"); + } + + let channel = client.subscribe(channelName, { id: channelStreamId }); + + if ( + !check(channel, { + "successful subscription": (obj) => obj, + }) + ) { + // Cooldown + sleep(randomIntBetween(5, 10) / 5); + fail("failed to subscribe"); + } + + subTrend.add(channel.ackDuration()); + + let subscribedAt = Date.now(); + + let result = channel.history({ since: ((Date.now() - 10000) / 1000) | 0 }); + check(result, { + "history received": (obj) => obj, + }); + historyTrend.add(channel.historyDuration()); + + for (let i = 0; ; i++) { + // Sampling + if (sender && randomIntBetween(1, 10) / 10 <= sendingRate) { + let start = Date.now(); + broadcastsSent.add(1); + // Create message via cable instead of a form + channel.perform("broadcast", { + ts: start, + content: `hello from ${__VU} numero ${i + 1}`, + }); + } + + sleep(randomIntBetween(5, 10) / 100); + + let incoming = channel.receiveAll(1); + + for (let message of incoming) { + let received = message.__timestamp__; + + if (message.action == "broadcastResult") { + acksRcvd.add(1); + let ts = message.ts; + rttTrend.add(received - ts); + } + + if (message.action == "broadcast") { + let ts = message.ts; + + // Historical message, shouldn't be taken into account for broadcast duration + if (ts < subscribedAt) { + historyRcvd.add(1); + continue; + } + + broadcastsRcvd.add(1); + broadcastTrend.add(received - ts); + } + } + + sleep(randomIntBetween(5, 10) / 10); + + if (i > iterations) break; + } + + sleep(randomIntBetween(5, 10) / 10); + + client.disconnect(); +} diff --git a/etc/k6/results/2023-10-broker.md b/etc/k6/results/2023-10-broker.md new file mode 100644 index 00000000..da4393d7 --- /dev/null +++ b/etc/k6/results/2023-10-broker.md @@ -0,0 +1,215 @@ +# Broker benchmarks + +Scenario: broadcast + history retrieval. + +Key metrics: rtt trend, broadcast trend, history trend. + +No RPC server (gobench-cable binary). + +Multi-node setup uses Traefik as a load balancer and 3 AnyCable-Go nodes. + +## NUM_CHANNELS=5, SENDING_RATE=0.7, SENDERS_RATIO=0.5 + +This configuration helps to see the difference between brokers. Having less channels or broadcast per seconds works fine for all brokers. + +Speaking of the number we have: + +- ~125-150 broadcast/s; +- ~120-150k historical messages received out of ~1kk in total (70-80k/s). + +## Multi-node, no broker, Redis pub/sub + +This setup doesn't use any broker; we can refer to it as baseline setup for multi-node tests. + +```sh +✓ successful connection +✓ successful subscription +✗ history received +↳ 0% — ✓ 0 / ✗ 3267 + +acks_rcvd............: 7198 49.775111/s +broadcast_duration...: avg=4.13ms min=0s med=3ms max=43ms p(90)=7ms p(95)=10ms +broadcasts_rcvd......: 1688013 11672.830561/s +broadcasts_sent......: 7198 49.775111/s +checks...............: 66.66% ✓ 6534 ✗ 3267 +data_received........: 271 MB 1.9 MB/s +data_sent............: 2.9 MB 20 kB/s +history_duration.....: avg=266.91µs min=0s med=0s max=17ms p(90)=0s p(95)=1ms +history_rcvd.........: 19 0.131387/s +iteration_duration...: avg=42.11s min=39.07s med=42.12s max=45.18s p(90)=43.37s p(95)=43.74s +iterations...........: 3267 22.591732/s +rtt..................: avg=2.93ms min=0s med=2ms max=33ms p(90)=6ms p(95)=8ms +suback...............: avg=335.47µs min=0s med=0s max=19ms p(90)=0s p(95)=2ms +vus..................: 6 min=6 max=1500 +vus_max..............: 1500 min=1500 max=1500 +ws_connecting........: avg=4.14ms min=594.08µs med=1.78ms max=113.94ms p(90)=8.48ms p(95)=11.67ms +ws_msgs_received.....: 1759570 12167.65657/s +ws_msgs_sent.........: 13732 94.958575/s +ws_sessions..........: 3267 22.591732/s +``` + +## Multi-node, Redis broker, Redis pub/sub + +```sh +✓ successful connection +✓ successful subscription +✓ history received + +acks_rcvd............: 7183 49.690276/s +broadcast_duration...: avg=4.21ms min=0s med=3ms max=44ms p(90)=8ms p(95)=10ms +broadcasts_rcvd......: 1673008 11573.469106/s +broadcasts_sent......: 7183 49.690276/s +checks...............: 100.00% ✓ 9798 ✗ 0 +data_received........: 406 MB 2.8 MB/s +data_sent............: 2.9 MB 20 kB/s +history_duration.....: avg=1.71ms min=0s med=1ms max=26ms p(90)=3ms p(95)=5ms +history_rcvd.........: 269789 1866.33576/s +iteration_duration...: avg=42.07s min=38.46s med=42.07s max=45.3s p(90)=43.3s p(95)=43.63s +iterations...........: 3266 22.593407/s +rtt..................: avg=3.15ms min=0s med=3ms max=27ms p(90)=6ms p(95)=8ms +suback...............: avg=132.57µs min=0s med=0s max=12ms p(90)=0s p(95)=1ms +vus..................: 2 min=2 max=1500 +vus_max..............: 1500 min=1500 max=1500 +ws_connecting........: avg=3.01ms min=584.7µs med=1.61ms max=30.99ms p(90)=6.93ms p(95)=9.54ms +ws_msgs_received.....: 2014672 13937.018921/s +ws_msgs_sent.........: 13715 94.877089/s +ws_sessions..........: 3266 22.593407/s +``` + +## Multi-node, embedded NATS broker and pub/sub + +```sh +✓ successful connection +✓ successful subscription +✓ history received + +acks_rcvd............: 7206 49.568904/s +broadcast_duration...: avg=4.78ms min=0s med=4ms max=221ms p(90)=9ms p(95)=11ms +broadcasts_rcvd......: 1692092 11639.626116/s +broadcasts_sent......: 7206 49.568904/s +checks...............: 100.00% ✓ 9792 ✗ 0 +data_received........: 407 MB 2.8 MB/s +data_sent............: 2.9 MB 20 kB/s +history_duration.....: avg=22.44ms min=0s med=3ms max=237ms p(90)=67.7ms p(95)=92ms +history_rcvd.........: 266712 1834.668541/s +iteration_duration...: avg=42.13s min=38.9s med=42.12s max=45.35s p(90)=43.42s p(95)=43.77s +iterations...........: 3264 22.452526/s +rtt..................: avg=1.92ms min=0s med=1ms max=63ms p(90)=5ms p(95)=7ms +suback...............: avg=181.37µs min=0s med=0s max=16ms p(90)=0s p(95)=1ms +vus..................: 2 min=2 max=1500 +vus_max..............: 1500 min=1500 max=1500 +ws_connecting........: avg=3.3ms min=473.95µs med=1.67ms max=44.94ms p(90)=7.85ms p(95)=10.64ms +ws_msgs_received.....: 2030319 13966.23473/s +ws_msgs_sent.........: 13734 94.473956/s +ws_sessions..........: 3264 22.452526/s +``` + +## Single node, memory broker + +```sh +✓ successful connection +✓ successful subscription +✓ history received + +acks_rcvd............: 7157 49.461266/s +broadcast_duration...: avg=1.77ms min=0s med=1ms max=16ms p(90)=3ms p(95)=4ms +broadcasts_rcvd......: 1671812 11553.715077/s +broadcasts_sent......: 7157 49.461266/s +checks...............: 100.00% ✓ 9786 ✗ 0 +data_received........: 404 MB 2.8 MB/s +data_sent............: 2.9 MB 20 kB/s +history_duration.....: avg=931.94µs min=0s med=1ms max=21ms p(90)=2ms p(95)=3ms +history_rcvd.........: 267642 1849.645421/s +iteration_duration...: avg=42.13s min=39.04s med=42.14s max=45.28s p(90)=43.34s p(95)=43.68s +iterations...........: 3262 22.543335/s +rtt..................: avg=585.72µs min=0s med=0s max=11ms p(90)=2ms p(95)=3ms +suback...............: avg=77.25µs min=0s med=0s max=8ms p(90)=0s p(95)=0s +vus..................: 6 min=6 max=1500 +vus_max..............: 1500 min=1500 max=1500 +ws_connecting........: avg=948.18µs min=167µs med=521.35µs max=11.82ms p(90)=2.06ms p(95)=3.08ms +ws_msgs_received.....: 2010091 13891.525299/s +ws_msgs_sent.........: 13681 94.547937/s +ws_sessions..........: 3262 22.543335/s +``` + +## Single node, Redis broker + +```sh +✓ successful connection +✓ successful subscription +✓ history received + +acks_rcvd............: 7181 49.690377/s +broadcast_duration...: avg=2.32ms min=0s med=2ms max=32ms p(90)=4ms p(95)=5ms +broadcasts_rcvd......: 1671940 11569.325909/s +broadcasts_sent......: 7181 49.690377/s +checks...............: 100.00% ✓ 9804 ✗ 0 +data_received........: 404 MB 2.8 MB/s +data_sent............: 2.9 MB 20 kB/s +history_duration.....: avg=1.55ms min=0s med=1ms max=18ms p(90)=3ms p(95)=4ms +history_rcvd.........: 269239 1863.053542/s +iteration_duration...: avg=42.11s min=38.69s med=42.11s max=45.73s p(90)=43.33s p(95)=43.69s +iterations...........: 3268 22.613585/s +rtt..................: avg=863.66µs min=0s med=1ms max=31ms p(90)=2ms p(95)=3ms +suback...............: avg=110.77µs min=0s med=0s max=19ms p(90)=0s p(95)=1ms +vus..................: 5 min=5 max=1500 +vus_max..............: 1500 min=1500 max=1500 +ws_connecting........: avg=876.93µs min=115.54µs med=524.25µs max=12.09ms p(90)=2.02ms p(95)=2.81ms +ws_msgs_received.....: 2012477 13925.740336/s +ws_msgs_sent.........: 13717 94.917547/s +ws_sessions..........: 3268 22.613585/s +``` + +## Single node, embedded NATS broker + +```sh +✓ successful connection +✓ successful subscription +✓ history received + +acks_rcvd............: 7205 49.640288/s +broadcast_duration...: avg=2.16ms min=0s med=2ms max=195ms p(90)=4ms p(95)=5ms +broadcasts_rcvd......: 1676679 11551.815128/s +broadcasts_sent......: 7205 49.640288/s +checks...............: 100.00% ✓ 9798 ✗ 0 +data_received........: 404 MB 2.8 MB/s +data_sent............: 2.9 MB 20 kB/s +history_duration.....: avg=20.16ms min=0s med=2ms max=213ms p(90)=60ms p(95)=86ms +history_rcvd.........: 268459 1849.601944/s +iteration_duration...: avg=42.1s min=39.17s med=42.09s max=45.55s p(90)=43.41s p(95)=43.78s +iterations...........: 3266 22.50176/s +rtt..................: avg=934.21µs min=0s med=1ms max=64ms p(90)=2ms p(95)=3ms +suback...............: avg=96.44µs min=0s med=0s max=20ms p(90)=0s p(95)=0s +vus..................: 1 min=1 max=1500 +vus_max..............: 1500 min=1500 max=1500 +ws_connecting........: avg=914.67µs min=177.33µs med=512.52µs max=31.01ms p(90)=2.13ms p(95)=3ms +ws_msgs_received.....: 2016894 13895.794378/s +ws_msgs_sent.........: 13737 94.643807/s +ws_sessions..........: 3266 22.50176/s +``` + +## Notes + +Commands used to run server with different configurations: + +```sh +# Single node, no broker +PORT=8080 ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info make run-gobench + +# Single node, memory +ANYCABLE_BROKER=memory PORT=8080 ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info make run-gobench + +# Single node, Redis +ANYCABLE_BROKER=redis PORT=8080 ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info make run-gobench + +# Multi-node +PORT=8081 ANYCABLE_BROKER=redis ANYCABLE_PUBSUB=redis ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info make run-gobench +PORT=8082 ANYCABLE_BROKER=redis ANYCABLE_PUBSUB=redis ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info make run-gobench +PORT=8083 ANYCABLE_BROKER=redis ANYCABLE_PUBSUB=redis ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info make run-gobench +(cd etc/traefik && traefik --configFile=traefik.yml) + +# Multi-node w/ eNATS +ANYCABLE_EMBED_NATS=1 ANYCABLE_ENATS_CLUSTER=nats://localhost:4342 ANYCABLE_ENATS_ADDR=nats://localhost:4242 ANYCABLE_ENATS_CLUSTER_ROUTES=nats://localhost:4342 PORT=8081 ANYCABLE_PRESETS=broker ANYCABLE_BROADCAST_ADAPTER=nats ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info ANYCABLE_ENATS_DEBUG=true make run-gobench +ANYCABLE_EMBED_NATS=1 ANYCABLE_ENATS_CLUSTER=nats://localhost:4343 ANYCABLE_ENATS_CLUSTER_ROUTES=nats://localhost:4342 ANYCABLE_ENATS_ADDR=nats://localhost:4243 PORT=8082 ANYCABLE_PRESETS=broker ANYCABLE_BROADCAST_ADAPTER=nats ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info ANYCABLE_ENATS_DEBUG=true make run-gobench +ANYCABLE_EMBED_NATS=1 ANYCABLE_ENATS_CLUSTER=nats://localhost:4344 ANYCABLE_ENATS_CLUSTER_ROUTES=nats://localhost:4342 ANYCABLE_ENATS_ADDR=nats://localhost:4244 PORT=8083 ANYCABLE_PRESETS=broker ANYCABLE_BROADCAST_ADAPTER=nats ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info ANYCABLE_ENATS_DEBUG=true make run-gobench +``` diff --git a/etc/traefik/rules.toml b/etc/traefik/rules.toml new file mode 100644 index 00000000..0fe1331a --- /dev/null +++ b/etc/traefik/rules.toml @@ -0,0 +1,13 @@ +# rules.toml + +[http.routers.websocket] + rule = "Host(`localhost`) && Path(`/cable`)" + service = "websocket" + +[http.services.websocket.loadBalancer] + [[http.services.websocket.loadBalancer.servers]] + url = "http://localhost:8081" + [[http.services.websocket.loadBalancer.servers]] + url = "http://localhost:8082" + [[http.services.websocket.loadBalancer.servers]] + url = "http://localhost:8083" diff --git a/etc/traefik/traefik.yml b/etc/traefik/traefik.yml new file mode 100644 index 00000000..334fc74a --- /dev/null +++ b/etc/traefik/traefik.yml @@ -0,0 +1,9 @@ +# traefik.yml + +entryPoints: + web: + address: ":8080" + +providers: + file: + filename: "./rules.toml"