Skip to content

Commit

Permalink
dev: add broker benchmarks
Browse files Browse the repository at this point in the history
  • Loading branch information
@palkan
palkan committed Nov 2, 2023
1 parent 9660cc4 commit 0ec3c62
Show file tree
Hide file tree
Showing 4 changed files with 402 additions and 0 deletions.
165 changes: 165 additions & 0 deletions etc/k6/broker.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
// Build k6 with xk6-cable like this:
// xk6 build v0.38.3 --with github.com/anycable/[email protected]

import { check, sleep, fail } from "k6";
import cable from "k6/x/cable";
import { randomIntBetween } from "https://jslib.k6.io/k6-utils/1.1.0/index.js";

const rampingOptions = {
scenarios: {
default: {
executor: "ramping-vus",
startVUs: 100,
stages: [
{ duration: "10s", target: 300 },
{ duration: "10s", target: 500 },
{ duration: "10s", target: 1000 },
{ duration: "10s", target: 1300 },
{ duration: "10s", target: 1500 },
{ duration: "30s", target: 1500 },
{ duration: "30s", target: 0 },
],
gracefulStop: "1m",
gracefulRampDown: "1m",
},
},
};

export const options = __ENV.SKIP_OPTIONS ? {} : rampingOptions;

import { Trend, Counter } from "k6/metrics";
let rttTrend = new Trend("rtt", true);
let subTrend = new Trend("suback", true);
let broadcastTrend = new Trend("broadcast_duration", true);
let historyTrend = new Trend("history_duration", true);
let historyRcvd = new Counter("history_rcvd");
let broadcastsSent = new Counter("broadcasts_sent");
let broadcastsRcvd = new Counter("broadcasts_rcvd");
let acksRcvd = new Counter("acks_rcvd");

// Load ENV from .env
function loadDotEnv() {
try {
let dotenv = open("./.env");
dotenv.split(/[\n\r]/m).forEach((line) => {
// Ignore comments
if (line[0] === "#") return;

let parts = line.split("=", 2);

__ENV[parts[0]] = parts[1];
});
} catch (_err) {}
}

loadDotEnv();

let config = __ENV;

config.URL = config.URL || "ws://localhost:8080/cable";

let url = config.URL;
let channelName = config.CHANNEL_ID || "BenchmarkChannel";

let numChannels = parseInt(config.NUM_CHANNELS || "5") || 5;
let channelStreamId = __VU % numChannels;

let sendersRatio = parseFloat(config.SENDERS_RATIO || "0.3") || 1;
let sendersMod = (1 / sendersRatio) | 0;
let sender = __VU % sendersMod == 0;

let sendingRate = parseFloat(config.SENDING_RATE || "0.3");

let iterations = (config.N || "20") | 0;

export default function () {
let cableOptions = {
receiveTimeoutMs: 15000,
logLevel: config.DEBUG === "true" ? "debug" : "info",
};

// Prevent from creating a lot of connections at once
sleep(randomIntBetween(2, 10) / 5);

let client = cable.connect(url, cableOptions);

if (
!check(client, {
"successful connection": (obj) => obj,
})
) {
// Cooldown
sleep(randomIntBetween(5, 10) / 5);
fail("connection failed");
}

let channel = client.subscribe(channelName, { id: channelStreamId });

if (
!check(channel, {
"successful subscription": (obj) => obj,
})
) {
// Cooldown
sleep(randomIntBetween(5, 10) / 5);
fail("failed to subscribe");
}

subTrend.add(channel.ackDuration());

let subscribedAt = Date.now();

let result = channel.history({ since: ((Date.now() - 10000) / 1000) | 0 });
check(result, {
"history received": (obj) => obj,
});
historyTrend.add(channel.historyDuration());

for (let i = 0; ; i++) {
// Sampling
if (sender && randomIntBetween(1, 10) / 10 <= sendingRate) {
let start = Date.now();
broadcastsSent.add(1);
// Create message via cable instead of a form
channel.perform("broadcast", {
ts: start,
content: `hello from ${__VU} numero ${i + 1}`,
});
}

sleep(randomIntBetween(5, 10) / 100);

let incoming = channel.receiveAll(1);

for (let message of incoming) {
let received = message.__timestamp__;

if (message.action == "broadcastResult") {
acksRcvd.add(1);
let ts = message.ts;
rttTrend.add(received - ts);
}

if (message.action == "broadcast") {
let ts = message.ts;

// Historical message, shouldn't be taken into account for broadcast duration
if (ts < subscribedAt) {
historyRcvd.add(1);
continue;
}

broadcastsRcvd.add(1);
broadcastTrend.add(received - ts);
}
}

sleep(randomIntBetween(5, 10) / 10);

if (i > iterations) break;
}

sleep(randomIntBetween(5, 10) / 10);

client.disconnect();
}
215 changes: 215 additions & 0 deletions etc/k6/results/2023-10-broker.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
# Broker benchmarks

Scenario: broadcast + history retrieval.

Key metrics: rtt trend, broadcast trend, history trend.

No RPC server (gobench-cable binary).

Multi-node setup uses Traefik as a load balancer and 3 AnyCable-Go nodes.

## NUM_CHANNELS=5, SENDING_RATE=0.7, SENDERS_RATIO=0.5

This configuration helps to see the difference between brokers. Having less channels or broadcast per seconds works fine for all brokers.

Speaking of the number we have:

- ~125-150 broadcast/s;
- ~120-150k historical messages received out of ~1kk in total (70-80k/s).

## Multi-node, no broker, Redis pub/sub

This setup doesn't use any broker; we can refer to it as baseline setup for multi-node tests.

```sh
✓ successful connection
✓ successful subscription
✗ history received
↳ 0% — ✓ 0 / ✗ 3267

acks_rcvd............: 7198 49.775111/s
broadcast_duration...: avg=4.13ms min=0s med=3ms max=43ms p(90)=7ms p(95)=10ms
broadcasts_rcvd......: 1688013 11672.830561/s
broadcasts_sent......: 7198 49.775111/s
checks...............: 66.66% ✓ 6534 ✗ 3267
data_received........: 271 MB 1.9 MB/s
data_sent............: 2.9 MB 20 kB/s
history_duration.....: avg=266.91µs min=0s med=0s max=17ms p(90)=0s p(95)=1ms
history_rcvd.........: 19 0.131387/s
iteration_duration...: avg=42.11s min=39.07s med=42.12s max=45.18s p(90)=43.37s p(95)=43.74s
iterations...........: 3267 22.591732/s
rtt..................: avg=2.93ms min=0s med=2ms max=33ms p(90)=6ms p(95)=8ms
suback...............: avg=335.47µs min=0s med=0s max=19ms p(90)=0s p(95)=2ms
vus..................: 6 min=6 max=1500
vus_max..............: 1500 min=1500 max=1500
ws_connecting........: avg=4.14ms min=594.08µs med=1.78ms max=113.94ms p(90)=8.48ms p(95)=11.67ms
ws_msgs_received.....: 1759570 12167.65657/s
ws_msgs_sent.........: 13732 94.958575/s
ws_sessions..........: 3267 22.591732/s
```

## Multi-node, Redis broker, Redis pub/sub

```sh
✓ successful connection
✓ successful subscription
✓ history received

acks_rcvd............: 7183 49.690276/s
broadcast_duration...: avg=4.21ms min=0s med=3ms max=44ms p(90)=8ms p(95)=10ms
broadcasts_rcvd......: 1673008 11573.469106/s
broadcasts_sent......: 7183 49.690276/s
checks...............: 100.00% ✓ 9798 ✗ 0
data_received........: 406 MB 2.8 MB/s
data_sent............: 2.9 MB 20 kB/s
history_duration.....: avg=1.71ms min=0s med=1ms max=26ms p(90)=3ms p(95)=5ms
history_rcvd.........: 269789 1866.33576/s
iteration_duration...: avg=42.07s min=38.46s med=42.07s max=45.3s p(90)=43.3s p(95)=43.63s
iterations...........: 3266 22.593407/s
rtt..................: avg=3.15ms min=0s med=3ms max=27ms p(90)=6ms p(95)=8ms
suback...............: avg=132.57µs min=0s med=0s max=12ms p(90)=0s p(95)=1ms
vus..................: 2 min=2 max=1500
vus_max..............: 1500 min=1500 max=1500
ws_connecting........: avg=3.01ms min=584.7µs med=1.61ms max=30.99ms p(90)=6.93ms p(95)=9.54ms
ws_msgs_received.....: 2014672 13937.018921/s
ws_msgs_sent.........: 13715 94.877089/s
ws_sessions..........: 3266 22.593407/s
```

## Multi-node, embedded NATS broker and pub/sub

```sh
✓ successful connection
✓ successful subscription
✓ history received

acks_rcvd............: 7206 49.568904/s
broadcast_duration...: avg=4.78ms min=0s med=4ms max=221ms p(90)=9ms p(95)=11ms
broadcasts_rcvd......: 1692092 11639.626116/s
broadcasts_sent......: 7206 49.568904/s
checks...............: 100.00% ✓ 9792 ✗ 0
data_received........: 407 MB 2.8 MB/s
data_sent............: 2.9 MB 20 kB/s
history_duration.....: avg=22.44ms min=0s med=3ms max=237ms p(90)=67.7ms p(95)=92ms
history_rcvd.........: 266712 1834.668541/s
iteration_duration...: avg=42.13s min=38.9s med=42.12s max=45.35s p(90)=43.42s p(95)=43.77s
iterations...........: 3264 22.452526/s
rtt..................: avg=1.92ms min=0s med=1ms max=63ms p(90)=5ms p(95)=7ms
suback...............: avg=181.37µs min=0s med=0s max=16ms p(90)=0s p(95)=1ms
vus..................: 2 min=2 max=1500
vus_max..............: 1500 min=1500 max=1500
ws_connecting........: avg=3.3ms min=473.95µs med=1.67ms max=44.94ms p(90)=7.85ms p(95)=10.64ms
ws_msgs_received.....: 2030319 13966.23473/s
ws_msgs_sent.........: 13734 94.473956/s
ws_sessions..........: 3264 22.452526/s
```

## Single node, memory broker

```sh
✓ successful connection
✓ successful subscription
✓ history received

acks_rcvd............: 7157 49.461266/s
broadcast_duration...: avg=1.77ms min=0s med=1ms max=16ms p(90)=3ms p(95)=4ms
broadcasts_rcvd......: 1671812 11553.715077/s
broadcasts_sent......: 7157 49.461266/s
checks...............: 100.00% ✓ 9786 ✗ 0
data_received........: 404 MB 2.8 MB/s
data_sent............: 2.9 MB 20 kB/s
history_duration.....: avg=931.94µs min=0s med=1ms max=21ms p(90)=2ms p(95)=3ms
history_rcvd.........: 267642 1849.645421/s
iteration_duration...: avg=42.13s min=39.04s med=42.14s max=45.28s p(90)=43.34s p(95)=43.68s
iterations...........: 3262 22.543335/s
rtt..................: avg=585.72µs min=0s med=0s max=11ms p(90)=2ms p(95)=3ms
suback...............: avg=77.25µs min=0s med=0s max=8ms p(90)=0s p(95)=0s
vus..................: 6 min=6 max=1500
vus_max..............: 1500 min=1500 max=1500
ws_connecting........: avg=948.18µs min=167µs med=521.35µs max=11.82ms p(90)=2.06ms p(95)=3.08ms
ws_msgs_received.....: 2010091 13891.525299/s
ws_msgs_sent.........: 13681 94.547937/s
ws_sessions..........: 3262 22.543335/s
```

## Single node, Redis broker

```sh
✓ successful connection
✓ successful subscription
✓ history received

acks_rcvd............: 7181 49.690377/s
broadcast_duration...: avg=2.32ms min=0s med=2ms max=32ms p(90)=4ms p(95)=5ms
broadcasts_rcvd......: 1671940 11569.325909/s
broadcasts_sent......: 7181 49.690377/s
checks...............: 100.00% ✓ 9804 ✗ 0
data_received........: 404 MB 2.8 MB/s
data_sent............: 2.9 MB 20 kB/s
history_duration.....: avg=1.55ms min=0s med=1ms max=18ms p(90)=3ms p(95)=4ms
history_rcvd.........: 269239 1863.053542/s
iteration_duration...: avg=42.11s min=38.69s med=42.11s max=45.73s p(90)=43.33s p(95)=43.69s
iterations...........: 3268 22.613585/s
rtt..................: avg=863.66µs min=0s med=1ms max=31ms p(90)=2ms p(95)=3ms
suback...............: avg=110.77µs min=0s med=0s max=19ms p(90)=0s p(95)=1ms
vus..................: 5 min=5 max=1500
vus_max..............: 1500 min=1500 max=1500
ws_connecting........: avg=876.93µs min=115.54µs med=524.25µs max=12.09ms p(90)=2.02ms p(95)=2.81ms
ws_msgs_received.....: 2012477 13925.740336/s
ws_msgs_sent.........: 13717 94.917547/s
ws_sessions..........: 3268 22.613585/s
```

## Single node, embedded NATS broker

```sh
✓ successful connection
✓ successful subscription
✓ history received

acks_rcvd............: 7205 49.640288/s
broadcast_duration...: avg=2.16ms min=0s med=2ms max=195ms p(90)=4ms p(95)=5ms
broadcasts_rcvd......: 1676679 11551.815128/s
broadcasts_sent......: 7205 49.640288/s
checks...............: 100.00% ✓ 9798 ✗ 0
data_received........: 404 MB 2.8 MB/s
data_sent............: 2.9 MB 20 kB/s
history_duration.....: avg=20.16ms min=0s med=2ms max=213ms p(90)=60ms p(95)=86ms
history_rcvd.........: 268459 1849.601944/s
iteration_duration...: avg=42.1s min=39.17s med=42.09s max=45.55s p(90)=43.41s p(95)=43.78s
iterations...........: 3266 22.50176/s
rtt..................: avg=934.21µs min=0s med=1ms max=64ms p(90)=2ms p(95)=3ms
suback...............: avg=96.44µs min=0s med=0s max=20ms p(90)=0s p(95)=0s
vus..................: 1 min=1 max=1500
vus_max..............: 1500 min=1500 max=1500
ws_connecting........: avg=914.67µs min=177.33µs med=512.52µs max=31.01ms p(90)=2.13ms p(95)=3ms
ws_msgs_received.....: 2016894 13895.794378/s
ws_msgs_sent.........: 13737 94.643807/s
ws_sessions..........: 3266 22.50176/s
```

## Notes

Commands used to run server with different configurations:

```sh
# Single node, no broker
PORT=8080 ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info make run-gobench

# Single node, memory
ANYCABLE_BROKER=memory PORT=8080 ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info make run-gobench

# Single node, Redis
ANYCABLE_BROKER=redis PORT=8080 ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info make run-gobench

# Multi-node
PORT=8081 ANYCABLE_BROKER=redis ANYCABLE_PUBSUB=redis ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info make run-gobench
PORT=8082 ANYCABLE_BROKER=redis ANYCABLE_PUBSUB=redis ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info make run-gobench
PORT=8083 ANYCABLE_BROKER=redis ANYCABLE_PUBSUB=redis ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info make run-gobench
(cd etc/traefik && traefik --configFile=traefik.yml)

# Multi-node w/ eNATS
ANYCABLE_EMBED_NATS=1 ANYCABLE_ENATS_CLUSTER=nats://localhost:4342 ANYCABLE_ENATS_ADDR=nats://localhost:4242 ANYCABLE_ENATS_CLUSTER_ROUTES=nats://localhost:4342 PORT=8081 ANYCABLE_PRESETS=broker ANYCABLE_BROADCAST_ADAPTER=nats ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info ANYCABLE_ENATS_DEBUG=true make run-gobench
ANYCABLE_EMBED_NATS=1 ANYCABLE_ENATS_CLUSTER=nats://localhost:4343 ANYCABLE_ENATS_CLUSTER_ROUTES=nats://localhost:4342 ANYCABLE_ENATS_ADDR=nats://localhost:4243 PORT=8082 ANYCABLE_PRESETS=broker ANYCABLE_BROADCAST_ADAPTER=nats ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info ANYCABLE_ENATS_DEBUG=true make run-gobench
ANYCABLE_EMBED_NATS=1 ANYCABLE_ENATS_CLUSTER=nats://localhost:4344 ANYCABLE_ENATS_CLUSTER_ROUTES=nats://localhost:4342 ANYCABLE_ENATS_ADDR=nats://localhost:4244 PORT=8083 ANYCABLE_PRESETS=broker ANYCABLE_BROADCAST_ADAPTER=nats ANYCABLE_DEBUG=0 ANYCABLE_LOG_LEVEL=info ANYCABLE_ENATS_DEBUG=true make run-gobench
```
13 changes: 13 additions & 0 deletions etc/traefik/rules.toml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# rules.toml

[http.routers.websocket]
rule = "Host(`localhost`) && Path(`/cable`)"
service = "websocket"

[http.services.websocket.loadBalancer]
[[http.services.websocket.loadBalancer.servers]]
url = "http://localhost:8081"
[[http.services.websocket.loadBalancer.servers]]
url = "http://localhost:8082"
[[http.services.websocket.loadBalancer.servers]]
url = "http://localhost:8083"
Loading

0 comments on commit 0ec3c62

Please sign in to comment.