v0.2

grapheo12 · Dec 10, 2024 · 73cb5ae · 73cb5ae
1 parent 8ddd8f8
commit 73cb5ae
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 24 deletions.
diff --git a/experiments/lan_experiment.sh b/experiments/lan_experiment.sh
@@ -7,7 +7,11 @@ set -o xtrace
 
 # ALL_CLIENTS="-c 1 -c 10 -c 100 -c 500 -c 700 -c 1000 -c 1200 -c 1500 -c 2000"
 # ALL_CLIENTS="-c 100 -c 200 -c 300 -c 500 -c 700 -c 900 -c 1000 -c 1200 -c 1500 -c 1800 -c 2000 -c 2500"
-ALL_CLIENTS="-c 900 -c 1500 -c 2500 -c 5000"
+ALL_CLIENTS="-c 500 -c 700"
+# ALL_CLIENTS="-c 10 -c 20 -c 30 -c 40"
+# ALL_CLIENTS="-c 50 -c 60 -c 70 -c 80"
+# ALL_CLIENTS="-c 60 -c 70 -c 90 -c 100 -c 1200"
+# ALL_CLIENTS="-c 900 -c 1500 -c 2500 -c 5000"
 # ALL_CLIENTS="-c 200 -c 300 -c 500 -c 1000 -c 1500"
 # ALL_CLIENTS="-c 1800 -c 2000 -c 2500"
 # ALL_CLIENTS="-c 1700 -c 2000 -c 2500 -c 3000"
@@ -20,14 +24,14 @@ ALL_CLIENTS="-c 900 -c 1500 -c 2500 -c 5000"
 
 start_time=$(date -Ins)
 # start_time='2024-08-07T10:39:54.859389+00:00'
-RUN_CMD="python3 scripts/run_remote_client_sweep.py -nt /tmp/local_template.json -ct scripts/local_client_template.json -ips ../nodelist.txt -i ../cluster_key.pem -r 2 -s 60 -up 10 -down 10 $ALL_CLIENTS"
+RUN_CMD="python3 scripts/run_remote_client_sweep.py -nt /tmp/local_template.json -ct scripts/local_client_template.json -ips ../nodelist.txt -i ../cluster_key.pem -r 1 -s 60 -up 10 -down 10 $ALL_CLIENTS"
 
 
 jq '.consensus_config.max_backlog_batch_size = 1000 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 50 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
 
-# Run pirateship
-make
-$RUN_CMD
+# # Run pirateship
+# make
+# $RUN_CMD
 
 # Run chained_pbft
 make chained_pbft_logger
@@ -48,7 +52,7 @@ end_time=$(date -Ins)
 # Plot together
 python3 scripts/plot_time_range_client_sweep.py \
     --path logs --end $end_time --start $start_time \
-    -r 2 -c 3 -l node1 -up 10 -down 10 -o plot.png \
-    --legend "pirateship+byz" \
-    --legend "pbft+onlybyz" \
+    -r 3 -c 3 -l node1 -up 30 -down 30 -o plot.png \
+    --legend "pbft+onlybyz"
     # --legend "signed_raft"
+    # --legend "pirateship+byz" \
diff --git a/experiments/sign_effect.sh b/experiments/sign_effect.sh
@@ -5,56 +5,56 @@
 
 set -o xtrace
 
-ALL_CLIENTS="-c 1200"
+ALL_CLIENTS="-c 1500"
 
 
 start_time=$(date -Ins)
 # start_time='2024-08-07T10:39:54.859389+00:00'
 RUN_CMD="python3 scripts/run_remote_client_sweep.py -nt /tmp/local_template.json -ct scripts/local_client_template.json -ips ../nodelist.txt -i ../cluster_key.pem -r 1 -s 120 -up 40 -down 20 $ALL_CLIENTS"
 
-jq '.consensus_config.max_backlog_batch_size = 1000 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 1 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
+jq '.consensus_config.max_backlog_batch_size = 1 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 1 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
 
 # Run pirateship
 make signed_raft_logger
 $RUN_CMD
 
-jq '.consensus_config.max_backlog_batch_size = 1000 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 2 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
+jq '.consensus_config.max_backlog_batch_size = 1 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 2 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
 
 # Run pirateship
 make signed_raft_logger
 $RUN_CMD
 
-jq '.consensus_config.max_backlog_batch_size = 1000 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 5 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
+jq '.consensus_config.max_backlog_batch_size = 1 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 5 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
 
 # Run pirateship
 make signed_raft_logger
 $RUN_CMD
 
-jq '.consensus_config.max_backlog_batch_size = 1000 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 10 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
+jq '.consensus_config.max_backlog_batch_size = 1 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 10 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
 
 # Run pirateship
 make signed_raft_logger
 $RUN_CMD
 
-# jq '.consensus_config.max_backlog_batch_size = 1000 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 10 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
+# jq '.consensus_config.max_backlog_batch_size = 1 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 10 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
 
 # # Run pirateship
 # make signed_raft_logger
 # $RUN_CMD
 
-# jq '.consensus_config.max_backlog_batch_size = 1000 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 20 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
+# jq '.consensus_config.max_backlog_batch_size = 1 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 20 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
 
 # # Run pirateship
 # make signed_raft_logger
 # $RUN_CMD
 
-# jq '.consensus_config.max_backlog_batch_size = 1000 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 50 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
+# jq '.consensus_config.max_backlog_batch_size = 1 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 50 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
 
 # # Run pirateship
 # make signed_raft_logger
 # $RUN_CMD
 
-jq '.consensus_config.max_backlog_batch_size = 1000 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 50 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
+jq '.consensus_config.max_backlog_batch_size = 1 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 50 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
 
 # Run pirateship
 make lucky_raft_logger

diff --git a/experiments/storage_test.sh b/experiments/storage_test.sh
@@ -5,12 +5,12 @@
 
 set -o xtrace
 
-ALL_CLIENTS="-c 1500"
+ALL_CLIENTS="-c 2000"
 
 
 start_time=$(date -Ins)
 # start_time='2023-10-30T20:35:31.196252+00:00'
-RUN_CMD="python3 scripts/run_remote_client_sweep.py -nt /tmp/local_template.json -ct scripts/local_client_template.json -ips ../nodelist.txt -i ../cluster_key.pem -r 3 -s 120 -up 40 -down 20 $ALL_CLIENTS"
+RUN_CMD="python3 scripts/run_remote_client_sweep.py -nt /tmp/local_template.json -ct scripts/local_client_template.json -ips ../nodelist.txt -i ../cluster_key.pem -r 1 -s 60 -up 10 -down 10 $ALL_CLIENTS"
 
 jq '.consensus_config.max_backlog_batch_size = 1000 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 50 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 4000' scripts/local_template.json > /tmp/local_template.json
 

diff --git a/experiments/wan_experiment.sh b/experiments/wan_experiment.sh
@@ -22,7 +22,7 @@ start_time=$(date -Ins)
 
 python3 experiments/setup_nodelist.py c1 ..
 
-jq '.consensus_config.batch_max_delay_ms = 25 | .consensus_config.signature_max_delay_ms = 2500 | .consensus_config.max_backlog_batch_size = 1000 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 50 | .consensus_config.liveness_u = 1 | .consensus_config.view_timeout_ms = 8000' scripts/local_template.json > /tmp/local_template.json
+jq '.consensus_config.batch_max_delay_ms = 2 | .consensus_config.signature_max_delay_ms = 2500 | .consensus_config.max_backlog_batch_size = 1000 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 50 | .consensus_config.liveness_u = 1 | .consensus_config.view_timeout_ms = 8000' scripts/local_template.json > /tmp/local_template.json
 
 python3 scripts/run_remote_client_sweep.py \
     -nt /tmp/local_template.json \
@@ -34,7 +34,7 @@ python3 experiments/setup_nodelist.py restore ..
 
 python3 experiments/setup_nodelist.py c2 ..
 
-jq '.consensus_config.batch_max_delay_ms = 25 | .consensus_config.signature_max_delay_ms = 2500 | .consensus_config.max_backlog_batch_size = 1000 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 50 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 8000' scripts/local_template.json > /tmp/local_template.json
+jq '.consensus_config.batch_max_delay_ms = 2 | .consensus_config.signature_max_delay_ms = 2500 | .consensus_config.max_backlog_batch_size = 1000 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 50 | .consensus_config.liveness_u = 2 | .consensus_config.view_timeout_ms = 8000' scripts/local_template.json > /tmp/local_template.json
 
 python3 scripts/run_remote_client_sweep.py \
     -nt /tmp/local_template.json \
@@ -47,7 +47,7 @@ python3 experiments/setup_nodelist.py restore ..
 
 python3 experiments/setup_nodelist.py c3 ..
 
-jq '.consensus_config.batch_max_delay_ms = 25 | .consensus_config.signature_max_delay_ms = 2500 | .consensus_config.max_backlog_batch_size = 1000 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 50 | .consensus_config.liveness_u = 3 | .consensus_config.view_timeout_ms = 8000' scripts/local_template.json > /tmp/local_template.json
+jq '.consensus_config.batch_max_delay_ms = 2 | .consensus_config.signature_max_delay_ms = 2500 | .consensus_config.max_backlog_batch_size = 1000 | .consensus_config.quorum_diversity_k = 3 | .consensus_config.signature_max_delay_blocks = 50 | .consensus_config.liveness_u = 3 | .consensus_config.view_timeout_ms = 8000' scripts/local_template.json > /tmp/local_template.json
 
 python3 scripts/run_remote_client_sweep.py \
     -nt /tmp/local_template.json \

diff --git a/scripts/plot_latency_cdf.py b/scripts/plot_latency_cdf.py
@@ -63,7 +63,7 @@ def plot_all_latency_cdfs(path, start, end, num_clients, ramp_up, ramp_down, out
             continue
         latencies, stat = parse_only_client_latencies(d, num_clients, ramp_up, ramp_down)
         latencies = [l / 1000.0 for l in latencies]
-        lat_p99 = quantiles(latencies, n=100)[97]
+        lat_p99 = quantiles(latencies, n=100)[90]
         lat_min = min(latencies)
 
         if lat_p99 > max_p99:
@@ -75,8 +75,9 @@ def plot_all_latency_cdfs(path, start, end, num_clients, ramp_up, ramp_down, out
         plt.plot(stat[0] / 1000.0, stat[1], label=legend[i])
 
     plt.xlim(min_min, max_p99)
+    # plt.xscale("log")
     plt.ylim(0, 1)
-    plt.legend()
+    plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.4), ncol=len(legend) //2)
 
     plt.xlabel("Latency (ms)")
     plt.ylabel("CDF")