Submit transactions to builder private mempool

Add an option in `submit-transactions` to submit to a builder. Add
a second transaction generator task in the demos to submit to the
builder. This both serves as a test of the builder API and allows
us to directly compare the latencies of transactions submitted via
the public and private mempools.

.env

@@ -21,9 +21,9 @@ ESPRESSO_ORCHESTRATOR_PORT=40001
 ESPRESSO_ORCHESTRATOR_NUM_NODES=5
 ESPRESSO_ORCHESTRATOR_START_DELAY=5s
 ESPRESSO_ORCHESTRATOR_NEXT_VIEW_TIMEOUT=30s
-ESPRESSO_ORCHESTRATOR_MIN_TRANSACTIONS=50
-ESPRESSO_ORCHESTRATOR_MIN_PROPOSE_TIME=1s
-ESPRESSO_ORCHESTRATOR_MAX_PROPOSE_TIME=2s
+ESPRESSO_ORCHESTRATOR_MIN_TRANSACTIONS=0
+ESPRESSO_ORCHESTRATOR_MIN_PROPOSE_TIME=0s
+ESPRESSO_ORCHESTRATOR_MAX_PROPOSE_TIME=20s
 ESPRESSO_SEQUENCER_CDN_ENDPOINT=marshal-0:${ESPRESSO_CDN_SERVER_PORT}
 ESPRESSO_SEQUENCER_ORCHESTRATOR_URL=http://orchestrator:${ESPRESSO_ORCHESTRATOR_PORT}
 ESPRESSO_SEQUENCER_API_PORT=50000
@@ -104,8 +104,9 @@ ESPRESSO_BUILDER_WEBSERVER_RESPONSE_TIMEOUT_DURATION=1s
 ESPRESSO_BUILDER_BUFFER_VIEW_NUM_COUNT=15
 
 # Load generator
-ESPRESSO_SUBMIT_TRANSACTIONS_DELAY=1s
-ESPRESSO_SUBMIT_TRANSACTIONS_PORT=44010
+ESPRESSO_SUBMIT_TRANSACTIONS_DELAY=2s
+ESPRESSO_SUBMIT_TRANSACTIONS_PUBLIC_PORT=44010
+ESPRESSO_SUBMIT_TRANSACTIONS_PRIVATE_PORT=44020
 
 # Query service stress test
 ESPRESSO_NASTY_CLIENT_PORT=44011

docker-compose.yaml

@@ -397,20 +397,38 @@ services:
       deploy-contracts:
         condition: service_completed_successfully
 
-  submit-transactions:
+  submit-transactions-public:
     image: ghcr.io/espressosystems/espresso-sequencer/submit-transactions:main
     ports:
-      - "$ESPRESSO_SUBMIT_TRANSACTIONS_PORT:$ESPRESSO_SUBMIT_TRANSACTIONS_PORT"
+      - "$ESPRESSO_SUBMIT_TRANSACTIONS_PUBLIC_PORT:8080"
     environment:
-      - ESPRESSO_SUBMIT_TRANSACTIONS_SUBMIT_URL=$ESPRESSO_SEQUENCER_URL
+      - ESPRESSO_SUBMIT_TRANSACTIONS_PORT=8080
       - ESPRESSO_SUBMIT_TRANSACTIONS_DELAY
-      - ESPRESSO_SUBMIT_TRANSACTIONS_PORT
+      - ESPRESSO_SEQUENCER_URL
+      - RUST_LOG
+      - RUST_LOG_FORMAT
+      - ASYNC_STD_THREAD_COUNT
+    depends_on:
+      sequencer0:
+        condition: service_healthy
+
+  submit-transactions-private:
+    image: ghcr.io/espressosystems/espresso-sequencer/submit-transactions:main
+    ports:
+      - "$ESPRESSO_SUBMIT_TRANSACTIONS_PRIVATE_PORT:8080"
+    environment:
+      - ESPRESSO_SUBMIT_TRANSACTIONS_PORT=8080
+      - ESPRESSO_SUBMIT_TRANSACTIONS_SUBMIT_URL=http://permissionless-builder:$ESPRESSO_BUILDER_SERVER_PORT/txn_submit
+      - ESPRESSO_SUBMIT_TRANSACTIONS_DELAY
+      - ESPRESSO_SEQUENCER_URL
       - RUST_LOG
       - RUST_LOG_FORMAT
       - ASYNC_STD_THREAD_COUNT
     depends_on:
       sequencer0:
         condition: service_healthy
+      permissionless-builder:
+        condition: service_healthy
 
   permissionless-builder:
     image: ghcr.io/espressosystems/espresso-sequencer/builder:main

process-compose.yaml

@@ -4,7 +4,6 @@ environment:
   - ESPRESSO_SEQUENCER_CDN_ENDPOINT=127.0.0.1:$ESPRESSO_CDN_SERVER_PORT
   - ESPRESSO_SEQUENCER_ORCHESTRATOR_URL=http://localhost:$ESPRESSO_ORCHESTRATOR_PORT
   - ESPRESSO_SEQUENCER_URL=http://localhost:$ESPRESSO_SEQUENCER_API_PORT
-  - ESPRESSO_SUBMIT_TRANSACTIONS_SUBMIT_URL=http://localhost:$ESPRESSO_SEQUENCER_API_PORT
   - ESPRESSO_SEQUENCER_L1_PROVIDER=http://localhost:$ESPRESSO_SEQUENCER_L1_PORT
   - ESPRESSO_DEMO_L1_HTTP_PROVIDER=$ESPRESSO_SEQUENCER_L1_PROVIDER
   - ESPRESSO_STATE_RELAY_SERVER_URL=http://localhost:$ESPRESSO_STATE_RELAY_SERVER_PORT
@@ -352,16 +351,36 @@ processes:
         path: /healthcheck
       failure_threshold: 100
 
-  submit-transactions:
+  submit-transactions-public:
     command: submit-transactions
+    environment:
+      - ESPRESSO_SUBMIT_TRANSACTIONS_PORT=$ESPRESSO_SUBMIT_TRANSACTIONS_PUBLIC_PORT
     depends_on:
       sequencer0:
         condition: process_healthy
     readiness_probe:
       http_get:
         scheme: http
         host: localhost
-        port: $ESPRESSO_SUBMIT_TRANSACTIONS_PORT
+        port: $ESPRESSO_SUBMIT_TRANSACTIONS_PUBLIC_PORT
+        path: /healthcheck
+      failure_threshold: 100
+
+  submit-transactions-private:
+    command: submit-transactions
+    environment:
+      - ESPRESSO_SUBMIT_TRANSACTIONS_PORT=$ESPRESSO_SUBMIT_TRANSACTIONS_PRIVATE_PORT
+      - ESPRESSO_SUBMIT_TRANSACTIONS_SUBMIT_URL=http://localhost:$ESPRESSO_BUILDER_SERVER_PORT/txn_submit
+    depends_on:
+      sequencer0:
+        condition: process_healthy
+      permissionless-builder:
+        condition: process_healthy
+    readiness_probe:
+      http_get:
+        scheme: http
+        host: localhost
+        port: $ESPRESSO_SUBMIT_TRANSACTIONS_PRIVATE_PORT
         path: /healthcheck
       failure_threshold: 100
 

sequencer/src/bin/submit-transactions.rs

@@ -104,11 +104,23 @@ struct Options {
     #[clap(short, long, env = "ESPRESSO_SUBMIT_TRANSACTIONS_PORT")]
     port: Option<u16>,
 
+    /// Alternative URL to submit transactions to, if not the query service URL.
+    #[clap(long, env = "ESPRESSO_SUBMIT_TRANSACTIONS_SUBMIT_URL")]
+    submit_url: Option<Url>,
+
     /// URL of the query service.
-    #[clap(env = "ESPRESSO_SUBMIT_TRANSACTIONS_SUBMIT_URL")]
+    #[clap(env = "ESPRESSO_SEQUENCER_URL")]
     url: Url,
 }
 
+impl Options {
+    fn submit_url(&self) -> Url {
+        self.submit_url
+            .clone()
+            .unwrap_or_else(|| self.url.join("submit").unwrap())
+    }
+}
+
 #[async_std::main]
 async fn main() {
     setup_backtrace();
@@ -221,7 +233,9 @@ async fn submit_transactions<Ver: StaticVersionType>(
     mut rng: ChaChaRng,
     _: Ver,
 ) {
-    let client = Client::<Error, Ver>::new(opt.url.clone());
+    let url = opt.submit_url();
+    tracing::info!(%url, "starting load generator task");
+    let client = Client::<Error, Ver>::new(url);
 
     // Create an exponential distribution for sampling delay times. The distribution should have
     // mean `opt.delay`, or parameter `\lambda = 1 / opt.delay`.
@@ -236,7 +250,7 @@ async fn submit_transactions<Ver: StaticVersionType>(
             tx.payload().len()
         );
         if let Err(err) = client
-            .post::<()>("submit/submit")
+            .post::<()>("submit")
             .body_binary(&tx)
             .unwrap()
             .send()