refactor(tree_backfiller): adjust thread management. add documentatio…

…n. general refactor to improve readability of the crate. (#124)

tree_backfiller/Cargo.toml

@@ -32,7 +32,6 @@ thiserror = "1.0.31"
 serde_json = "1.0.81"
 cadence = "0.29.0"
 cadence-macros = "0.29.0"
-hyper = "0.14.23"
 anchor-client = "0.28.0"
 tokio = { version = "1.26.0", features = ["full", "tracing"] }
 sqlx = { version = "0.6.2", features = [
@@ -69,29 +68,15 @@ mpl-bubblegum = "1.0.1-beta.3"
 spl-account-compression = { version = "0.2.0", features = ["no-entrypoint"] }
 spl-concurrent-merkle-tree = "0.2.0"
 uuid = "1.0.0"
-async-trait = "0.1.53"
-num-traits = "0.2.15"
-blockbuster = "0.9.0-beta.1"
 figment = { version = "0.10.6", features = ["env", "toml", "yaml"] }
 solana-sdk = "~1.16.16"
 solana-client = "~1.16.16"
-spl-token = { version = ">= 3.5.0, < 5.0", features = ["no-entrypoint"] }
 solana-transaction-status = "~1.16.16"
 solana-account-decoder = "~1.16.16"
-solana-geyser-plugin-interface = "~1.16.16"
-solana-sdk-macro = "~1.16.16"
-rand = "0.8.5"
 rust-crypto = "0.2.36"
 url = "2.3.1"
 anchor-lang = "0.28.0"
 borsh = "~0.10.3"
-stretto = { version = "0.7", features = ["async"] }
-tokio-stream = "0.1.12"
-tracing-subscriber = { version = "0.3.16", features = [
-  "json",
-  "env-filter",
-  "ansi",
-] }
 clap = { version = "4.2.2", features = ["derive", "cargo", "env"] }
 
 [lints]

tree_backfiller/README.md

@@ -1,6 +1,6 @@
 # Tree Backfiller
 
-The Tree Backfiller crawls all trees on-chain and backfills any transactions related to a tree that have not already been observed.
+The tree backfiller CLI assists in detecting and indexing missing updates within a compression tree that are managed by the MPL bubblegum program.
 
 ## Commands
 
@@ -10,34 +10,59 @@ Command line arguments can also be set through environment variables.
 
 The `run` command initiates the crawling and backfilling process. It requires the Solana RPC URL, the database URL, and the messenger Redis URL.
 
+```mermaid
+flowchart
+    start((Start)) -->init[Initialize RPC, DB]
+    init --> fetchTreesDB[Fetch Trees from DB]
+    fetchTreesDB --> findGapsDB[Find Gaps in DB]
+    findGapsDB --> enqueueGapFills[Enqueue Gap Fills]
+    enqueueGapFills --> gapWorkerManager[Gap Worker Manager]
+    gapWorkerManager --> crawlSignatures[Crawl Solana RPC]
+    crawlSignatures --> enqueueSignatures[Enqueue Signatures]
+    enqueueSignatures --> transactionWorkerManager[Transaction Worker Manager]
+    transactionWorkerManager --> fetchTransactionsRPC[Fetch Transactions RPC]
+    fetchTransactionsRPC --> processTransactions[Push Transaction to Messenger]
+    processTransactions ---> Finished
 ```
-Usage: das-tree-backfiller run [OPTIONS] --solana-rpc-url <SOLANA_RPC_URL> --database-url <DATABASE_URL> --messenger-redis-url <MESSENGER_REDIS_URL>
+
+```
+Usage: das-tree-backfiller run [OPTIONS] --database-url <DATABASE_URL> --messenger-redis-url <MESSENGER_REDIS_URL> --solana-rpc-url <SOLANA_RPC_URL>
 
 Options:
-      --solana-rpc-url <SOLANA_RPC_URL>
-          Solana RPC URL [env: SOLANA_RPC_URL=]
       --tree-crawler-count <TREE_CRAWLER_COUNT>
-          Number of tree crawler workers [env: TREE_CRAWLER_COUNT=] [default: 100]
+          Number of tree crawler workers [env: TREE_CRAWLER_COUNT=] [default: 20]
       --signature-channel-size <SIGNATURE_CHANNEL_SIZE>
-          The size of the signature channel. This is the number of signatures that can be queued up. [env: SIGNATURE_CHANNEL_SIZE=] [default: 10000]
-      --queue-channel-size <QUEUE_CHANNEL_SIZE>
-          [env: QUEUE_CHANNEL_SIZE=] [default: 1000]
+          The size of the signature channel [env: SIGNATURE_CHANNEL_SIZE=] [default: 10000]
+      --gap-channel-size <GAP_CHANNEL_SIZE>
+          The size of the signature channel [env: GAP_CHANNEL_SIZE=] [default: 1000]
+      --transaction-worker-count <TRANSACTION_WORKER_COUNT>
+          The number of transaction workers [env: TRANSACTION_WORKER_COUNT=] [default: 100]
+      --gap-worker-count <GAP_WORKER_COUNT>
+          The number of gap workers [env: GAP_WORKER_COUNT=] [default: 25]
+      --only-trees <ONLY_TREES>
+          The list of trees to crawl. If not specified, all trees will be crawled [env: ONLY_TREES=]
       --database-url <DATABASE_URL>
-          [env: DATABASE_URL=postgres://solana:solana@localhost:5432/solana]
+          The database URL [env: DATABASE_URL=]
       --database-max-connections <DATABASE_MAX_CONNECTIONS>
-          [env: DATABASE_MAX_CONNECTIONS=] [default: 125]
+          The maximum number of connections to the database [env: DATABASE_MAX_CONNECTIONS=] [default: 125]
       --database-min-connections <DATABASE_MIN_CONNECTIONS>
-          [env: DATABASE_MIN_CONNECTIONS=] [default: 5]
+          The minimum number of connections to the database [env: DATABASE_MIN_CONNECTIONS=] [default: 5]
       --messenger-redis-url <MESSENGER_REDIS_URL>
-          [env: MESSENGER_REDIS_URL=redis://localhost:6379]
+          [env: MESSENGER_REDIS_URL=]
       --messenger-redis-batch-size <MESSENGER_REDIS_BATCH_SIZE>
           [env: MESSENGER_REDIS_BATCH_SIZE=] [default: 100]
-      --messenger-stream-max-buffer-size <MESSENGER_STREAM_MAX_BUFFER_SIZE>
-          [env: MESSENGER_STREAM_MAX_BUFFER_SIZE=] [default: 10000000]
+      --messenger-queue-connections <MESSENGER_QUEUE_CONNECTIONS>
+          [env: MESSENGER_QUEUE_CONNECTIONS=] [default: 25]
+      --messenger-queue-stream <MESSENGER_QUEUE_STREAM>
+          [env: MESSENGER_QUEUE_STREAM=] [default: TXNFILL]
       --metrics-host <METRICS_HOST>
           [env: METRICS_HOST=] [default: 127.0.0.1]
       --metrics-port <METRICS_PORT>
           [env: METRICS_PORT=] [default: 8125]
+      --metrics-prefix <METRICS_PREFIX>
+          [env: METRICS_PREFIX=] [default: das.backfiller]
+      --solana-rpc-url <SOLANA_RPC_URL>
+          [env: SOLANA_RPC_URL=]
   -h, --help
           Print help
 ```
@@ -49,8 +74,11 @@ The Tree Backfiller provides several metrics for monitoring performance and stat
 Metric | Description
 --- | ---
 transaction.failed | Count of failed transaction
+transaction.succeeded | Count of successfully queued transaction
 transaction.queued | Time for a transaction to be queued
+gap.failed | Count of failed gap crawling
+gap.succeeded | Count of successfully crawled gaps
+gap.queued | Time for a gap to be queued
+tree.succeeded | Count of completed tree crawl
 tree.crawled | Time to crawl a tree
-tree.completed | Count of completed tree crawl
-tree.failed | Count of failed tree crawls
 job.completed | Time to complete the job