Bubblegum verify

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 members = [
-  "backfill",
+  "bubblegum",
   "blockbuster",
   "core",
   "das_api",
@@ -48,7 +48,7 @@ cargo-lock = "9.0.0"
 chrono = "0.4.19"
 clap = "4.2.2"
 das-core = { path = "core" }
-das-backfill = { path = "backfill" }
+das-bubblegum = { path = "bubblegum" }
 das_api = { path = "das_api" }
 derive_more = { version = "0.99.17" }
 digital_asset_types = { path = "digital_asset_types" }
@@ -78,7 +78,7 @@ lru = "0.12.3"
 metrics = "0.20.1"
 migration = { path = "migration" }
 mime_guess = "2.0.4"
-mpl-bubblegum = "1.2.0"
+mpl-bubblegum = "1.4.0"
 mpl-core = { version = "0.8.0-beta.1", features = ["serde"] }
 mpl-token-metadata = "4.1.1"
 nft_ingester = { path = "nft_ingester" }
@@ -116,7 +116,7 @@ solana-program = "~1.18"
 solana-sdk = "~1.18"
 solana-transaction-status = "~1.18"
 solana-zk-token-sdk = "1.17.16"
-spl-account-compression = "0.3.0"
+spl-account-compression = "0.4.0"
 spl-associated-token-account = ">= 1.1.3, < 3.0"
 spl-concurrent-merkle-tree = "0.2.0"
 spl-noop = "0.2.0"
@@ -125,6 +125,7 @@ spl-token = ">= 3.5.0, < 5.0"
 spl-token-2022 = { version = "1.0", features = ["no-entrypoint"] }
 spl-token-group-interface = "0.1.0"
 spl-token-metadata-interface = "0.2.0"
+sha3 = "0.10.8"
 sqlx = "0.6.2"
 stretto = "0.8.4"
 thiserror = "1.0.31"

backfill/Cargo.toml → bubblegum/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "das-backfill"
+name = "das-bubblegum"
 version = { workspace = true }
 edition = { workspace = true }
 repository = { workspace = true }
@@ -28,6 +28,7 @@ num-traits = { workspace = true }
 sea-orm = { workspace = true }
 serde_json = { workspace = true }
 solana-sdk = { workspace = true }
+sha3 = { workspace = true }
 solana-transaction-status = { workspace = true }
 spl-account-compression = { workspace = true, features = ["no-entrypoint"] }
 spl-token = { workspace = true, features = ["no-entrypoint"] }

backfill/src/gap.rs → bubblegum/src/backfill/gap.rs

@@ -1,5 +1,4 @@
-use super::ErrorKind;
-use crate::Rpc;
+use crate::{error::ErrorKind, Rpc};
 use anyhow::Result;
 use clap::Args;
 use sea_orm::{DatabaseConnection, DbBackend, FromQueryResult, Statement, Value};

bubblegum/src/backfill/mod.rs

@@ -0,0 +1,2 @@
+pub mod gap;
+pub mod worker;

backfill/src/worker/gap.rs → bubblegum/src/backfill/worker/gap.rs

@@ -9,8 +9,7 @@ use tokio::{
     task::JoinHandle,
 };
 
-use crate::gap::TreeGapFill;
-use crate::BubblegumBackfillContext;
+use crate::{backfill::gap::TreeGapFill, BubblegumContext};
 
 #[derive(Parser, Debug, Clone)]
 pub struct GapWorkerArgs {
@@ -26,7 +25,7 @@ pub struct GapWorkerArgs {
 impl GapWorkerArgs {
     pub fn start(
         &self,
-        context: BubblegumBackfillContext,
+        context: BubblegumContext,
         forward: Sender<Signature>,
     ) -> Result<(JoinHandle<()>, Sender<TreeGapFill>)> {
         let (gap_sender, mut gap_receiver) = channel::<TreeGapFill>(self.gap_channel_size);

backfill/src/worker/program_transformer.rs → bubblegum/src/backfill/worker/program_transformer.rs

@@ -9,7 +9,7 @@ use std::sync::Arc;
 use tokio::sync::mpsc::{channel, Sender, UnboundedSender};
 use tokio::task::JoinHandle;
 
-use crate::BubblegumBackfillContext;
+use crate::BubblegumContext;
 
 #[derive(Parser, Debug, Clone)]
 pub struct ProgramTransformerWorkerArgs {
@@ -22,7 +22,7 @@ pub struct ProgramTransformerWorkerArgs {
 impl ProgramTransformerWorkerArgs {
     pub fn start(
         &self,
-        context: BubblegumBackfillContext,
+        context: BubblegumContext,
         forwarder: UnboundedSender<DownloadMetadataInfo>,
     ) -> Result<(JoinHandle<()>, Sender<TransactionInfo>)> {
         let (sender, mut receiver) =

backfill/src/worker/transaction.rs → bubblegum/src/backfill/worker/transaction.rs

@@ -1,4 +1,4 @@
-use crate::error::ErrorKind;
+use crate::{error::ErrorKind, BubblegumContext};
 use anyhow::Result;
 use clap::Parser;
 use das_core::Rpc;
@@ -143,11 +143,13 @@ pub struct SignatureWorkerArgs {
     pub signature_worker_count: usize,
 }
 
+type TransactionSender = Sender<TransactionInfo>;
+
 impl SignatureWorkerArgs {
     pub fn start(
         &self,
-        context: crate::BubblegumBackfillContext,
-        forwarder: Sender<TransactionInfo>,
+        context: BubblegumContext,
+        forwarder: TransactionSender,
     ) -> Result<(JoinHandle<()>, Sender<Signature>)> {
         let (sig_sender, mut sig_receiver) = channel::<Signature>(self.signature_channel_size);
         let worker_count = self.signature_worker_count;

backfill/src/worker/tree.rs → bubblegum/src/backfill/worker/tree.rs

@@ -1,7 +1,7 @@
 use crate::{
-    gap::{TreeGapFill, TreeGapModel},
+    backfill::gap::{TreeGapFill, TreeGapModel},
     tree::TreeResponse,
-    BubblegumBackfillContext,
+    BubblegumContext,
 };
 use anyhow::Result;
 use clap::Parser;
@@ -32,11 +32,7 @@ pub struct TreeWorkerArgs {
     pub force: bool,
 }
 impl TreeWorkerArgs {
-    pub fn start(
-        &self,
-        context: BubblegumBackfillContext,
-        tree: TreeResponse,
-    ) -> JoinHandle<Result<()>> {
+    pub fn start(&self, context: BubblegumContext, tree: TreeResponse) -> JoinHandle<Result<()>> {
         let db_pool = context.database_pool.clone();
         let metadata_json_download_db_pool = context.database_pool.clone();
 

backfill/src/lib.rs → bubblegum/src/lib.rs

@@ -1,12 +1,14 @@
+mod backfill;
 mod error;
-mod gap;
 mod tree;
-mod worker;
 
 use das_core::{MetadataJsonDownloadWorkerArgs, Rpc};
 pub use error::ErrorKind;
+mod verify;
+pub use verify::ProofReport;
 
 use anyhow::Result;
+use backfill::worker::{ProgramTransformerWorkerArgs, SignatureWorkerArgs, TreeWorkerArgs};
 use clap::Parser;
 use digital_asset_types::dao::cl_audits_v2;
 use futures::{stream::FuturesUnordered, StreamExt};
@@ -19,16 +21,14 @@ use solana_sdk::signature::Signature;
 use std::str::FromStr;
 use tracing::error;
 use tree::TreeResponse;
-use worker::ProgramTransformerWorkerArgs;
-use worker::{SignatureWorkerArgs, TreeWorkerArgs};
 
 #[derive(Clone)]
-pub struct BubblegumBackfillContext {
+pub struct BubblegumContext {
     pub database_pool: sqlx::PgPool,
     pub solana_rpc: Rpc,
 }
 
-impl BubblegumBackfillContext {
+impl BubblegumContext {
     pub const fn new(database_pool: sqlx::PgPool, solana_rpc: Rpc) -> Self {
         Self {
             database_pool,
@@ -38,7 +38,7 @@ impl BubblegumBackfillContext {
 }
 
 #[derive(Debug, Parser, Clone)]
-pub struct BubblegumBackfillArgs {
+pub struct BackfillArgs {
     /// Number of tree crawler workers
     #[arg(long, env, default_value = "20")]
     pub tree_crawler_count: usize,
@@ -51,10 +51,7 @@ pub struct BubblegumBackfillArgs {
     pub tree_worker: TreeWorkerArgs,
 }
 
-pub async fn start_bubblegum_backfill(
-    context: BubblegumBackfillContext,
-    args: BubblegumBackfillArgs,
-) -> Result<()> {
+pub async fn start_backfill(context: BubblegumContext, args: BackfillArgs) -> Result<()> {
     let trees = if let Some(ref only_trees) = args.only_trees {
         TreeResponse::find(&context.solana_rpc, only_trees.clone()).await?
     } else {
@@ -99,7 +96,7 @@ pub struct BubblegumReplayArgs {
 }
 
 pub async fn start_bubblegum_replay(
-    context: BubblegumBackfillContext,
+    context: BubblegumContext,
     args: BubblegumReplayArgs,
 ) -> Result<()> {
     let pubkey = Pubkey::from_str(&args.tree)
@@ -154,3 +151,34 @@ pub async fn start_bubblegum_replay(
 
     Ok(())
 }
+
+#[derive(Debug, Parser, Clone)]
+pub struct VerifyArgs {
+    /// The list of trees to verify. If not specified, all trees will be crawled.
+    #[arg(long, env, use_value_delimiter = true)]
+    pub only_trees: Option<Vec<String>>,
+
+    #[arg(long, env, default_value = "20")]
+    pub max_concurrency: usize,
+}
+
+pub async fn verify_bubblegum(
+    context: BubblegumContext,
+    args: VerifyArgs,
+) -> Result<Vec<verify::ProofReport>> {
+    let trees = if let Some(ref only_trees) = args.only_trees {
+        TreeResponse::find(&context.solana_rpc, only_trees.clone()).await?
+    } else {
+        TreeResponse::all(&context.solana_rpc).await?
+    };
+
+    let mut reports = Vec::new();
+
+    for tree in trees {
+        let report = verify::check(context.clone(), tree, args.max_concurrency).await?;
+
+        reports.push(report);
+    }
+
+    Ok(reports)
+}