Skip to content

Commit

Permalink
feat: move backfill to bubblegum crate and verfiy to it
Browse files Browse the repository at this point in the history
  • Loading branch information
kespinola committed Sep 14, 2024
1 parent 4c3f649 commit a665b39
Show file tree
Hide file tree
Showing 19 changed files with 187 additions and 43 deletions.
11 changes: 8 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[workspace]
members = [
"backfill",
"bubblegum",
"blockbuster",
"core",
"das_api",
Expand Down Expand Up @@ -48,7 +48,12 @@ cargo-lock = "9.0.0"
chrono = "0.4.19"
clap = "4.2.2"
das-core = { path = "core" }
<<<<<<< Updated upstream
das-backfill = { path = "backfill" }
=======
das-metadata-json = { path = "metadata_json" }
das-bubblegum = { path = "bubblegum" }
>>>>>>> Stashed changes
das_api = { path = "das_api" }
derive_more = { version = "0.99.17" }
digital_asset_types = { path = "digital_asset_types" }
Expand Down Expand Up @@ -78,7 +83,7 @@ lru = "0.12.3"
metrics = "0.20.1"
migration = { path = "migration" }
mime_guess = "2.0.4"
mpl-bubblegum = "1.2.0"
mpl-bubblegum = "1.4.0"
mpl-core = { version = "0.8.0-beta.1", features = ["serde"] }
mpl-token-metadata = "4.1.1"
nft_ingester = { path = "nft_ingester" }
Expand Down Expand Up @@ -116,7 +121,7 @@ solana-program = "~1.18"
solana-sdk = "~1.18"
solana-transaction-status = "~1.18"
solana-zk-token-sdk = "1.17.16"
spl-account-compression = "0.3.0"
spl-account-compression = "0.4.0"
spl-associated-token-account = ">= 1.1.3, < 3.0"
spl-concurrent-merkle-tree = "0.2.0"
spl-noop = "0.2.0"
Expand Down
2 changes: 1 addition & 1 deletion backfill/Cargo.toml → bubblegum/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[package]
name = "das-backfill"
name = "das-bubblegum"
version = { workspace = true }
edition = { workspace = true }
repository = { workspace = true }
Expand Down
File renamed without changes.
3 changes: 1 addition & 2 deletions backfill/src/gap.rs → bubblegum/src/backfill/gap.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
use super::ErrorKind;
use crate::Rpc;
use crate::{error::ErrorKind, Rpc};
use anyhow::Result;
use clap::Args;
use sea_orm::{DatabaseConnection, DbBackend, FromQueryResult, Statement, Value};
Expand Down
2 changes: 2 additions & 0 deletions bubblegum/src/backfill/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
pub mod gap;
pub mod worker;
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@ use tokio::{
task::JoinHandle,
};

use crate::gap::TreeGapFill;
use crate::BubblegumBackfillContext;
use crate::{backfill::gap::TreeGapFill, BubblegumContext};

#[derive(Parser, Debug, Clone)]
pub struct GapWorkerArgs {
Expand All @@ -26,7 +25,7 @@ pub struct GapWorkerArgs {
impl GapWorkerArgs {
pub fn start(
&self,
context: BubblegumBackfillContext,
context: BubblegumContext,
forward: Sender<Signature>,
) -> Result<(JoinHandle<()>, Sender<TreeGapFill>)> {
let (gap_sender, mut gap_receiver) = channel::<TreeGapFill>(self.gap_channel_size);
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use program_transformers::{ProgramTransformer, TransactionInfo};
use tokio::sync::mpsc::{channel, Sender, UnboundedSender};
use tokio::task::JoinHandle;

use crate::BubblegumBackfillContext;
use crate::BubblegumContext;

#[derive(Parser, Debug, Clone)]
pub struct ProgramTransformerWorkerArgs {
Expand All @@ -17,7 +17,7 @@ pub struct ProgramTransformerWorkerArgs {
impl ProgramTransformerWorkerArgs {
pub fn start(
&self,
context: BubblegumBackfillContext,
context: BubblegumContext,
forwarder: UnboundedSender<DownloadMetadataInfo>,
) -> Result<(JoinHandle<()>, Sender<TransactionInfo>)> {
let (sender, mut receiver) =
Expand All @@ -35,7 +35,7 @@ impl ProgramTransformerWorkerArgs {
transactions.push(gap);
}

transactions.sort_by(|a, b| b.signature.cmp(&a.signature));
transactions.sort_by(|(a, _), (b, _)| a.cmp(b));

for transaction in transactions {
if let Err(e) = program_transformer.handle_transaction(&transaction).await {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use crate::error::ErrorKind;
use crate::{error::ErrorKind, BubblegumContext};
use anyhow::Result;
use clap::Parser;
use das_core::Rpc;
Expand Down Expand Up @@ -143,11 +143,13 @@ pub struct SignatureWorkerArgs {
pub signature_worker_count: usize,
}

type TransactionSender = Sender<(Option<i64>, TransactionInfo)>;

impl SignatureWorkerArgs {
pub fn start(
&self,
context: crate::BubblegumBackfillContext,
forwarder: Sender<TransactionInfo>,
context: BubblegumContext,
forwarder: TransactionSender
) -> Result<(JoinHandle<()>, Sender<Signature>)> {
let (sig_sender, mut sig_receiver) = channel::<Signature>(self.signature_channel_size);
let worker_count = self.signature_worker_count;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use crate::{
gap::{TreeGapFill, TreeGapModel},
backfill::gap::{TreeGapFill, TreeGapModel},
tree::TreeResponse,
BubblegumBackfillContext,
BubblegumContext,
};
use anyhow::Result;
use clap::Parser;
Expand Down Expand Up @@ -32,11 +32,7 @@ pub struct TreeWorkerArgs {
pub force: bool,
}
impl TreeWorkerArgs {
pub fn start(
&self,
context: BubblegumBackfillContext,
tree: TreeResponse,
) -> JoinHandle<Result<()>> {
pub fn start(&self, context: BubblegumContext, tree: TreeResponse) -> JoinHandle<Result<()>> {
let db_pool = context.database_pool.clone();
let metadata_json_download_db_pool = context.database_pool.clone();

Expand Down
File renamed without changes.
40 changes: 28 additions & 12 deletions backfill/src/lib.rs → bubblegum/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,24 +1,22 @@
mod backfill;
mod error;
mod gap;
mod tree;
mod worker;

pub use error::ErrorKind;
mod verify;

use anyhow::Result;
use backfill::worker::TreeWorkerArgs;
use clap::Parser;
use das_core::Rpc;
use futures::{stream::FuturesUnordered, StreamExt};
use tree::TreeResponse;
use worker::TreeWorkerArgs;

#[derive(Clone)]
pub struct BubblegumBackfillContext {
pub struct BubblegumContext {
pub database_pool: sqlx::PgPool,
pub solana_rpc: Rpc,
}

impl BubblegumBackfillContext {
impl BubblegumContext {
pub const fn new(database_pool: sqlx::PgPool, solana_rpc: Rpc) -> Self {
Self {
database_pool,
Expand All @@ -28,7 +26,7 @@ impl BubblegumBackfillContext {
}

#[derive(Debug, Parser, Clone)]
pub struct BubblegumBackfillArgs {
pub struct BackfillArgs {
/// Number of tree crawler workers
#[arg(long, env, default_value = "20")]
pub tree_crawler_count: usize,
Expand All @@ -41,10 +39,14 @@ pub struct BubblegumBackfillArgs {
pub tree_worker: TreeWorkerArgs,
}

pub async fn start_bubblegum_backfill(
context: BubblegumBackfillContext,
args: BubblegumBackfillArgs,
) -> Result<()> {
#[derive(Debug, Parser, Clone)]
pub struct VerifyArgs {
/// The list of trees to verify. If not specified, all trees will be crawled.
#[arg(long, env, use_value_delimiter = true)]
pub only_trees: Option<Vec<String>>,
}

pub async fn start_backfill(context: BubblegumContext, args: BackfillArgs) -> Result<()> {
let trees = if let Some(ref only_trees) = args.only_trees {
TreeResponse::find(&context.solana_rpc, only_trees.clone()).await?
} else {
Expand All @@ -67,3 +69,17 @@ pub async fn start_bubblegum_backfill(

Ok(())
}

pub async fn verify_bubblegum(context: BubblegumContext, args: VerifyArgs) -> Result<()> {
let trees = if let Some(ref only_trees) = args.only_trees {
TreeResponse::find(&context.solana_rpc, only_trees.clone()).await?
} else {
TreeResponse::all(&context.solana_rpc).await?
};

for tree in trees {
verify::check(context.clone(), tree).await?;
}

Ok(())
}
11 changes: 8 additions & 3 deletions backfill/src/tree.rs → bubblegum/src/tree.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use super::ErrorKind;
use super::error::ErrorKind;
use anyhow::Result;
use borsh::BorshDeserialize;
use das_core::Rpc;
Expand All @@ -10,7 +10,7 @@ use spl_account_compression::state::{
};
use std::str::FromStr;

#[derive(Debug, Clone)]
#[derive(Clone)]
pub struct TreeHeaderResponse {
pub max_depth: u32,
pub max_buffer_size: u32,
Expand All @@ -33,10 +33,11 @@ impl TryFrom<ConcurrentMerkleTreeHeader> for TreeHeaderResponse {
}
}

#[derive(Debug, Clone)]
pub struct TreeResponse {
pub pubkey: Pubkey,
pub tree_header: TreeHeaderResponse,
pub concurrent_merkle_tree_header: ConcurrentMerkleTreeHeader,
pub bytes: Vec<u8>,
pub seq: u64,
}

Expand All @@ -47,6 +48,8 @@ impl TreeResponse {
let (header_bytes, rest) = bytes.split_at(CONCURRENT_MERKLE_TREE_HEADER_SIZE_V1);
let header: ConcurrentMerkleTreeHeader =
ConcurrentMerkleTreeHeader::try_from_slice(header_bytes)?;
let concurrent_merkle_tree_header: ConcurrentMerkleTreeHeader =
ConcurrentMerkleTreeHeader::try_from_slice(header_bytes)?;

let merkle_tree_size = merkle_tree_get_size(&header)?;
let (tree_bytes, _canopy_bytes) = rest.split_at(merkle_tree_size);
Expand All @@ -63,6 +66,8 @@ impl TreeResponse {
Ok(Self {
pubkey,
tree_header,
concurrent_merkle_tree_header,
bytes: tree_bytes.to_vec(),
seq,
})
}
Expand Down
86 changes: 86 additions & 0 deletions bubblegum/src/verify.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
use super::BubblegumContext;
use crate::error::ErrorKind;
use crate::tree::TreeResponse;
use anyhow::{anyhow, Result};
use digital_asset_types::dapi::get_proof_for_asset;
use digital_asset_types::rpc::AssetProof;
use mpl_bubblegum::accounts::TreeConfig;
use sea_orm::SqlxPostgresConnector;
use solana_sdk::pubkey::Pubkey;
use spl_account_compression::concurrent_tree_wrapper::{merkle_tree_prove_leaf, ProveLeafArgs};

trait TryFromAssetProof {
fn try_from_asset_proof(proof: AssetProof) -> Result<Self, anyhow::Error>
where
Self: Sized;
}

impl TryFromAssetProof for ProveLeafArgs {
fn try_from_asset_proof(proof: AssetProof) -> Result<Self, anyhow::Error> {
Ok(ProveLeafArgs {
current_root: bs58::decode(&proof.root)
.into_vec()
.map_err(|e| anyhow!(e))?
.try_into()
.map_err(|_| anyhow!("Invalid root length"))?,
leaf: bs58::decode(&proof.leaf)
.into_vec()
.map_err(|e| anyhow!(e))?
.try_into()
.map_err(|_| anyhow!("Invalid leaf length"))?,
proof_vec: proof
.proof
.iter()
.map(|p| {
bs58::decode(p)
.into_vec()
.map_err(|e| anyhow!(e))
.and_then(|v| v.try_into().map_err(|_| anyhow!("Invalid proof length")))
})
.collect::<Result<Vec<[u8; 32]>>>()?,
index: proof.node_index as u32,
})
}
}
pub async fn check(context: BubblegumContext, tree: TreeResponse) -> Result<()> {
let TreeResponse {
pubkey,
concurrent_merkle_tree_header,
bytes,
..
} = tree;

let db = SqlxPostgresConnector::from_sqlx_postgres_pool(context.database_pool);

let (tree_config_pubkey, _) = TreeConfig::find_pda(&tree.pubkey);

let account = context.solana_rpc.get_account(&tree_config_pubkey).await?;
let account = account
.value
.ok_or_else(|| ErrorKind::Generic("Account not found".to_string()))?;

let tree_config = TreeConfig::from_bytes(account.data.as_slice())?;

for i in 0..tree_config.num_minted {
let (asset, _) = Pubkey::find_program_address(
&[b"asset", &pubkey.to_bytes(), &i.to_le_bytes()],
&mpl_bubblegum::ID,
);
let proof = get_proof_for_asset(&db, asset.to_bytes().to_vec()).await?;

let prove_leaf_args = ProveLeafArgs::try_from_asset_proof(proof)?;

let result = merkle_tree_prove_leaf(
&concurrent_merkle_tree_header,
pubkey,
&bytes,
&prove_leaf_args,
);

if let Err(e) = &result {
log::error!("Proof not valid for asset pubkey {}: {:?}", asset, e);
}
}

Ok(())
}
2 changes: 1 addition & 1 deletion ops/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ bs58 = { workspace = true }
cadence = { workspace = true }
cadence-macros = { workspace = true }
clap = { workspace = true, features = ["derive", "cargo", "env"] }
das-backfill = { workspace = true }
das-bubblegum = { workspace = true }
das-core = { workspace = true }
digital_asset_types = { workspace = true }
env_logger = { workspace = true }
Expand Down
8 changes: 4 additions & 4 deletions ops/src/bubblegum/backfiller.rs
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
use anyhow::Result;
use clap::Parser;
use das_backfill::{start_bubblegum_backfill, BubblegumBackfillArgs, BubblegumBackfillContext};
use das_bubblegum::{start_backfill, BackfillArgs, BubblegumContext};
use das_core::{connect_db, PoolArgs, Rpc, SolanaRpcArgs};

#[derive(Debug, Parser, Clone)]
pub struct Args {
/// Backfill Bubblegum Args
#[clap(flatten)]
pub backfill_bubblegum: BubblegumBackfillArgs,
pub backfill_bubblegum: BackfillArgs,

/// Database configuration
#[clap(flatten)]
Expand Down Expand Up @@ -47,7 +47,7 @@ pub async fn run(config: Args) -> Result<()> {
let database_pool = connect_db(&config.database).await?;

let solana_rpc = Rpc::from_config(&config.solana);
let context = BubblegumBackfillContext::new(database_pool, solana_rpc);
let context = BubblegumContext::new(database_pool, solana_rpc);

start_bubblegum_backfill(context, config.backfill_bubblegum).await
start_backfill(context, config.backfill_bubblegum).await
}
Loading

0 comments on commit a665b39

Please sign in to comment.