Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bubblegum Update Metadata Version 2 #135

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
09a599d
Add code to index Bubblegum Update Metadata
danenbm Oct 15, 2023
5ef2eb9
Update rust toolchain file
danenbm Oct 15, 2023
7dcf74b
Merge branch 'main' into danenbm/update-metadata-parsing
danenbm Oct 17, 2023
2084aa7
Fix moved variable after merge
danenbm Oct 17, 2023
269bf0d
Add code from mintV1 that allows for empty URI
danenbm Oct 17, 2023
bdf8e3c
Ordering using asset.seq initially applied to update_metadata
danenbm Oct 18, 2023
0240ce5
Add simple check for whether asset was decompressed to Bubblegum tran…
danenbm Oct 19, 2023
c26c693
Don't prevent sequence number update when already decompressed
danenbm Oct 19, 2023
dd891e8
Add sequence number to downloading metadata background task
danenbm Oct 19, 2023
97fe275
Add sequence number migration (Sea ORM not regenerated yet)
danenbm Oct 20, 2023
660a6ba
Regenerate Sea-ORM types
danenbm Oct 20, 2023
213dac5
Use new sequence numbers for Bubblegum Update Metadata
danenbm Oct 20, 2023
026c16a
Extra condition to protect out of order creator verification
danenbm Oct 21, 2023
e797adc
Remove base_info_seq for each creator and add creators_added_seq to a…
danenbm Oct 23, 2023
f2b1218
Regenerate Sea-ORM types
danenbm Oct 23, 2023
77020fa
Change creator metadata updates to use new creators_added_seq
danenbm Oct 23, 2023
5c15538
Use update_metadata_seq to protect multiple items from multiple tables
danenbm Oct 23, 2023
ccc9257
Refactor to use update_metadata_seq
danenbm Oct 23, 2023
2077511
Modify unprotected_upsert_asset_base_info to update info not covered …
danenbm Oct 29, 2023
a2999e2
Use updated version of Blockbuster that correctly parses
danenbm Nov 2, 2023
d817795
Change download metadata seq check to use lte
danenbm Nov 27, 2023
30d83ea
Index verified for token metadata collection
danenbm Nov 28, 2023
c7028bb
Update to use latest blockbuster beta release
danenbm Nov 29, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ Because this is a multi component system the easiest way to develop or locally t
#### Regenerating DB Types
Edit the init.sql, then run `docker compose up db`
Then with a local `DATABASE_URL` var exported like this `export DATABASE_URL=postgres://solana:solana@localhost/solana` you can run
` sea-orm-cli generate entity -o ./digital_asset_types/src/dao/generated/ --database-url $DATABASE_URL --with-serde both --expanded-format`
`sea-orm-cli generate entity -o ./digital_asset_types/src/dao/generated/ --database-url $DATABASE_URL --with-serde both --expanded-format`

If you need to install `sea-orm-cli` run `cargo install sea-orm-cli`.

Expand Down
2 changes: 1 addition & 1 deletion das_api/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ schemars = "0.8.6"
schemars_derive = "0.8.6"
open-rpc-derive = { version = "0.0.4"}
open-rpc-schema = { version = "0.0.4"}
blockbuster = "0.9.0-beta.1"
blockbuster = "0.9.0-beta.2"
anchor-lang = "0.28.0"
mpl-token-metadata = { version = "=2.0.0-beta.1", features = ["serde-feature"] }
mpl-candy-machine-core = { version = "2.0.1", features = ["no-entrypoint"] }
Expand Down
2 changes: 1 addition & 1 deletion digital_asset_types/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ solana-sdk = "~1.16.16"
num-traits = "0.2.15"
num-derive = "0.3.3"
thiserror = "1.0.31"
blockbuster = "0.9.0-beta.1"
blockbuster = "0.9.0-beta.2"
jsonpath_lib = "0.3.0"
mime_guess = "2.0.4"
url = "2.3.1"
Expand Down
3 changes: 3 additions & 0 deletions digital_asset_types/src/dao/generated/asset.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ pub struct Model {
pub owner_delegate_seq: Option<i64>,
pub was_decompressed: bool,
pub leaf_seq: Option<i64>,
pub update_metadata_seq: Option<i64>,
}

#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
Expand Down Expand Up @@ -78,6 +79,7 @@ pub enum Column {
OwnerDelegateSeq,
WasDecompressed,
LeafSeq,
UpdateMetadataSeq,
}

#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
Expand Down Expand Up @@ -133,6 +135,7 @@ impl ColumnTrait for Column {
Self::OwnerDelegateSeq => ColumnType::BigInteger.def().null(),
Self::WasDecompressed => ColumnType::Boolean.def(),
Self::LeafSeq => ColumnType::BigInteger.def().null(),
Self::UpdateMetadataSeq => ColumnType::BigInteger.def().null(),
}
}
}
Expand Down
6 changes: 3 additions & 3 deletions digital_asset_types/src/dao/generated/asset_creators.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ pub struct Model {
pub creator: Vec<u8>,
pub share: i32,
pub verified: bool,
pub seq: Option<i64>,
pub verified_seq: Option<i64>,
pub slot_updated: Option<i64>,
pub position: i16,
}
Expand All @@ -31,7 +31,7 @@ pub enum Column {
Creator,
Share,
Verified,
Seq,
VerifiedSeq,
SlotUpdated,
Position,
}
Expand Down Expand Up @@ -62,7 +62,7 @@ impl ColumnTrait for Column {
Self::Creator => ColumnType::Binary.def(),
Self::Share => ColumnType::Integer.def(),
Self::Verified => ColumnType::Boolean.def(),
Self::Seq => ColumnType::BigInteger.def().null(),
Self::VerifiedSeq => ColumnType::BigInteger.def().null(),
Self::SlotUpdated => ColumnType::BigInteger.def().null(),
Self::Position => ColumnType::SmallInteger.def(),
}
Expand Down
13 changes: 8 additions & 5 deletions digital_asset_types/src/dao/generated/asset_data.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,9 @@ pub struct Model {
pub metadata: Json,
pub slot_updated: i64,
pub reindex: Option<bool>,
pub raw_name: Vec<u8>,
pub raw_symbol: Vec<u8>,
pub raw_name: Option<Vec<u8>>,
pub raw_symbol: Option<Vec<u8>>,
pub download_metadata_seq: Option<i64>,
}

#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
Expand All @@ -40,6 +41,7 @@ pub enum Column {
Reindex,
RawName,
RawSymbol,
DownloadMetadataSeq,
}

#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
Expand Down Expand Up @@ -70,9 +72,10 @@ impl ColumnTrait for Column {
Self::MetadataMutability => Mutability::db_type(),
Self::Metadata => ColumnType::JsonBinary.def(),
Self::SlotUpdated => ColumnType::BigInteger.def(),
Self::Reindex => ColumnType::Boolean.def(),
Self::RawName => ColumnType::Binary.def(),
Self::RawSymbol => ColumnType::Binary.def(),
Self::Reindex => ColumnType::Boolean.def().null(),
Self::RawName => ColumnType::Binary.def().null(),
Self::RawSymbol => ColumnType::Binary.def().null(),
Self::DownloadMetadataSeq => ColumnType::BigInteger.def().null(),
}
}
}
Expand Down
6 changes: 3 additions & 3 deletions digital_asset_types/src/dao/generated/asset_grouping.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3
//! SeaORM Entity. Generated by sea-orm-codegen 0.9.3

use sea_orm::entity::prelude::*;
use serde::{Deserialize, Serialize};
Expand All @@ -12,15 +12,15 @@ impl EntityName for Entity {
}
}

#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel, Eq, Serialize, Deserialize)]
#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel, Serialize, Deserialize)]
pub struct Model {
pub id: i64,
pub asset_id: Vec<u8>,
pub group_key: String,
pub group_value: Option<String>,
pub seq: Option<i64>,
pub slot_updated: Option<i64>,
pub verified: Option<bool>,
pub verified: bool,
pub group_info_seq: Option<i64>,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we need a migration to set the default value for thid? since this used to be optional we might have null values in. db

Copy link
Contributor Author

@danenbm danenbm Dec 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you look at m20230720_120101_add_asset_grouping_verified.rs it was not nullable from that point. All I did for this was regenerate the SeaORM objects based on the existing migrations. Do we need a special migration to check whether people have it as Optional in their databases? My guess is that it wasn't ever nullable and the SeaORM was somehow not updated?

}

Expand Down
2 changes: 1 addition & 1 deletion digital_asset_types/src/dao/generated/cl_audits.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ pub struct Model {
pub seq: i64,
pub level: i64,
pub hash: Vec<u8>,
pub created_at: Option<DateTime>,
pub created_at: DateTime,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same as above , does this need a migration since it has changed from option

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah I have same response. If you look at m20230919_072154_cl_audits.rs its not nullable in the original migration that added the table.

pub tx: String,
}

Expand Down
112 changes: 56 additions & 56 deletions digital_asset_types/src/dao/generated/sea_orm_active_enums.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,62 +3,6 @@
use sea_orm::entity::prelude::*;
use serde::{Deserialize, Serialize};

#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "mutability")]
pub enum Mutability {
#[sea_orm(string_value = "immutable")]
Immutable,
#[sea_orm(string_value = "mutable")]
Mutable,
#[sea_orm(string_value = "unknown")]
Unknown,
}
#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(
rs_type = "String",
db_type = "Enum",
enum_name = "v1_account_attachments"
)]
pub enum V1AccountAttachments {
#[sea_orm(string_value = "edition")]
Edition,
#[sea_orm(string_value = "edition_marker")]
EditionMarker,
#[sea_orm(string_value = "master_edition_v1")]
MasterEditionV1,
#[sea_orm(string_value = "master_edition_v2")]
MasterEditionV2,
#[sea_orm(string_value = "unknown")]
Unknown,
}
#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "task_status")]
pub enum TaskStatus {
#[sea_orm(string_value = "failed")]
Failed,
#[sea_orm(string_value = "pending")]
Pending,
#[sea_orm(string_value = "running")]
Running,
#[sea_orm(string_value = "success")]
Success,
}
#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(
rs_type = "String",
db_type = "Enum",
enum_name = "royalty_target_type"
)]
pub enum RoyaltyTargetType {
#[sea_orm(string_value = "creators")]
Creators,
#[sea_orm(string_value = "fanout")]
Fanout,
#[sea_orm(string_value = "single")]
Single,
#[sea_orm(string_value = "unknown")]
Unknown,
}
#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(
rs_type = "String",
Expand Down Expand Up @@ -98,6 +42,24 @@ pub enum ChainMutability {
Unknown,
}
#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(
rs_type = "String",
db_type = "Enum",
enum_name = "v1_account_attachments"
)]
pub enum V1AccountAttachments {
#[sea_orm(string_value = "edition")]
Edition,
#[sea_orm(string_value = "edition_marker")]
EditionMarker,
#[sea_orm(string_value = "master_edition_v1")]
MasterEditionV1,
#[sea_orm(string_value = "master_edition_v2")]
MasterEditionV2,
#[sea_orm(string_value = "unknown")]
Unknown,
}
#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(
rs_type = "String",
db_type = "Enum",
Expand All @@ -114,6 +76,32 @@ pub enum SpecificationVersions {
V2,
}
#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "mutability")]
pub enum Mutability {
#[sea_orm(string_value = "immutable")]
Immutable,
#[sea_orm(string_value = "mutable")]
Mutable,
#[sea_orm(string_value = "unknown")]
Unknown,
}
#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(
rs_type = "String",
db_type = "Enum",
enum_name = "royalty_target_type"
)]
pub enum RoyaltyTargetType {
#[sea_orm(string_value = "creators")]
Creators,
#[sea_orm(string_value = "fanout")]
Fanout,
#[sea_orm(string_value = "single")]
Single,
#[sea_orm(string_value = "unknown")]
Unknown,
}
#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "owner_type")]
pub enum OwnerType {
#[sea_orm(string_value = "single")]
Expand All @@ -123,3 +111,15 @@ pub enum OwnerType {
#[sea_orm(string_value = "unknown")]
Unknown,
}
#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "task_status")]
pub enum TaskStatus {
#[sea_orm(string_value = "failed")]
Failed,
#[sea_orm(string_value = "pending")]
Pending,
#[sea_orm(string_value = "running")]
Running,
#[sea_orm(string_value = "success")]
Success,
}
10 changes: 6 additions & 4 deletions digital_asset_types/tests/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,9 @@ pub fn create_asset_data(
metadata: JsonValue::String("processing".to_string()),
slot_updated: 0,
reindex: None,
raw_name: metadata.name.into_bytes().to_vec().clone(),
raw_symbol: metadata.symbol.into_bytes().to_vec().clone(),
raw_name: Some(metadata.name.into_bytes().to_vec().clone()),
raw_symbol: Some(metadata.symbol.into_bytes().to_vec().clone()),
download_metadata_seq: Some(0),
},
)
}
Expand Down Expand Up @@ -157,6 +158,7 @@ pub fn create_asset(
owner_delegate_seq: Some(0),
was_decompressed: false,
leaf_seq: Some(0),
update_metadata_seq: Some(0),
},
)
}
Expand All @@ -182,7 +184,7 @@ pub fn create_asset_creator(
creator,
share,
verified,
seq: Some(0),
verified_seq: Some(0),
slot_updated: Some(0),
position: 0,
},
Expand Down Expand Up @@ -231,7 +233,7 @@ pub fn create_asset_grouping(
id: row_num,
group_key: "collection".to_string(),
slot_updated: Some(0),
verified: Some(false),
verified: false,
group_info_seq: Some(0),
},
)
Expand Down
5 changes: 3 additions & 2 deletions digital_asset_types/tests/json_parsing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,9 @@ pub async fn parse_onchain_json(json: serde_json::Value) -> Content {
metadata: json,
slot_updated: 0,
reindex: None,
raw_name: String::from("Handalf").into_bytes().to_vec(),
raw_symbol: String::from("").into_bytes().to_vec(),
raw_name: Some(String::from("Handalf").into_bytes().to_vec()),
raw_symbol: Some(String::from("").into_bytes().to_vec()),
download_metadata_seq: Some(0),
};

v1_content_from_json(&asset_data).unwrap()
Expand Down
2 changes: 2 additions & 0 deletions migration/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ mod m20230724_120101_add_group_info_seq;
mod m20230726_013107_remove_not_null_constraint_from_group_value;
mod m20230918_182123_add_raw_name_symbol;
mod m20230919_072154_cl_audits;
mod m20231019_120101_add_seq_numbers_bgum_update_metadata;

pub struct Migrator;

Expand Down Expand Up @@ -67,6 +68,7 @@ impl MigratorTrait for Migrator {
Box::new(m20230726_013107_remove_not_null_constraint_from_group_value::Migration),
Box::new(m20230918_182123_add_raw_name_symbol::Migration),
Box::new(m20230919_072154_cl_audits::Migration),
Box::new(m20231019_120101_add_seq_numbers_bgum_update_metadata::Migration),
]
}
}
Loading
Loading