From 0a39d7d3ccb59b162c379573d1d4d530b1942422 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 18 Sep 2023 14:19:47 +0200 Subject: [PATCH 1/3] feat: create worker queue using redis for uploading metadata json in the background --- Cargo.lock | 1053 +++++++++++------ api/Cargo.toml | 5 +- api/src/background_worker/job.rs | 45 + api/src/background_worker/job_queue.rs | 79 ++ api/src/background_worker/mod.rs | 4 + .../tasks/metadata_json_upload_task.rs | 1 + api/src/background_worker/tasks/mod.rs | 8 + api/src/background_worker/worker.rs | 80 ++ api/src/entities/job_trackings.rs | 19 + api/src/entities/mod.rs | 1 + api/src/lib.rs | 1 + docker-compose.yaml | 8 + migration/src/lib.rs | 8 + ...20230914_154759_add_job_trackings_table.rs | 56 + 14 files changed, 1030 insertions(+), 338 deletions(-) create mode 100644 api/src/background_worker/job.rs create mode 100644 api/src/background_worker/job_queue.rs create mode 100644 api/src/background_worker/mod.rs create mode 100644 api/src/background_worker/tasks/metadata_json_upload_task.rs create mode 100644 api/src/background_worker/tasks/mod.rs create mode 100644 api/src/background_worker/worker.rs create mode 100644 api/src/entities/job_trackings.rs create mode 100644 migration/src/m20230914_154759_add_job_trackings_table.rs diff --git a/Cargo.lock b/Cargo.lock index aecea8a..a9850dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,9 +14,9 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.21.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" dependencies = [ "gimli", ] @@ -52,9 +52,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f2135563fb5c609d2b2b87c1e8ce7bc41b0b45430fa9661f457981503dd5bf0" +checksum = "86b8f9420f797f2d9e935edf629310eb938a0d839f984e25327f3c7eed22300c" dependencies = [ "memchr", ] @@ -88,23 +88,24 @@ dependencies = [ [[package]] name = "anstream" -version = "0.5.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is-terminal", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.3" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" [[package]] name = "anstyle-parse" @@ -126,9 +127,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c" dependencies = [ "anstyle", "windows-sys", @@ -136,9 +137,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" dependencies = [ "backtrace", ] @@ -173,7 +174,7 @@ dependencies = [ "async-stream", "async-trait", "base64 0.13.1", - "bytes 1.5.0", + "bytes 1.4.0", "chrono", "fast_chemail", "fnv", @@ -238,7 +239,7 @@ dependencies = [ "futures-util", "poem", "serde_json", - "tokio-util", + "tokio-util 0.7.8", ] [[package]] @@ -247,7 +248,7 @@ version = "5.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d461325bfb04058070712296601dfe5e5bd6cdff84780a0a8c569ffb15c87eb3" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "indexmap 1.9.3", "serde", "serde_json", @@ -272,18 +273,27 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", ] [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", +] + +[[package]] +name = "atoi" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c57d12312ff59c811c0643f4d80830505833c9ffaebd193d819392b265be8e" +dependencies = [ + "num-traits", ] [[package]] @@ -315,9 +325,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" dependencies = [ "addr2line", "cc", @@ -328,6 +338,19 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "bae" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b8de67cc41132507eeece2584804efcb15f85ba516e34c944b7667f480397a" +dependencies = [ + "heck 0.3.3", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "base-x" version = "0.2.11" @@ -348,9 +371,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.4" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" [[package]] name = "base64ct" @@ -386,9 +409,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" dependencies = [ "serde", ] @@ -407,24 +430,24 @@ dependencies = [ [[package]] name = "blake2b_simd" -version = "1.0.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23285ad32269793932e830392f2fe2f83e26488fd3ec778883a93c8323735780" +checksum = "3c2f0dc9a68c6317d884f97cc36cf5a3d20ba14ce404227df55e1af708ab04bc" dependencies = [ "arrayref", "arrayvec", - "constant_time_eq", + "constant_time_eq 0.2.6", ] [[package]] name = "blake2s_simd" -version = "1.0.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94230421e395b9920d23df13ea5d77a20e1725331f90fbbf6df6040b33f756ae" +checksum = "6637f448b9e61dfadbdcbae9a885fadee1f3eaffb1f8d3c1965d3ade8bdfd44f" dependencies = [ "arrayref", "arrayvec", - "constant_time_eq", + "constant_time_eq 0.2.6", ] [[package]] @@ -437,7 +460,7 @@ dependencies = [ "arrayvec", "cc", "cfg-if", - "constant_time_eq", + "constant_time_eq 0.3.0", "digest 0.10.7", ] @@ -566,9 +589,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "bv" @@ -604,22 +627,22 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.14.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" +checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.5.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1" +checksum = "fdde5c9cd29ebd706ce1b35600920a33550e402fc998a2e53ad3b42c3c47a192" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", ] [[package]] @@ -636,18 +659,18 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "bytes" -version = "1.5.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" dependencies = [ "serde", ] [[package]] name = "cc" -version = "1.0.83" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" dependencies = [ "jobserver", "libc", @@ -661,17 +684,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "serde", + "time 0.1.45", "wasm-bindgen", - "windows-targets", + "winapi", ] [[package]] @@ -689,43 +713,45 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.4" +version = "4.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" +checksum = "c27cdf28c0f604ba3f512b0c9a409f8de8513e4816705deb0498b627e7c3a3fd" dependencies = [ "clap_builder", "clap_derive", + "once_cell", ] [[package]] name = "clap_builder" -version = "4.4.4" +version = "4.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" +checksum = "08a9f1ab5e9f01a9b81f202e8562eb9a10de70abf9eaeac1be465c28b75aa4aa" dependencies = [ "anstream", "anstyle", "clap_lex", + "once_cell", "strsim", ] [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", ] [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" [[package]] name = "colorchoice" @@ -733,6 +759,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "combine" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +dependencies = [ + "bytes 1.4.0", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util 0.7.8", +] + [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -759,6 +799,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" +[[package]] +name = "constant_time_eq" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a53c0a4d288377e7415b53dcfc3c04da5cdc2cc95c8d5ac178b58f0b861ad6" + [[package]] name = "constant_time_eq" version = "0.3.0" @@ -927,7 +973,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.37", + "syn 2.0.28", ] [[package]] @@ -949,7 +995,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.37", + "syn 2.0.28", ] [[package]] @@ -991,9 +1037,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "7684a49fb1af197853ef7b2ee694bc1f5b4179556f1e5710e1760c5db6f5e929" dependencies = [ "serde", ] @@ -1042,6 +1088,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "dtoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" + [[package]] name = "duct" version = "0.13.6" @@ -1065,9 +1117,9 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.33" +version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ "cfg-if", ] @@ -1080,9 +1132,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.3" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" dependencies = [ "errno-dragonfly", "libc", @@ -1146,12 +1198,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "835a3dc7d1ec9e75e2b5fb4ba75396837112d2060b03f7d43bc1897c7f7211da" -[[package]] -name = "finl_unicode" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" - [[package]] name = "fixedbitset" version = "0.4.2" @@ -1248,6 +1294,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a604f7a68fbf8103337523b1fadc8ade7361ee3f112f7c680ad179651616aed5" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot 0.11.2", +] + [[package]] name = "futures-intrusive" version = "0.5.0" @@ -1256,7 +1313,7 @@ checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" dependencies = [ "futures-core", "lock_api", - "parking_lot", + "parking_lot 0.12.1", ] [[package]] @@ -1273,7 +1330,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", ] [[package]] @@ -1361,9 +1418,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" [[package]] name = "glob" @@ -1373,11 +1430,11 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.21" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "fnv", "futures-core", "futures-sink", @@ -1386,15 +1443,15 @@ dependencies = [ "indexmap 1.9.3", "slab", "tokio", - "tokio-util", + "tokio-util 0.7.8", "tracing", ] [[package]] name = "handlebars" -version = "4.4.0" +version = "4.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39b3bc2a8f715298032cf5087e58573809374b08160aa7d750582bdb82d2683" +checksum = "83c3372087601b532857d332f5957cbae686da52bb7810bf038c3e3c3cc2fa0d" dependencies = [ "log", "pest", @@ -1443,26 +1500,27 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.8.4" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +checksum = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f" dependencies = [ "hashbrown 0.14.0", ] [[package]] name = "headers" -version = "0.3.9" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" +checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" dependencies = [ - "base64 0.21.4", - "bytes 1.5.0", + "base64 0.13.1", + "bitflags 1.3.2", + "bytes 1.4.0", "headers-core", "http", "httpdate", "mime", - "sha1", + "sha1 0.10.5", ] [[package]] @@ -1474,6 +1532,15 @@ dependencies = [ "http", ] +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "heck" version = "0.4.1" @@ -1485,9 +1552,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "hex" @@ -1539,8 +1606,8 @@ dependencies = [ [[package]] name = "holaplex-hub-core" -version = "0.5.5" -source = "git+https://github.com/holaplex/hub-core?branch=stable#ffd5b04a3f411d2bae8af317f2b8626012ec3c0b" +version = "0.5.6" +source = "git+https://github.com/holaplex/hub-core?branch=stable#1d20d2fbb974c59340dbaca00387988d1bb69272" dependencies = [ "anyhow", "async-trait", @@ -1565,13 +1632,13 @@ dependencies = [ "rand 0.8.5", "rdkafka", "reqwest", - "sea-orm", + "sea-orm 0.10.7", "serde_json", "serde_with", "strum 0.24.1", "thiserror", "tokio", - "toml 0.7.8", + "toml 0.7.6", "tracing", "tracing-loki", "tracing-subscriber", @@ -1582,7 +1649,7 @@ dependencies = [ [[package]] name = "holaplex-hub-core-build" version = "0.2.1" -source = "git+https://github.com/holaplex/hub-core?branch=stable#ffd5b04a3f411d2bae8af317f2b8626012ec3c0b" +source = "git+https://github.com/holaplex/hub-core?branch=stable#1d20d2fbb974c59340dbaca00387988d1bb69272" dependencies = [ "anyhow", "dotenv", @@ -1601,18 +1668,18 @@ dependencies = [ [[package]] name = "holaplex-hub-core-macros" version = "0.1.0" -source = "git+https://github.com/holaplex/hub-core?branch=stable#ffd5b04a3f411d2bae8af317f2b8626012ec3c0b" +source = "git+https://github.com/holaplex/hub-core?branch=stable#1d20d2fbb974c59340dbaca00387988d1bb69272" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", "thiserror", ] [[package]] name = "holaplex-hub-core-schemas" version = "0.3.2" -source = "git+https://github.com/holaplex/hub-core?branch=stable#ffd5b04a3f411d2bae8af317f2b8626012ec3c0b" +source = "git+https://github.com/holaplex/hub-core?branch=stable#1d20d2fbb974c59340dbaca00387988d1bb69272" dependencies = [ "holaplex-hub-core-build", "prost", @@ -1630,8 +1697,9 @@ dependencies = [ "poem", "prost", "prost-types", + "redis", "reqwest", - "sea-orm", + "sea-orm 0.12.2", "serde", "serde_json", "solana-program", @@ -1664,9 +1732,9 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "fnv", - "itoa", + "itoa 1.0.9", ] [[package]] @@ -1675,7 +1743,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "http", "pin-project-lite", ] @@ -1688,9 +1756,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.3" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" @@ -1698,7 +1766,7 @@ version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "futures-channel", "futures-core", "futures-util", @@ -1707,7 +1775,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa", + "itoa 1.0.9", "pin-project-lite", "socket2 0.4.9", "tokio", @@ -1722,7 +1790,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "hyper", "native-tls", "tokio", @@ -1797,7 +1865,7 @@ checksum = "ce243b1bfa62ffc028f1cc3b6034ec63d649f3031bc8a4fbbb004e1ac17d1f68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", ] [[package]] @@ -1816,23 +1884,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] -name = "itertools" -version = "0.10.5" +name = "is-terminal" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "either", + "hermit-abi", + "rustix", + "windows-sys", ] [[package]] name = "itertools" -version = "0.11.0" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + [[package]] name = "itoa" version = "1.0.9" @@ -1877,9 +1953,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.148" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libm" @@ -1960,9 +2036,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.7" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" +checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" [[package]] name = "lock_api" @@ -1976,9 +2052,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" [[package]] name = "loki-api" @@ -2025,9 +2101,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.3" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memmap2" @@ -2084,7 +2160,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "encoding_rs", "futures-util", "http", @@ -2184,9 +2260,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" dependencies = [ "autocfg", "num-integer", @@ -2285,9 +2361,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.1" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" dependencies = [ "memchr", ] @@ -2306,11 +2382,11 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e" dependencies = [ - "bitflags 2.4.0", + "bitflags 1.3.2", "cfg-if", "foreign-types", "libc", @@ -2327,7 +2403,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", ] [[package]] @@ -2338,9 +2414,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac" dependencies = [ "cc", "libc", @@ -2428,6 +2504,16 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "ouroboros" +version = "0.15.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1358bd1558bd2a083fed428ffeda486fbfb323e698cdda7794259d592ca72db" +dependencies = [ + "aliasable", + "ouroboros_macro 0.15.6", +] + [[package]] name = "ouroboros" version = "0.17.2" @@ -2435,21 +2521,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2ba07320d39dfea882faa70554b4bd342a5f273ed59ba7c1c6b4c840492c954" dependencies = [ "aliasable", - "ouroboros_macro", + "ouroboros_macro 0.17.2", "static_assertions", ] +[[package]] +name = "ouroboros_macro" +version = "0.15.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f7d21ccd03305a674437ee1248f3ab5d4b1db095cf1caf49f1713ddf61956b7" +dependencies = [ + "Inflector", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "ouroboros_macro" version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec4c6225c69b4ca778c0aea097321a64c421cf4577b331c61b229267edabb6f8" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", ] [[package]] @@ -2458,6 +2557,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -2465,7 +2575,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.8", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -2476,7 +2600,7 @@ checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.3.5", "smallvec", "windows-targets", ] @@ -2504,20 +2628,19 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pest" -version = "2.7.3" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a4d085fd991ac8d5b05a147b437791b4260b76326baf0fc60cf7c9c27ecd33" +checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a" dependencies = [ - "memchr", "thiserror", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.7.3" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bee7be22ce7918f641a33f08e3f43388c7656772244e2bbb2477f44cc9021a" +checksum = "666d00490d4ac815001da55838c500eafb0320019bbaa44444137c48b443a853" dependencies = [ "pest", "pest_generator", @@ -2525,22 +2648,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.3" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1511785c5e98d79a05e8a6bc34b4ac2168a0e3e92161862030ad84daa223141" +checksum = "68ca01446f50dbda87c1786af8770d535423fa8a53aec03b8f4e3d7eb10e0929" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", ] [[package]] name = "pest_meta" -version = "2.7.3" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42f0394d3123e33353ca5e1e89092e533d2cc490389f2bd6131c43c634ebc5f" +checksum = "56af0a30af74d0445c0bf6d9d051c979b516a1a5af790d251daee76005420a48" dependencies = [ "once_cell", "pest", @@ -2549,12 +2672,12 @@ dependencies = [ [[package]] name = "petgraph" -version = "0.6.4" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" dependencies = [ "fixedbitset", - "indexmap 2.0.0", + "indexmap 1.9.3", ] [[package]] @@ -2594,14 +2717,14 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", ] [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" [[package]] name = "pin-utils" @@ -2638,20 +2761,20 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "poem" -version = "1.3.58" +version = "1.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc7ae19f3e791ae8108b08801abb3708d64d3a16490c720e0b81040cae87b5d" +checksum = "f0d92c532a37a9e98c0e9a0411e6852b8acccf9ec07d5e6e450b01cbf947d90b" dependencies = [ "anyhow", "async-trait", - "base64 0.21.4", - "bytes 1.5.0", + "base64 0.21.2", + "bytes 1.4.0", "futures-util", "headers", "http", "hyper", "mime", - "parking_lot", + "parking_lot 0.12.1", "percent-encoding", "pin-project-lite", "poem-derive", @@ -2666,20 +2789,20 @@ dependencies = [ "tokio", "tokio-stream", "tokio-tungstenite", - "tokio-util", + "tokio-util 0.7.8", "tracing", ] [[package]] name = "poem-derive" -version = "1.3.58" +version = "1.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2550a0bce7273b278894ef3ccc5a6869e7031b6870042f3cc6826ed9faa980a6" +checksum = "f5dd58846a1f582215370384c3090c62c9ef188e9d798ffc67ea90d0a1a8a3b8" dependencies = [ "proc-macro-crate 1.1.3", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", ] [[package]] @@ -2743,9 +2866,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -2760,7 +2883,7 @@ dependencies = [ "fnv", "lazy_static", "memchr", - "parking_lot", + "parking_lot 0.12.1", "protobuf", "thiserror", ] @@ -2771,7 +2894,7 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "prost-derive", ] @@ -2781,9 +2904,9 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ - "bytes 1.5.0", - "heck", - "itertools 0.10.5", + "bytes 1.4.0", + "heck 0.4.1", + "itertools", "lazy_static", "log", "multimap", @@ -2804,7 +2927,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools", "proc-macro2", "quote", "syn 1.0.109", @@ -2847,9 +2970,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ "proc-macro2", ] @@ -2951,9 +3074,9 @@ dependencies = [ [[package]] name = "rdkafka-sys" -version = "4.6.0+2.2.0" +version = "4.5.0+1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad63c279fca41a27c231c450a2d2ad18288032e9cbb159ad16c9d96eba35aaaf" +checksum = "1bb0676c2112342ac7165decdedbc4e7086c0af384479ccce534546b10687a5d" dependencies = [ "libc", "libz-sys", @@ -2964,6 +3087,35 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "redis" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4f0ceb2ec0dd769483ecd283f6615aa83dcd0be556d5294c6e659caefe7cc54" +dependencies = [ + "async-trait", + "bytes 1.4.0", + "combine", + "dtoa", + "futures-util", + "itoa 0.4.8", + "percent-encoding", + "pin-project-lite", + "sha1 0.6.1", + "tokio", + "tokio-util 0.6.10", + "url", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.3.5" @@ -2975,14 +3127,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.5" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.8", - "regex-syntax 0.7.5", + "regex-automata 0.3.6", + "regex-syntax 0.7.4", ] [[package]] @@ -2996,13 +3148,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.8" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax 0.7.4", ] [[package]] @@ -3013,9 +3165,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "rend" @@ -3028,12 +3180,12 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.20" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64 0.21.4", - "bytes 1.5.0", + "base64 0.21.2", + "bytes 1.4.0", "encoding_rs", "futures-core", "futures-util", @@ -3055,7 +3207,7 @@ dependencies = [ "serde_urlencoded", "tokio", "tokio-native-tls", - "tokio-util", + "tokio-util 0.7.8", "tower-service", "url", "wasm-bindgen", @@ -3141,13 +3293,14 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.32.0" +version = "1.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c4216490d5a413bc6d10fa4742bd7d4955941d062c0ef873141d6b0e7b30fd" +checksum = "4a2ab0025103a60ecaaf3abf24db1db240a4e1c15837090d2c32f625ac98abea" dependencies = [ "arrayvec", "borsh 0.10.3", - "bytes 1.5.0", + "byteorder", + "bytes 1.4.0", "num-traits", "rand 0.8.5", "rkyv", @@ -3172,11 +3325,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.13" +version = "0.38.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" +checksum = "172891ebdceb05aa0005f533a6cbfca599ddd7d966f6f5d4d9b2e70478e70399" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.3.3", "errno", "libc", "linux-raw-sys", @@ -3200,7 +3353,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.4", + "base64 0.21.2", ] [[package]] @@ -3268,11 +3421,39 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bd3534a9978d0aa7edd2808dc1f8f31c4d0ecd31ddf71d997b3c98e9f3c9114" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", +] + +[[package]] +name = "sea-orm" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88694d01b528a94f90ad87f8d2f546d060d070eee180315c67d158cb69476034" +dependencies = [ + "async-stream", + "async-trait", + "chrono", + "futures", + "futures-util", + "log", + "ouroboros 0.15.6", + "rust_decimal", + "sea-orm-macros 0.10.7", + "sea-query 0.27.2", + "sea-query-binder 0.2.2", + "sea-strum", + "serde", + "serde_json", + "sqlx 0.6.3", + "thiserror", + "time 0.3.25", + "tracing", + "url", + "uuid", ] [[package]] @@ -3287,17 +3468,17 @@ dependencies = [ "chrono", "futures", "log", - "ouroboros", + "ouroboros 0.17.2", "rust_decimal", - "sea-orm-macros", - "sea-query", - "sea-query-binder", + "sea-orm-macros 0.12.2", + "sea-query 0.30.1", + "sea-query-binder 0.5.0", "serde", "serde_json", - "sqlx", + "sqlx 0.7.1", "strum 0.25.0", "thiserror", - "time", + "time 0.3.25", "tracing", "url", "uuid", @@ -3320,17 +3501,30 @@ dependencies = [ "url", ] +[[package]] +name = "sea-orm-macros" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216195de9c6b2474fd0efab486173dccd0eff21f28cc54aa4c0205d52fb3af0" +dependencies = [ + "bae", + "heck 0.3.3", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "sea-orm-macros" version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd90e73d5f5b184bad525767da29fbfec132b4e62ebd6f60d2f2737ec6468f62" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "sea-bae", - "syn 2.0.37", + "syn 2.0.28", "unicode-ident", ] @@ -3344,13 +3538,27 @@ dependencies = [ "clap", "dotenvy", "futures", - "sea-orm", + "sea-orm 0.12.2", "sea-orm-cli", "sea-schema", "tracing", "tracing-subscriber", ] +[[package]] +name = "sea-query" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0fc4d8e44e1d51c739a68d336252a18bc59553778075d5e32649be6ec92ed" +dependencies = [ + "chrono", + "rust_decimal", + "sea-query-derive 0.2.0", + "serde_json", + "time 0.3.25", + "uuid", +] + [[package]] name = "sea-query" version = "0.30.1" @@ -3363,9 +3571,24 @@ dependencies = [ "inherent", "ordered-float", "rust_decimal", - "sea-query-derive", + "sea-query-derive 0.4.0", + "serde_json", + "time 0.3.25", + "uuid", +] + +[[package]] +name = "sea-query-binder" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c2585b89c985cfacfe0ec9fc9e7bb055b776c1a2581c4e3c6185af2b8bf8865" +dependencies = [ + "chrono", + "rust_decimal", + "sea-query 0.27.2", "serde_json", - "time", + "sqlx 0.6.3", + "time 0.3.25", "uuid", ] @@ -3378,20 +3601,33 @@ dependencies = [ "bigdecimal", "chrono", "rust_decimal", - "sea-query", + "sea-query 0.30.1", "serde_json", - "sqlx", - "time", + "sqlx 0.7.1", + "time 0.3.25", "uuid", ] +[[package]] +name = "sea-query-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cdc022b4f606353fe5dc85b09713a04e433323b70163e81513b141c6ae6eb5" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "syn 1.0.109", + "thiserror", +] + [[package]] name = "sea-query-derive" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd78f2e0ee8e537e9195d1049b752e0433e2cac125426bccb7b5c3e508096117" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "syn 1.0.109", @@ -3405,7 +3641,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cd9561232bd1b82ea748b581f15909d11de0db6563ddcf28c5d908aee8282f1" dependencies = [ "futures", - "sea-query", + "sea-query 0.30.1", "sea-schema-derive", ] @@ -3415,9 +3651,31 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f686050f76bffc4f635cda8aea6df5548666b830b52387e8bc7de11056d11e" dependencies = [ - "heck", + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "sea-strum" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391d06a6007842cfe79ac6f7f53911b76dfd69fc9a6769f1cf6569d12ce20e1b" +dependencies = [ + "sea-strum_macros", +] + +[[package]] +name = "sea-strum_macros" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b4397b825df6ccf1e98bcdabef3bbcfc47ff5853983467850eeab878384f21" +dependencies = [ + "heck 0.3.3", "proc-macro2", "quote", + "rustversion", "syn 1.0.109", ] @@ -3458,9 +3716,9 @@ checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" dependencies = [ "serde_derive", ] @@ -3476,22 +3734,22 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", ] [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" dependencies = [ - "itoa", + "itoa 1.0.9", "ryu", "serde", ] @@ -3512,7 +3770,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa", + "itoa 1.0.9", "ryu", "serde", ] @@ -3530,7 +3788,7 @@ dependencies = [ "serde", "serde_json", "serde_with_macros", - "time", + "time 0.3.25", ] [[package]] @@ -3542,7 +3800,16 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", +] + +[[package]] +name = "sha1" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" +dependencies = [ + "sha1_smol", ] [[package]] @@ -3556,6 +3823,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.9.9" @@ -3627,9 +3900,9 @@ checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" [[package]] name = "slab" -version = "0.4.9" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg", ] @@ -3658,9 +3931,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" dependencies = [ "libc", "windows-sys", @@ -3716,14 +3989,14 @@ dependencies = [ "console_log", "curve25519-dalek", "getrandom 0.1.16", - "itertools 0.10.5", + "itertools", "js-sys", "lazy_static", "libsecp256k1", "log", "num-derive", "num-traits", - "parking_lot", + "parking_lot 0.12.1", "rand 0.7.3", "rustc_version", "rustversion", @@ -3779,28 +4052,84 @@ dependencies = [ [[package]] name = "sqlformat" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85" +checksum = "0c12bc9199d1db8234678b7051747c07f517cdcf019262d1847b94ec8b1aee3e" dependencies = [ - "itertools 0.11.0", + "itertools", "nom", "unicode_categories", ] +[[package]] +name = "sqlx" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8de3b03a925878ed54a954f621e64bf55a3c1bd29652d0d1a17830405350188" +dependencies = [ + "sqlx-core 0.6.3", + "sqlx-macros 0.6.3", +] + [[package]] name = "sqlx" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e58421b6bc416714d5115a2ca953718f6c621a51b68e4f4922aea5a4391a721" dependencies = [ - "sqlx-core", - "sqlx-macros", + "sqlx-core 0.7.1", + "sqlx-macros 0.7.1", "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", ] +[[package]] +name = "sqlx-core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8241483a83a3f33aa5fff7e7d9def398ff9990b2752b6c6112b83c6d246029" +dependencies = [ + "ahash 0.7.6", + "atoi 1.0.0", + "bitflags 1.3.2", + "byteorder", + "bytes 1.4.0", + "chrono", + "crc", + "crossbeam-queue", + "dotenvy", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive 0.4.2", + "futures-util", + "hashlink", + "hex", + "indexmap 1.9.3", + "itoa 1.0.9", + "libc", + "log", + "memchr", + "num-bigint", + "once_cell", + "paste", + "percent-encoding", + "rust_decimal", + "serde", + "serde_json", + "sha2 0.10.7", + "smallvec", + "sqlformat", + "sqlx-rt", + "stringprep", + "thiserror", + "time 0.3.25", + "url", + "uuid", +] + [[package]] name = "sqlx-core" version = "0.7.1" @@ -3808,10 +4137,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53" dependencies = [ "ahash 0.8.3", - "atoi", + "atoi 2.0.0", "bigdecimal", "byteorder", - "bytes 1.5.0", + "bytes 1.4.0", "chrono", "crc", "crossbeam-queue", @@ -3820,7 +4149,7 @@ dependencies = [ "event-listener", "futures-channel", "futures-core", - "futures-intrusive", + "futures-intrusive 0.5.0", "futures-io", "futures-util", "hashlink", @@ -3840,7 +4169,7 @@ dependencies = [ "smallvec", "sqlformat", "thiserror", - "time", + "time 0.3.25", "tokio", "tokio-stream", "tracing", @@ -3849,6 +4178,26 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "sqlx-macros" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9966e64ae989e7e575b19d7265cb79d7fc3cbbdf179835cb0d716f294c2049c9" +dependencies = [ + "dotenvy", + "either", + "heck 0.4.1", + "once_cell", + "proc-macro2", + "quote", + "serde_json", + "sha2 0.10.7", + "sqlx-core 0.6.3", + "sqlx-rt", + "syn 1.0.109", + "url", +] + [[package]] name = "sqlx-macros" version = "0.7.1" @@ -3857,7 +4206,7 @@ checksum = "208e3165167afd7f3881b16c1ef3f2af69fa75980897aac8874a0696516d12c2" dependencies = [ "proc-macro2", "quote", - "sqlx-core", + "sqlx-core 0.7.1", "sqlx-macros-core", "syn 1.0.109", ] @@ -3870,7 +4219,7 @@ checksum = "8a4a8336d278c62231d87f24e8a7a74898156e34c1c18942857be2acb29c7dfc" dependencies = [ "dotenvy", "either", - "heck", + "heck 0.4.1", "hex", "once_cell", "proc-macro2", @@ -3878,7 +4227,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.7", - "sqlx-core", + "sqlx-core 0.7.1", "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", @@ -3894,12 +4243,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482" dependencies = [ - "atoi", - "base64 0.21.4", + "atoi 2.0.0", + "base64 0.21.2", "bigdecimal", - "bitflags 2.4.0", + "bitflags 2.3.3", "byteorder", - "bytes 1.5.0", + "bytes 1.4.0", "chrono", "crc", "digest 0.10.7", @@ -3913,7 +4262,7 @@ dependencies = [ "hex", "hkdf", "hmac 0.12.1", - "itoa", + "itoa 1.0.9", "log", "md-5", "memchr", @@ -3923,13 +4272,13 @@ dependencies = [ "rsa", "rust_decimal", "serde", - "sha1", + "sha1 0.10.5", "sha2 0.10.7", "smallvec", - "sqlx-core", + "sqlx-core 0.7.1", "stringprep", "thiserror", - "time", + "time 0.3.25", "tracing", "uuid", "whoami", @@ -3941,10 +4290,10 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e" dependencies = [ - "atoi", - "base64 0.21.4", + "atoi 2.0.0", + "base64 0.21.2", "bigdecimal", - "bitflags 2.4.0", + "bitflags 2.3.3", "byteorder", "chrono", "crc", @@ -3958,7 +4307,7 @@ dependencies = [ "hkdf", "hmac 0.12.1", "home", - "itoa", + "itoa 1.0.9", "log", "md-5", "memchr", @@ -3968,38 +4317,44 @@ dependencies = [ "rust_decimal", "serde", "serde_json", - "sha1", + "sha1 0.10.5", "sha2 0.10.7", "smallvec", - "sqlx-core", + "sqlx-core 0.7.1", "stringprep", "thiserror", - "time", + "time 0.3.25", "tracing", "uuid", "whoami", ] +[[package]] +name = "sqlx-rt" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804d3f245f894e61b1e6263c84b23ca675d96753b5abfd5cc8597d86806e8024" + [[package]] name = "sqlx-sqlite" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4c21bf34c7cae5b283efb3ac1bcc7670df7561124dc2f8bdc0b59be40f79a2" dependencies = [ - "atoi", + "atoi 2.0.0", "chrono", "flume", "futures-channel", "futures-core", "futures-executor", - "futures-intrusive", + "futures-intrusive 0.5.0", "futures-util", "libsqlite3-sys", "log", "percent-encoding", "serde", - "sqlx-core", - "time", + "sqlx-core 0.7.1", + "time 0.3.25", "tracing", "url", "uuid", @@ -4025,11 +4380,10 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "stringprep" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +checksum = "db3737bde7edce97102e0e2b15365bf7a20bfdb5f60f4f9e8d7004258a51a8da" dependencies = [ - "finl_unicode", "unicode-bidi", "unicode-normalization", ] @@ -4061,7 +4415,7 @@ version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "rustversion", @@ -4087,9 +4441,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.37" +version = "2.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ "proc-macro2", "quote", @@ -4116,35 +4470,35 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.8.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" dependencies = [ "cfg-if", "fastrand 2.0.0", - "redox_syscall", + "redox_syscall 0.3.5", "rustix", "windows-sys", ] [[package]] name = "thiserror" -version = "1.0.48" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.48" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", ] [[package]] @@ -4159,12 +4513,23 @@ dependencies = [ [[package]] name = "time" -version = "0.3.28" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "time" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea" dependencies = [ "deranged", - "itoa", + "itoa 1.0.9", "serde", "time-core", "time-macros", @@ -4178,9 +4543,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.14" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" +checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd" dependencies = [ "time-core", ] @@ -4207,13 +4572,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ "backtrace", - "bytes 1.5.0", + "bytes 1.4.0", "libc", "mio", "num_cpus", - "parking_lot", + "parking_lot 0.12.1", "pin-project-lite", - "socket2 0.5.4", + "socket2 0.5.3", "tokio-macros", "windows-sys", ] @@ -4226,7 +4591,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", ] [[package]] @@ -4252,9 +4617,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.20.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2dbec703c26b00d74844519606ef15d09a7d6857860f84ad223dec002ddea2" +checksum = "ec509ac96e9a0c43427c74f003127d953a265737636129424288d27cb5c4b12c" dependencies = [ "futures-util", "log", @@ -4262,13 +4627,27 @@ dependencies = [ "tungstenite", ] +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes 1.4.0", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "futures-core", "futures-io", "futures-sink", @@ -4289,9 +4668,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.8" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" dependencies = [ "serde", "serde_spanned", @@ -4310,9 +4689,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.19.15" +version = "0.19.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" dependencies = [ "indexmap 2.0.0", "serde", @@ -4348,7 +4727,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", ] [[package]] @@ -4429,18 +4808,18 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "tungstenite" -version = "0.20.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e862a1c4128df0112ab625f55cd5c934bcb4312ba80b39ae4b4835a3fd58e649" +checksum = "15fba1a6d6bb030745759a9a2a588bfe8490fc8b4751a277db3a0be1c9ebbf67" dependencies = [ "byteorder", - "bytes 1.5.0", + "bytes 1.4.0", "data-encoding", "http", "httparse", "log", "rand 0.8.5", - "sha1", + "sha1 0.10.5", "thiserror", "url", "utf-8", @@ -4448,9 +4827,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.17.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "ucd-trie" @@ -4475,9 +4854,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-normalization" @@ -4508,9 +4887,9 @@ checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" [[package]] name = "unsigned-varint" -version = "0.7.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" +checksum = "d86a8dc7f45e4c1b0d30e43038c38f274e77af056aa5f74b93c2cf9eb3c1c836" [[package]] name = "untrusted" @@ -4520,9 +4899,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.4.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" dependencies = [ "form_urlencoded", "idna", @@ -4591,6 +4970,12 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -4618,7 +5003,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", "wasm-bindgen-shared", ] @@ -4652,7 +5037,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.28", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4665,9 +5050,9 @@ checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "wasm-streams" -version = "0.3.0" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" dependencies = [ "futures-util", "js-sys", @@ -4697,14 +5082,13 @@ dependencies = [ [[package]] name = "which" -version = "4.4.2" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" dependencies = [ "either", - "home", + "libc", "once_cell", - "rustix", ] [[package]] @@ -4755,9 +5139,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.5" +version = "0.48.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -4770,63 +5154,62 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.5" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" [[package]] name = "windows_aarch64_msvc" -version = "0.48.5" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" [[package]] name = "windows_i686_gnu" -version = "0.48.5" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" [[package]] name = "windows_i686_msvc" -version = "0.48.5" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" [[package]] name = "windows_x86_64_gnu" -version = "0.48.5" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.5" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" [[package]] name = "windows_x86_64_msvc" -version = "0.48.5" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.5.15" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +checksum = "acaaa1190073b2b101e15083c38ee8ec891b5e05cbee516521e94ec008f61e64" dependencies = [ "memchr", ] [[package]] name = "winreg" -version = "0.50.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ - "cfg-if", - "windows-sys", + "winapi", ] [[package]] diff --git a/api/Cargo.toml b/api/Cargo.toml index 21dfc72..68d7bf1 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -2,9 +2,7 @@ name = "holaplex-hub-nfts" version = "0.1.1" publish = false -authors = [ - "Holaplex ", -] +authors = ["Holaplex "] edition = "2021" description = "Holaplex Hub nfts service" readme = "./README.md" @@ -29,6 +27,7 @@ async-graphql = { version = "5.0.4", features = [ "dataloader", "apollo_tracing", ] } +redis = { version = "0.20.0", features = ["aio"] } serde = { version = "1.0.152", features = ["derive"] } serde_json = "1.0.93" solana-program = "1" diff --git a/api/src/background_worker/job.rs b/api/src/background_worker/job.rs new file mode 100644 index 0000000..5be17d8 --- /dev/null +++ b/api/src/background_worker/job.rs @@ -0,0 +1,45 @@ +use super::tasks::BackgroundTask; +use serde::{Deserialize, Serialize}; + +#[derive(Debug)] +pub struct Job { + pub id: i64, + pub task: T, +} + +impl Serialize for Job { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let payload = self.task.payload(); + let payload_str = serde_json::to_string(&payload).map_err(serde::ser::Error::custom)?; + + let mut state = serializer.serialize_struct("Job", 3)?; + state.serialize_field("id", &self.id)?; + state.serialize_field("task", &payload_str)?; + state.end() + } +} + +impl<'de, T: BackgroundTask + for<'a> Deserialize<'a>> Deserialize<'de> for Job { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + struct JobHelper { + id: i64, + task: String, + } + + let helper = JobHelper::deserialize(deserializer)?; + + let task: T = serde_json::from_str(&helper.task).map_err(serde::de::Error::custom)?; + + Ok(Job { + id: helper.id, + task, + }) + } +} diff --git a/api/src/background_worker/job_queue.rs b/api/src/background_worker/job_queue.rs new file mode 100644 index 0000000..95f2ba4 --- /dev/null +++ b/api/src/background_worker/job_queue.rs @@ -0,0 +1,79 @@ +use super::{job::Job, tasks::BackgroundTask}; +use crate::db::Connection; +use redis::AsyncCommands; +use redis::Client; +use std::error::Error; +use std::fmt; +use std::sync::{Arc, Mutex}; + +#[derive(Debug)] +struct LockError(String); + +impl fmt::Display for LockError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl Error for LockError {} + +pub struct JobQueue { + client: Arc>, + db_pool: Arc, +} + +impl JobQueue { + pub async fn new(redis_url: &str, db_pool: Connection) -> Self { + let client = Client::open(redis_url).expect("Failed to create Redis client"); + Self { + client: Arc::new(Mutex::new(client)), + db_pool: Arc::new(db_pool), + } + } + + pub async fn enqueue( + &self, + job: &Job, + ) -> Result<(), Box> { + let client_guard = self + .client + .lock() + .map_err(|e| Box::new(LockError(e.to_string())) as Box)?; + let mut conn = client_guard.get_async_connection().await?; + + let payload = serde_json::to_string(&job.task.payload())?; + + redis::cmd("LPUSH") + .arg("job_queue") + .arg(payload) + .query_async(&mut conn) + .await?; + Ok(()) + } + + pub async fn dequeue( + &self, + ) -> Result>, Box> { + let client_guard = self + .client + .lock() + .map_err(|e| Box::new(LockError(e.to_string())) as Box)?; + let mut conn = client_guard.get_async_connection()?; + + let payload: Option = redis::cmd("RPOP") + .arg("job_queue") + .query_async(&mut conn) + .await?; + + if let Some(payload) = payload { + let task: Box = serde_json::from_str(&payload)?; + let job = Job { + id: generate_unique_id(), // You would need to implement this function + task, + }; + Ok(Some(job)) + } else { + Ok(None) + } + } +} diff --git a/api/src/background_worker/mod.rs b/api/src/background_worker/mod.rs new file mode 100644 index 0000000..b5d8123 --- /dev/null +++ b/api/src/background_worker/mod.rs @@ -0,0 +1,4 @@ +mod job_queue; +mod worker; +mod job; +mod tasks; \ No newline at end of file diff --git a/api/src/background_worker/tasks/metadata_json_upload_task.rs b/api/src/background_worker/tasks/metadata_json_upload_task.rs new file mode 100644 index 0000000..a33c3e6 --- /dev/null +++ b/api/src/background_worker/tasks/metadata_json_upload_task.rs @@ -0,0 +1 @@ +use super::BackgroundTask; diff --git a/api/src/background_worker/tasks/mod.rs b/api/src/background_worker/tasks/mod.rs new file mode 100644 index 0000000..54eb749 --- /dev/null +++ b/api/src/background_worker/tasks/mod.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value as Json; + +pub trait BackgroundTask: Send + Sync + std::fmt::Debug { + fn process(&self) -> Result<(), Box>; + fn payload(&self) -> Json; +} + diff --git a/api/src/background_worker/worker.rs b/api/src/background_worker/worker.rs new file mode 100644 index 0000000..1a2f7f1 --- /dev/null +++ b/api/src/background_worker/worker.rs @@ -0,0 +1,80 @@ +use super::job_queue::JobQueue; +use crate::db::Connection; +use hub_core::{ + tokio, + tracing::{error, info}, +}; +use std::sync::Arc; + +pub struct Worker { + job_queue: Arc, + db_pool: Arc, +} + +impl Worker { + pub fn new(job_queue: Arc, db_pool: Connection) -> Self { + Self { + job_queue, + db_pool: Arc::new(db_pool), + } + } + + pub async fn start(&self) { + loop { + if let Ok(Some(mut job)) = self.job_queue.dequeue().await { + let job_queue_clone = self.job_queue.clone(); + let job_id = job.id; + let db_pool_clone = Arc::clone(&self.db_pool); + tokio::spawn(async move { + if JobTracking::find_by_id(job_id, &db_pool_clone) + .await? + .is_none() + { + + // Create a new record in the database + JobTracking::create( + job_id, + "JobType", + job.task.payload(), + "processing", + &db_pool_clone, + ) + .await?; + + // sora elle espinola + // Process the job using the trait method + match job.task.process() { + Ok(_) => { + // Update the job status in the database to "completed" + JobTracking::update_status(&job, "completed", &db_pool_clone) + .await + .unwrap(); + }, + Err(e) => { + println!("Job processing failed: {}", e); + + // Re-queue the job and update the job status in the database to "queued" + job_queue_clone + .enqueue(&job) + .await + .expect("Failed to re-queue job"); + JobTracking::update_status(&job, "queued", &db_pool_clone) + .await + .unwrap(); + }, + } + } else { + info!("Duplicate job detected, skipping: {}", job_id); + } + Ok::<(), sea_orm::DbErr>(()) + }) + .await + .unwrap_or_else(|e| { + error!("An error occurred: {}", e); + + Ok::<(), sea_orm::DbErr>(()) + }); + } + } + } +} diff --git a/api/src/entities/job_trackings.rs b/api/src/entities/job_trackings.rs new file mode 100644 index 0000000..dfc1b15 --- /dev/null +++ b/api/src/entities/job_trackings.rs @@ -0,0 +1,19 @@ +use sea_orm::entity::prelude::*; +use serde_json::Value as Json; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "job_trackings")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i64, + pub job_type: String, + pub payload: Json, + pub status: String, + pub created_at: DateTimeWithTimeZone, + pub updated_at: DateTimeWithTimeZone, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} \ No newline at end of file diff --git a/api/src/entities/mod.rs b/api/src/entities/mod.rs index 7fc278d..4815af4 100644 --- a/api/src/entities/mod.rs +++ b/api/src/entities/mod.rs @@ -19,3 +19,4 @@ pub mod sea_orm_active_enums; pub mod switch_collection_histories; pub mod transfer_charges; pub mod update_histories; +pub mod job_trackings; diff --git a/api/src/lib.rs b/api/src/lib.rs index 51ea412..a365c0a 100644 --- a/api/src/lib.rs +++ b/api/src/lib.rs @@ -15,6 +15,7 @@ pub mod mutations; pub mod nft_storage; pub mod objects; pub mod queries; +pub mod background_worker; use async_graphql::{ dataloader::DataLoader, diff --git a/docker-compose.yaml b/docker-compose.yaml index d372f52..609d786 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,5 +1,11 @@ version: '3.8' services: + redis: + image: redis:7.2.1 + ports: + - 6379:6379 + volumes: + - holaplex_hub_nfts_redis:/data db: image: postgres:15.1 container_name: db @@ -28,4 +34,6 @@ services: - 29092:29092 volumes: holaplex_hub_nfts: + driver: local + holaplex_hub_nfts_redis: driver: local \ No newline at end of file diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 46e80fd..52ecf9e 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -53,11 +53,15 @@ mod m20230725_144506_drop_solana_collections_table; mod m20230807_090847_create_histories_table; mod m20230818_163948_downcase_polygon_addresses; mod m20230821_131630_create_switch_collection_histories_table; +<<<<<<< HEAD mod m20230905_100852_add_type_to_drop; mod m20230910_204731_add_queued_variant_to_mints_status; mod m20230910_212742_make_owner_address_optional_for_mint; mod m20230911_144938_make_compressed_column_optional; mod m20230915_111128_create_mints_creation_status_idx; +======= +mod m20230914_154759_add_job_trackings_table; +>>>>>>> 502f5ad (feat: create worker queue using redis for uploading metadata json in the background) pub struct Migrator; @@ -118,11 +122,15 @@ impl MigratorTrait for Migrator { Box::new(m20230807_090847_create_histories_table::Migration), Box::new(m20230818_163948_downcase_polygon_addresses::Migration), Box::new(m20230821_131630_create_switch_collection_histories_table::Migration), +<<<<<<< HEAD Box::new(m20230905_100852_add_type_to_drop::Migration), Box::new(m20230910_204731_add_queued_variant_to_mints_status::Migration), Box::new(m20230910_212742_make_owner_address_optional_for_mint::Migration), Box::new(m20230911_144938_make_compressed_column_optional::Migration), Box::new(m20230915_111128_create_mints_creation_status_idx::Migration), +======= + Box::new(m20230914_154759_add_job_trackings_table::Migration), +>>>>>>> 502f5ad (feat: create worker queue using redis for uploading metadata json in the background) ] } } diff --git a/migration/src/m20230914_154759_add_job_trackings_table.rs b/migration/src/m20230914_154759_add_job_trackings_table.rs new file mode 100644 index 0000000..d4827cf --- /dev/null +++ b/migration/src/m20230914_154759_add_job_trackings_table.rs @@ -0,0 +1,56 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(JobTrackings::Table) + .if_not_exists() + .col( + ColumnDef::new(JobTrackings::Id) + .integer() + .not_null() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new(JobTrackings::JobType).string().not_null()) + .col(ColumnDef::new(JobTrackings::Status).string().not_null()) + .col(ColumnDef::new(JobTrackings::Payload).json().not_null()) + .col( + ColumnDef::new(JobTrackings::CreatedAt) + .timestamp_with_time_zone() + .not_null(), + ) + .col( + ColumnDef::new(JobTrackings::UpdatedAt) + .timestamp_with_time_zone() + .not_null(), + ) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(JobTrackings::Table).to_owned()) + .await + } +} + +/// Learn more at https://docs.rs/sea-query#iden +#[derive(Iden)] +enum JobTrackings { + Table, + Id, + Status, + JobType, + Payload, + CreatedAt, + UpdatedAt, +} From ae649183158ec537298ee610317265c54256788c Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Tue, 26 Sep 2023 15:06:00 +0200 Subject: [PATCH 2/3] refactor: update all mutations that upload metadata json to use metadata json upload job queue. also modify these mutations to use transactions and re-organize structure of mutations to make more readable. --- .env | 1 + Cargo.lock | 452 ++--------- api/Cargo.toml | 2 +- api/src/background_worker/job.rs | 58 +- api/src/background_worker/job_queue.rs | 129 +++- api/src/background_worker/mod.rs | 8 +- .../tasks/metadata_json_upload_task.rs | 717 +++++++++++++++++- api/src/background_worker/tasks/mod.rs | 66 +- api/src/background_worker/worker.rs | 134 ++-- api/src/collection.rs | 58 -- api/src/entities/collections.rs | 6 + api/src/entities/drops.rs | 8 + api/src/entities/job_trackings.rs | 35 +- api/src/entities/metadata_jsons.rs | 14 +- api/src/entities/mod.rs | 2 +- api/src/entities/sea_orm_active_enums.rs | 14 +- api/src/events.rs | 26 +- api/src/handlers.rs | 4 +- api/src/lib.rs | 15 +- api/src/main.rs | 34 +- api/src/metadata_json.rs | 170 ----- api/src/mutations/collection.rs | 253 +++--- api/src/mutations/drop.rs | 364 +++++---- api/src/mutations/mint.rs | 524 +++++++------ api/src/nft_storage.rs | 2 +- api/src/objects/metadata_json.rs | 178 +++-- migration/Cargo.toml | 11 +- migration/src/lib.rs | 10 +- ...lable_metadata_jsons_identifier_and_uri.rs | 39 + 29 files changed, 1902 insertions(+), 1432 deletions(-) delete mode 100644 api/src/collection.rs delete mode 100644 api/src/metadata_json.rs create mode 100644 migration/src/m20230922_150621_nullable_metadata_jsons_identifier_and_uri.rs diff --git a/.env b/.env index fd6cf4f..ba460bd 100644 --- a/.env +++ b/.env @@ -10,3 +10,4 @@ IPFS_ENDPOINT=https://nftstorage.link/ipfs CREDIT_SHEET=credits.toml ASSET_CDN=https://assets.holaplex.tools NFT_STORAGE_AUTH_TOKEN="" +REDIS_URL=redis://localhost:6379 \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index a9850dd..46b0024 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -239,7 +239,7 @@ dependencies = [ "futures-util", "poem", "serde_json", - "tokio-util 0.7.8", + "tokio-util", ] [[package]] @@ -287,15 +287,6 @@ dependencies = [ "syn 2.0.28", ] -[[package]] -name = "atoi" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c57d12312ff59c811c0643f4d80830505833c9ffaebd193d819392b265be8e" -dependencies = [ - "num-traits", -] - [[package]] name = "atoi" version = "2.0.0" @@ -338,19 +329,6 @@ dependencies = [ "rustc-demangle", ] -[[package]] -name = "bae" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b8de67cc41132507eeece2584804efcb15f85ba516e34c944b7667f480397a" -dependencies = [ - "heck 0.3.3", - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "base-x" version = "0.2.11" @@ -741,7 +719,7 @@ version = "4.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" dependencies = [ - "heck 0.4.1", + "heck", "proc-macro2", "quote", "syn 2.0.28", @@ -770,7 +748,7 @@ dependencies = [ "memchr", "pin-project-lite", "tokio", - "tokio-util 0.7.8", + "tokio-util", ] [[package]] @@ -1088,12 +1066,6 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" -[[package]] -name = "dtoa" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" - [[package]] name = "duct" version = "0.13.6" @@ -1294,17 +1266,6 @@ dependencies = [ "futures-util", ] -[[package]] -name = "futures-intrusive" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a604f7a68fbf8103337523b1fadc8ade7361ee3f112f7c680ad179651616aed5" -dependencies = [ - "futures-core", - "lock_api", - "parking_lot 0.11.2", -] - [[package]] name = "futures-intrusive" version = "0.5.0" @@ -1313,7 +1274,7 @@ checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" dependencies = [ "futures-core", "lock_api", - "parking_lot 0.12.1", + "parking_lot", ] [[package]] @@ -1443,7 +1404,7 @@ dependencies = [ "indexmap 1.9.3", "slab", "tokio", - "tokio-util 0.7.8", + "tokio-util", "tracing", ] @@ -1520,7 +1481,7 @@ dependencies = [ "http", "httpdate", "mime", - "sha1 0.10.5", + "sha1", ] [[package]] @@ -1532,15 +1493,6 @@ dependencies = [ "http", ] -[[package]] -name = "heck" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "heck" version = "0.4.1" @@ -1632,7 +1584,7 @@ dependencies = [ "rand 0.8.5", "rdkafka", "reqwest", - "sea-orm 0.10.7", + "sea-orm", "serde_json", "serde_with", "strum 0.24.1", @@ -1699,7 +1651,7 @@ dependencies = [ "prost-types", "redis", "reqwest", - "sea-orm 0.12.2", + "sea-orm", "serde", "serde_json", "solana-program", @@ -1734,7 +1686,7 @@ checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes 1.4.0", "fnv", - "itoa 1.0.9", + "itoa", ] [[package]] @@ -1775,7 +1727,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.9", + "itoa", "pin-project-lite", "socket2 0.4.9", "tokio", @@ -1903,12 +1855,6 @@ dependencies = [ "either", ] -[[package]] -name = "itoa" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - [[package]] name = "itoa" version = "1.0.9" @@ -2504,16 +2450,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "ouroboros" -version = "0.15.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1358bd1558bd2a083fed428ffeda486fbfb323e698cdda7794259d592ca72db" -dependencies = [ - "aliasable", - "ouroboros_macro 0.15.6", -] - [[package]] name = "ouroboros" version = "0.17.2" @@ -2521,30 +2457,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2ba07320d39dfea882faa70554b4bd342a5f273ed59ba7c1c6b4c840492c954" dependencies = [ "aliasable", - "ouroboros_macro 0.17.2", + "ouroboros_macro", "static_assertions", ] -[[package]] -name = "ouroboros_macro" -version = "0.15.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f7d21ccd03305a674437ee1248f3ab5d4b1db095cf1caf49f1713ddf61956b7" -dependencies = [ - "Inflector", - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "ouroboros_macro" version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec4c6225c69b4ca778c0aea097321a64c421cf4577b331c61b229267edabb6f8" dependencies = [ - "heck 0.4.1", + "heck", "proc-macro-error", "proc-macro2", "quote", @@ -2557,17 +2480,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] - [[package]] name = "parking_lot" version = "0.12.1" @@ -2575,21 +2487,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.8", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", + "parking_lot_core", ] [[package]] @@ -2600,7 +2498,7 @@ checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall", "smallvec", "windows-targets", ] @@ -2774,7 +2672,7 @@ dependencies = [ "http", "hyper", "mime", - "parking_lot 0.12.1", + "parking_lot", "percent-encoding", "pin-project-lite", "poem-derive", @@ -2789,7 +2687,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-tungstenite", - "tokio-util 0.7.8", + "tokio-util", "tracing", ] @@ -2883,7 +2781,7 @@ dependencies = [ "fnv", "lazy_static", "memchr", - "parking_lot 0.12.1", + "parking_lot", "protobuf", "thiserror", ] @@ -2905,7 +2803,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ "bytes 1.4.0", - "heck 0.4.1", + "heck", "itertools", "lazy_static", "log", @@ -3089,33 +2987,25 @@ dependencies = [ [[package]] name = "redis" -version = "0.20.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4f0ceb2ec0dd769483ecd283f6615aa83dcd0be556d5294c6e659caefe7cc54" +checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba" dependencies = [ "async-trait", "bytes 1.4.0", "combine", - "dtoa", "futures-util", - "itoa 0.4.8", + "itoa", "percent-encoding", "pin-project-lite", - "sha1 0.6.1", + "ryu", + "sha1_smol", + "socket2 0.4.9", "tokio", - "tokio-util 0.6.10", + "tokio-util", "url", ] -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.3.5" @@ -3207,7 +3097,7 @@ dependencies = [ "serde_urlencoded", "tokio", "tokio-native-tls", - "tokio-util 0.7.8", + "tokio-util", "tower-service", "url", "wasm-bindgen", @@ -3421,41 +3311,13 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bd3534a9978d0aa7edd2808dc1f8f31c4d0ecd31ddf71d997b3c98e9f3c9114" dependencies = [ - "heck 0.4.1", + "heck", "proc-macro-error", "proc-macro2", "quote", "syn 2.0.28", ] -[[package]] -name = "sea-orm" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88694d01b528a94f90ad87f8d2f546d060d070eee180315c67d158cb69476034" -dependencies = [ - "async-stream", - "async-trait", - "chrono", - "futures", - "futures-util", - "log", - "ouroboros 0.15.6", - "rust_decimal", - "sea-orm-macros 0.10.7", - "sea-query 0.27.2", - "sea-query-binder 0.2.2", - "sea-strum", - "serde", - "serde_json", - "sqlx 0.6.3", - "thiserror", - "time 0.3.25", - "tracing", - "url", - "uuid", -] - [[package]] name = "sea-orm" version = "0.12.2" @@ -3468,14 +3330,14 @@ dependencies = [ "chrono", "futures", "log", - "ouroboros 0.17.2", + "ouroboros", "rust_decimal", - "sea-orm-macros 0.12.2", - "sea-query 0.30.1", - "sea-query-binder 0.5.0", + "sea-orm-macros", + "sea-query", + "sea-query-binder", "serde", "serde_json", - "sqlx 0.7.1", + "sqlx", "strum 0.25.0", "thiserror", "time 0.3.25", @@ -3501,26 +3363,13 @@ dependencies = [ "url", ] -[[package]] -name = "sea-orm-macros" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7216195de9c6b2474fd0efab486173dccd0eff21f28cc54aa4c0205d52fb3af0" -dependencies = [ - "bae", - "heck 0.3.3", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "sea-orm-macros" version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd90e73d5f5b184bad525767da29fbfec132b4e62ebd6f60d2f2737ec6468f62" dependencies = [ - "heck 0.4.1", + "heck", "proc-macro2", "quote", "sea-bae", @@ -3538,27 +3387,13 @@ dependencies = [ "clap", "dotenvy", "futures", - "sea-orm 0.12.2", + "sea-orm", "sea-orm-cli", "sea-schema", "tracing", "tracing-subscriber", ] -[[package]] -name = "sea-query" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4f0fc4d8e44e1d51c739a68d336252a18bc59553778075d5e32649be6ec92ed" -dependencies = [ - "chrono", - "rust_decimal", - "sea-query-derive 0.2.0", - "serde_json", - "time 0.3.25", - "uuid", -] - [[package]] name = "sea-query" version = "0.30.1" @@ -3571,23 +3406,8 @@ dependencies = [ "inherent", "ordered-float", "rust_decimal", - "sea-query-derive 0.4.0", - "serde_json", - "time 0.3.25", - "uuid", -] - -[[package]] -name = "sea-query-binder" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c2585b89c985cfacfe0ec9fc9e7bb055b776c1a2581c4e3c6185af2b8bf8865" -dependencies = [ - "chrono", - "rust_decimal", - "sea-query 0.27.2", + "sea-query-derive", "serde_json", - "sqlx 0.6.3", "time 0.3.25", "uuid", ] @@ -3601,33 +3421,20 @@ dependencies = [ "bigdecimal", "chrono", "rust_decimal", - "sea-query 0.30.1", + "sea-query", "serde_json", - "sqlx 0.7.1", + "sqlx", "time 0.3.25", "uuid", ] -[[package]] -name = "sea-query-derive" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34cdc022b4f606353fe5dc85b09713a04e433323b70163e81513b141c6ae6eb5" -dependencies = [ - "heck 0.3.3", - "proc-macro2", - "quote", - "syn 1.0.109", - "thiserror", -] - [[package]] name = "sea-query-derive" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd78f2e0ee8e537e9195d1049b752e0433e2cac125426bccb7b5c3e508096117" dependencies = [ - "heck 0.4.1", + "heck", "proc-macro2", "quote", "syn 1.0.109", @@ -3641,7 +3448,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cd9561232bd1b82ea748b581f15909d11de0db6563ddcf28c5d908aee8282f1" dependencies = [ "futures", - "sea-query 0.30.1", + "sea-query", "sea-schema-derive", ] @@ -3651,34 +3458,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f686050f76bffc4f635cda8aea6df5548666b830b52387e8bc7de11056d11e" dependencies = [ - "heck 0.4.1", + "heck", "proc-macro2", "quote", "syn 1.0.109", ] -[[package]] -name = "sea-strum" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "391d06a6007842cfe79ac6f7f53911b76dfd69fc9a6769f1cf6569d12ce20e1b" -dependencies = [ - "sea-strum_macros", -] - -[[package]] -name = "sea-strum_macros" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69b4397b825df6ccf1e98bcdabef3bbcfc47ff5853983467850eeab878384f21" -dependencies = [ - "heck 0.3.3", - "proc-macro2", - "quote", - "rustversion", - "syn 1.0.109", -] - [[package]] name = "seahash" version = "4.1.0" @@ -3749,7 +3534,7 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" dependencies = [ - "itoa 1.0.9", + "itoa", "ryu", "serde", ] @@ -3770,7 +3555,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.9", + "itoa", "ryu", "serde", ] @@ -3803,15 +3588,6 @@ dependencies = [ "syn 2.0.28", ] -[[package]] -name = "sha1" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" -dependencies = [ - "sha1_smol", -] - [[package]] name = "sha1" version = "0.10.5" @@ -3996,7 +3772,7 @@ dependencies = [ "log", "num-derive", "num-traits", - "parking_lot 0.12.1", + "parking_lot", "rand 0.7.3", "rustc_version", "rustversion", @@ -4061,75 +3837,19 @@ dependencies = [ "unicode_categories", ] -[[package]] -name = "sqlx" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8de3b03a925878ed54a954f621e64bf55a3c1bd29652d0d1a17830405350188" -dependencies = [ - "sqlx-core 0.6.3", - "sqlx-macros 0.6.3", -] - [[package]] name = "sqlx" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e58421b6bc416714d5115a2ca953718f6c621a51b68e4f4922aea5a4391a721" dependencies = [ - "sqlx-core 0.7.1", - "sqlx-macros 0.7.1", + "sqlx-core", + "sqlx-macros", "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", ] -[[package]] -name = "sqlx-core" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8241483a83a3f33aa5fff7e7d9def398ff9990b2752b6c6112b83c6d246029" -dependencies = [ - "ahash 0.7.6", - "atoi 1.0.0", - "bitflags 1.3.2", - "byteorder", - "bytes 1.4.0", - "chrono", - "crc", - "crossbeam-queue", - "dotenvy", - "either", - "event-listener", - "futures-channel", - "futures-core", - "futures-intrusive 0.4.2", - "futures-util", - "hashlink", - "hex", - "indexmap 1.9.3", - "itoa 1.0.9", - "libc", - "log", - "memchr", - "num-bigint", - "once_cell", - "paste", - "percent-encoding", - "rust_decimal", - "serde", - "serde_json", - "sha2 0.10.7", - "smallvec", - "sqlformat", - "sqlx-rt", - "stringprep", - "thiserror", - "time 0.3.25", - "url", - "uuid", -] - [[package]] name = "sqlx-core" version = "0.7.1" @@ -4137,7 +3857,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53" dependencies = [ "ahash 0.8.3", - "atoi 2.0.0", + "atoi", "bigdecimal", "byteorder", "bytes 1.4.0", @@ -4149,7 +3869,7 @@ dependencies = [ "event-listener", "futures-channel", "futures-core", - "futures-intrusive 0.5.0", + "futures-intrusive", "futures-io", "futures-util", "hashlink", @@ -4178,26 +3898,6 @@ dependencies = [ "webpki-roots", ] -[[package]] -name = "sqlx-macros" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9966e64ae989e7e575b19d7265cb79d7fc3cbbdf179835cb0d716f294c2049c9" -dependencies = [ - "dotenvy", - "either", - "heck 0.4.1", - "once_cell", - "proc-macro2", - "quote", - "serde_json", - "sha2 0.10.7", - "sqlx-core 0.6.3", - "sqlx-rt", - "syn 1.0.109", - "url", -] - [[package]] name = "sqlx-macros" version = "0.7.1" @@ -4206,7 +3906,7 @@ checksum = "208e3165167afd7f3881b16c1ef3f2af69fa75980897aac8874a0696516d12c2" dependencies = [ "proc-macro2", "quote", - "sqlx-core 0.7.1", + "sqlx-core", "sqlx-macros-core", "syn 1.0.109", ] @@ -4219,7 +3919,7 @@ checksum = "8a4a8336d278c62231d87f24e8a7a74898156e34c1c18942857be2acb29c7dfc" dependencies = [ "dotenvy", "either", - "heck 0.4.1", + "heck", "hex", "once_cell", "proc-macro2", @@ -4227,7 +3927,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.7", - "sqlx-core 0.7.1", + "sqlx-core", "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", @@ -4243,7 +3943,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482" dependencies = [ - "atoi 2.0.0", + "atoi", "base64 0.21.2", "bigdecimal", "bitflags 2.3.3", @@ -4262,7 +3962,7 @@ dependencies = [ "hex", "hkdf", "hmac 0.12.1", - "itoa 1.0.9", + "itoa", "log", "md-5", "memchr", @@ -4272,10 +3972,10 @@ dependencies = [ "rsa", "rust_decimal", "serde", - "sha1 0.10.5", + "sha1", "sha2 0.10.7", "smallvec", - "sqlx-core 0.7.1", + "sqlx-core", "stringprep", "thiserror", "time 0.3.25", @@ -4290,7 +3990,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e" dependencies = [ - "atoi 2.0.0", + "atoi", "base64 0.21.2", "bigdecimal", "bitflags 2.3.3", @@ -4307,7 +4007,7 @@ dependencies = [ "hkdf", "hmac 0.12.1", "home", - "itoa 1.0.9", + "itoa", "log", "md-5", "memchr", @@ -4317,10 +4017,10 @@ dependencies = [ "rust_decimal", "serde", "serde_json", - "sha1 0.10.5", + "sha1", "sha2 0.10.7", "smallvec", - "sqlx-core 0.7.1", + "sqlx-core", "stringprep", "thiserror", "time 0.3.25", @@ -4329,31 +4029,25 @@ dependencies = [ "whoami", ] -[[package]] -name = "sqlx-rt" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804d3f245f894e61b1e6263c84b23ca675d96753b5abfd5cc8597d86806e8024" - [[package]] name = "sqlx-sqlite" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4c21bf34c7cae5b283efb3ac1bcc7670df7561124dc2f8bdc0b59be40f79a2" dependencies = [ - "atoi 2.0.0", + "atoi", "chrono", "flume", "futures-channel", "futures-core", "futures-executor", - "futures-intrusive 0.5.0", + "futures-intrusive", "futures-util", "libsqlite3-sys", "log", "percent-encoding", "serde", - "sqlx-core 0.7.1", + "sqlx-core", "time 0.3.25", "tracing", "url", @@ -4415,7 +4109,7 @@ version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ - "heck 0.4.1", + "heck", "proc-macro2", "quote", "rustversion", @@ -4476,7 +4170,7 @@ checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" dependencies = [ "cfg-if", "fastrand 2.0.0", - "redox_syscall 0.3.5", + "redox_syscall", "rustix", "windows-sys", ] @@ -4529,7 +4223,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea" dependencies = [ "deranged", - "itoa 1.0.9", + "itoa", "serde", "time-core", "time-macros", @@ -4576,7 +4270,7 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot 0.12.1", + "parking_lot", "pin-project-lite", "socket2 0.5.3", "tokio-macros", @@ -4627,20 +4321,6 @@ dependencies = [ "tungstenite", ] -[[package]] -name = "tokio-util" -version = "0.6.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" -dependencies = [ - "bytes 1.4.0", - "futures-core", - "futures-sink", - "log", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.8" @@ -4819,7 +4499,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "sha1 0.10.5", + "sha1", "thiserror", "url", "utf-8", diff --git a/api/Cargo.toml b/api/Cargo.toml index 68d7bf1..421db1c 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -27,7 +27,7 @@ async-graphql = { version = "5.0.4", features = [ "dataloader", "apollo_tracing", ] } -redis = { version = "0.20.0", features = ["aio"] } +redis = { version = "0.23.3", features = ["tokio-comp"] } serde = { version = "1.0.152", features = ["derive"] } serde_json = "1.0.93" solana-program = "1" diff --git a/api/src/background_worker/job.rs b/api/src/background_worker/job.rs index 5be17d8..32d6b21 100644 --- a/api/src/background_worker/job.rs +++ b/api/src/background_worker/job.rs @@ -1,45 +1,43 @@ +use serde::{ + de::{Deserialize, Deserializer, Error as DeError}, + Serialize, +}; + use super::tasks::BackgroundTask; -use serde::{Deserialize, Serialize}; -#[derive(Debug)] -pub struct Job { +#[derive(Serialize, Debug)] +pub struct Job> { pub id: i64, pub task: T, + _context_marker: std::marker::PhantomData, } -impl Serialize for Job { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - let payload = self.task.payload(); - let payload_str = serde_json::to_string(&payload).map_err(serde::ser::Error::custom)?; - - let mut state = serializer.serialize_struct("Job", 3)?; - state.serialize_field("id", &self.id)?; - state.serialize_field("task", &payload_str)?; - state.end() +impl Job +where + T: Serialize + Send + Sync + BackgroundTask, +{ + #[must_use] + pub fn new(id: i64, task: T) -> Self { + Self { + id, + task, + _context_marker: std::marker::PhantomData, + } } } -impl<'de, T: BackgroundTask + for<'a> Deserialize<'a>> Deserialize<'de> for Job { +impl<'de, C, T> Deserialize<'de> for Job +where + C: Clone, + T: Serialize + Send + Sync + BackgroundTask, + T: for<'a> Deserialize<'a>, +{ fn deserialize(deserializer: D) -> Result where - D: serde::Deserializer<'de>, + D: Deserializer<'de>, { - #[derive(Deserialize)] - struct JobHelper { - id: i64, - task: String, - } - - let helper = JobHelper::deserialize(deserializer)?; + let (id, task) = Deserialize::deserialize(deserializer).map_err(DeError::custom)?; - let task: T = serde_json::from_str(&helper.task).map_err(serde::de::Error::custom)?; - - Ok(Job { - id: helper.id, - task, - }) + Ok(Job::new(id, task)) } } diff --git a/api/src/background_worker/job_queue.rs b/api/src/background_worker/job_queue.rs index 95f2ba4..08bcd1a 100644 --- a/api/src/background_worker/job_queue.rs +++ b/api/src/background_worker/job_queue.rs @@ -1,10 +1,13 @@ +use std::{error::Error as StdError, fmt, sync::Arc}; + +use hub_core::{prelude::*, thiserror, tokio::sync::Mutex}; +use redis::{Client, RedisError}; +use sea_orm::{error::DbErr, ActiveModelTrait}; +use serde::{Deserialize, Serialize}; +use serde_json::Error as SerdeJsonError; + use super::{job::Job, tasks::BackgroundTask}; -use crate::db::Connection; -use redis::AsyncCommands; -use redis::Client; -use std::error::Error; -use std::fmt; -use std::sync::{Arc, Mutex}; +use crate::{db::Connection, entities::job_trackings}; #[derive(Debug)] struct LockError(String); @@ -15,65 +18,109 @@ impl fmt::Display for LockError { } } -impl Error for LockError {} +impl StdError for LockError {} +// Job queue errors +#[derive(thiserror::Error, Debug)] +pub enum JobQueueError { + #[error("Redis error: {0}")] + RedisConnection(#[from] RedisError), + #[error("Database error: {0}")] + Database(#[from] DbErr), + #[error("Serialization error: {0}")] + Serde(#[from] SerdeJsonError), + #[error("Background task error: {0}")] + BackgroundTask(#[from] Error), +} +#[derive(Clone)] pub struct JobQueue { client: Arc>, - db_pool: Arc, + db_pool: Connection, } impl JobQueue { - pub async fn new(redis_url: &str, db_pool: Connection) -> Self { - let client = Client::open(redis_url).expect("Failed to create Redis client"); - Self { - client: Arc::new(Mutex::new(client)), - db_pool: Arc::new(db_pool), - } + pub fn new(client: Arc>, db_pool: Connection) -> Self { + Self { client, db_pool } } - pub async fn enqueue( - &self, - job: &Job, - ) -> Result<(), Box> { - let client_guard = self - .client - .lock() - .map_err(|e| Box::new(LockError(e.to_string())) as Box)?; + /// Enqueue a job + /// # Arguments + /// * `self` - The job queue + /// * `task` - The task to enqueue + /// # Returns + /// * `Result<(), JobQueueError>` - The result of the operation + /// # Errors + /// * `JobQueueError` - The error that occurred + pub async fn enqueue(&self, task: T) -> Result<(), JobQueueError> + where + T: Serialize + for<'de> Deserialize<'de> + Send + Sync + BackgroundTask, + C: Clone, + { + let client_guard = self.client.lock().await; let mut conn = client_guard.get_async_connection().await?; + let db_conn = self.db_pool.get(); - let payload = serde_json::to_string(&job.task.payload())?; + let payload = task.payload()?; + let new_job_tracking = job_trackings::Entity::create(task.name(), payload, "queued"); + + let new_job_tracking = new_job_tracking.insert(db_conn).await?; + + let job_to_enqueue = Job::new(new_job_tracking.id, task); + + let payload = serde_json::to_string(&job_to_enqueue)?; redis::cmd("LPUSH") .arg("job_queue") .arg(payload) .query_async(&mut conn) .await?; + Ok(()) } - pub async fn dequeue( - &self, - ) -> Result>, Box> { - let client_guard = self - .client - .lock() - .map_err(|e| Box::new(LockError(e.to_string())) as Box)?; - let mut conn = client_guard.get_async_connection()?; + /// Dequeue a job + /// # Arguments + /// * `self` - The job queue + /// # Returns + /// * `Result>, JobQueueError>` - The result of the operation + /// # Errors + /// * `JobQueueError` - The error that occurred + pub async fn dequeue(&self) -> Result>, JobQueueError> + where + T: Serialize + for<'de> Deserialize<'de> + Send + Sync + BackgroundTask, + C: Clone, + { + let client_guard = self.client.lock().await; + let mut conn = client_guard.get_async_connection().await?; + let db_conn = self.db_pool.get(); - let payload: Option = redis::cmd("RPOP") + let res: Option = redis::cmd("BRPOP") .arg("job_queue") + .arg(0) .query_async(&mut conn) .await?; - if let Some(payload) = payload { - let task: Box = serde_json::from_str(&payload)?; - let job = Job { - id: generate_unique_id(), // You would need to implement this function - task, - }; - Ok(Some(job)) - } else { - Ok(None) + if let Some(job_data) = res { + let job: Job = serde_json::from_str(&job_data)?; + + let job_tracking = job_trackings::Entity::find_by_id(job.id) + .one(db_conn) + .await?; + + if let Some(job_tracking) = job_tracking { + if job_tracking.status == "completed" || job_tracking.status == "processing" { + return Ok(None); + } + + let job_tracking_am = + job_trackings::Entity::update_status(job_tracking, "processing"); + + job_tracking_am.save(db_conn).await?; + } + + return Ok(Some(job)); } + + Ok(None) } } diff --git a/api/src/background_worker/mod.rs b/api/src/background_worker/mod.rs index b5d8123..4d75334 100644 --- a/api/src/background_worker/mod.rs +++ b/api/src/background_worker/mod.rs @@ -1,4 +1,4 @@ -mod job_queue; -mod worker; -mod job; -mod tasks; \ No newline at end of file +pub mod job; +pub mod job_queue; +pub mod tasks; +pub mod worker; diff --git a/api/src/background_worker/tasks/metadata_json_upload_task.rs b/api/src/background_worker/tasks/metadata_json_upload_task.rs index a33c3e6..836e7ec 100644 --- a/api/src/background_worker/tasks/metadata_json_upload_task.rs +++ b/api/src/background_worker/tasks/metadata_json_upload_task.rs @@ -1 +1,716 @@ -use super::BackgroundTask; +use hub_core::{anyhow::Result, producer::Producer}; +use sea_orm::{prelude::*, Set}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::{BackgroundTask, BackgroundTaskError}; +use crate::{ + blockchains::{polygon::Polygon, solana::Solana, CollectionEvent, DropEvent}, + db::Connection, + entities::{ + collection_creators, collection_mints, collections, drops, metadata_jsons, mint_creators, + sea_orm_active_enums::Blockchain as BlockchainEnum, update_histories, + }, + mutations::collection::fetch_owner, + nft_storage::NftStorageClient, + objects::MetadataJsonInput, + proto::{ + CreateEditionTransaction, EditionInfo, MasterEdition, MetaplexMasterEditionTransaction, + MetaplexMetadata, MintMetaplexMetadataTransaction, NftEventKey, NftEvents, + UpdateEdtionTransaction, UpdateSolanaMintPayload, + }, +}; + +#[async_trait::async_trait] +trait After { + async fn after( + &self, + db: Connection, + context: Context, + identifier: String, + ) -> Result<(), BackgroundTaskError>; +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct CreateDrop { + pub drop_id: Uuid, +} + +#[async_trait::async_trait] +impl After for CreateDrop { + async fn after( + &self, + db: Connection, + context: Context, + identifier: String, + ) -> Result<(), BackgroundTaskError> { + let metadata_uri = context + .nft_storage + .ipfs_endpoint + .join(&identifier)? + .to_string(); + + let conn = db.get(); + let (drop, collection) = drops::Entity::find_by_id_with_collection(self.drop_id) + .one(conn) + .await? + .ok_or(BackgroundTaskError::RecordNotFound)?; + + let collection = collection.ok_or(BackgroundTaskError::RecordNotFound)?; + let supply = collection.supply; + let seller_fee_basis_points = collection.seller_fee_basis_points; + + let metadata_json = metadata_jsons::Entity::find_by_id(collection.id) + .one(conn) + .await? + .ok_or(BackgroundTaskError::RecordNotFound)?; + let creators = collection_creators::Entity::find() + .filter(collection_creators::Column::CollectionId.eq(collection.id)) + .all(conn) + .await?; + + let owner_address = fetch_owner(conn, collection.project_id, collection.blockchain) + .await + .map_err(|_| BackgroundTaskError::NoProjectWallet)?; + + let mut metadata_json_am: metadata_jsons::ActiveModel = metadata_json.clone().into(); + + metadata_json_am.uri = Set(Some(metadata_uri.clone())); + metadata_json_am.identifier = Set(Some(identifier.clone())); + + metadata_json_am.update(conn).await?; + + let event_key = NftEventKey { + id: collection.id.to_string(), + user_id: collection.created_by.to_string(), + project_id: collection.project_id.to_string(), + }; + + match collection.blockchain { + BlockchainEnum::Solana => { + context + .solana + .event() + .create_drop( + drop.drop_type, + event_key, + MetaplexMasterEditionTransaction { + master_edition: Some(MasterEdition { + owner_address, + supply, + name: metadata_json.name, + symbol: metadata_json.symbol, + metadata_uri, + seller_fee_basis_points: seller_fee_basis_points.into(), + creators: creators.into_iter().map(Into::into).collect(), + }), + }, + ) + .await?; + }, + BlockchainEnum::Polygon => { + context + .polygon + .create_drop(drop.drop_type, event_key, CreateEditionTransaction { + amount: supply.ok_or(BackgroundTaskError::NoSupply)?, + edition_info: Some(EditionInfo { + creator: creators + .get(0) + .ok_or(BackgroundTaskError::NoCreator)? + .address + .clone(), + collection: metadata_json.name, + uri: metadata_uri, + description: metadata_json.description, + image_uri: metadata_json.image, + }), + fee_receiver: owner_address, + fee_numerator: seller_fee_basis_points.into(), + }) + .await?; + }, + BlockchainEnum::Ethereum => { + return Err(BackgroundTaskError::BlockchainNotSupported); + }, + }; + + Ok(()) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct MintToCollection { + pub collection_mint_id: Uuid, +} + +#[async_trait::async_trait] +impl After for MintToCollection { + async fn after( + &self, + db: Connection, + context: Context, + identifier: String, + ) -> Result<(), BackgroundTaskError> { + let metadata_uri = context + .nft_storage + .ipfs_endpoint + .join(&identifier)? + .to_string(); + + let conn = db.get(); + let (collection_mint, collection) = + collection_mints::Entity::find_by_id_with_collection(self.collection_mint_id) + .one(conn) + .await? + .ok_or(BackgroundTaskError::RecordNotFound)?; + + let collection = collection.ok_or(BackgroundTaskError::RecordNotFound)?; + let seller_fee_basis_points = collection.seller_fee_basis_points; + + let metadata_json = metadata_jsons::Entity::find_by_id(collection_mint.id) + .one(conn) + .await? + .ok_or(BackgroundTaskError::RecordNotFound)?; + let creators = mint_creators::Entity::find() + .filter(mint_creators::Column::CollectionMintId.eq(collection_mint.id)) + .all(conn) + .await?; + + let owner_address = fetch_owner(conn, collection.project_id, collection.blockchain) + .await + .map_err(|_| BackgroundTaskError::NoProjectWallet)?; + + let mut metadata_json_am: metadata_jsons::ActiveModel = metadata_json.clone().into(); + + metadata_json_am.uri = Set(Some(metadata_uri.clone())); + metadata_json_am.identifier = Set(Some(identifier.clone())); + + metadata_json_am.update(conn).await?; + + let event_key = NftEventKey { + id: collection_mint.id.to_string(), + user_id: collection_mint.created_by.to_string(), + project_id: collection.project_id.to_string(), + }; + + let recipient_address = collection_mint.owner.ok_or(BackgroundTaskError::NoOwner)?; + let compressed = collection_mint.compressed.unwrap_or_default(); + + match collection.blockchain { + BlockchainEnum::Solana => { + context + .solana + .event() + .mint_to_collection(event_key, MintMetaplexMetadataTransaction { + metadata: Some(MetaplexMetadata { + owner_address, + name: metadata_json.name, + symbol: metadata_json.symbol, + metadata_uri, + seller_fee_basis_points: seller_fee_basis_points.into(), + creators: creators.into_iter().map(Into::into).collect(), + }), + recipient_address, + compressed, + collection_id: collection.id.to_string(), + }) + .await?; + }, + BlockchainEnum::Ethereum | BlockchainEnum::Polygon => { + return Err(BackgroundTaskError::BlockchainNotSupported); + }, + }; + + Ok(()) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct CreateCollection { + pub collection_id: Uuid, +} + +#[async_trait::async_trait] +impl After for CreateCollection { + async fn after( + &self, + db: Connection, + context: Context, + identifier: String, + ) -> Result<(), BackgroundTaskError> { + let metadata_uri = context + .nft_storage + .ipfs_endpoint + .join(&identifier)? + .to_string(); + + let conn = db.get(); + + let collection = collections::Entity::find_by_id(self.collection_id) + .one(conn) + .await? + .ok_or(BackgroundTaskError::RecordNotFound)?; + + let seller_fee_basis_points = collection.seller_fee_basis_points; + + let metadata_json = metadata_jsons::Entity::find_by_id(collection.id) + .one(conn) + .await? + .ok_or(BackgroundTaskError::RecordNotFound)?; + let creators = collection_creators::Entity::find() + .filter(collection_creators::Column::CollectionId.eq(collection.id)) + .all(conn) + .await?; + + let owner_address = fetch_owner(conn, collection.project_id, collection.blockchain) + .await + .map_err(|_| BackgroundTaskError::NoProjectWallet)?; + + let mut metadata_json_am: metadata_jsons::ActiveModel = metadata_json.clone().into(); + + metadata_json_am.uri = Set(Some(metadata_uri.clone())); + metadata_json_am.identifier = Set(Some(identifier.clone())); + + metadata_json_am.update(conn).await?; + + let event_key = NftEventKey { + id: collection.id.to_string(), + user_id: collection.created_by.to_string(), + project_id: collection.project_id.to_string(), + }; + + match collection.blockchain { + BlockchainEnum::Solana => { + context + .solana + .event() + .create_collection(event_key, MetaplexMasterEditionTransaction { + master_edition: Some(MasterEdition { + owner_address, + supply: Some(0), + name: metadata_json.name, + symbol: metadata_json.symbol, + metadata_uri, + seller_fee_basis_points: seller_fee_basis_points.into(), + creators: creators.into_iter().map(Into::into).collect(), + }), + }) + .await?; + }, + BlockchainEnum::Ethereum | BlockchainEnum::Polygon => { + return Err(BackgroundTaskError::BlockchainNotSupported); + }, + }; + + Ok(()) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct QueueMintToDrop { + pub drop_id: Uuid, + pub collection_mint_id: Uuid, +} + +#[async_trait::async_trait] +impl After for QueueMintToDrop { + async fn after( + &self, + db: Connection, + context: Context, + identifier: String, + ) -> Result<(), BackgroundTaskError> { + let metadata_uri = context + .nft_storage + .ipfs_endpoint + .join(&identifier)? + .to_string(); + + let conn = db.get(); + + let metadata_json = metadata_jsons::Entity::find_by_id(self.collection_mint_id) + .one(conn) + .await? + .ok_or(BackgroundTaskError::RecordNotFound)?; + + let mut metadata_json_am: metadata_jsons::ActiveModel = metadata_json.clone().into(); + + metadata_json_am.uri = Set(Some(metadata_uri.clone())); + metadata_json_am.identifier = Set(Some(identifier.clone())); + + metadata_json_am.update(conn).await?; + + Ok(()) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct UpdateMint { + pub update_history_id: Uuid, +} + +#[async_trait::async_trait] +impl After for UpdateMint { + async fn after( + &self, + db: Connection, + context: Context, + identifier: String, + ) -> Result<(), BackgroundTaskError> { + let metadata_uri = context + .nft_storage + .ipfs_endpoint + .join(&identifier)? + .to_string(); + + let conn = db.get(); + + let update_history = update_histories::Entity::find() + .filter(update_histories::Column::Id.eq(self.update_history_id)) + .one(conn) + .await? + .ok_or(BackgroundTaskError::RecordNotFound)?; + + let (collection_mint, collection) = + collection_mints::Entity::find_by_id_with_collection(update_history.mint_id) + .one(conn) + .await? + .ok_or(BackgroundTaskError::RecordNotFound)?; + + let metadata_json = metadata_jsons::Entity::find_by_id(collection_mint.id) + .one(conn) + .await? + .ok_or(BackgroundTaskError::RecordNotFound)?; + let creators = mint_creators::Entity::find() + .filter(mint_creators::Column::CollectionMintId.eq(collection_mint.id)) + .all(conn) + .await?; + + let collection = collection.ok_or(BackgroundTaskError::RecordNotFound)?; + + let owner_address = fetch_owner(conn, collection.project_id, collection.blockchain) + .await + .map_err(|_| BackgroundTaskError::NoProjectWallet)?; + + let mut metadata_json_am: metadata_jsons::ActiveModel = metadata_json.clone().into(); + + metadata_json_am.uri = Set(Some(metadata_uri.clone())); + metadata_json_am.identifier = Set(Some(identifier.clone())); + + metadata_json_am.update(conn).await?; + + match collection.blockchain { + BlockchainEnum::Solana => { + context + .solana + .event() + .update_collection_mint( + NftEventKey { + id: update_history.id.to_string(), + project_id: collection.project_id.to_string(), + user_id: update_history.created_by.to_string(), + }, + UpdateSolanaMintPayload { + metadata: Some(MetaplexMetadata { + owner_address, + name: metadata_json.name, + symbol: metadata_json.symbol, + metadata_uri, + seller_fee_basis_points: collection.seller_fee_basis_points.into(), + creators: creators.into_iter().map(Into::into).collect(), + }), + collection_id: collection.id.to_string(), + mint_id: update_history.mint_id.to_string(), + }, + ) + .await?; + }, + BlockchainEnum::Ethereum | BlockchainEnum::Polygon => { + return Err(BackgroundTaskError::BlockchainNotSupported); + }, + }; + + Ok(()) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct PatchCollection { + pub collection_id: Uuid, + pub updated_by_id: Uuid, +} + +#[async_trait::async_trait] +impl After for PatchCollection { + async fn after( + &self, + db: Connection, + context: Context, + identifier: String, + ) -> Result<(), BackgroundTaskError> { + let metadata_uri = context + .nft_storage + .ipfs_endpoint + .join(&identifier)? + .to_string(); + + let conn = db.get(); + + let collection = collections::Entity::find_by_id(self.collection_id) + .one(conn) + .await? + .ok_or(BackgroundTaskError::RecordNotFound)?; + + let seller_fee_basis_points = collection.seller_fee_basis_points; + + let metadata_json = metadata_jsons::Entity::find_by_id(collection.id) + .one(conn) + .await? + .ok_or(BackgroundTaskError::RecordNotFound)?; + let creators = collection_creators::Entity::find() + .filter(collection_creators::Column::CollectionId.eq(collection.id)) + .all(conn) + .await?; + + let owner_address = fetch_owner(conn, collection.project_id, collection.blockchain) + .await + .map_err(|_| BackgroundTaskError::NoProjectWallet)?; + + let mut metadata_json_am: metadata_jsons::ActiveModel = metadata_json.clone().into(); + + metadata_json_am.uri = Set(Some(metadata_uri.clone())); + metadata_json_am.identifier = Set(Some(identifier.clone())); + + metadata_json_am.update(conn).await?; + + let event_key = NftEventKey { + id: collection.id.to_string(), + user_id: self.updated_by_id.to_string(), + project_id: collection.project_id.to_string(), + }; + + match collection.blockchain { + BlockchainEnum::Solana => { + context + .solana + .event() + .update_collection(event_key, MetaplexMasterEditionTransaction { + master_edition: Some(MasterEdition { + owner_address, + supply: Some(0), + name: metadata_json.name, + symbol: metadata_json.symbol, + metadata_uri, + seller_fee_basis_points: seller_fee_basis_points.into(), + creators: creators.into_iter().map(Into::into).collect(), + }), + }) + .await?; + }, + BlockchainEnum::Polygon | BlockchainEnum::Ethereum => { + return Err(BackgroundTaskError::BlockchainNotSupported); + }, + }; + + Ok(()) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct PatchDrop { + pub drop_id: Uuid, + pub updated_by_id: Uuid, +} + +#[async_trait::async_trait] +impl After for PatchDrop { + async fn after( + &self, + db: Connection, + context: Context, + identifier: String, + ) -> Result<(), BackgroundTaskError> { + let metadata_uri = context + .nft_storage + .ipfs_endpoint + .join(&identifier)? + .to_string(); + + let conn = db.get(); + + let (drop, collection) = drops::Entity::find_by_id_with_collection(self.drop_id) + .one(conn) + .await? + .ok_or(BackgroundTaskError::RecordNotFound)?; + + let collection = collection.ok_or(BackgroundTaskError::RecordNotFound)?; + + let seller_fee_basis_points = collection.seller_fee_basis_points; + + let metadata_json = metadata_jsons::Entity::find_by_id(collection.id) + .one(conn) + .await? + .ok_or(BackgroundTaskError::RecordNotFound)?; + let creators = collection_creators::Entity::find() + .filter(collection_creators::Column::CollectionId.eq(collection.id)) + .all(conn) + .await?; + + let owner_address = fetch_owner(conn, collection.project_id, collection.blockchain) + .await + .map_err(|_| BackgroundTaskError::NoProjectWallet)?; + + let mut metadata_json_am: metadata_jsons::ActiveModel = metadata_json.clone().into(); + + metadata_json_am.uri = Set(Some(metadata_uri.clone())); + metadata_json_am.identifier = Set(Some(identifier.clone())); + + metadata_json_am.update(conn).await?; + + let event_key = NftEventKey { + id: collection.id.to_string(), + user_id: self.updated_by_id.to_string(), + project_id: collection.project_id.to_string(), + }; + + match collection.blockchain { + BlockchainEnum::Solana => { + context + .solana + .event() + .update_drop( + drop.drop_type, + event_key, + MetaplexMasterEditionTransaction { + master_edition: Some(MasterEdition { + owner_address, + supply: collection.supply.map(TryInto::try_into).transpose()?, + name: metadata_json.name, + symbol: metadata_json.symbol, + metadata_uri, + seller_fee_basis_points: seller_fee_basis_points.into(), + creators: creators.into_iter().map(Into::into).collect(), + }), + }, + ) + .await?; + }, + BlockchainEnum::Polygon => { + context + .polygon + .event() + .update_drop(drop.drop_type, event_key, UpdateEdtionTransaction { + edition_info: Some(EditionInfo { + description: metadata_json.description, + image_uri: metadata_json.image, + collection: metadata_json.name, + uri: metadata_uri, + creator: creators + .get(0) + .ok_or(BackgroundTaskError::NoCreator)? + .address + .clone(), + }), + }) + .await?; + }, + BlockchainEnum::Ethereum => { + return Err(BackgroundTaskError::BlockchainNotSupported); + }, + }; + + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Caller { + CreateDrop(CreateDrop), + PatchDrop(PatchDrop), + MintToCollection(MintToCollection), + CreateCollection(CreateCollection), + PatchCollection(PatchCollection), + QueueMintToDrop(QueueMintToDrop), + UpdateMint(UpdateMint), +} + +#[async_trait::async_trait] +impl After for Caller { + async fn after( + &self, + db: Connection, + context: Context, + identifier: String, + ) -> Result<(), BackgroundTaskError> { + match self { + Self::CreateDrop(inner) => inner.after(db, context, identifier).await, + Self::MintToCollection(inner) => inner.after(db, context, identifier).await, + Self::CreateCollection(inner) => inner.after(db, context, identifier).await, + Self::QueueMintToDrop(inner) => inner.after(db, context, identifier).await, + Self::UpdateMint(inner) => inner.after(db, context, identifier).await, + Self::PatchCollection(inner) => inner.after(db, context, identifier).await, + Self::PatchDrop(inner) => inner.after(db, context, identifier).await, + } + } +} + +#[derive(Clone, Serialize, Debug, Deserialize)] +pub struct MetadataJsonUploadTask { + pub metadata_json: MetadataJsonInput, + pub caller: Caller, +} + +impl MetadataJsonUploadTask { + #[must_use] + pub fn new(metadata_json: MetadataJsonInput, caller: Caller) -> Self { + Self { + metadata_json, + caller, + } + } +} + +#[derive(Clone)] +pub struct Context { + nft_storage: NftStorageClient, + solana: Solana, + polygon: Polygon, + producer: Producer, +} + +impl Context { + #[must_use] + pub fn new( + nft_storage: NftStorageClient, + solana: Solana, + polygon: Polygon, + producer: Producer, + ) -> Self { + Self { + nft_storage, + solana, + polygon, + producer, + } + } +} + +#[async_trait::async_trait] +impl BackgroundTask for MetadataJsonUploadTask { + fn name(&self) -> &'static str { + "MetadataJsonUploadTask" + } + + fn payload(&self) -> Result { + serde_json::to_value(self).map_err(Into::into) + } + + async fn process(&self, db: Connection, context: Context) -> Result<(), BackgroundTaskError> { + let response = context.nft_storage.upload(&self.metadata_json).await?; + let cid = response.value.cid; + + self.caller.after(db, context.clone(), cid).await?; + + Ok(()) + } +} diff --git a/api/src/background_worker/tasks/mod.rs b/api/src/background_worker/tasks/mod.rs index 54eb749..0715ffa 100644 --- a/api/src/background_worker/tasks/mod.rs +++ b/api/src/background_worker/tasks/mod.rs @@ -1,8 +1,66 @@ -use serde::{Deserialize, Serialize}; +use hub_core::{anyhow::Result, prelude::*, producer::SendError, thiserror, url}; +use sea_orm::error::DbErr; use serde_json::Value as Json; -pub trait BackgroundTask: Send + Sync + std::fmt::Debug { - fn process(&self) -> Result<(), Box>; - fn payload(&self) -> Json; +use crate::db::Connection; + +mod metadata_json_upload_task; + +#[derive(thiserror::Error, Debug)] +pub enum BackgroundTaskError { + #[error("Uri string parse error: {0}")] + UrlParse(#[from] url::ParseError), + #[error("Hub core error: {0}")] + HubCore(#[from] Error), + #[error("Database error: {0}")] + Database(#[from] DbErr), + #[error("Blockchain not supported")] + BlockchainNotSupported, + #[error("Db record not found")] + RecordNotFound, + #[error("No supply")] + NoSupply, + #[error("No owner")] + NoOwner, + #[error("No project wallet")] + NoProjectWallet, + #[error("Unable to send nft event")] + ProducerSend(#[from] SendError), + #[error("Unable to convert value: {0}")] + Conversion(#[from] std::convert::Infallible), + #[error("No creator")] + NoCreator, +} + +#[async_trait::async_trait] +pub trait BackgroundTask: Send + Sync + std::fmt::Debug { + /// Process the task + /// # Arguments + /// * `self` - The task + /// * `db` - The database connection + /// * `context` - The context + /// # Returns + /// * `Result<(), BackgroundTaskError>` - The result of the operation + /// # Error + /// * `BackgroundTaskError` - The error that occurred + async fn process(&self, db: Connection, context: C) -> Result<(), BackgroundTaskError>; + /// Get the payload of the task + /// # Arguments + /// * `self` - The task + /// # Returns + /// * `Result` - The payload of the task + /// # Error + /// * `anyhow::Error` - Unable to serialize the payload + fn payload(&self) -> Result; + fn name(&self) -> &'static str; } +pub use metadata_json_upload_task::{ + Caller as MetadataJsonUploadCaller, Context as MetadataJsonUploadContext, + CreateCollection as MetadataJsonUploadCreateCollection, + CreateDrop as MetadataJsonUploadCreateDrop, MetadataJsonUploadTask, + MintToCollection as MetadataJsonUploadMintToCollection, + PatchCollection as MetadataJsonUploadPatchCollection, PatchDrop as MetadataJsonUploadPatchDrop, + QueueMintToDrop as MetadataJsonUploadQueueMintToDrop, + UpdateMint as MetadataJsonUploadUpdateMint, +}; diff --git a/api/src/background_worker/worker.rs b/api/src/background_worker/worker.rs index 1a2f7f1..8cf70aa 100644 --- a/api/src/background_worker/worker.rs +++ b/api/src/background_worker/worker.rs @@ -1,79 +1,95 @@ -use super::job_queue::JobQueue; -use crate::db::Connection; use hub_core::{ - tokio, + thiserror, tokio, tracing::{error, info}, }; -use std::sync::Arc; +use sea_orm::{error::DbErr, ActiveModelTrait}; +use serde::{Deserialize, Serialize}; -pub struct Worker { - job_queue: Arc, - db_pool: Arc, +use super::{ + job_queue::{JobQueue, JobQueueError}, + tasks::BackgroundTask, +}; +use crate::{db::Connection, entities::job_trackings}; + +#[derive(thiserror::Error, Debug)] +pub enum WorkerError { + #[error("Job queue error: {0}")] + JobQueue(#[from] JobQueueError), + #[error("Database error: {0}")] + Database(#[from] DbErr), +} +pub struct Worker> { + job_queue: JobQueue, + db_pool: Connection, + context: C, + _task_marker: std::marker::PhantomData, } -impl Worker { - pub fn new(job_queue: Arc, db_pool: Connection) -> Self { +impl Worker +where + T: Serialize + for<'de> Deserialize<'de> + Send + Sync + BackgroundTask, + C: Clone, +{ + pub fn new(job_queue: JobQueue, db_pool: Connection, context: C) -> Self { Self { job_queue, - db_pool: Arc::new(db_pool), + db_pool, + context, + _task_marker: std::marker::PhantomData, } } - pub async fn start(&self) { + /// Start the worker + /// # Arguments + /// * `self` - The worker + /// # Returns + /// * `Result<(), WorkerError>` - The result of the operation + /// # Errors + /// * `WorkerError` - The error that occurred + pub async fn start(&self) -> Result<(), WorkerError> { loop { - if let Ok(Some(mut job)) = self.job_queue.dequeue().await { - let job_queue_clone = self.job_queue.clone(); - let job_id = job.id; - let db_pool_clone = Arc::clone(&self.db_pool); - tokio::spawn(async move { - if JobTracking::find_by_id(job_id, &db_pool_clone) - .await? - .is_none() + // Dequeue the next job to process + let job_option = self.job_queue.dequeue::().await?; + let db_conn = self.db_pool.get(); + + if let Some(job) = job_option { + // Process the job + let model = job_trackings::Entity::find_by_id(job.id) + .one(db_conn) + .await?; + + if let Some(model) = model { + match job + .task + .process(self.db_pool.clone(), self.context.clone()) + .await { + Ok(_) => { + // If successful, update the status in the job_trackings table to "completed" + let job_tracking_am = + job_trackings::Entity::update_status(model, "completed"); - // Create a new record in the database - JobTracking::create( - job_id, - "JobType", - job.task.payload(), - "processing", - &db_pool_clone, - ) - .await?; + job_tracking_am.update(db_conn).await?; - // sora elle espinola - // Process the job using the trait method - match job.task.process() { - Ok(_) => { - // Update the job status in the database to "completed" - JobTracking::update_status(&job, "completed", &db_pool_clone) - .await - .unwrap(); - }, - Err(e) => { - println!("Job processing failed: {}", e); + info!("Successfully processed job {}", job.id); + }, + Err(e) => { + // If an error occurs, update the status in the job_trackings table to "failed" + let job_tracking_am = + job_trackings::Entity::update_status(model, "failed"); - // Re-queue the job and update the job status in the database to "queued" - job_queue_clone - .enqueue(&job) - .await - .expect("Failed to re-queue job"); - JobTracking::update_status(&job, "queued", &db_pool_clone) - .await - .unwrap(); - }, - } - } else { - info!("Duplicate job detected, skipping: {}", job_id); - } - Ok::<(), sea_orm::DbErr>(()) - }) - .await - .unwrap_or_else(|e| { - error!("An error occurred: {}", e); + job_tracking_am.update(db_conn).await?; - Ok::<(), sea_orm::DbErr>(()) - }); + // Log the error (or handle it in some other way) + error!("Error processing job {}: {}", job.id, e); + }, + } + } else { + error!("Job tracking record not found for job {}", job.id); + } + } else { + // If no job was dequeued, you might want to add a delay here to avoid busy-waiting + tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } } diff --git a/api/src/collection.rs b/api/src/collection.rs deleted file mode 100644 index 0bb22d6..0000000 --- a/api/src/collection.rs +++ /dev/null @@ -1,58 +0,0 @@ -use hub_core::anyhow::Result; -use sea_orm::{prelude::*, Set}; - -use crate::{ - db::Connection, - entities::{ - collection_creators::ActiveModel as CollectionCreatorActiveModel, - collections::{ActiveModel, Model}, - }, - objects::Creator, -}; - -#[derive(Debug, Clone)] -pub struct Collection { - collection: ActiveModel, - creators: Option>, -} - -impl Collection { - #[must_use] - pub fn new(collection: ActiveModel) -> Self { - Self { - collection, - creators: None, - } - } - - pub fn creators(&mut self, creators: Vec) -> &Collection { - self.creators = Some(creators); - - self - } - - /// Res - /// - /// # Errors - /// This function fails if unable to save `collection` or `collection_creators` to the db - pub async fn save(&self, db: &Connection) -> Result { - let conn = db.get(); - - let collection = self.collection.clone().insert(conn).await?; - - let creators = self.creators.clone().unwrap_or_default(); - - for creator in creators { - let am = CollectionCreatorActiveModel { - collection_id: Set(collection.id), - address: Set(creator.address), - verified: Set(creator.verified.unwrap_or_default()), - share: Set(creator.share.try_into()?), - }; - - am.insert(conn).await?; - } - - Ok(collection) - } -} diff --git a/api/src/entities/collections.rs b/api/src/entities/collections.rs index a414d18..2c87027 100644 --- a/api/src/entities/collections.rs +++ b/api/src/entities/collections.rs @@ -64,3 +64,9 @@ impl Related for Entity { Relation::MintHistories.def() } } + +impl Entity { + pub fn find_by_id(id: Uuid) -> Select { + Self::find().filter(Column::Id.eq(id)) + } +} diff --git a/api/src/entities/drops.rs b/api/src/entities/drops.rs index ed5a48e..0310732 100644 --- a/api/src/entities/drops.rs +++ b/api/src/entities/drops.rs @@ -42,3 +42,11 @@ impl Related for Entity { } impl ActiveModelBehavior for ActiveModel {} + +impl Entity { + pub fn find_by_id_with_collection( + id: Uuid, + ) -> sea_orm::SelectTwo { + Self::find_by_id(id).select_also(super::collections::Entity) + } +} diff --git a/api/src/entities/job_trackings.rs b/api/src/entities/job_trackings.rs index dfc1b15..a087037 100644 --- a/api/src/entities/job_trackings.rs +++ b/api/src/entities/job_trackings.rs @@ -1,4 +1,5 @@ -use sea_orm::entity::prelude::*; +use hub_core::chrono; +use sea_orm::{entity::prelude::*, Set}; use serde_json::Value as Json; #[derive(Clone, Debug, PartialEq, DeriveEntityModel)] @@ -16,4 +17,34 @@ pub struct Model { #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation {} -impl ActiveModelBehavior for ActiveModel {} \ No newline at end of file +impl ActiveModelBehavior for ActiveModel {} + +impl Entity { + // Find a job tracking record by its ID + pub fn find_by_id(id: i64) -> Select { + Self::find().filter(Column::Id.eq(id)) + } + + // Create a new job tracking record + pub fn create(job_type: &str, payload: Json, status: &str) -> ActiveModel { + let now: DateTimeWithTimeZone = chrono::Utc::now().into(); + + ActiveModel { + job_type: Set(job_type.to_string()), + payload: Set(payload), + status: Set(status.to_string()), + created_at: Set(now), + updated_at: Set(now), + ..Default::default() + } + } + + // Update the status of an existing job tracking record + pub fn update_status(model: Model, new_status: &str) -> ActiveModel { + let mut active_model: ActiveModel = model.into(); + + active_model.status = Set(new_status.to_string()); + + active_model + } +} diff --git a/api/src/entities/metadata_jsons.rs b/api/src/entities/metadata_jsons.rs index 3afb243..01e7c48 100644 --- a/api/src/entities/metadata_jsons.rs +++ b/api/src/entities/metadata_jsons.rs @@ -11,11 +11,13 @@ use sea_orm::entity::prelude::*; pub struct Model { #[sea_orm(primary_key, auto_increment = false)] pub id: Uuid, - pub identifier: String, + #[sea_orm(nullable)] + pub identifier: Option, /// The assigned name of the NFT. pub name: String, /// The URI for the complete metadata JSON. - pub uri: String, + #[sea_orm(nullable)] + pub uri: Option, /// The symbol of the NFT. pub symbol: String, /// The description of the NFT. @@ -23,8 +25,10 @@ pub struct Model { /// The image URI for the NFT. pub image: String, /// An optional animated version of the NFT art. + #[sea_orm(nullable)] pub animation_url: Option, /// An optional URL where viewers can find more information on the NFT, such as the collection's homepage or Twitter page. + #[sea_orm(nullable)] pub external_url: Option, } @@ -49,3 +53,9 @@ impl Related for Entity { } impl ActiveModelBehavior for ActiveModel {} + +impl Entity { + pub fn find_by_id(id: Uuid) -> Select { + Self::find().filter(Column::Id.eq(id)) + } +} diff --git a/api/src/entities/mod.rs b/api/src/entities/mod.rs index 4815af4..7ac87a3 100644 --- a/api/src/entities/mod.rs +++ b/api/src/entities/mod.rs @@ -8,6 +8,7 @@ pub mod collection_mints; pub mod collections; pub mod customer_wallets; pub mod drops; +pub mod job_trackings; pub mod metadata_json_attributes; pub mod metadata_json_files; pub mod metadata_jsons; @@ -19,4 +20,3 @@ pub mod sea_orm_active_enums; pub mod switch_collection_histories; pub mod transfer_charges; pub mod update_histories; -pub mod job_trackings; diff --git a/api/src/entities/sea_orm_active_enums.rs b/api/src/entities/sea_orm_active_enums.rs index 69989a4..20e827f 100644 --- a/api/src/entities/sea_orm_active_enums.rs +++ b/api/src/entities/sea_orm_active_enums.rs @@ -47,7 +47,19 @@ pub enum CreationStatus { Queued, } -#[derive(Debug, Clone, Default, PartialEq, Eq, EnumIter, DeriveActiveEnum, Enum, Copy)] +#[derive( + Debug, + Clone, + Default, + PartialEq, + Eq, + EnumIter, + DeriveActiveEnum, + Enum, + Copy, + Serialize, + Deserialize, +)] #[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "drop_type")] pub enum DropType { #[default] diff --git a/api/src/events.rs b/api/src/events.rs index bd73902..9cacdb5 100644 --- a/api/src/events.rs +++ b/api/src/events.rs @@ -1,5 +1,5 @@ use hub_core::{ - chrono::{DateTime, NaiveDateTime, Offset, Utc}, + chrono::{NaiveDateTime, TimeZone, Utc}, credits::{CreditsClient, TransactionId}, metrics::KeyValue, prelude::*, @@ -365,13 +365,13 @@ impl Processor { let metadata_json = metadata_jsons::ActiveModel { id: Set(id.parse()?), name: Set(name), - uri: Set(uri), + uri: Set(Some(uri)), symbol: Set(symbol), description: Set(description.unwrap_or_default()), image: Set(image), animation_url: Set(None), external_url: Set(None), - identifier: Set(String::new()), + identifier: Set(Some(String::new())), }; let json_model = metadata_json.insert(self.db.get()).await?; @@ -420,8 +420,10 @@ impl Processor { image, } = metadata.ok_or(ProcessorErrorKind::MissingCollectionMetadata)?; + let id = id.parse()?; + let mint_am = collection_mints::ActiveModel { - id: Set(id.parse()?), + id: Set(id), collection_id: Set(collection_id.parse()?), address: Set(Some(mint_address)), owner: Set(Some(owner)), @@ -440,15 +442,15 @@ impl Processor { let mint_model = mint_am.insert(self.db.get()).await?; let metadata_json = metadata_jsons::ActiveModel { - id: Set(id.parse()?), + id: Set(id), name: Set(name), - uri: Set(uri), + uri: Set(Some(uri)), symbol: Set(symbol), description: Set(description.unwrap_or_default()), image: Set(image), animation_url: Set(None), external_url: Set(None), - identifier: Set(String::new()), + identifier: Set(Some(String::new())), }; let json_model = metadata_json.insert(self.db.get()).await?; @@ -526,10 +528,10 @@ impl Processor { let created_at = timestamp .and_then(|t| { - Some(DateTime::from_naive_utc_and_offset( - NaiveDateTime::from_timestamp_opt(t.seconds, t.nanos.try_into().ok()?)?, - Utc.fix(), - )) + let naive_datetime = + NaiveDateTime::from_timestamp_opt(t.seconds, t.nanos.try_into().ok()?)?; + + Some(Utc.from_utc_datetime(&naive_datetime)) }) .ok_or(ProcessorErrorKind::InvalidTimestamp)?; @@ -556,7 +558,7 @@ impl Processor { collection_mint_id: Set(mint.id), sender: Set(mint.owner.ok_or(ProcessorErrorKind::RecordMissingOwner)?), recipient: Set(new_owner.clone()), - created_at: Set(created_at), + created_at: Set(created_at.into()), ..Default::default() }; diff --git a/api/src/handlers.rs b/api/src/handlers.rs index 5c3aa37..f79bba5 100644 --- a/api/src/handlers.rs +++ b/api/src/handlers.rs @@ -42,8 +42,8 @@ pub async fn graphql_handler( .data(state.credits.clone()) .data(state.solana.clone()) .data(state.polygon.clone()) - .data(state.nft_storage.clone()) - .data(state.asset_proxy.clone()), + .data(state.asset_proxy.clone()) + .data(state.metadata_json_upload_job_queue.clone()), ) .await .into()) diff --git a/api/src/lib.rs b/api/src/lib.rs index a365c0a..7d823ea 100644 --- a/api/src/lib.rs +++ b/api/src/lib.rs @@ -2,26 +2,25 @@ #![warn(clippy::pedantic, clippy::cargo)] #![allow(clippy::module_name_repetitions)] +pub mod background_worker; pub mod blockchains; -pub mod collection; pub mod dataloaders; pub mod db; pub mod entities; pub mod events; pub mod handlers; -pub mod metadata_json; pub mod metrics; pub mod mutations; pub mod nft_storage; pub mod objects; pub mod queries; -pub mod background_worker; use async_graphql::{ dataloader::DataLoader, extensions::{ApolloTracing, Logger}, EmptySubscription, Schema, }; +use background_worker::job_queue::JobQueue; use blockchains::{polygon::Polygon, solana::Solana}; use dataloaders::{ CollectionDropLoader, CollectionLoader, CollectionMintHistoriesLoader, CollectionMintLoader, @@ -45,7 +44,6 @@ use hub_core::{ }; use metrics::Metrics; use mutations::Mutation; -use nft_storage::NftStorageClient; use poem::{async_trait, FromRequest, Request, RequestBody}; use queries::Query; @@ -117,6 +115,9 @@ pub struct Args { #[command(flatten)] pub nft_storage: nft_storage::NftStorageArgs, + + #[arg(long, env)] + pub redis_url: String, } #[derive(Debug, Clone, Copy)] @@ -236,8 +237,8 @@ pub struct AppState { pub credits: CreditsClient, pub solana: Solana, pub polygon: Polygon, - pub nft_storage: NftStorageClient, pub asset_proxy: AssetProxy, + pub metadata_json_upload_job_queue: JobQueue, } impl AppState { @@ -250,8 +251,8 @@ impl AppState { credits: CreditsClient, solana: Solana, polygon: Polygon, - nft_storage: NftStorageClient, asset_proxy: AssetProxy, + metadata_json_upload_job_queue: JobQueue, ) -> Self { Self { schema, @@ -260,8 +261,8 @@ impl AppState { credits, solana, polygon, - nft_storage, asset_proxy, + metadata_json_upload_job_queue, } } } diff --git a/api/src/main.rs b/api/src/main.rs index b7c3369..7f67077 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -1,6 +1,13 @@ //! +use std::sync::Arc; + use holaplex_hub_nfts::{ + background_worker::{ + job_queue::JobQueue, + tasks::{MetadataJsonUploadContext, MetadataJsonUploadTask}, + worker::Worker, + }, blockchains::{polygon::Polygon, solana::Solana}, build_schema, db::Connection, @@ -12,6 +19,7 @@ use holaplex_hub_nfts::{ }; use hub_core::{prelude::*, tokio}; use poem::{get, listener::TcpListener, middleware::AddData, post, EndpointExt, Route, Server}; +use redis::Client as RedisClient; pub fn main() { let opts = hub_core::StartConfig { @@ -23,6 +31,7 @@ pub fn main() { port, db, nft_storage, + redis_url, } = args; common.rt.block_on(async move { @@ -52,19 +61,38 @@ pub fn main() { let solana = Solana::new(producer.clone()); let polygon = Polygon::new(producer.clone()); + let redis_client = RedisClient::open(redis_url)?; + let redis_client = Arc::new(tokio::sync::Mutex::new(redis_client)); + + let metadata_json_upload_task_context = MetadataJsonUploadContext::new( + nft_storage, + solana.clone(), + polygon.clone(), + producer.clone(), + ); + + let job_queue = JobQueue::new(redis_client, connection.clone()); + let worker = Worker::::new( + job_queue.clone(), + connection.clone(), + metadata_json_upload_task_context, + ); + let state = AppState::new( schema, connection.clone(), producer.clone(), credits.clone(), - solana, - polygon, - nft_storage, + solana.clone(), + polygon.clone(), common.asset_proxy, + job_queue.clone(), ); let cons = common.consumer_cfg.build::().await?; + tokio::spawn(async move { worker.start().await }); + tokio::spawn(async move { cons.consume( |b| { diff --git a/api/src/metadata_json.rs b/api/src/metadata_json.rs deleted file mode 100644 index 9c95854..0000000 --- a/api/src/metadata_json.rs +++ /dev/null @@ -1,170 +0,0 @@ -use hub_core::{anyhow::Result, prelude::anyhow, uuid::Uuid}; -use metadata_jsons::Column as MetadataJsonColumn; -use sea_orm::{prelude::*, sea_query::OnConflict, Set, TransactionTrait}; - -use crate::{ - db::Connection, - entities::{ - metadata_json_attributes, metadata_json_files, metadata_jsons, - prelude::{MetadataJsonAttributes, MetadataJsonFiles, MetadataJsons}, - }, - nft_storage::NftStorageClient, - objects::MetadataJsonInput, -}; -#[derive(Clone, Debug)] -pub struct MetadataJson { - pub metadata_json: MetadataJsonInput, - pub uri: Option, - pub identifier: Option, -} - -impl MetadataJson { - #[must_use] - pub fn new(metadata_json: MetadataJsonInput) -> Self { - Self { - metadata_json, - uri: None, - identifier: None, - } - } - - /// Fetches metadata from the database and constructs a `Self` instance. - /// - /// # Arguments - /// - /// * `id` - The ID of the metadata to fetch. - /// * `db` - The database connection to use. - /// - /// # Errors - /// - /// This function fails if there is no matching `metadata_json` entry in the database - /// or if it is unable to fetch related data from the database - pub async fn fetch(id: Uuid, db: &Connection) -> Result { - let (metadata_json_model, attributes) = metadata_jsons::Entity::find_by_id(id) - .find_with_related(MetadataJsonAttributes) - .all(db.get()) - .await? - .first() - .map(ToOwned::to_owned) - .ok_or(anyhow!("no metadata_json entry found in db"))?; - - let files = metadata_json_files::Entity::find() - .filter(metadata_json_files::Column::MetadataJsonId.eq(id)) - .all(db.get()) - .await?; - - let metadata_json = (metadata_json_model.clone(), attributes, Some(files)).into(); - - Ok(Self { - metadata_json, - uri: Some(metadata_json_model.uri.clone()), - identifier: Some(metadata_json_model.identifier), - }) - } - - /// Res - /// - /// # Errors - /// This function fails if unable to upload `metadata_json` to nft.storage - pub async fn upload(&mut self, nft_storage: &NftStorageClient) -> Result<&Self> { - let response = nft_storage.upload(self.metadata_json.clone()).await?; - let cid = response.value.cid; - - let uri = nft_storage.ipfs_endpoint.join(&cid)?.to_string(); - - self.uri = Some(uri); - self.identifier = Some(cid); - - Ok(self) - } - - /// Res - /// - /// # Errors - /// This function fails if unable to save `metadata_json` to the db - pub async fn save(&self, id: Uuid, db: &Connection) -> Result { - let payload = self.metadata_json.clone(); - let identifier = self - .identifier - .clone() - .ok_or_else(|| anyhow!("no identifier. call #upload before #save"))?; - let uri = self - .uri - .clone() - .ok_or_else(|| anyhow!("no uri. call #upload before #save"))?; - - let metadata_json_active_model = metadata_jsons::ActiveModel { - id: Set(id), - identifier: Set(identifier), - name: Set(payload.name), - uri: Set(uri), - symbol: Set(payload.symbol), - description: Set(payload.description), - image: Set(payload.image), - animation_url: Set(payload.animation_url), - external_url: Set(payload.external_url), - }; - - let metadata_json = MetadataJsons::insert(metadata_json_active_model) - .on_conflict( - OnConflict::column(MetadataJsonColumn::Id) - .update_columns([ - MetadataJsonColumn::Identifier, - MetadataJsonColumn::Name, - MetadataJsonColumn::Uri, - MetadataJsonColumn::Symbol, - MetadataJsonColumn::Description, - MetadataJsonColumn::Image, - MetadataJsonColumn::AnimationUrl, - MetadataJsonColumn::ExternalUrl, - ]) - .clone(), - ) - .exec_with_returning(db.get()) - .await?; - - let tx = db.get().clone().begin().await?; - - MetadataJsonAttributes::delete_many() - .filter(metadata_json_attributes::Column::MetadataJsonId.eq(metadata_json.id)) - .exec(&tx) - .await?; - - for attribute in payload.attributes { - let am = metadata_json_attributes::ActiveModel { - metadata_json_id: Set(metadata_json.id), - trait_type: Set(attribute.trait_type), - value: Set(attribute.value), - ..Default::default() - }; - - am.insert(&tx).await?; - } - - tx.commit().await?; - - if let Some(files) = payload.properties.unwrap_or_default().files { - let tx = db.get().clone().begin().await?; - - MetadataJsonFiles::delete_many() - .filter(metadata_json_files::Column::MetadataJsonId.eq(metadata_json.id)) - .exec(&tx) - .await?; - - for file in files { - let metadata_json_file_am = metadata_json_files::ActiveModel { - metadata_json_id: Set(metadata_json.id), - uri: Set(file.uri), - file_type: Set(file.file_type), - ..Default::default() - }; - - metadata_json_file_am.insert(&tx).await?; - } - - tx.commit().await?; - } - - Ok(metadata_json) - } -} diff --git a/api/src/mutations/collection.rs b/api/src/mutations/collection.rs index d1c0476..ab23c2f 100644 --- a/api/src/mutations/collection.rs +++ b/api/src/mutations/collection.rs @@ -12,8 +12,14 @@ use sea_orm::{prelude::*, ModelTrait, Set, TransactionTrait}; use serde::{Deserialize, Serialize}; use crate::{ - blockchains::{polygon::Polygon, solana::Solana, CollectionEvent}, - collection::Collection, + background_worker::{ + job_queue::JobQueue, + tasks::{ + MetadataJsonUploadCaller, MetadataJsonUploadCreateCollection, + MetadataJsonUploadPatchCollection, MetadataJsonUploadTask, + }, + }, + blockchains::{solana::Solana, CollectionEvent}, entities::{ collection_creators, collection_mints, collections, metadata_jsons, prelude::{CollectionCreators, CollectionMints, Collections, Drops, MetadataJsons}, @@ -21,14 +27,13 @@ use crate::{ sea_orm_active_enums::{Blockchain, Blockchain as BlockchainEnum, CreationStatus}, switch_collection_histories, }, - metadata_json::MetadataJson, objects::{Collection as CollectionObject, CollectionMint, Creator, MetadataJsonInput}, proto::{ nft_events::Event as NftEvent, CollectionCreation, CollectionImport, CreationStatus as NftCreationStatus, Creator as ProtoCreator, MasterEdition, MetaplexMasterEditionTransaction, NftEventKey, NftEvents, }, - Actions, AppContext, NftStorageClient, + Actions, AppContext, }; #[derive(Default)] @@ -61,9 +66,8 @@ impl Mutation { let conn = db.get(); let credits = ctx.data::>()?; - let solana = ctx.data::()?; - let nft_storage = ctx.data::()?; let nfts_producer = ctx.data::>()?; + let metadata_json_upload_job_queue = ctx.data::()?; let owner_address = fetch_owner(conn, input.project, input.blockchain).await?; @@ -83,6 +87,8 @@ impl Mutation { ) .await?; + let tx = conn.begin().await?; + let collection_am = collections::ActiveModel { blockchain: Set(input.blockchain), supply: Set(Some(0)), @@ -94,48 +100,33 @@ impl Mutation { ..Default::default() }; - let collection = Collection::new(collection_am) - .creators(input.creators.clone()) - .save(db) - .await?; + let collection = collection_am.insert(&tx).await?; - let metadata_json = MetadataJson::new(input.metadata_json) - .upload(nft_storage) - .await? - .save(collection.id, db) - .await?; + for creator in input.creators { + let am = collection_creators::ActiveModel { + collection_id: Set(collection.id), + address: Set(creator.address), + verified: Set(creator.verified.unwrap_or_default()), + share: Set(creator.share.try_into()?), + }; - let event_key = NftEventKey { - id: collection.id.to_string(), - user_id: user_id.to_string(), - project_id: input.project.to_string(), - }; + am.insert(&tx).await?; + } - match input.blockchain { - BlockchainEnum::Solana => { - solana - .event() - .create_collection(event_key, MetaplexMasterEditionTransaction { - master_edition: Some(MasterEdition { - owner_address, - supply: Some(0), - name: metadata_json.name, - symbol: metadata_json.symbol, - metadata_uri: metadata_json.uri, - seller_fee_basis_points: 0, - creators: input - .creators - .into_iter() - .map(TryFrom::try_from) - .collect::>()?, - }), - }) - .await?; - }, - BlockchainEnum::Ethereum | BlockchainEnum::Polygon => { - return Err(Error::new("blockchain not supported as this time")); - }, - }; + input.metadata_json.save(collection.id, &tx).await?; + + tx.commit().await?; + + metadata_json_upload_job_queue + .enqueue(MetadataJsonUploadTask { + metadata_json: input.metadata_json, + caller: MetadataJsonUploadCaller::CreateCollection( + MetadataJsonUploadCreateCollection { + collection_id: collection.id, + }, + ), + }) + .await?; nfts_producer .send( @@ -185,7 +176,7 @@ impl Mutation { let collection = Collections::find() .filter(collections::Column::Id.eq(input.id)) - .one(db.get()) + .one(conn) .await? .ok_or(Error::new("collection not found"))?; @@ -197,6 +188,10 @@ impl Mutation { .one(conn) .await? .ok_or(Error::new("metadata json not found"))?; + let metadata_uri = metadata_json + .uri + .ok_or(Error::new("metadata uri not found"))?; + let creators = CollectionCreators::find() .filter(collection_creators::Column::CollectionId.eq(collection.id)) .all(conn) @@ -229,7 +224,7 @@ impl Mutation { owner_address, name: metadata_json.name, symbol: metadata_json.symbol, - metadata_uri: metadata_json.uri, + metadata_uri, seller_fee_basis_points: 0, supply: Some(0), creators: creators @@ -264,7 +259,7 @@ impl Mutation { let AppContext { db, user_id, .. } = ctx.data::()?; let user_id = user_id.0.ok_or(Error::new("X-USER-ID header not found"))?; - let txn = db.get().begin().await?; + let conn = db.get(); validate_solana_address(&input.collection)?; @@ -274,9 +269,11 @@ impl Mutation { .eq(input.collection.clone()) .and(collections::Column::ProjectId.eq(input.project)), ) - .one(db.get()) + .one(conn) .await?; + let txn = conn.begin().await?; + if let Some(collection) = collection.clone() { let mints = CollectionMints::find() .filter(collection_mints::Column::CollectionId.eq(collection.id)) @@ -334,22 +331,22 @@ impl Mutation { input: PatchCollectionInput, ) -> Result { let PatchCollectionInput { - id: _, metadata_json, creators, + .. } = input; - let AppContext { db, user_id, .. } = ctx.data::()?; - let conn = db.get(); - let nft_storage = ctx.data::()?; + let solana = ctx.data::()?; - let _polygon = ctx.data::()?; + let metadata_json_upload_job_queue = ctx.data::()?; let user_id = user_id.0.ok_or(Error::new("X-USER-ID header not found"))?; + let conn = db.get(); + let collection = Collections::find() .filter(collections::Column::Id.eq(input.id)) - .one(db.get()) + .one(conn) .await? .ok_or(Error::new("collection not found"))?; @@ -366,13 +363,21 @@ impl Mutation { validate_json(collection.blockchain, metadata_json)?; } + let metadata_json_model = metadata_jsons::Entity::find() + .filter(metadata_jsons::Column::Id.eq(collection.id)) + .one(conn) + .await? + .ok_or(Error::new("metadata json not found"))?; + let current_creators = collection_creators::Entity::find() .filter(collection_creators::Column::CollectionId.eq(collection.id)) .all(conn) .await?; - if let Some(creators) = creators.clone() { - let creators = creators + let tx = conn.begin().await?; + + let creators: Vec = if let Some(creators) = creators { + let creator_ams = creators .clone() .into_iter() .map(|creator| { @@ -385,77 +390,74 @@ impl Mutation { }) .collect::>>()?; - conn.transaction::<_, (), DbErr>(|txn| { - Box::pin(async move { - collection_creators::Entity::delete_many() - .filter(collection_creators::Column::CollectionId.eq(collection.id)) - .exec(txn) - .await?; - - collection_creators::Entity::insert_many(creators) - .exec(txn) - .await?; - - Ok(()) - }) - }) - .await?; - } - - let metadata_json_model = metadata_jsons::Entity::find() - .filter(metadata_jsons::Column::Id.eq(collection.id)) - .one(conn) - .await? - .ok_or(Error::new("metadata json not found"))?; + collection_creators::Entity::delete_many() + .filter(collection_creators::Column::CollectionId.eq(collection.id)) + .exec(&tx) + .await?; - let metadata_json_model = if let Some(metadata_json) = metadata_json { - metadata_json_model.clone().delete(conn).await?; + collection_creators::Entity::insert_many(creator_ams) + .exec(&tx) + .await?; - MetadataJson::new(metadata_json.clone()) - .upload(nft_storage) - .await? - .save(collection.id, db) - .await? + creators + .into_iter() + .map(TryFrom::try_from) + .collect::>()? } else { - metadata_json_model + current_creators.into_iter().map(Into::into).collect() }; - let event_key = NftEventKey { - id: collection.id.to_string(), - user_id: user_id.to_string(), - project_id: collection.project_id.to_string(), - }; + if let Some(metadata_json) = metadata_json { + metadata_json_model.delete(&tx).await?; - match collection.blockchain { - BlockchainEnum::Solana => { - let creators = if let Some(creators) = creators.clone() { - creators - .into_iter() - .map(TryFrom::try_from) - .collect::>()? - } else { - current_creators.into_iter().map(Into::into).collect() - }; + metadata_json.save(collection.id, &tx).await?; - solana - .event() - .update_collection(event_key, MetaplexMasterEditionTransaction { - master_edition: Some(MasterEdition { - owner_address, - supply: Some(0), - name: metadata_json_model.name, - symbol: metadata_json_model.symbol, - metadata_uri: metadata_json_model.uri, - seller_fee_basis_points: 0, - creators, - }), - }) - .await?; - }, - BlockchainEnum::Polygon | BlockchainEnum::Ethereum => { - return Err(Error::new("blockchain not supported yet")); - }, - }; + metadata_json_upload_job_queue + .enqueue(MetadataJsonUploadTask { + metadata_json, + caller: MetadataJsonUploadCaller::PatchCollection( + MetadataJsonUploadPatchCollection { + collection_id: collection.id, + updated_by_id: user_id, + }, + ), + }) + .await?; + } else { + let event_key = NftEventKey { + id: collection.id.to_string(), + user_id: user_id.to_string(), + project_id: collection.project_id.to_string(), + }; + + let metadata_uri = metadata_json_model + .uri + .ok_or(Error::new("metadata uri not found"))?; + + match collection.blockchain { + BlockchainEnum::Solana => { + solana + .event() + .update_collection(event_key, MetaplexMasterEditionTransaction { + master_edition: Some(MasterEdition { + owner_address, + supply: Some(0), + name: metadata_json_model.name, + symbol: metadata_json_model.symbol, + metadata_uri, + seller_fee_basis_points: 0, + creators, + }), + }) + .await?; + }, + BlockchainEnum::Polygon | BlockchainEnum::Ethereum => { + return Err(Error::new("blockchain not supported as this time")); + }, + }; + } + + tx.commit().await?; Ok(PatchCollectionPayload { collection: collection.into(), @@ -485,6 +487,7 @@ impl Mutation { } = ctx.data::()?; let credits = ctx.data::>()?; let solana = ctx.data::()?; + let conn = db.get(); let user_id = user_id.0.ok_or(Error::new("X-USER-ID header not found"))?; let org_id = organization_id @@ -492,7 +495,7 @@ impl Mutation { .ok_or(Error::new("X-ORG-ID header not found"))?; let balance = balance.0.ok_or(Error::new("X-BALANCE header not found"))?; let (mint, collection) = CollectionMints::find_by_id_with_collection(mint) - .one(db.get()) + .one(conn) .await? .ok_or(Error::new("Mint not found"))?; @@ -500,7 +503,7 @@ impl Mutation { let new_collection = Collections::find() .filter(collections::Column::Address.eq(collection_address.to_string())) - .one(db.get()) + .one(conn) .await? .ok_or(Error::new("Collection not found"))?; @@ -514,7 +517,7 @@ impl Mutation { if new_collection .find_related(Drops) - .one(db.get()) + .one(conn) .await? .is_some() { @@ -551,7 +554,7 @@ impl Mutation { ..Default::default() }; - let history = history_am.insert(db.get()).await?; + let history = history_am.insert(conn).await?; solana .event() diff --git a/api/src/mutations/drop.rs b/api/src/mutations/drop.rs index 63882aa..c8eb56f 100644 --- a/api/src/mutations/drop.rs +++ b/api/src/mutations/drop.rs @@ -9,21 +9,26 @@ use serde::{Deserialize, Serialize}; use super::collection::{validate_creators, validate_json, validate_solana_creator_verification}; use crate::{ + background_worker::{ + job_queue::JobQueue, + tasks::{ + MetadataJsonUploadCaller, MetadataJsonUploadCreateDrop, MetadataJsonUploadPatchDrop, + MetadataJsonUploadTask, + }, + }, blockchains::{polygon::Polygon, solana::Solana, DropEvent}, - collection::Collection, entities::{ collection_creators, collections, drops, metadata_jsons, prelude::{CollectionCreators, Collections, Drops, MetadataJsons}, project_wallets, sea_orm_active_enums::{Blockchain as BlockchainEnum, CreationStatus, DropType}, }, - metadata_json::MetadataJson, objects::{Creator, Drop, MetadataJsonInput}, proto::{ self, nft_events::Event as NftEvent, CreationStatus as NftCreationStatus, EditionInfo, NftEventKey, NftEvents, }, - Actions, AppContext, NftStorageClient, + Actions, AppContext, }; #[derive(Default)] @@ -57,10 +62,8 @@ impl Mutation { let conn = db.get(); let credits = ctx.data::>()?; - let solana = ctx.data::()?; - let polygon = ctx.data::()?; - let nft_storage = ctx.data::()?; let nfts_producer = ctx.data::>()?; + let metadata_json_upload_job_queue = ctx.data::()?; let owner_address = fetch_owner(conn, input.project, input.blockchain).await?; let supply = if input.drop_type == DropType::Open { @@ -86,6 +89,8 @@ impl Mutation { let seller_fee_basis_points = input.seller_fee_basis_points.unwrap_or_default(); + let tx = conn.begin().await?; + let collection_am = collections::ActiveModel { blockchain: Set(input.blockchain), supply: Set(supply), @@ -97,23 +102,20 @@ impl Mutation { ..Default::default() }; - let collection = Collection::new(collection_am) - .creators(input.creators.clone()) - .save(db) - .await?; + let collection = collection_am.insert(&tx).await?; - let metadata_jsons::Model { - name, - symbol, - uri, - description, - image, - .. - } = MetadataJson::new(input.metadata_json) - .upload(nft_storage) - .await? - .save(collection.id, db) - .await?; + for creator in input.creators { + let am = collection_creators::ActiveModel { + collection_id: Set(collection.id), + address: Set(creator.address), + verified: Set(creator.verified.unwrap_or_default()), + share: Set(creator.share.try_into()?), + }; + + am.insert(&tx).await?; + } + + input.metadata_json.save(collection.id, &tx).await?; let drop = drops::ActiveModel { project_id: Set(input.project), @@ -129,67 +131,18 @@ impl Mutation { ..Default::default() }; - let drop_model = drop.insert(conn).await?; - let event_key = NftEventKey { - id: collection.id.to_string(), - user_id: user_id.to_string(), - project_id: input.project.to_string(), - }; + let drop_model = drop.insert(&tx).await?; - let payload = proto::MetaplexMasterEditionTransaction { - master_edition: Some(proto::MasterEdition { - owner_address: owner_address.clone(), - supply, - name: name.clone(), - symbol, - metadata_uri: uri.clone(), - seller_fee_basis_points: seller_fee_basis_points.into(), - creators: input - .creators - .clone() - .into_iter() - .map(TryFrom::try_from) - .collect::>()?, - }), - }; + tx.commit().await?; - match input.blockchain { - BlockchainEnum::Solana => { - solana - .event() - .create_drop(input.drop_type, event_key, payload) - .await?; - }, - BlockchainEnum::Polygon => { - let amount = input.supply.ok_or(Error::new("supply is required"))?; - polygon - .create_drop( - input.drop_type, - event_key, - proto::CreateEditionTransaction { - amount: amount.try_into()?, - edition_info: Some(proto::EditionInfo { - creator: input - .creators - .get(0) - .ok_or(Error::new("creator is required"))? - .clone() - .address, - collection: name, - uri, - description, - image_uri: image, - }), - fee_receiver: owner_address, - fee_numerator: seller_fee_basis_points.into(), - }, - ) - .await?; - }, - BlockchainEnum::Ethereum => { - return Err(Error::new("blockchain not supported as this time")); - }, - }; + metadata_json_upload_job_queue + .enqueue(MetadataJsonUploadTask::new( + input.metadata_json, + MetadataJsonUploadCaller::CreateDrop(MetadataJsonUploadCreateDrop { + drop_id: drop_model.id, + }), + )) + .await?; nfts_producer .send( @@ -261,6 +214,10 @@ impl Mutation { .one(conn) .await? .ok_or(Error::new("metadata json not found"))?; + + let metadata_uri = metadata_json + .uri + .ok_or(Error::new("metadata uri not found"))?; let creators = CollectionCreators::find() .filter(collection_creators::Column::CollectionId.eq(collection.id)) .all(conn) @@ -297,7 +254,7 @@ impl Mutation { supply: collection.supply.map(TryInto::try_into).transpose()?, name: metadata_json.name, symbol: metadata_json.symbol, - metadata_uri: metadata_json.uri, + metadata_uri, seller_fee_basis_points: collection.seller_fee_basis_points.into(), creators: creators .into_iter() @@ -462,16 +419,13 @@ impl Mutation { let AppContext { db, user_id, .. } = ctx.data::()?; let conn = db.get(); - let nft_storage = ctx.data::()?; + let metadata_json_upload_job_queue = ctx.data::()?; let solana = ctx.data::()?; let polygon = ctx.data::()?; let user_id = user_id.0.ok_or(Error::new("X-USER-ID header not found"))?; - let (drop_model, collection_model) = Drops::find() - .join(JoinType::InnerJoin, drops::Relation::Collections.def()) - .select_also(Collections) - .filter(drops::Column::Id.eq(id)) + let (drop_model, collection_model) = drops::Entity::find_by_id_with_collection(id) .one(conn) .await? .ok_or(Error::new("drop not found"))?; @@ -493,17 +447,25 @@ impl Mutation { validate_json(collection.blockchain, metadata_json)?; } + let current_creators = collection_creators::Entity::find() + .filter(collection_creators::Column::CollectionId.eq(collection.id)) + .all(conn) + .await?; + + let metadata_json_model = metadata_jsons::Entity::find() + .filter(metadata_jsons::Column::Id.eq(collection.id)) + .one(conn) + .await? + .ok_or(Error::new("metadata json not found"))?; + + let tx = conn.begin().await?; + let mut collection_am: collections::ActiveModel = collection.into(); if let Some(seller_fee_basis_points) = seller_fee_basis_points { collection_am.seller_fee_basis_points = Set(seller_fee_basis_points.try_into()?); } - let collection = collection_am.update(conn).await?; - - let current_creators = collection_creators::Entity::find() - .filter(collection_creators::Column::CollectionId.eq(collection.id)) - .all(conn) - .await?; + let collection = collection_am.update(&tx).await?; let mut drop_am = drops::ActiveModel::from(drop_model.clone()); @@ -522,8 +484,8 @@ impl Mutation { }) .transpose()?); - if let Some(creators) = creators.clone() { - let creators = creators + let creators = if let Some(creators) = creators { + let creator_ams = creators .clone() .into_iter() .map(|creator| { @@ -536,111 +498,102 @@ impl Mutation { }) .collect::>>()?; - conn.transaction::<_, (), DbErr>(|txn| { - Box::pin(async move { - collection_creators::Entity::delete_many() - .filter(collection_creators::Column::CollectionId.eq(collection.id)) - .exec(txn) - .await?; - - collection_creators::Entity::insert_many(creators) - .exec(txn) - .await?; + collection_creators::Entity::delete_many() + .filter(collection_creators::Column::CollectionId.eq(collection.id)) + .exec(&tx) + .await?; - Ok(()) - }) - }) - .await?; - } + collection_creators::Entity::insert_many(creator_ams) + .exec(&tx) + .await?; - let drop_model = drop_am.update(conn).await?; - - let metadata_json_model = metadata_jsons::Entity::find() - .filter(metadata_jsons::Column::Id.eq(collection.id)) - .one(conn) - .await? - .ok_or(Error::new("metadata json not found"))?; - - let metadata_json_model = if let Some(metadata_json) = metadata_json { - metadata_json_model.clone().delete(conn).await?; - - MetadataJson::new(metadata_json.clone()) - .upload(nft_storage) - .await? - .save(collection.id, db) - .await? + creators + .into_iter() + .map(TryFrom::try_from) + .collect::>()? } else { - metadata_json_model + current_creators.into_iter().map(Into::into).collect() }; - let event_key = NftEventKey { - id: collection.id.to_string(), - user_id: user_id.to_string(), - project_id: drop_model.project_id.to_string(), - }; + let drop_model = drop_am.update(&tx).await?; - match collection.blockchain { - BlockchainEnum::Solana => { - let creators = if let Some(creators) = creators.clone() { - creators - .into_iter() - .map(TryFrom::try_from) - .collect::>()? - } else { - current_creators.into_iter().map(Into::into).collect() - }; + if let Some(metadata_json) = metadata_json { + metadata_json_model.delete(&tx).await?; - solana - .event() - .update_drop( - drop_model.drop_type, - event_key, - proto::MetaplexMasterEditionTransaction { - master_edition: Some(proto::MasterEdition { - owner_address, - supply: collection.supply.map(TryInto::try_into).transpose()?, - name: metadata_json_model.name, - symbol: metadata_json_model.symbol, - metadata_uri: metadata_json_model.uri, - seller_fee_basis_points: collection.seller_fee_basis_points.into(), - creators, - }), - }, - ) - .await?; - }, - BlockchainEnum::Polygon => { - let creator = if let Some(creators) = creators { - creators[0].address.clone() - } else { - current_creators - .get(0) - .ok_or(Error::new("No current creator found in db"))? - .address - .clone() - }; + metadata_json.save(collection.id, &tx).await?; - polygon - .event() - .update_drop( - drop_model.drop_type, - event_key, - proto::UpdateEdtionTransaction { - edition_info: Some(EditionInfo { - description: metadata_json_model.description, - image_uri: metadata_json_model.image, - collection: metadata_json_model.name, - uri: metadata_json_model.uri, - creator, - }), - }, - ) - .await?; - }, - BlockchainEnum::Ethereum => { - return Err(Error::new("blockchain not supported yet")); - }, - }; + metadata_json_upload_job_queue + .enqueue(MetadataJsonUploadTask { + metadata_json, + caller: MetadataJsonUploadCaller::PatchDrop(MetadataJsonUploadPatchDrop { + drop_id: drop_model.id, + updated_by_id: user_id, + }), + }) + .await?; + } else { + let event_key = NftEventKey { + id: collection.id.to_string(), + user_id: user_id.to_string(), + project_id: drop_model.project_id.to_string(), + }; + + let metadata_uri = metadata_json_model + .uri + .ok_or(Error::new("metadata uri not found"))?; + + match collection.blockchain { + BlockchainEnum::Solana => { + solana + .event() + .update_drop( + drop_model.drop_type, + event_key, + proto::MetaplexMasterEditionTransaction { + master_edition: Some(proto::MasterEdition { + owner_address, + supply: collection.supply.map(TryInto::try_into).transpose()?, + name: metadata_json_model.name, + symbol: metadata_json_model.symbol, + metadata_uri, + seller_fee_basis_points: collection + .seller_fee_basis_points + .into(), + creators, + }), + }, + ) + .await?; + }, + BlockchainEnum::Polygon => { + polygon + .event() + .update_drop( + drop_model.drop_type, + event_key, + proto::UpdateEdtionTransaction { + edition_info: Some(EditionInfo { + description: metadata_json_model.description, + image_uri: metadata_json_model.image, + collection: metadata_json_model.name, + uri: metadata_uri, + creator: creators + .get(0) + .ok_or(Error::new("no creator found"))? + .address + .clone(), + }), + }, + ) + .await?; + }, + BlockchainEnum::Ethereum => { + return Err(Error::new("blockchain not supported yet")); + }, + }; + } + + tx.commit().await?; Ok(PatchDropPayload { drop: Drop::new(drop_model, collection), @@ -710,11 +663,38 @@ impl CreateDropInput { validate_end_time(&self.end_time)?; validate_creators(self.blockchain, &self.creators)?; validate_json(self.blockchain, &self.metadata_json)?; + validate_polygon_supply(self.blockchain, self.supply)?; + validate_polygon_creator(self.blockchain, &self.creators)?; Ok(()) } } +/// Validates the creators for polygon drops. +/// # Returns +/// - Ok(()) if the creators are provided for polygon drops. +/// # Errors +/// - Err with an appropriate error message if the creators are not provided for polygon drops. +fn validate_polygon_creator(blockchain: BlockchainEnum, creators: &[Creator]) -> Result<()> { + if blockchain == BlockchainEnum::Polygon && creators.len() > 1 { + return Err(Error::new("Only one creator is allowed for polygon drops")); + } + + Ok(()) +} + +/// Validates the supply for polygon drops. +/// # Returns +/// - Ok(()) if the supply is provided for polygon drops. +/// # Errors +/// - Err with an appropriate error message if the supply is not provided for polygon drops. +fn validate_polygon_supply(blockchain: BlockchainEnum, supply: Option) -> Result<()> { + if blockchain == BlockchainEnum::Polygon && supply.is_none() { + return Err(Error::new("Supply is required for polygon drops")); + } + + Ok(()) +} /// Validates the end time of the NFT drop. /// # Returns /// - Ok(()) if the end time is in the future or if it's not provided. diff --git a/api/src/mutations/mint.rs b/api/src/mutations/mint.rs index 68208ff..6915bf0 100644 --- a/api/src/mutations/mint.rs +++ b/api/src/mutations/mint.rs @@ -16,25 +16,33 @@ use super::collection::{ fetch_owner, validate_creators, validate_json, validate_solana_creator_verification, }; use crate::{ + background_worker::{ + job_queue::JobQueue, + tasks::{ + MetadataJsonUploadCaller, MetadataJsonUploadMintToCollection, + MetadataJsonUploadQueueMintToDrop, MetadataJsonUploadTask, + MetadataJsonUploadUpdateMint, + }, + }, blockchains::{ polygon::Polygon, solana::{MintDropTransaction, Solana}, CollectionEvent, DropEvent, }, entities::{ - collection_creators, collection_mints, collections, drops, mint_creators, mint_histories, - prelude::{CollectionCreators, CollectionMints, Collections, Drops}, + collection_creators, collection_mints, collections, drops, metadata_json_attributes, + metadata_json_files, metadata_jsons, mint_creators, mint_histories, + prelude::{CollectionCreators, CollectionMints, Collections}, project_wallets, sea_orm_active_enums::{Blockchain as BlockchainEnum, CreationStatus}, update_histories, }, - metadata_json::MetadataJson, objects::{CollectionMint, Creator, MetadataJsonInput}, proto::{ self, nft_events::Event as NftEvent, CreationStatus as NftCreationStatus, MetaplexMetadata, MintCollectionCreation, MintCreation, NftEventKey, NftEvents, RetryUpdateSolanaMintPayload, }, - Actions, AppContext, NftStorageClient, OrganizationId, UserID, + Actions, AppContext, OrganizationId, UserID, }; #[derive(Default)] @@ -72,42 +80,23 @@ impl Mutation { .0 .ok_or(Error::new("X-CREDIT-BALANCE header not found"))?; - let drop_model = Drops::find() - .join(JoinType::InnerJoin, drops::Relation::Collections.def()) - .select_also(Collections) - .filter(drops::Column::Id.eq(input.drop)) + let (drop_model, collection) = drops::Entity::find_by_id_with_collection(input.drop) .one(conn) - .await?; + .await? + .ok_or(Error::new("drop not found"))?; - let (drop_model, collection_model) = drop_model.ok_or(Error::new("drop not found"))?; + let collection = collection.ok_or(Error::new("collection not found"))?; // Call check_drop_status to check that drop is currently running check_drop_status(&drop_model)?; - let collection = collection_model.ok_or(Error::new("collection not found"))?; - if collection.supply == Some(collection.total_mints) { return Err(Error::new("Collection is sold out")); } let edition = collection.total_mints.add(1); - // Fetch the project wallet address which will sign the transaction by hub-treasuries - let wallet = project_wallets::Entity::find() - .filter( - project_wallets::Column::ProjectId - .eq(drop_model.project_id) - .and(project_wallets::Column::Blockchain.eq(collection.blockchain)), - ) - .one(conn) - .await?; - - let owner_address = wallet - .ok_or(Error::new(format!( - "no project wallet found for {:?} blockchain", - collection.blockchain - )))? - .wallet_address; + let owner_address = fetch_owner(conn, collection.project_id, collection.blockchain).await?; let TransactionId(credits_deduction_id) = credits .submit_pending_deduction( @@ -119,6 +108,8 @@ impl Mutation { ) .await?; + let tx = conn.begin().await?; + // insert a collection mint record into database let collection_mint_active_model = collection_mints::ActiveModel { collection_id: Set(collection.id), @@ -132,6 +123,100 @@ impl Mutation { }; let collection_mint_model = collection_mint_active_model.insert(conn).await?; + + let mut collection_am = collections::ActiveModel::from(collection.clone()); + collection_am.total_mints = Set(edition); + collection_am.update(&tx).await?; + + // inserts a mint histories record in the database + let mint_history_am = mint_histories::ActiveModel { + mint_id: Set(collection_mint_model.id), + wallet: Set(input.recipient.clone()), + collection_id: Set(collection.id), + tx_signature: Set(None), + status: Set(CreationStatus::Pending), + created_at: Set(Utc::now().into()), + ..Default::default() + }; + + mint_history_am.insert(&tx).await?; + + if collection.blockchain == BlockchainEnum::Solana { + let collection_metadata_json = + metadata_jsons::Entity::find_by_id(collection_mint_model.id) + .one(conn) + .await? + .ok_or(Error::new("metadata json not found"))?; + + let creators = mint_creators::Entity::find() + .filter(mint_creators::Column::CollectionMintId.eq(collection_mint_model.id)) + .all(conn) + .await?; + + let files = metadata_json_files::Entity::find() + .filter(metadata_json_files::Column::MetadataJsonId.eq(collection_metadata_json.id)) + .all(conn) + .await?; + let attributes = metadata_json_attributes::Entity::find() + .filter( + metadata_json_attributes::Column::MetadataJsonId + .eq(collection_metadata_json.id), + ) + .all(conn) + .await?; + + let mut metadata_json_am = metadata_jsons::ActiveModel::from(collection_metadata_json); + + metadata_json_am.id = Set(collection_mint_model.id); + + let collection_mint_metadata_json = metadata_json_am.insert(&tx).await?; + + // TODO: Look into bug regarding creators not being saved to the database when minting editions + let creators: Vec = creators + .into_iter() + .map(|creator| mint_creators::ActiveModel { + collection_mint_id: Set(collection_mint_model.id), + address: Set(creator.address), + verified: Set(creator.verified), + share: Set(creator.share), + }) + .collect(); + + mint_creators::Entity::insert_many(creators) + .exec(&tx) + .await?; + + let files: Vec = files + .into_iter() + .map(|file| metadata_json_files::ActiveModel { + metadata_json_id: Set(collection_mint_metadata_json.id), + uri: Set(file.uri), + file_type: Set(file.file_type), + ..Default::default() + }) + .collect(); + + metadata_json_files::Entity::insert_many(files) + .exec(&tx) + .await?; + + let attributes: Vec = attributes + .into_iter() + .map(|attribute| metadata_json_attributes::ActiveModel { + metadata_json_id: Set(collection_mint_metadata_json.id), + trait_type: Set(attribute.trait_type), + value: Set(attribute.value), + ..Default::default() + }) + .collect(); + + metadata_json_attributes::Entity::insert_many(attributes) + .exec(&tx) + .await?; + } + + tx.commit().await?; + let event_key = NftEventKey { id: collection_mint_model.id.to_string(), user_id: user_id.to_string(), @@ -140,11 +225,6 @@ impl Mutation { match collection.blockchain { BlockchainEnum::Solana => { - MetadataJson::fetch(collection.id, db) - .await? - .save(collection_mint_model.id, db) - .await?; - solana .event() .mint_drop( @@ -173,23 +253,6 @@ impl Mutation { }, }; - let mut collection_am = collections::ActiveModel::from(collection.clone()); - collection_am.total_mints = Set(edition); - collection_am.update(conn).await?; - - // inserts a mint histories record in the database - let purchase_am = mint_histories::ActiveModel { - mint_id: Set(collection_mint_model.id), - wallet: Set(input.recipient), - collection_id: Set(collection.id), - tx_signature: Set(None), - status: Set(CreationStatus::Pending), - created_at: Set(Utc::now().into()), - ..Default::default() - }; - - purchase_am.insert(conn).await?; - nfts_producer .send( Some(&NftEvents { @@ -360,9 +423,8 @@ impl Mutation { } = ctx.data::()?; let credits = ctx.data::>()?; let conn = db.get(); - let solana = ctx.data::()?; let nfts_producer = ctx.data::>()?; - let nft_storage = ctx.data::()?; + let metadata_json_upload_job_queue = ctx.data::()?; let UserID(id) = user_id; let OrganizationId(org) = organization_id; @@ -413,27 +475,28 @@ impl Mutation { ) .await?; + let tx = conn.begin().await?; + // insert a collection mint record into database let collection_mint_active_model = collection_mints::ActiveModel { collection_id: Set(collection.id), owner: Set(Some(input.recipient.clone())), creation_status: Set(CreationStatus::Pending), - seller_fee_basis_points: Set(collection.seller_fee_basis_points), + seller_fee_basis_points: Set(seller_fee_basis_points.try_into()?), created_by: Set(user_id), compressed: Set(Some(compressed)), credits_deduction_id: Set(Some(credits_deduction_id)), ..Default::default() }; - let collection_mint_model = collection_mint_active_model.insert(conn).await?; + let collection_mint_model = collection_mint_active_model.insert(&tx).await?; - let metadata_json = MetadataJson::new(input.metadata_json) - .upload(nft_storage) - .await? - .save(collection_mint_model.id, db) + input + .metadata_json + .save(collection_mint_model.id, &tx) .await?; - for creator in creators.clone() { + for creator in creators { let am = mint_creators::ActiveModel { collection_mint_id: Set(collection_mint_model.id), address: Set(creator.address), @@ -441,45 +504,12 @@ impl Mutation { share: Set(creator.share.try_into()?), }; - am.insert(conn).await?; + am.insert(&tx).await?; } - let event_key = NftEventKey { - id: collection_mint_model.id.to_string(), - user_id: user_id.to_string(), - project_id: collection.project_id.to_string(), - }; - - match collection.blockchain { - BlockchainEnum::Solana => { - solana - .event() - .mint_to_collection(event_key, proto::MintMetaplexMetadataTransaction { - metadata: Some(MetaplexMetadata { - owner_address, - name: metadata_json.name, - symbol: metadata_json.symbol, - metadata_uri: metadata_json.uri, - seller_fee_basis_points: seller_fee_basis_points.into(), - creators: creators - .into_iter() - .map(TryFrom::try_from) - .collect::>()?, - }), - recipient_address: input.recipient.to_string(), - compressed, - collection_id: collection.id.to_string(), - }) - .await?; - }, - BlockchainEnum::Ethereum | BlockchainEnum::Polygon => { - return Err(Error::new("blockchain not supported as this time")); - }, - }; - let mut collection_am = collections::ActiveModel::from(collection.clone()); collection_am.total_mints = Set(collection.total_mints.add(1)); - collection_am.update(conn).await?; + collection_am.update(&tx).await?; let mint_history_am = mint_histories::ActiveModel { mint_id: Set(collection_mint_model.id), @@ -491,7 +521,20 @@ impl Mutation { ..Default::default() }; - mint_history_am.insert(conn).await?; + mint_history_am.insert(&tx).await?; + + tx.commit().await?; + + metadata_json_upload_job_queue + .enqueue(MetadataJsonUploadTask { + caller: MetadataJsonUploadCaller::MintToCollection( + MetadataJsonUploadMintToCollection { + collection_mint_id: collection_mint_model.id, + }, + ), + metadata_json: input.metadata_json, + }) + .await?; nfts_producer .send( @@ -531,8 +574,7 @@ impl Mutation { } = ctx.data::()?; let credits = ctx.data::>()?; let conn = db.get(); - let solana = ctx.data::()?; - let nft_storage = ctx.data::()?; + let metadata_json_upload_job_queue = ctx.data::()?; let UserID(id) = user_id; let OrganizationId(org) = organization_id; @@ -545,9 +587,7 @@ impl Mutation { let creators = input.creators; - let (mint, collection) = CollectionMints::find() - .find_also_related(Collections) - .filter(collection_mints::Column::Id.eq(input.id)) + let (mint, collection) = CollectionMints::find_by_id_with_collection(input.id) .one(conn) .await? .ok_or(Error::new("Mint not found"))?; @@ -589,7 +629,7 @@ impl Mutation { }) .collect::>>()?; - let deduction_id = credits + let TransactionId(deduction_id) = credits .submit_pending_deduction( org_id, user_id, @@ -599,81 +639,47 @@ impl Mutation { ) .await?; + let tx = conn.begin().await?; + let mut mint_am: collection_mints::ActiveModel = mint.clone().into(); let update_history_am = update_histories::ActiveModel { mint_id: Set(mint.id), txn_signature: Set(None), - credit_deduction_id: Set(deduction_id.0), + credit_deduction_id: Set(deduction_id), created_by: Set(user_id), status: Set(CreationStatus::Pending), ..Default::default() }; - let update_history = update_history_am.insert(db.get()).await?; - let sfbp = mint.seller_fee_basis_points.try_into()?; - conn.transaction::<_, (), DbErr>(|txn| { - Box::pin(async move { - mint_creators::Entity::delete_many() - .filter(mint_creators::Column::CollectionMintId.eq(mint.id)) - .exec(txn) - .await?; + let update_history = update_history_am.insert(&tx).await?; - mint_creators::Entity::insert_many(creators_am) - .exec(txn) - .await?; + mint_creators::Entity::delete_many() + .filter(mint_creators::Column::CollectionMintId.eq(mint.id)) + .exec(&tx) + .await?; - if let Some(sfbp) = input.seller_fee_basis_points { - mint_am.seller_fee_basis_points = Set(sfbp.try_into().unwrap_or_default()); - mint_am.update(txn).await?; - } + mint_creators::Entity::insert_many(creators_am) + .exec(&tx) + .await?; - Ok(()) - }) - }) - .await?; + if let Some(sfbp) = input.seller_fee_basis_points { + mint_am.seller_fee_basis_points = Set(sfbp.try_into().unwrap_or_default()); + mint_am.update(&tx).await?; + } - let metadata_json = MetadataJson::new(input.metadata_json) - .upload(nft_storage) - .await? - .save(mint.id, db) - .await?; + input.metadata_json.save(mint.id, &tx).await?; - match collection.blockchain { - BlockchainEnum::Solana => { - solana - .event() - .update_collection_mint( - NftEventKey { - id: update_history.id.to_string(), - project_id: collection.project_id.to_string(), - user_id: user_id.to_string(), - }, - proto::UpdateSolanaMintPayload { - metadata: Some(MetaplexMetadata { - owner_address, - name: metadata_json.name, - symbol: metadata_json.symbol, - metadata_uri: metadata_json.uri, - seller_fee_basis_points: input - .seller_fee_basis_points - .unwrap_or(sfbp) - .try_into()?, - creators: creators - .into_iter() - .map(TryFrom::try_from) - .collect::>()?, - }), - collection_id: collection.id.to_string(), - mint_id: update_history.mint_id.to_string(), - }, - ) - .await?; - }, - BlockchainEnum::Ethereum | BlockchainEnum::Polygon => { - return Err(Error::new("blockchain not supported as this time")); - }, - }; + tx.commit().await?; + + metadata_json_upload_job_queue + .enqueue(MetadataJsonUploadTask { + caller: MetadataJsonUploadCaller::UpdateMint(MetadataJsonUploadUpdateMint { + update_history_id: update_history.id, + }), + metadata_json: input.metadata_json, + }) + .await?; Ok(UpdateMintPayload { collection_mint: mint.into(), @@ -707,15 +713,15 @@ impl Mutation { .await? .ok_or(Error::new("Update history not found"))?; + let collection = collection.ok_or(Error::new("Collection not found"))?; + if update_history.status == CreationStatus::Created { return Err(Error::new("Mint already updated")); } let mut update_history_am = update_histories::ActiveModel::from(update_history.clone()); update_history_am.status = Set(CreationStatus::Pending); - update_history_am.update(db.get()).await?; - - let collection = collection.ok_or(Error::new("Collection not found"))?; + update_history_am.update(conn).await?; match collection.blockchain { BlockchainEnum::Solana => { @@ -796,15 +802,14 @@ impl Mutation { let owner_address = fetch_owner(conn, project_id, blockchain).await?; - let MetadataJson { - metadata_json, uri, .. - } = MetadataJson::fetch(collection_mint_model.id, db).await?; + let metadata_json = metadata_jsons::Entity::find_by_id(collection_mint_model.id) + .one(conn) + .await? + .ok_or(Error::new("metadata json not found"))?; - let event_key = NftEventKey { - id: collection_mint_model.id.to_string(), - user_id: user_id.to_string(), - project_id: project_id.to_string(), - }; + let metadata_uri = metadata_json + .uri + .ok_or(Error::new("metadata uri not found"))?; let creators = mint_creators::Entity::find_by_collection_mint_id(collection_mint_model.id) .all(conn) @@ -820,6 +825,12 @@ impl Mutation { ) .await?; + let event_key = NftEventKey { + id: collection_mint_model.id.to_string(), + user_id: user_id.to_string(), + project_id: project_id.to_string(), + }; + match collection.blockchain { BlockchainEnum::Solana => { solana @@ -829,7 +840,7 @@ impl Mutation { owner_address, name: metadata_json.name, symbol: metadata_json.symbol, - metadata_uri: uri.ok_or(Error::new("metadata uri not found"))?, + metadata_uri, seller_fee_basis_points: collection_mint_model .seller_fee_basis_points .into(), @@ -866,25 +877,32 @@ impl Mutation { let AppContext { db, user_id, .. } = ctx.data::()?; let conn = db.get(); - let nft_storage = ctx.data::()?; + + let metadata_json_upload_job_queue = ctx.data::()?; let nfts_producer = ctx.data::>()?; let UserID(id) = user_id; let user_id = id.ok_or(Error::new("X-USER-ID header not found"))?; - let (drop, collection) = drops::Entity::find_by_id(input.drop) - .find_also_related(Collections) + let (drop, collection) = drops::Entity::find_by_id_with_collection(input.drop) .one(conn) .await? .ok_or(Error::new("drop not found"))?; let collection_model = collection.ok_or(Error::new("collection not found"))?; + let creators = CollectionCreators::find() + .filter(collection_creators::Column::CollectionId.eq(collection_model.id)) + .all(conn) + .await?; + + let tx = conn.begin().await?; + let mut collection_am: collections::ActiveModel = collection_model.clone().into(); collection_am.supply = Set(collection_model.supply.map(|supply| supply.add(1))); - collection_am.update(conn).await?; + collection_am.update(&tx).await?; let mint = collection_mints::ActiveModel { collection_id: Set(drop.collection_id), @@ -896,18 +914,9 @@ impl Mutation { ..Default::default() }; - let mint_model = mint.insert(conn).await?; - - MetadataJson::new(input.metadata_json) - .upload(nft_storage) - .await? - .save(mint_model.id, db) - .await?; + let mint_model = mint.insert(&tx).await?; - let creators = CollectionCreators::find() - .filter(collection_creators::Column::CollectionId.eq(collection_model.id)) - .all(conn) - .await?; + input.metadata_json.save(mint_model.id, &tx).await?; let mint_creators: Vec<_> = creators .iter() @@ -920,7 +929,21 @@ impl Mutation { .collect(); mint_creators::Entity::insert_many(mint_creators) - .exec(conn) + .exec(&tx) + .await?; + + tx.commit().await?; + + metadata_json_upload_job_queue + .enqueue(MetadataJsonUploadTask { + caller: MetadataJsonUploadCaller::QueueMintToDrop( + MetadataJsonUploadQueueMintToDrop { + drop_id: drop.id, + collection_mint_id: mint_model.id, + }, + ), + metadata_json: input.metadata_json, + }) .await?; nfts_producer @@ -934,7 +957,7 @@ impl Mutation { Some(&NftEventKey { id: mint_model.id.to_string(), project_id: drop.project_id.to_string(), - user_id: user_id.to_string(), + user_id: mint_model.created_by.to_string(), }), ) .await?; @@ -957,10 +980,12 @@ impl Mutation { balance, .. } = ctx.data::()?; + let credits = ctx.data::>()?; - let conn = db.get(); - let solana = ctx.data::()?; let nfts_producer = ctx.data::>()?; + let solana = ctx.data::()?; + + let conn = db.get(); let UserID(id) = user_id; let OrganizationId(org) = organization_id; @@ -982,14 +1007,25 @@ impl Mutation { let collection = collection.ok_or(Error::new("collection not found"))?; + let drop = drops::Entity::find() + .filter(drops::Column::CollectionId.eq(collection.id)) + .one(conn) + .await? + .ok_or(Error::new("drop not found"))?; + let project_id = collection.project_id; let blockchain = collection.blockchain; let owner_address = fetch_owner(conn, project_id, blockchain).await?; - let MetadataJson { - metadata_json, uri, .. - } = MetadataJson::fetch(mint.id, db).await?; + let metadata_json = metadata_jsons::Entity::find_by_id(mint.id) + .one(conn) + .await? + .ok_or(Error::new("metadata json not found"))?; + + let metadata_uri = metadata_json + .uri + .ok_or(Error::new("No metadata json uri found"))?; let event_key = NftEventKey { id: mint.id.to_string(), @@ -1017,9 +1053,13 @@ impl Mutation { ) .await?; + let tx = conn.begin().await?; + let mut collection_am = collections::ActiveModel::from(collection.clone()); + collection_am.total_mints = Set(collection.total_mints.add(1)); - collection_am.update(conn).await?; + + collection_am.update(&tx).await?; let mut mint_am: collection_mints::ActiveModel = mint.into(); @@ -1029,7 +1069,21 @@ impl Mutation { mint_am.owner = Set(Some(input.recipient.clone())); mint_am.seller_fee_basis_points = Set(collection.seller_fee_basis_points); - let mint = mint_am.update(conn).await?; + let mint = mint_am.update(&tx).await?; + + let mint_history_am = mint_histories::ActiveModel { + mint_id: Set(mint.id), + wallet: Set(input.recipient.clone()), + collection_id: Set(collection.id), + tx_signature: Set(None), + status: Set(CreationStatus::Pending), + created_at: Set(Utc::now().into()), + ..Default::default() + }; + + mint_history_am.insert(&tx).await?; + + tx.commit().await?; match collection.blockchain { BlockchainEnum::Solana => { @@ -1042,8 +1096,7 @@ impl Mutation { owner_address, name: metadata_json.name, symbol: metadata_json.symbol, - metadata_uri: uri - .ok_or(Error::new("No metadata json uri found"))?, + metadata_uri, seller_fee_basis_points: mint.seller_fee_basis_points.into(), creators: creators.into_iter().map(Into::into).collect(), }), @@ -1059,12 +1112,6 @@ impl Mutation { }, }; - let drop = drops::Entity::find() - .filter(drops::Column::CollectionId.eq(collection.id)) - .one(conn) - .await? - .ok_or(Error::new("drop not found"))?; - nfts_producer .send( Some(&NftEvents { @@ -1081,18 +1128,6 @@ impl Mutation { ) .await?; - let mint_history_am = mint_histories::ActiveModel { - mint_id: Set(mint.id), - wallet: Set(input.recipient), - collection_id: Set(collection.id), - tx_signature: Set(None), - status: Set(CreationStatus::Pending), - created_at: Set(Utc::now().into()), - ..Default::default() - }; - - mint_history_am.insert(conn).await?; - Ok(MintQueuedPayload { collection_mint: mint.into(), }) @@ -1150,15 +1185,14 @@ impl Mutation { let owner_address = fetch_owner(conn, project_id, blockchain).await?; - let MetadataJson { - metadata_json, uri, .. - } = MetadataJson::fetch(mint.id, db).await?; + let metadata_json = metadata_jsons::Entity::find_by_id(mint.id) + .one(conn) + .await? + .ok_or(Error::new("metadata json not found"))?; - let event_key = NftEventKey { - id: mint.id.to_string(), - user_id: user_id.to_string(), - project_id: project_id.to_string(), - }; + let metadata_uri = metadata_json + .uri + .ok_or(Error::new("No metadata json uri found"))?; let creators = mint_creators::Entity::find_by_collection_mint_id(mint.id) .all(conn) @@ -1180,9 +1214,11 @@ impl Mutation { ) .await?; + let tx = conn.begin().await?; + let mut collection_am = collections::ActiveModel::from(collection.clone()); collection_am.total_mints = Set(collection.total_mints.add(1)); - collection_am.update(conn).await?; + collection_am.update(&tx).await?; let mut mint_am: collection_mints::ActiveModel = mint.into(); @@ -1192,7 +1228,27 @@ impl Mutation { mint_am.owner = Set(Some(input.recipient.clone())); mint_am.seller_fee_basis_points = Set(collection.seller_fee_basis_points); - let mint = mint_am.update(conn).await?; + let mint = mint_am.update(&tx).await?; + + let mint_history_am = mint_histories::ActiveModel { + mint_id: Set(mint.id), + wallet: Set(input.recipient.clone()), + collection_id: Set(collection.id), + tx_signature: Set(None), + status: Set(CreationStatus::Pending), + created_at: Set(Utc::now().into()), + ..Default::default() + }; + + mint_history_am.insert(&tx).await?; + + tx.commit().await?; + + let event_key = NftEventKey { + id: mint.id.to_string(), + user_id: user_id.to_string(), + project_id: project_id.to_string(), + }; match collection.blockchain { BlockchainEnum::Solana => { @@ -1205,7 +1261,7 @@ impl Mutation { owner_address, name: metadata_json.name, symbol: metadata_json.symbol, - metadata_uri: uri.ok_or(Error::new("No metadata json uri"))?, + metadata_uri, seller_fee_basis_points: mint.seller_fee_basis_points.into(), creators: creators.into_iter().map(Into::into).collect(), }), @@ -1221,18 +1277,6 @@ impl Mutation { }, }; - let mint_history_am = mint_histories::ActiveModel { - mint_id: Set(mint.id), - wallet: Set(input.recipient), - collection_id: Set(collection.id), - tx_signature: Set(None), - status: Set(CreationStatus::Pending), - created_at: Set(Utc::now().into()), - ..Default::default() - }; - - mint_history_am.insert(conn).await?; - nfts_producer .send( Some(&NftEvents { diff --git a/api/src/nft_storage.rs b/api/src/nft_storage.rs index 0807762..c534050 100644 --- a/api/src/nft_storage.rs +++ b/api/src/nft_storage.rs @@ -68,7 +68,7 @@ impl NftStorageClient { /// /// # Errors /// If the upload fails - pub async fn upload(&self, data: impl Serialize) -> Result { + pub async fn upload(&self, data: &impl Serialize) -> Result { self.post("/upload".to_string(), data) .await? .json() diff --git a/api/src/objects/metadata_json.rs b/api/src/objects/metadata_json.rs index 75ca582..a62fc6d 100644 --- a/api/src/objects/metadata_json.rs +++ b/api/src/objects/metadata_json.rs @@ -1,14 +1,11 @@ use async_graphql::{ComplexObject, Context, InputObject, Result, SimpleObject}; use hub_core::{assets::AssetProxy, uuid::Uuid}; use reqwest::Url; +use sea_orm::{prelude::*, sea_query::OnConflict, DatabaseTransaction, Set}; use serde::{Deserialize, Serialize}; use crate::{ - entities::{ - metadata_json_attributes::{self, Model as MetadataJsonAttributeModel}, - metadata_json_files::Model as MetadataJsonFileModel, - metadata_jsons::{self, Model as MetadataJsonModel}, - }, + entities::{metadata_json_attributes, metadata_json_files, metadata_jsons}, AppContext, }; @@ -18,12 +15,14 @@ use crate::{ #[derive(Clone, Debug, PartialEq, Eq, SimpleObject)] #[graphql(complex, concrete(name = "MetadataJson", params()))] pub struct MetadataJson { + // The id of the metadata json. pub id: Uuid, - pub identifier: String, + // The assigned identifier of the metadata json uri. + pub identifier: Option, /// The assigned name of the NFT. pub name: String, /// The URI for the complete metadata JSON. - pub uri: String, + pub uri: Option, /// The symbol of the NFT. pub symbol: String, /// The description of the NFT. @@ -62,34 +61,6 @@ impl MetadataJson { } } -impl From for MetadataJson { - fn from( - metadata_jsons::Model { - id, - identifier, - name, - uri, - symbol, - description, - image, - animation_url, - external_url, - }: metadata_jsons::Model, - ) -> Self { - Self { - id, - identifier, - name, - uri, - symbol, - description, - image_original: image, - animation_url, - external_url, - } - } -} - #[derive(Clone, Debug, Serialize, Deserialize, InputObject)] pub struct MetadataJsonInput { pub name: String, @@ -136,52 +107,109 @@ pub struct Collection { pub family: Option, } -impl - From<( - MetadataJsonModel, - Vec, - Option>, - )> for MetadataJsonInput -{ - fn from( - (metadata_json, attributes, files): ( - MetadataJsonModel, - Vec, - Option>, - ), - ) -> Self { - let input = MetadataJsonInput { - name: metadata_json.name, - symbol: metadata_json.symbol, - description: metadata_json.description, - image: metadata_json.image, - animation_url: metadata_json.animation_url, - collection: None, - attributes: attributes.iter().map(|a| a.clone().into()).collect(), - external_url: metadata_json.external_url, - properties: Some(Property { - files: files.map(|files| files.iter().map(|f| f.clone().into()).collect()), - category: None, - }), +impl MetadataJsonInput { + /// Saves the metadata json to the database. If the metadata json already exists, it will update the existing record. + /// # Arguments + /// * `id` - The id of the metadata json + /// * `tx` - The database transaction to use + /// # Returns + /// Returns Ok with () + /// # Errors + /// Returns Err if unable to save records to the database + pub async fn save(&self, id: Uuid, tx: &DatabaseTransaction) -> Result<()> { + let metadata_json = self.clone(); + let metadata_json_active_model = metadata_jsons::ActiveModel { + id: Set(id), + identifier: Set(None), + name: Set(metadata_json.name), + uri: Set(None), + symbol: Set(metadata_json.symbol), + description: Set(metadata_json.description), + image: Set(metadata_json.image), + animation_url: Set(metadata_json.animation_url), + external_url: Set(metadata_json.external_url), }; - input - } -} -impl From for Attribute { - fn from(attribute: MetadataJsonAttributeModel) -> Self { - Attribute { - trait_type: attribute.trait_type, - value: attribute.value, + let metadata_json_model = metadata_jsons::Entity::insert(metadata_json_active_model) + .on_conflict( + OnConflict::column(metadata_jsons::Column::Id) + .update_columns([ + metadata_jsons::Column::Identifier, + metadata_jsons::Column::Name, + metadata_jsons::Column::Uri, + metadata_jsons::Column::Symbol, + metadata_jsons::Column::Description, + metadata_jsons::Column::Image, + metadata_jsons::Column::AnimationUrl, + metadata_jsons::Column::ExternalUrl, + ]) + .clone(), + ) + .exec_with_returning(tx) + .await?; + + metadata_json_attributes::Entity::delete_many() + .filter(metadata_json_attributes::Column::MetadataJsonId.eq(metadata_json_model.id)) + .exec(tx) + .await?; + + for attribute in metadata_json.attributes { + let am = metadata_json_attributes::ActiveModel { + metadata_json_id: Set(metadata_json_model.id), + trait_type: Set(attribute.trait_type), + value: Set(attribute.value), + ..Default::default() + }; + + am.insert(tx).await?; + } + + if let Some(files) = metadata_json.properties.unwrap_or_default().files { + metadata_json_files::Entity::delete_many() + .filter(metadata_json_files::Column::MetadataJsonId.eq(metadata_json_model.id)) + .exec(tx) + .await?; + + for file in files { + let metadata_json_file_am = metadata_json_files::ActiveModel { + metadata_json_id: Set(metadata_json_model.id), + uri: Set(file.uri), + file_type: Set(file.file_type), + ..Default::default() + }; + + metadata_json_file_am.insert(tx).await?; + } } + + Ok(()) } } -impl From for File { - fn from(file: MetadataJsonFileModel) -> Self { - File { - uri: file.uri, - file_type: file.file_type, +impl From for MetadataJson { + fn from( + metadata_jsons::Model { + id, + identifier, + name, + uri, + symbol, + description, + image, + animation_url, + external_url, + }: metadata_jsons::Model, + ) -> Self { + Self { + id, + identifier, + name, + uri, + symbol, + description, + image_original: image, + animation_url, + external_url, } } } diff --git a/migration/Cargo.toml b/migration/Cargo.toml index 6ab8d30..3001e3a 100644 --- a/migration/Cargo.toml +++ b/migration/Cargo.toml @@ -2,9 +2,7 @@ name = "migration" version = "0.1.0" publish = false -authors = [ - "Holaplex ", -] +authors = ["Holaplex "] edition = "2021" description = "Holaplex Hub Nfts migrations" readme = "./README.md" @@ -22,11 +20,8 @@ name = "migration" [dependencies] -tokio = { version = "1.32.0", features = ["macros"] } +tokio = { version = "1.32.0", features = ["macros", "rt-multi-thread"] } [dependencies.sea-orm-migration] version = "0.12.2" -features = [ - "runtime-tokio-rustls", - "sqlx-postgres", -] \ No newline at end of file +features = ["runtime-tokio-rustls", "sqlx-postgres"] diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 52ecf9e..a778326 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -53,15 +53,13 @@ mod m20230725_144506_drop_solana_collections_table; mod m20230807_090847_create_histories_table; mod m20230818_163948_downcase_polygon_addresses; mod m20230821_131630_create_switch_collection_histories_table; -<<<<<<< HEAD mod m20230905_100852_add_type_to_drop; mod m20230910_204731_add_queued_variant_to_mints_status; mod m20230910_212742_make_owner_address_optional_for_mint; mod m20230911_144938_make_compressed_column_optional; -mod m20230915_111128_create_mints_creation_status_idx; -======= mod m20230914_154759_add_job_trackings_table; ->>>>>>> 502f5ad (feat: create worker queue using redis for uploading metadata json in the background) +mod m20230915_111128_create_mints_creation_status_idx; +mod m20230922_150621_nullable_metadata_jsons_identifier_and_uri; pub struct Migrator; @@ -122,15 +120,13 @@ impl MigratorTrait for Migrator { Box::new(m20230807_090847_create_histories_table::Migration), Box::new(m20230818_163948_downcase_polygon_addresses::Migration), Box::new(m20230821_131630_create_switch_collection_histories_table::Migration), -<<<<<<< HEAD Box::new(m20230905_100852_add_type_to_drop::Migration), Box::new(m20230910_204731_add_queued_variant_to_mints_status::Migration), Box::new(m20230910_212742_make_owner_address_optional_for_mint::Migration), Box::new(m20230911_144938_make_compressed_column_optional::Migration), Box::new(m20230915_111128_create_mints_creation_status_idx::Migration), -======= Box::new(m20230914_154759_add_job_trackings_table::Migration), ->>>>>>> 502f5ad (feat: create worker queue using redis for uploading metadata json in the background) + Box::new(m20230922_150621_nullable_metadata_jsons_identifier_and_uri::Migration), ] } } diff --git a/migration/src/m20230922_150621_nullable_metadata_jsons_identifier_and_uri.rs b/migration/src/m20230922_150621_nullable_metadata_jsons_identifier_and_uri.rs new file mode 100644 index 0000000..58669d4 --- /dev/null +++ b/migration/src/m20230922_150621_nullable_metadata_jsons_identifier_and_uri.rs @@ -0,0 +1,39 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(MetadataJsons::Table) + .modify_column(ColumnDef::new(MetadataJsons::Uri).null()) + .modify_column(ColumnDef::new(MetadataJsons::Identifier).null()) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(MetadataJsons::Table) + .modify_column(ColumnDef::new(MetadataJsons::Uri).not_null()) + .modify_column(ColumnDef::new(MetadataJsons::Identifier).not_null()) + .to_owned(), + ) + .await + } +} + +/// Learn more at https://docs.rs/sea-query#iden +#[derive(Iden)] +enum MetadataJsons { + Table, + Uri, + Identifier, +} From d5dce4826a4dc6b660ed71568758f923f52a9621 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Thu, 28 Sep 2023 15:35:01 +0200 Subject: [PATCH 3/3] feat: run each task in a dedicated green thread --- api/src/background_worker/job.rs | 31 +---- api/src/background_worker/job_queue.rs | 27 +++-- .../tasks/metadata_json_upload_task.rs | 17 +-- api/src/background_worker/tasks/mod.rs | 2 +- api/src/background_worker/worker.rs | 106 +++++++++++------- api/src/blockchains/polygon.rs | 2 +- api/src/blockchains/solana.rs | 2 +- api/src/entities/drops.rs | 5 +- api/src/entities/job_trackings.rs | 6 +- api/src/main.rs | 11 +- api/src/mutations/collection.rs | 19 ++-- api/src/mutations/drop.rs | 8 +- api/src/mutations/mint.rs | 81 ++++++------- api/src/nft_storage.rs | 24 +++- 14 files changed, 173 insertions(+), 168 deletions(-) diff --git a/api/src/background_worker/job.rs b/api/src/background_worker/job.rs index 32d6b21..4989384 100644 --- a/api/src/background_worker/job.rs +++ b/api/src/background_worker/job.rs @@ -1,23 +1,20 @@ -use serde::{ - de::{Deserialize, Deserializer, Error as DeError}, - Serialize, -}; +use serde::{Deserialize, Serialize}; use super::tasks::BackgroundTask; -#[derive(Serialize, Debug)] -pub struct Job> { - pub id: i64, +#[derive(Serialize, Deserialize, Debug)] +pub struct Job> { + pub id: i32, pub task: T, _context_marker: std::marker::PhantomData, } -impl Job +impl Job where T: Serialize + Send + Sync + BackgroundTask, { #[must_use] - pub fn new(id: i64, task: T) -> Self { + pub fn new(id: i32, task: T) -> Self { Self { id, task, @@ -25,19 +22,3 @@ where } } } - -impl<'de, C, T> Deserialize<'de> for Job -where - C: Clone, - T: Serialize + Send + Sync + BackgroundTask, - T: for<'a> Deserialize<'a>, -{ - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let (id, task) = Deserialize::deserialize(deserializer).map_err(DeError::custom)?; - - Ok(Job::new(id, task)) - } -} diff --git a/api/src/background_worker/job_queue.rs b/api/src/background_worker/job_queue.rs index 08bcd1a..bcac193 100644 --- a/api/src/background_worker/job_queue.rs +++ b/api/src/background_worker/job_queue.rs @@ -1,6 +1,6 @@ -use std::{error::Error as StdError, fmt, sync::Arc}; +use std::{error::Error as StdError, fmt}; -use hub_core::{prelude::*, thiserror, tokio::sync::Mutex}; +use hub_core::{prelude::*, thiserror}; use redis::{Client, RedisError}; use sea_orm::{error::DbErr, ActiveModelTrait}; use serde::{Deserialize, Serialize}; @@ -32,14 +32,15 @@ pub enum JobQueueError { #[error("Background task error: {0}")] BackgroundTask(#[from] Error), } -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct JobQueue { - client: Arc>, + client: Client, db_pool: Connection, } impl JobQueue { - pub fn new(client: Arc>, db_pool: Connection) -> Self { + #[must_use] + pub fn new(client: Client, db_pool: Connection) -> Self { Self { client, db_pool } } @@ -54,10 +55,9 @@ impl JobQueue { pub async fn enqueue(&self, task: T) -> Result<(), JobQueueError> where T: Serialize + for<'de> Deserialize<'de> + Send + Sync + BackgroundTask, - C: Clone, + C: Clone + std::fmt::Debug + Send + Sync, { - let client_guard = self.client.lock().await; - let mut conn = client_guard.get_async_connection().await?; + let mut conn = self.client.get_async_connection().await?; let db_conn = self.db_pool.get(); let payload = task.payload()?; @@ -88,20 +88,19 @@ impl JobQueue { pub async fn dequeue(&self) -> Result>, JobQueueError> where T: Serialize + for<'de> Deserialize<'de> + Send + Sync + BackgroundTask, - C: Clone, + C: Clone + std::fmt::Debug + Send + Sync, { - let client_guard = self.client.lock().await; - let mut conn = client_guard.get_async_connection().await?; + let mut conn = self.client.get_async_connection().await?; let db_conn = self.db_pool.get(); - let res: Option = redis::cmd("BRPOP") + let res: Option<(String, String)> = redis::cmd("BRPOP") .arg("job_queue") .arg(0) .query_async(&mut conn) .await?; - if let Some(job_data) = res { - let job: Job = serde_json::from_str(&job_data)?; + if let Some((_, job_details)) = res { + let job: Job = serde_json::from_str(&job_details)?; let job_tracking = job_trackings::Entity::find_by_id(job.id) .one(db_conn) diff --git a/api/src/background_worker/tasks/metadata_json_upload_task.rs b/api/src/background_worker/tasks/metadata_json_upload_task.rs index 836e7ec..e162949 100644 --- a/api/src/background_worker/tasks/metadata_json_upload_task.rs +++ b/api/src/background_worker/tasks/metadata_json_upload_task.rs @@ -1,4 +1,4 @@ -use hub_core::{anyhow::Result, producer::Producer}; +use hub_core::anyhow::Result; use sea_orm::{prelude::*, Set}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -16,8 +16,8 @@ use crate::{ objects::MetadataJsonInput, proto::{ CreateEditionTransaction, EditionInfo, MasterEdition, MetaplexMasterEditionTransaction, - MetaplexMetadata, MintMetaplexMetadataTransaction, NftEventKey, NftEvents, - UpdateEdtionTransaction, UpdateSolanaMintPayload, + MetaplexMetadata, MintMetaplexMetadataTransaction, NftEventKey, UpdateEdtionTransaction, + UpdateSolanaMintPayload, }, }; @@ -670,27 +670,20 @@ impl MetadataJsonUploadTask { } } -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct Context { nft_storage: NftStorageClient, solana: Solana, polygon: Polygon, - producer: Producer, } impl Context { #[must_use] - pub fn new( - nft_storage: NftStorageClient, - solana: Solana, - polygon: Polygon, - producer: Producer, - ) -> Self { + pub fn new(nft_storage: NftStorageClient, solana: Solana, polygon: Polygon) -> Self { Self { nft_storage, solana, polygon, - producer, } } } diff --git a/api/src/background_worker/tasks/mod.rs b/api/src/background_worker/tasks/mod.rs index 0715ffa..a113bfb 100644 --- a/api/src/background_worker/tasks/mod.rs +++ b/api/src/background_worker/tasks/mod.rs @@ -33,7 +33,7 @@ pub enum BackgroundTaskError { } #[async_trait::async_trait] -pub trait BackgroundTask: Send + Sync + std::fmt::Debug { +pub trait BackgroundTask: Send + Sync + std::fmt::Debug { /// Process the task /// # Arguments /// * `self` - The task diff --git a/api/src/background_worker/worker.rs b/api/src/background_worker/worker.rs index 8cf70aa..a021e74 100644 --- a/api/src/background_worker/worker.rs +++ b/api/src/background_worker/worker.rs @@ -18,7 +18,7 @@ pub enum WorkerError { #[error("Database error: {0}")] Database(#[from] DbErr), } -pub struct Worker> { +pub struct Worker> { job_queue: JobQueue, db_pool: Connection, context: C, @@ -27,8 +27,8 @@ pub struct Worker> { impl Worker where - T: Serialize + for<'de> Deserialize<'de> + Send + Sync + BackgroundTask, - C: Clone, + T: 'static + Serialize + for<'de> Deserialize<'de> + Send + Sync + BackgroundTask, + C: 'static + Clone + std::fmt::Debug + Send + Sync, { pub fn new(job_queue: JobQueue, db_pool: Connection, context: C) -> Self { Self { @@ -40,57 +40,83 @@ where } /// Start the worker + /// + /// This method starts the worker by continuously dequeuing jobs from the job queue and processing them. + /// Each job is processed in a separate asynchronous task. If a job is found, it is processed and its status is updated in the database. + /// If no job is found, the worker sleeps for a short duration before trying to dequeue the next job. + /// Errors during job processing or database operations are logged. + /// /// # Arguments - /// * `self` - The worker + /// * `self` - A reference to the worker instance. + /// /// # Returns - /// * `Result<(), WorkerError>` - The result of the operation + /// * `Result<(), WorkerError>` - This method returns a `Result` type. If the worker starts successfully, it returns `Ok(())`. + /// If an error occurs while dequeuing a job from the job queue, it returns `Err(WorkerError)`. + /// /// # Errors - /// * `WorkerError` - The error that occurred + /// * `WorkerError::JobQueue(JobQueueError)` - This error occurs when there is an issue dequeuing a job from the job queue. + /// * `WorkerError::Database(DbErr)` - This error occurs when there is a database operation error. pub async fn start(&self) -> Result<(), WorkerError> { + let db_pool = self.db_pool.clone(); + let context = self.context.clone(); + let job_queue = self.job_queue.clone(); + loop { // Dequeue the next job to process - let job_option = self.job_queue.dequeue::().await?; - let db_conn = self.db_pool.get(); + let job_option = job_queue.dequeue::().await?; - if let Some(job) = job_option { - // Process the job - let model = job_trackings::Entity::find_by_id(job.id) - .one(db_conn) - .await?; + tokio::spawn({ + let db_pool = db_pool.clone(); + let context = context.clone(); + async move { + let db_conn = db_pool.get(); + let db_pool_process = db_pool.clone(); - if let Some(model) = model { - match job - .task - .process(self.db_pool.clone(), self.context.clone()) - .await - { - Ok(_) => { - // If successful, update the status in the job_trackings table to "completed" - let job_tracking_am = - job_trackings::Entity::update_status(model, "completed"); + if let Some(job) = job_option { + // Process the job + let job_tracking_result = + job_trackings::Entity::find_by_id(job.id).one(db_conn).await; - job_tracking_am.update(db_conn).await?; + // Handle the error explicitly here + let model = match job_tracking_result { + Ok(model) => model, + Err(e) => { + error!("Error finding job tracking: {}", e); + return; + }, + }; - info!("Successfully processed job {}", job.id); - }, - Err(e) => { - // If an error occurs, update the status in the job_trackings table to "failed" - let job_tracking_am = - job_trackings::Entity::update_status(model, "failed"); + let Some(model) = model + else { + error!("Job tracking not found"); + return; + }; - job_tracking_am.update(db_conn).await?; + let result = job.task.process(db_pool_process, context).await; - // Log the error (or handle it in some other way) - error!("Error processing job {}: {}", job.id, e); - }, + match result { + Ok(_) => { + let job_tracking_am = + job_trackings::Entity::update_status(model, "completed"); + if let Err(e) = job_tracking_am.update(db_conn).await { + error!("Error updating job tracking: {}", e); + } + info!("Successfully processed job {}", job.id); + }, + Err(e) => { + let job_tracking_am = + job_trackings::Entity::update_status(model, "failed"); + if let Err(e) = job_tracking_am.update(db_conn).await { + error!("Error updating job tracking: {}", e); + } + error!("Error processing job {}: {}", job.id, e); + }, + } + } else { + tokio::time::sleep(std::time::Duration::from_millis(250)).await; } - } else { - error!("Job tracking record not found for job {}", job.id); } - } else { - // If no job was dequeued, you might want to add a delay here to avoid busy-waiting - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - } + }); } } } diff --git a/api/src/blockchains/polygon.rs b/api/src/blockchains/polygon.rs index 8696a33..151fb1c 100644 --- a/api/src/blockchains/polygon.rs +++ b/api/src/blockchains/polygon.rs @@ -13,7 +13,7 @@ use crate::{ }, }; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct Polygon { producer: Producer, } diff --git a/api/src/blockchains/solana.rs b/api/src/blockchains/solana.rs index 838775a..15a82a1 100644 --- a/api/src/blockchains/solana.rs +++ b/api/src/blockchains/solana.rs @@ -19,7 +19,7 @@ use crate::{ }, }; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct Solana { producer: Producer, } diff --git a/api/src/entities/drops.rs b/api/src/entities/drops.rs index 0310732..ff0e7af 100644 --- a/api/src/entities/drops.rs +++ b/api/src/entities/drops.rs @@ -47,6 +47,9 @@ impl Entity { pub fn find_by_id_with_collection( id: Uuid, ) -> sea_orm::SelectTwo { - Self::find_by_id(id).select_also(super::collections::Entity) + Self::find() + .filter(Column::Id.eq(id)) + .inner_join(super::collections::Entity) + .select_also(super::collections::Entity) } } diff --git a/api/src/entities/job_trackings.rs b/api/src/entities/job_trackings.rs index a087037..ce3bea0 100644 --- a/api/src/entities/job_trackings.rs +++ b/api/src/entities/job_trackings.rs @@ -6,7 +6,7 @@ use serde_json::Value as Json; #[sea_orm(table_name = "job_trackings")] pub struct Model { #[sea_orm(primary_key)] - pub id: i64, + pub id: i32, pub job_type: String, pub payload: Json, pub status: String, @@ -21,7 +21,7 @@ impl ActiveModelBehavior for ActiveModel {} impl Entity { // Find a job tracking record by its ID - pub fn find_by_id(id: i64) -> Select { + pub fn find_by_id(id: i32) -> Select { Self::find().filter(Column::Id.eq(id)) } @@ -41,9 +41,11 @@ impl Entity { // Update the status of an existing job tracking record pub fn update_status(model: Model, new_status: &str) -> ActiveModel { + let now: DateTimeWithTimeZone = chrono::Utc::now().into(); let mut active_model: ActiveModel = model.into(); active_model.status = Set(new_status.to_string()); + active_model.updated_at = Set(now); active_model } diff --git a/api/src/main.rs b/api/src/main.rs index 7f67077..9cdb0b9 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -1,7 +1,5 @@ //! -use std::sync::Arc; - use holaplex_hub_nfts::{ background_worker::{ job_queue::JobQueue, @@ -62,14 +60,9 @@ pub fn main() { let polygon = Polygon::new(producer.clone()); let redis_client = RedisClient::open(redis_url)?; - let redis_client = Arc::new(tokio::sync::Mutex::new(redis_client)); - let metadata_json_upload_task_context = MetadataJsonUploadContext::new( - nft_storage, - solana.clone(), - polygon.clone(), - producer.clone(), - ); + let metadata_json_upload_task_context = + MetadataJsonUploadContext::new(nft_storage, solana.clone(), polygon.clone()); let job_queue = JobQueue::new(redis_client, connection.clone()); let worker = Worker::::new( diff --git a/api/src/mutations/collection.rs b/api/src/mutations/collection.rs index ab23c2f..b9e8e68 100644 --- a/api/src/mutations/collection.rs +++ b/api/src/mutations/collection.rs @@ -272,17 +272,18 @@ impl Mutation { .one(conn) .await?; - let txn = conn.begin().await?; - if let Some(collection) = collection.clone() { + let txn = conn.begin().await?; + let mints = CollectionMints::find() .filter(collection_mints::Column::CollectionId.eq(collection.id)) .all(&txn) .await?; - if let Some(collection_json) = - MetadataJsons::find_by_id(collection.id).one(&txn).await? - { + let collection_json_response = + MetadataJsons::find_by_id(collection.id).one(&txn).await?; + + if let Some(collection_json) = collection_json_response { collection_json.delete(&txn).await?; } @@ -395,9 +396,11 @@ impl Mutation { .exec(&tx) .await?; - collection_creators::Entity::insert_many(creator_ams) - .exec(&tx) - .await?; + if !creator_ams.is_empty() { + collection_creators::Entity::insert_many(creator_ams) + .exec(&tx) + .await?; + } creators .into_iter() diff --git a/api/src/mutations/drop.rs b/api/src/mutations/drop.rs index c8eb56f..b782290 100644 --- a/api/src/mutations/drop.rs +++ b/api/src/mutations/drop.rs @@ -503,9 +503,11 @@ impl Mutation { .exec(&tx) .await?; - collection_creators::Entity::insert_many(creator_ams) - .exec(&tx) - .await?; + if !creator_ams.is_empty() { + collection_creators::Entity::insert_many(creator_ams) + .exec(&tx) + .await?; + } creators .into_iter() diff --git a/api/src/mutations/mint.rs b/api/src/mutations/mint.rs index 6915bf0..733373b 100644 --- a/api/src/mutations/mint.rs +++ b/api/src/mutations/mint.rs @@ -33,7 +33,6 @@ use crate::{ collection_creators, collection_mints, collections, drops, metadata_json_attributes, metadata_json_files, metadata_jsons, mint_creators, mint_histories, prelude::{CollectionCreators, CollectionMints, Collections}, - project_wallets, sea_orm_active_enums::{Blockchain as BlockchainEnum, CreationStatus}, update_histories, }, @@ -142,14 +141,13 @@ impl Mutation { mint_history_am.insert(&tx).await?; if collection.blockchain == BlockchainEnum::Solana { - let collection_metadata_json = - metadata_jsons::Entity::find_by_id(collection_mint_model.id) - .one(conn) - .await? - .ok_or(Error::new("metadata json not found"))?; - - let creators = mint_creators::Entity::find() - .filter(mint_creators::Column::CollectionMintId.eq(collection_mint_model.id)) + let collection_metadata_json = metadata_jsons::Entity::find_by_id(collection.id) + .one(conn) + .await? + .ok_or(Error::new("metadata json not found"))?; + + let creators = collection_creators::Entity::find() + .filter(collection_creators::Column::CollectionId.eq(collection.id)) .all(conn) .await?; @@ -171,7 +169,6 @@ impl Mutation { let collection_mint_metadata_json = metadata_json_am.insert(&tx).await?; - // TODO: Look into bug regarding creators not being saved to the database when minting editions let creators: Vec = creators .into_iter() .map(|creator| mint_creators::ActiveModel { @@ -182,9 +179,11 @@ impl Mutation { }) .collect(); - mint_creators::Entity::insert_many(creators) - .exec(&tx) - .await?; + if !creators.is_empty() { + mint_creators::Entity::insert_many(creators) + .exec(&tx) + .await?; + } let files: Vec = files .into_iter() @@ -196,9 +195,11 @@ impl Mutation { }) .collect(); - metadata_json_files::Entity::insert_many(files) - .exec(&tx) - .await?; + if !files.is_empty() { + metadata_json_files::Entity::insert_many(files) + .exec(&tx) + .await?; + } let attributes: Vec = attributes .into_iter() @@ -210,9 +211,11 @@ impl Mutation { }) .collect(); - metadata_json_attributes::Entity::insert_many(attributes) - .exec(&tx) - .await?; + if !attributes.is_empty() { + metadata_json_attributes::Entity::insert_many(attributes) + .exec(&tx) + .await?; + } } tx.commit().await?; @@ -335,22 +338,7 @@ impl Mutation { let edition = collection_mint_model.edition; let project_id = drop_model.project_id; - // Fetch the project wallet address which will sign the transaction by hub-treasuries - let wallet = project_wallets::Entity::find() - .filter( - project_wallets::Column::ProjectId - .eq(project_id) - .and(project_wallets::Column::Blockchain.eq(collection.blockchain)), - ) - .one(conn) - .await?; - - let owner_address = wallet - .ok_or(Error::new(format!( - "no project wallet found for {:?} blockchain", - collection.blockchain - )))? - .wallet_address; + let owner_address = fetch_owner(conn, project_id, collection.blockchain).await?; let TransactionId(_) = credits .submit_pending_deduction( @@ -437,12 +425,11 @@ impl Mutation { let creators = input.creators; - let collection = Collections::find() - .filter(collections::Column::Id.eq(input.collection)) + let collection = Collections::find_by_id(input.collection) .one(conn) - .await?; + .await? + .ok_or(Error::new("collection not found"))?; - let collection = collection.ok_or(Error::new("collection not found"))?; let blockchain = collection.blockchain; let compressed = input.compressed.unwrap_or_default(); @@ -659,9 +646,11 @@ impl Mutation { .exec(&tx) .await?; - mint_creators::Entity::insert_many(creators_am) - .exec(&tx) - .await?; + if !creators_am.is_empty() { + mint_creators::Entity::insert_many(creators_am) + .exec(&tx) + .await?; + } if let Some(sfbp) = input.seller_fee_basis_points { mint_am.seller_fee_basis_points = Set(sfbp.try_into().unwrap_or_default()); @@ -928,9 +917,11 @@ impl Mutation { }) .collect(); - mint_creators::Entity::insert_many(mint_creators) - .exec(&tx) - .await?; + if !mint_creators.is_empty() { + mint_creators::Entity::insert_many(mint_creators) + .exec(&tx) + .await?; + } tx.commit().await?; diff --git a/api/src/nft_storage.rs b/api/src/nft_storage.rs index c534050..ab115fd 100644 --- a/api/src/nft_storage.rs +++ b/api/src/nft_storage.rs @@ -1,6 +1,11 @@ use std::collections::HashMap; -use hub_core::{anyhow::Result, clap, prelude::*}; +use hub_core::{ + anyhow::Result, + backon::{ExponentialBuilder, Retryable}, + clap, + prelude::*, +}; use reqwest::Response; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -69,11 +74,18 @@ impl NftStorageClient { /// # Errors /// If the upload fails pub async fn upload(&self, data: &impl Serialize) -> Result { - self.post("/upload".to_string(), data) - .await? - .json() - .await - .context("failed to parse response") + let post = || self.post("/upload".to_string(), data); + + post.retry( + &ExponentialBuilder::default() + .with_jitter() + .with_min_delay(Duration::from_millis(30)) + .with_max_times(15), + ) + .await? + .json() + .await + .context("failed to parse response") } }