From 5a43490630d1f629c8c3008cc73253bd6a02f0b4 Mon Sep 17 00:00:00 2001 From: raykast Date: Wed, 27 Sep 2023 22:10:14 -0700 Subject: [PATCH 1/8] Added non-idempotent handler for upload drop cmd. --- Cargo.toml | 14 +- src/cache.rs | 44 +- src/cli.rs | 15 +- src/commands/upload_drop.rs | 621 +++++++++++++++++++++++-- src/common/concurrent.rs | 39 ++ src/common/metadata_json.rs | 86 ++++ src/common/toposort.rs | 128 +++++ src/common/url_permissive.rs | 85 ++++ src/config.rs | 17 +- src/main.rs | 8 + src/queries/queue-mint-to-drop.graphql | 3 +- 11 files changed, 1005 insertions(+), 55 deletions(-) create mode 100644 src/common/concurrent.rs create mode 100644 src/common/metadata_json.rs create mode 100644 src/common/toposort.rs create mode 100644 src/common/url_permissive.rs diff --git a/Cargo.toml b/Cargo.toml index 79b38b1..0f346fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,16 +12,24 @@ path = "src/main.rs" [dependencies] anyhow = "1.0.75" clap = { version = "4.4.4", features = ["cargo", "derive", "env", "wrap_help"] } +crossbeam = "0.8.2" directories = "5.0.1" -graphql_client = { version = "0.13.0", features = ["reqwest"] } +dispose = "0.5.0" +env_logger = "0.10.0" +futures-util = "0.3.28" +graphql_client = "0.13.0" +infer = "0.15.0" itertools = "0.11.0" log = "0.4.20" -reqwest = { version = "0.11.20", features = ["brotli", "deflate", "gzip", "json", "multipart"] } +num_cpus = "1.16.0" +parking_lot = "0.12.1" +reqwest = { version = "0.11.20", features = ["brotli", "deflate", "gzip", "json", "multipart", "stream"] } ron = "0.8.1" rpassword = "7.2.0" rustyline = { version = "12.0.0", default-features = false } serde = { version = "1.0.188", features = ["derive"] } -tokio = { version = "1.32.0", features = ["rt-multi-thread"] } +serde_json = { version = "1.0.107", features = ["float_roundtrip", "preserve_order"] } +tokio = { version = "1.32.0", features = ["parking_lot", "rt-multi-thread"] } toml = { version = "0.8.0", features = ["preserve_order"] } url = { version = "2.4.1", features = ["serde"] } uuid = { version = "1.4.1", features = ["serde"] } diff --git a/src/cache.rs b/src/cache.rs index a7ed383..aaab510 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -1,4 +1,7 @@ +#![allow(dead_code)] // TODO: replace with RocksDB? + use std::{ + collections::HashMap, fs::{self, File}, io::ErrorKind, path::{Path, PathBuf}, @@ -6,6 +9,7 @@ use std::{ use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; +use url::Url; const SWAPFILE_NAME: &str = ".hub.cache~"; const FILE_NAME: &str = ".hub.cache"; @@ -62,15 +66,37 @@ impl CacheItem { } } -#[derive(Debug, Default, Serialize, Deserialize)] pub struct Cache { - asset_uploads: CacheItem, + path: PathBuf, + contents: CacheContents, } impl Cache { pub fn load(path: impl AsRef) -> Result { let path = cache_file(&path); - let file = match File::open(&path) { + let contents = CacheContents::load(&path)?; + Ok(Self { path, contents }) + } + + #[inline] + pub fn json_uploads_mut(&mut self) -> &mut JsonUploads { self.contents.json_uploads.as_mut() } + + #[inline] + pub fn asset_uploads_mut(&mut self) -> &mut AssetUploads { + self.contents.asset_uploads.as_mut() + } +} + +#[derive(Debug, Default, Serialize, Deserialize)] +struct CacheContents { + json_uploads: CacheItem, + asset_uploads: CacheItem, +} + +impl CacheContents { + fn load(path: impl AsRef) -> Result { + let path = path.as_ref(); + let file = match File::open(path) { Ok(f) => f, Err(e) if e.kind() == ErrorKind::NotFound => return Ok(Self::default()), Err(e) => return Err(e).with_context(|| format!("Error opening cache file {path:?}")), @@ -79,7 +105,7 @@ impl Cache { ron::de::from_reader(file).with_context(|| format!("Error parsing cache file {path:?}")) } - pub fn save(&self, path: impl AsRef) -> Result<()> { + fn save(&self, path: impl AsRef) -> Result<()> { let swap = swap_file(&path); let path = cache_file(path); let file = File::create(&swap) @@ -90,10 +116,14 @@ impl Cache { fs::rename(swap, &path).with_context(|| format!("Error writing to cache file {path:?}")) } +} - #[inline] - pub fn asset_uploads_mut(&mut self) -> &mut AssetUploads { self.asset_uploads.as_mut() } +#[derive(Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct JsonUploads { + uploaded: HashMap, } #[derive(Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct AssetUploads {} +pub struct AssetUploads { + uploaded: HashMap, +} diff --git a/src/cli.rs b/src/cli.rs index 5c77a82..20fb1a9 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -100,7 +100,16 @@ pub struct UploadDrop { #[arg(short = 'd', long = "drop")] pub drop_id: Uuid, - /// Path to the structured asset directory - #[arg(short = 'p', long = "assets")] - pub asset_dir: PathBuf, + /// Specify a search path for assets + #[arg(short = 'I', long = "include")] + pub include_dirs: Vec, + + /// Limit the number of concurrently-running jobs + #[arg(short = 'j', long = "jobs", default_value_t = 4)] + pub jobs: u16, + + /// Path to a directory containing metadata JSON files to upload + #[arg(required = true)] + pub input_dirs: Vec, + } diff --git a/src/commands/upload_drop.rs b/src/commands/upload_drop.rs index 6075c93..ea82978 100644 --- a/src/commands/upload_drop.rs +++ b/src/commands/upload_drop.rs @@ -1,53 +1,618 @@ -use anyhow::Result; +use std::{ + borrow::Cow, + collections::{HashMap, HashSet}, + fmt::Write, + fs::{self, File}, + io::{self, prelude::*}, + iter, + num::NonZeroUsize, + path::PathBuf, + sync::{Arc, OnceLock}, +}; + +use anyhow::{Context as _, Result}; +use crossbeam::{ + channel::{self, Sender}, + queue::ArrayQueue, +}; +use futures_util::FutureExt; use graphql_client::GraphQLQuery; +use itertools::Either; +use log::{error, info, trace, warn}; +use reqwest::multipart; +use serde::{Deserialize, Serialize}; +use tokio::task::JoinHandle; +use url::Url; +use uuid::Uuid; + +use crate::{ + cli::UploadDrop, + common::{ + concurrent, + metadata_json::{self, MetadataJson}, + toposort::{Dependencies, Dependency, PendingFail}, + url_permissive::PermissiveUrl, + }, + config::Config, + runtime, +}; + +type UploadResponse = Vec; -use crate::{cache::Cache, cli::UploadDrop, config::Config, runtime}; +#[derive(Debug, Serialize, Deserialize)] +struct UploadedAsset { + name: String, + url: Url, +} #[allow(clippy::upper_case_acronyms)] -type UUID = uuid::Uuid; +type UUID = Uuid; #[derive(GraphQLQuery)] #[graphql( schema_path = "src/queries/schema.graphql", query_path = "src/queries/queue-mint-to-drop.graphql", + variables_derives = "Debug", response_derives = "Debug" )] struct QueueMintToDrop; +impl From for queue_mint_to_drop::MetadataJsonInput { + fn from(value: MetadataJson) -> Self { + let MetadataJson { + name, + symbol, + description, + image, + animation_url, + collection, + attributes, + external_url, + properties, + } = value; + Self { + name, + symbol, + description, + image: image.into(), + animation_url: animation_url.map(Into::into), + collection: collection.into(), + attributes: attributes.into_iter().map(Into::into).collect(), + external_url: external_url.map(Into::into), + properties: properties.into(), + } + } +} + +impl From for Option { + #[inline] + fn from(value: metadata_json::Collection) -> Self { + if value.is_empty() { + None + } else { + let metadata_json::Collection { name, family } = value; + Some(queue_mint_to_drop::MetadataJsonCollectionInput { name, family }) + } + } +} + +impl From for queue_mint_to_drop::MetadataJsonAttributeInput { + #[inline] + fn from(value: metadata_json::Attribute) -> Self { + let metadata_json::Attribute { trait_type, value } = value; + Self { trait_type, value } + } +} + +impl From for Option { + #[inline] + fn from(value: metadata_json::Properties) -> Self { + if value.is_empty() { + None + } else { + let metadata_json::Properties { files, category } = value; + Some(queue_mint_to_drop::MetadataJsonPropertyInput { + files: if files.is_empty() { + None + } else { + Some(files.into_iter().map(Into::into).collect()) + }, + category, + }) + } + } +} + +impl From for queue_mint_to_drop::MetadataJsonFileInput { + fn from(value: metadata_json::File) -> Self { + let metadata_json::File { uri, ty } = value; + Self { + uri: Some(uri.into()), + file_type: ty, + } + } +} + pub fn run(config: &Config, args: UploadDrop) -> Result<()> { - let UploadDrop { drop_id, asset_dir } = args; - let mut cache = Cache::load(&asset_dir)?; + let UploadDrop { + drop_id, + include_dirs, + jobs, + input_dirs, + } = args; + let include_dirs: HashSet<_> = include_dirs.into_iter().collect(); + + let (tx, rx) = channel::unbounded(); + for path in input_dirs + .iter() + .flat_map(|d| match fs::read_dir(d) { + Ok(r) => { + trace!("Traversing directory {r:?}"); - let uploads = cache.asset_uploads_mut(); + Either::Left(r.map(move |f| { + let f = f.with_context(|| format!("Error reading JSON directory {d:?}"))?; + let path = f.path(); + + Ok(if path.extension().map_or(false, |p| p == "json") { + Some(path) + } else { + None + }) + })) + }, + Err(e) => Either::Right( + [Err(e).context(format!("Error opening JSON directory {d:?}"))].into_iter(), + ), + }) + .filter_map(Result::transpose) + { + tx.send(Job::ScanJson(ScanJsonJob { path })) + .context("Error seeding initial job queue")?; + } + + info!("Processing {} JSON files(s)...", rx.len()); + + let ctx = Context { + include_dirs: include_dirs + .into_iter() + .collect::>() + .into_boxed_slice() + .into(), + drop_id, + graphql_endpoint: config.graphql_endpoint().clone(), + upload_endpoint: config.upload_endpoint()?, + client: config.graphql_client()?, + q: tx, + }; runtime()?.block_on(async move { - let client = config.graphql_client()?; + let res = concurrent::try_run( + jobs.into(), + |e| error!("{e:?}"), + || { + let job = match rx.try_recv() { + Ok(j) => Some(j), + Err(channel::TryRecvError::Empty) => None, + Err(e) => return Err(e).context("Error getting job from queue"), + }; + + let Some(job) = job else { + return Ok(None); + }; + + trace!("Submitting job: {job:?}"); + + Ok(Some(job.run(ctx.clone()).map(|f| { + f.context("Worker task panicked").and_then(|r| r) + }))) + }, + ) + .await; + + debug_assert!(rx.is_empty(), "Trailing jobs in queue"); + + res + })?; + + Ok(()) +} + +#[derive(Clone)] +struct Context { + include_dirs: Arc<[PathBuf]>, + drop_id: Uuid, + graphql_endpoint: Url, + upload_endpoint: Url, + client: reqwest::Client, + q: Sender, +} + +impl Context { + fn resolve_file io::Result, T>( + &self, + mut open: F, + ) -> Result, (&PathBuf, io::Error)> { + static NIL: OnceLock = OnceLock::new(); + + [NIL.get_or_init(PathBuf::new)] + .into_iter() + .chain(&*self.include_dirs) + .find_map(|d| { + let opened = match open(d) { + Ok(o) => o, + Err(e) if e.kind() == io::ErrorKind::NotFound => return None, + Err(e) => return Some(Err((d, e))), + }; + + Some(Ok((d, opened))) + }) + .transpose() + } +} + +#[derive(Debug)] +struct ScanJsonJob { + path: Result, +} + +impl ScanJsonJob { + fn run(self, ctx: Context) -> JoinHandle> { + tokio::task::spawn_blocking(move || { + let Self { path } = self; + let path = path?; + let json_file = File::open(&path).with_context(|| format!("Error opening {path:?}"))?; + let json: MetadataJson = serde_json::from_reader(json_file) + .with_context(|| format!("Error parsing {path:?}"))?; + + let mut seen = HashSet::new(); + let local_urls = json + .files() + .filter_map(|u| { + if !seen.insert(u) { + return None; + } + + let url = u.clone(); + trace!("{url:?} -> {:?}", url.to_file_path()); + let path = url.to_file_path()?; + let (include_dir, file) = match ctx + .resolve_file(|d| File::open(d.join(&path))) + .map_err(|(d, e)| { + anyhow::Error::new(e) + .context(format!("Error opening {:?}", d.join(&path))) + }) + .and_then(|f| f.with_context(|| format!("Unable to resolve path {path:?}"))) + { + Ok(f) => f, + Err(e) => return Some(Err(e)), + }; + let path = include_dir.join(path); + + let ty = if let Some(ty) = + json.properties.find_file(&url).and_then(|f| f.ty.clone()) + { + ty.into() + } else { + static INFER: infer::Infer = infer::Infer::new(); + + let limit = file + .metadata() + .map(|m| { + usize::try_from(std::cmp::min(m.len(), 8192)).unwrap_or_default() + + 1 + }) + .unwrap_or(0); + let mut bytes = Vec::with_capacity(limit); + + match file + .take(8192) + .read_to_end(&mut bytes) + .with_context(|| format!("Error reading signature of {path:?}")) + .and_then(|_| { + INFER + .get(&bytes) + .with_context(|| format!("Cannot infer MIME type for {path:?}")) + }) { + Ok(t) => t.mime_type().into(), + Err(e) => return Some(Err(e)), + } + }; + + Some(Ok((url, path, ty))) + }) + .collect::>(); - let res = config - .post_graphql::(&client, queue_mint_to_drop::Variables { + if let Some(dep_count) = NonZeroUsize::new(local_urls.len()) { + let rewrites = Arc::new(ArrayQueue::new(dep_count.get())); + let deps = Dependencies::new(dep_count, QueueJsonJob { + path, + json, + rewrites: Some(Arc::clone(&rewrites)), + }); + + debug_assert!(rewrites.capacity() == deps.len()); + for (res, dep) in local_urls.into_iter().zip(deps) { + ctx.q + .send(Job::UploadAsset(UploadAssetJob { + asset: res.map(|(source_url, path, mime_type)| Asset { + source_url, + path, + mime_type, + }), + rewrites: Arc::clone(&rewrites), + dep, + })) + .context("Error submitting asset upload job")?; + } + } else { + ctx.q + .send(Job::QueueJson(QueueJsonJob { + path, + json, + rewrites: None, + })) + .context("Error submitting JSON queue job")?; + } + + Ok(()) + }) + } +} + +struct FileRewrite { + source_url: PermissiveUrl, + dest_url: Url, + mime_type: Cow<'static, str>, +} + +type FileRewrites = Arc>; + +#[derive(Debug)] +struct Asset { + source_url: PermissiveUrl, + path: PathBuf, + mime_type: Cow<'static, str>, +} + +#[derive(Debug)] +struct UploadAssetJob { + asset: Result, + rewrites: FileRewrites, + dep: Dependency, +} + +impl UploadAssetJob { + fn run(self, ctx: Context) -> JoinHandle> { + tokio::spawn(async move { + let Self { + asset, + rewrites, + dep, + } = self; + let Asset { + path, + mime_type, + source_url, + } = asset?; + + let file = tokio::fs::File::open(&path) + .await + .with_context(|| format!("Error opening {path:?}"))?; + let name = path + .file_name() + .with_context(|| format!("Error resolving file name for {path:?}"))? + .to_string_lossy() + .into_owned(); + + let mut uploads = ctx + .client + .post(ctx.upload_endpoint) + .multipart( + multipart::Form::new().part( + "FIXME", // TODO + multipart::Part::stream(file) + .file_name(name.clone()) + .mime_str(&mime_type) + .with_context(|| { + format!("Invalid MIME type {:?} for {path:?}", mime_type.as_ref()) + })?, + ), + ) + .send() + .await + .with_context(|| format!("Error sending POST request for {path:?}"))? + .error_for_status() + .with_context(|| format!("POST request for {path:?} returned an error"))? + .json::() + .await + .with_context(|| format!("Error deserializing upload response JSON for {path:?}"))? + .into_iter(); + + if uploads.len() > 1 { + warn!("Trailing values in response data for {path:?}"); + } + + let upload = uploads + .find(|u| u.name == name) + .with_context(|| format!("Missing upload response data for {path:?}"))?; + + rewrites + .push(FileRewrite { + source_url, + dest_url: upload.url, + mime_type, + }) + .unwrap_or_else(|_: FileRewrite| { + unreachable!("Too many file rewrites for {path:?}") + }); + + info!("Successfully uploaded {path:?}"); + + dep.ok(|j| ctx.q.send(Job::QueueJson(j))) + .transpose() + .context("Error submitting JSON queue job")?; + + Ok(()) + }) + } +} + +#[derive(Debug)] +struct QueueJsonJob { + path: PathBuf, + json: MetadataJson, + rewrites: Option, +} + +impl QueueJsonJob { + fn format_errors(errors: Option>, f: impl FnOnce(String)) -> bool { + let mut errs = errors.into_iter().flatten().peekable(); + + if errs.peek().is_some() { + let mut s = String::new(); + + for err in errs { + write!(s, "\n {err}").unwrap(); + } + + f(s); + true + } else { + false + } + } + + fn run(self, ctx: Context) -> JoinHandle> { + tokio::spawn(async move { + let Self { + path, + mut json, + rewrites, + } = self; + + let rewrites: HashMap<_, _> = rewrites + .into_iter() + .flat_map(|r| iter::from_fn(move || r.pop())) + .map( + |FileRewrite { + source_url, + dest_url, + mime_type, + }| (source_url, (dest_url, mime_type)), + ) + .collect(); + + for file in json.files_mut() { + if let Some((url, _)) = rewrites.get(file) { + *file = PermissiveUrl::Url(url.clone()); + } + } + + let seen_files: HashMap<_, _> = json + .properties + .files + .iter() + .enumerate() + .map(|(i, f)| (f.uri.clone(), i)) + .collect(); + + for (uri, ty) in rewrites.into_values() { + let uri = PermissiveUrl::Url(uri); + if let Some(idx) = seen_files.get(&uri) { + json.properties.files[*idx].ty = Some(ty.into_owned()); + } else { + json.properties.files.push(metadata_json::File { + uri, + ty: Some(ty.into_owned()), + }); + } + } + + let input = queue_mint_to_drop::Variables { in_: queue_mint_to_drop::QueueMintToDropInput { - drop: drop_id, - metadata_json: queue_mint_to_drop::MetadataJsonInput { - name: "hi".into(), - symbol: "HI".into(), - description: "help".into(), - image: "data:text/plain;help".into(), - animation_url: None, - collection: None, - attributes: vec![], - external_url: None, - properties: None, - }, + drop: ctx.drop_id, + metadata_json: json.into(), }, - }) - .await; + }; - println!("{res:?}"); + trace!( + "GraphQL input for {path:?}: {}", + serde_json::to_string(&input).map_or_else(|e| e.to_string(), |j| j.to_string()) + ); - Result::<_>::Ok(()) - })?; + let res = ctx + .client + .post(ctx.graphql_endpoint) + .json(&QueueMintToDrop::build_query(input)) + .send() + .await + .with_context(|| format!("Error sending queueMintToDrop mutation for {path:?}"))? + .error_for_status() + .with_context(|| { + format!("queueMintToDrop mutation for {path:?} returned an error") + })? + .json::::ResponseData>>() + .await + .with_context(|| { + format!("Error parsing queueMintToDrop mutation response for {path:?}") + })?; + + trace!("GraphQL response for {path:?}: {res:?}"); + + if let Some(data) = res.data { + Self::format_errors(res.errors, |s| { + warn!("queueMintToDrop mutation for {path:?} returned one or more errors:{s}"); + }); + + let queue_mint_to_drop::ResponseData { + queue_mint_to_drop: + queue_mint_to_drop::QueueMintToDropQueueMintToDrop { + collection_mint: + queue_mint_to_drop::QueueMintToDropQueueMintToDropCollectionMint { + id, + collection, + }, + }, + } = data; + + info!("Mint successfully queued for {path:?}"); + } else { + let had_errs = Self::format_errors(res.errors, |s| { + error!("queueMintToDrop mutation for {path:?} returned one or more errors:{s}"); + }); + + if !had_errs { + error!("queueMintToDrop mutation for {path:?} returned no data"); + } + } - println!("{:?}", config.upload_endpoint()); + Ok(()) + }) + } +} + +impl PendingFail for QueueJsonJob { + fn failed(self) { + warn!("Skipping {:?} due to failed dependencies", self.path); + } +} + +// The cost of shuffling these around is probably less than the cost of allocation +#[allow(clippy::large_enum_variant)] +#[derive(Debug)] +enum Job { + ScanJson(ScanJsonJob), + UploadAsset(UploadAssetJob), + QueueJson(QueueJsonJob), +} - cache.save(asset_dir) +impl Job { + #[inline] + fn run(self, ctx: Context) -> JoinHandle> { + match self { + Job::ScanJson(j) => j.run(ctx), + Job::UploadAsset(j) => j.run(ctx), + Job::QueueJson(j) => j.run(ctx), + } + } } diff --git a/src/common/concurrent.rs b/src/common/concurrent.rs new file mode 100644 index 0000000..c557a7c --- /dev/null +++ b/src/common/concurrent.rs @@ -0,0 +1,39 @@ +use std::future::Future; + +use futures_util::{stream::FuturesUnordered, StreamExt}; + +pub async fn try_run< + F: Future>, + E, + G: FnMut() -> Result, E>, + H: FnMut(E), +>( + jobs: usize, + mut err: H, + mut get_job: G, +) -> Result<(), E> { + let mut futures = FuturesUnordered::new(); + + 'run: loop { + 'pull: while futures.len() < jobs { + let Some(job) = get_job()? else { + let false = futures.is_empty() else { + break 'run; + }; + + break 'pull; + }; + + futures.push(job); + } + + match futures.next().await { + None | Some(Ok(())) => (), + Some(Err(e)) => err(e), + } + } + + debug_assert!(futures.is_empty(), "Not all futures were yielded"); + + Ok(()) +} diff --git a/src/common/metadata_json.rs b/src/common/metadata_json.rs new file mode 100644 index 0000000..0ba3d2f --- /dev/null +++ b/src/common/metadata_json.rs @@ -0,0 +1,86 @@ +use serde::{Deserialize, Serialize}; + +use super::url_permissive::PermissiveUrl; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MetadataJson { + pub name: String, + pub symbol: String, + pub description: String, + pub image: PermissiveUrl, + #[serde(skip_serializing_if = "Option::is_none")] + pub animation_url: Option, + #[serde(default, skip_serializing_if = "Collection::is_empty")] + pub collection: Collection, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub attributes: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub external_url: Option, + #[serde(default, skip_serializing_if = "Properties::is_empty")] + pub properties: Properties, +} + +impl MetadataJson { + pub fn files(&self) -> impl Iterator { + [&self.image] + .into_iter() + .chain(self.animation_url.iter()) + .chain(self.external_url.iter()) + .chain(self.properties.files.iter().map(|f| &f.uri)) + } + + pub fn files_mut(&mut self) -> impl Iterator { + [&mut self.image] + .into_iter() + .chain(self.animation_url.iter_mut()) + .chain(self.external_url.iter_mut()) + .chain(self.properties.files.iter_mut().map(|f| &mut f.uri)) + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct Collection { + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub family: Option, +} + +impl Collection { + pub fn is_empty(&self) -> bool { + let Self { name, family } = self; + name.is_none() && family.is_none() + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Attribute { + pub trait_type: String, + pub value: String, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct Properties { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub files: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub category: Option, +} + +impl Properties { + pub fn is_empty(&self) -> bool { + let Self { files, category } = self; + files.is_empty() && category.is_none() + } + + pub fn find_file(&self, url: &U) -> Option<&File> where PermissiveUrl: PartialEq { + self.files.iter().find(|f| f.uri == *url) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct File { + pub uri: PermissiveUrl, + #[serde(rename = "type", skip_serializing_if = "Option::is_none")] + pub ty: Option, +} diff --git a/src/common/toposort.rs b/src/common/toposort.rs new file mode 100644 index 0000000..ed38f4b --- /dev/null +++ b/src/common/toposort.rs @@ -0,0 +1,128 @@ +use std::{ + iter, + sync::{ + atomic::{self, AtomicUsize}, + Arc, + }, num::NonZeroUsize, +}; + +use dispose::{Disposable, Dispose}; +use log::error; +use parking_lot::Mutex; + +pub trait PendingFail { + fn failed(self); +} + +#[derive(Debug)] +struct Pending { + unmet_deps: AtomicUsize, + failed_deps: AtomicUsize, + node: Mutex>, +} + +impl Drop for Pending { + fn drop(&mut self) { + let Self { + unmet_deps, + failed_deps, + node, + } = self; + let unmet_deps = unmet_deps.get_mut(); + let failed_deps = failed_deps.get_mut(); + let node = node.get_mut(); + + assert!( + unmet_deps <= failed_deps, + "Pending node dropped with unmet dependencies!", + ); + + if *failed_deps == 0 { + assert!( + node.is_none(), + "Pending node with no failed dependencies did not yield!", + ); + } else { + node.take() + .unwrap_or_else(|| unreachable!("Pending node with failed dependencies yielded!")) + .failed(); + } + } +} + +#[derive(Debug)] +#[repr(transparent)] +#[must_use = "Dependency iterator must be exhausted"] +pub struct Dependencies(iter::Take>>>); + +impl Dependencies { + pub fn new(deps: NonZeroUsize, node: T) -> Self { + let deps = deps.get(); + let this = Pending { + unmet_deps: deps.into(), + failed_deps: 0.into(), + node: Some(node).into(), + }; + + Self(std::iter::repeat(Arc::new(this)).take(deps)) + } +} + +impl Iterator for Dependencies { + type Item = Dependency; + + #[inline] + fn next(&mut self) -> Option { + self.0.next().map(|a| Dependency(Dependent(a).into())) + } +} + +impl ExactSizeIterator for Dependencies { + fn len(&self) -> usize { + // TODO: TrustedLen pls + self.0.size_hint().0 + } +} + +impl Drop for Dependencies { + fn drop(&mut self) { + assert!( + self.0.next().is_none(), + "Dependency iterator was not exhausted!", + ); + } +} + +#[derive(Debug)] +#[repr(transparent)] +struct Dependent(Arc>); + +impl Dispose for Dependent { + fn dispose(self) { self.0.failed_deps.fetch_add(1, atomic::Ordering::SeqCst); } +} + +#[derive(Debug)] +#[repr(transparent)] +pub struct Dependency(Disposable>); + +impl Dependency { + #[must_use] + pub fn ok(self, submit: impl FnOnce(T) -> U) -> Option { + let inner = unsafe { Disposable::leak(self.0) }; + + match inner.0.unmet_deps.fetch_sub(1, atomic::Ordering::SeqCst) { + 0 => panic!("Too many calls to next_dep!"), + 1 => Some(submit( + inner + .0 + .node + .try_lock() + .and_then(|mut j| j.take()) + .unwrap_or_else(|| { + unreachable!("Contended pending node - this should not happen!") + }), + )), + _ => None, + } + } +} diff --git a/src/common/url_permissive.rs b/src/common/url_permissive.rs new file mode 100644 index 0000000..00e585c --- /dev/null +++ b/src/common/url_permissive.rs @@ -0,0 +1,85 @@ +use std::{fmt, path::PathBuf, str::FromStr}; + +use serde::{ + de::{self, Visitor}, + Deserialize, Serialize, +}; +use url::Url; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum PermissiveUrl { + Url(Url), + Path(PathBuf), +} + +impl PermissiveUrl { + pub fn to_file_path(&self) -> Option { + match self { + Self::Url(u) => u.to_file_path().ok(), + Self::Path(p) => Some(p.clone()), + } + } +} + +impl From for String { + #[inline] + fn from(val: PermissiveUrl) -> Self { match val { + PermissiveUrl::Url(u) => u.into(), + PermissiveUrl::Path(p) => p.to_string_lossy().into_owned(), + }} +} + +impl FromStr for PermissiveUrl { + type Err = url::ParseError; + + fn from_str(s: &str) -> Result { + Url::parse(s) + .map(Self::Url) + .or_else(|e| PathBuf::from_str(s).map(Self::Path).map_err(|_| e)) + } +} + +impl PartialEq for PermissiveUrl where Url: PartialEq, PathBuf: PartialEq { + fn eq(&self, other: &Rhs) -> bool { + match self { + PermissiveUrl::Url(u) => u.eq(other), + PermissiveUrl::Path(p) => p.eq(other), + } + } +} + +impl Serialize for PermissiveUrl { + #[inline] + fn serialize(&self, serializer: S) -> Result + where S: serde::Serializer { + match self { + PermissiveUrl::Url(u) => u.serialize(serializer), + PermissiveUrl::Path(p) => p.serialize(serializer), + } + } +} + +impl<'de> Deserialize<'de> for PermissiveUrl { + fn deserialize(deserializer: D) -> Result + where D: serde::Deserializer<'de> { + struct PermissiveUrlVisitor; + + impl<'de> Visitor<'de> for PermissiveUrlVisitor { + type Value = PermissiveUrl; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a string representing an URL") + } + + fn visit_str(self, s: &str) -> Result + where E: de::Error { + PermissiveUrl::from_str(s).map_err(|err| { + let err_s = format!("{}", err); + de::Error::invalid_value(de::Unexpected::Str(s), &err_s.as_str()) + }) + } + } + + deserializer.deserialize_str(PermissiveUrlVisitor) + } +} diff --git a/src/config.rs b/src/config.rs index c75795f..c310957 100644 --- a/src/config.rs +++ b/src/config.rs @@ -172,17 +172,6 @@ impl Config { .context("Error building HTTP client") } - #[inline] - pub fn post_graphql<'a, Q: graphql_client::GraphQLQuery + 'a>( - &self, - client: &'a Client, - vars: Q::Variables, - ) -> impl std::future::Future< - Output = Result, reqwest::Error>, - > + 'a { - graphql_client::reqwest::post_graphql::(client, self.graphql_endpoint.clone(), vars) - } - pub fn hub_endpoint(&self) -> Result> { self.hub_endpoint.as_ref().map(Cow::Borrowed).map_or_else( || { @@ -210,7 +199,9 @@ impl Config { .ok()?; if let Some("graphql") = url.path_segments()?.last() { - url.path_segments_mut().unwrap().pop(); + let mut segs = url.path_segments_mut().unwrap(); + segs.pop(); + segs.push("api"); } Some(Cow::Owned(url)) @@ -224,6 +215,8 @@ impl Config { ) } + pub fn graphql_endpoint(&self) -> &Url { &self.graphql_endpoint } + pub fn upload_endpoint(&self) -> Result { let mut url = self.hub_endpoint()?.into_owned(); url.path_segments_mut() diff --git a/src/main.rs b/src/main.rs index ec2d578..906cc33 100644 --- a/src/main.rs +++ b/src/main.rs @@ -20,6 +20,13 @@ mod commands { pub mod upload_drop; } +mod common { + pub mod concurrent; + pub mod metadata_json; + pub mod toposort; + pub mod url_permissive; +} + use anyhow::{Context, Result}; use cli::{Opts, Subcommand, UploadSubcommand}; use config::{Config, ConfigLocation}; @@ -47,6 +54,7 @@ fn runtime() -> Result { } fn run() -> Result<()> { + env_logger::init(); let Opts { config, subcmd } = clap::Parser::parse(); let config = |w| ConfigLocation::new(config, w); diff --git a/src/queries/queue-mint-to-drop.graphql b/src/queries/queue-mint-to-drop.graphql index 98b9eef..9cbc07c 100644 --- a/src/queries/queue-mint-to-drop.graphql +++ b/src/queries/queue-mint-to-drop.graphql @@ -1,8 +1,7 @@ mutation QueueMintToDrop($in: QueueMintToDropInput!) { queueMintToDrop(input: $in) { collectionMint { - id - creationStatus + id # TODO: what's the minimal set of fields needed for caching? } } } From 947929edc54abfd265c6084fdccb0cbd1ff990e6 Mon Sep 17 00:00:00 2001 From: raykast Date: Wed, 27 Sep 2023 23:34:42 -0700 Subject: [PATCH 2/8] Removed Cargo.lock from the gitignore. --- .gitignore | 6 - Cargo.lock | 2010 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 2010 insertions(+), 6 deletions(-) create mode 100644 Cargo.lock diff --git a/.gitignore b/.gitignore index 196e176..5f3f0cd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,17 +3,11 @@ debug/ target/ -# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries -# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html -Cargo.lock - # These are backup files generated by rustfmt **/*.rs.bk # MSVC Windows builds of rustc generate these, which store debugging information *.pdb - # Added by cargo - /target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..085cde9 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2010 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "anstream" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" + +[[package]] +name = "anstyle-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +dependencies = [ + "anstyle", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "ascii" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" + +[[package]] +name = "async-compression" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb42b2197bf15ccb092b62c74515dbd8b86d0effd934795f6687c93b6e679a2c" +dependencies = [ + "brotli", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +dependencies = [ + "serde", +] + +[[package]] +name = "brotli" +version = "3.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfb" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f" +dependencies = [ + "byteorder", + "fnv", + "uuid", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "4.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", + "terminal_size", +] + +[[package]] +name = "clap_derive" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "clap_lex" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" + +[[package]] +name = "clipboard-win" +version = "4.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7191c27c2357d9b7ef96baac1773290d4ca63b24205b82a3fd8a0637afcf0362" +dependencies = [ + "error-code", + "str-buf", + "winapi", +] + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "combine" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3da6baa321ec19e1cc41d31bf599f00c783d0517095cdaf0332e3fe8d20680" +dependencies = [ + "ascii", + "byteorder", + "either", + "memchr", + "unreachable", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "directories" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys", +] + +[[package]] +name = "dispose" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec25f7683c53e023449962c76a1ae45b3192a000ad965c27659409a7d194e452" +dependencies = [ + "dispose-derive", +] + +[[package]] +name = "dispose-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baeea09a9ce2d11936dd39a3bafb0af104544fa3755f5a3f6f93d18d87c878ae" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_logger" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +dependencies = [ + "humantime", + "is-terminal", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "error-code" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21" +dependencies = [ + "libc", + "str-buf", +] + +[[package]] +name = "fastrand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + +[[package]] +name = "flate2" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" + +[[package]] +name = "graphql-introspection-query" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2a4732cf5140bd6c082434494f785a19cfb566ab07d1382c3671f5812fed6d" +dependencies = [ + "serde", +] + +[[package]] +name = "graphql-parser" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ebc8013b4426d5b81a4364c419a95ed0b404af2b82e2457de52d9348f0e474" +dependencies = [ + "combine", + "thiserror", +] + +[[package]] +name = "graphql_client" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cdf7b487d864c2939b23902291a5041bc4a84418268f25fda1c8d4e15ad8fa" +dependencies = [ + "graphql_query_derive", + "serde", + "serde_json", +] + +[[package]] +name = "graphql_client_codegen" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a40f793251171991c4eb75bd84bc640afa8b68ff6907bc89d3b712a22f700506" +dependencies = [ + "graphql-introspection-query", + "graphql-parser", + "heck", + "lazy_static", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 1.0.109", +] + +[[package]] +name = "graphql_query_derive" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00bda454f3d313f909298f626115092d348bc231025699f557b27e248475f48c" +dependencies = [ + "graphql_client_codegen", + "proc-macro2", + "syn 1.0.109", +] + +[[package]] +name = "h2" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap 1.9.3", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" + +[[package]] +name = "holaplex-hub-cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "crossbeam", + "directories", + "dispose", + "env_logger", + "futures-util", + "graphql_client", + "infer", + "itertools", + "log", + "num_cpus", + "parking_lot", + "reqwest", + "ron", + "rpassword", + "rustyline", + "serde", + "serde_json", + "tokio", + "toml", + "url", + "uuid", +] + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.4.9", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad227c3af19d4914570ad36d30409928b75967c298feb9ea1969db3a610bb14e" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + +[[package]] +name = "infer" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb33622da908807a06f9513c19b3c1ad50fab3e4137d82a78107d502075aa199" +dependencies = [ + "cfb", +] + +[[package]] +name = "ipnet" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" + +[[package]] +name = "is-terminal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +dependencies = [ + "hermit-abi", + "rustix", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "js-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "linux-raw-sys" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" + +[[package]] +name = "lock_api" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "memchr" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "openssl" +version = "0.10.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +dependencies = [ + "bitflags 2.4.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db7e971c2c2bba161b2d2fdf37080177eff520b3bc044787c7f1f5f9e78d869b" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.3.5", + "smallvec", + "windows-targets", +] + +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall 0.2.16", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + +[[package]] +name = "reqwest" +version = "0.11.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +dependencies = [ + "async-compression", + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "mime_guess", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "winreg", +] + +[[package]] +name = "ron" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" +dependencies = [ + "base64", + "bitflags 2.4.0", + "serde", + "serde_derive", +] + +[[package]] +name = "rpassword" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6678cf63ab3491898c0d021b493c94c9b221d91295294a2a5746eacbe5928322" +dependencies = [ + "libc", + "rtoolbox", + "winapi", +] + +[[package]] +name = "rtoolbox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034e22c514f5c0cb8a10ff341b9b048b5ceb21591f31c8f44c43b960f9b3524a" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustix" +version = "0.38.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bfe0f2582b4931a45d1fa608f8a8722e8b3c7ac54dd6d5f3b3212791fedef49" +dependencies = [ + "bitflags 2.4.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustyline" +version = "12.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "994eca4bca05c87e86e15d90fc7a91d1be64b4482b38cb2d27474568fe7c9db9" +dependencies = [ + "bitflags 2.4.0", + "cfg-if", + "clipboard-win", + "libc", + "log", + "memchr", + "nix", + "scopeguard", + "unicode-segmentation", + "unicode-width", + "utf8parse", + "winapi", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "serde_json" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +dependencies = [ + "indexmap 2.0.1", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_spanned" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" + +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "str-buf" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall 0.3.5", + "rustix", + "windows-sys", +] + +[[package]] +name = "termcolor" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "terminal_size" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" +dependencies = [ + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "socket2 0.5.3", + "windows-sys", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "toml" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c226a7bba6d859b63c92c4b4fe69c5b6b72d0cb897dbc8e6012298e6154cb56e" +dependencies = [ + "indexmap 2.0.1", + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ff63e60a958cefbb518ae1fd6566af80d9d4be430a33f3723dfc47d1d411d95" +dependencies = [ + "indexmap 2.0.1", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + +[[package]] +name = "unicase" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +dependencies = [ + "void", +] + +[[package]] +name = "url" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "uuid" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +dependencies = [ + "serde", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.37", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + +[[package]] +name = "wasm-streams" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "winnow" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +dependencies = [ + "memchr", +] + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys", +] From 55fe13bbc59fa51cba324dcb403758720ef4614b Mon Sep 17 00:00:00 2001 From: raykast Date: Sat, 30 Sep 2023 01:27:32 -0700 Subject: [PATCH 3/8] Add rocksdb and set up caching for upload drop. --- Cargo.lock | 304 ++++++++++++++++++++- Cargo.toml | 7 + build.rs | 7 + src/cache.rs | 493 +++++++++++++++++++++++++++++------ src/cli.rs | 53 +++- src/commands/upload_drop.rs | 407 +++++++++++++++++++---------- src/common/metadata_json.rs | 3 +- src/common/tokio.rs | 8 + src/common/toposort.rs | 4 +- src/common/url_permissive.rs | 16 +- src/main.rs | 61 +++-- src/proto/asset-upload.proto | 7 + src/proto/drop-mint.proto | 8 + 13 files changed, 1122 insertions(+), 256 deletions(-) create mode 100644 build.rs create mode 100644 src/common/tokio.rs create mode 100644 src/proto/asset-upload.proto create mode 100644 src/proto/drop-mint.proto diff --git a/Cargo.lock b/Cargo.lock index 085cde9..eeb88ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -142,6 +142,27 @@ version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" +[[package]] +name = "bindgen" +version = "0.65.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.37", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -196,15 +217,36 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ + "jobserver", "libc", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfb" version = "0.7.3" @@ -222,6 +264,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clang-sys" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.4.4" @@ -385,6 +438,19 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if", + "hashbrown 0.14.0", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "directories" version = "5.0.1" @@ -498,6 +564,12 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "flate2" version = "1.0.27" @@ -616,6 +688,12 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "graphql-introspection-query" version = "0.2.0" @@ -724,6 +802,7 @@ dependencies = [ "anyhow", "clap", "crossbeam", + "dashmap", "directories", "dispose", "env_logger", @@ -734,7 +813,10 @@ dependencies = [ "log", "num_cpus", "parking_lot", + "prost", + "prost-build", "reqwest", + "rocksdb", "ron", "rpassword", "rustyline", @@ -744,6 +826,16 @@ dependencies = [ "toml", "url", "uuid", + "xxhash-rust", +] + +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys", ] [[package]] @@ -894,6 +986,15 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "jobserver" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.64" @@ -909,17 +1010,60 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if", + "winapi", +] + +[[package]] +name = "librocksdb-sys" +version = "0.11.0+8.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", + "lz4-sys", + "pkg-config", +] + +[[package]] +name = "libz-sys" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" [[package]] name = "lock_api" @@ -937,6 +1081,16 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "lz4-sys" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "memchr" version = "2.6.3" @@ -968,6 +1122,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.1" @@ -988,6 +1148,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + [[package]] name = "native-tls" version = "0.2.11" @@ -1017,6 +1183,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num_cpus" version = "1.16.0" @@ -1115,12 +1291,28 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap 2.0.1", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -1139,6 +1331,16 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "prettyplease" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +dependencies = [ + "proc-macro2", + "syn 2.0.37", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1172,6 +1374,60 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" +dependencies = [ + "bytes", + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.37", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "prost-types" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.33" @@ -1280,6 +1536,16 @@ dependencies = [ "winreg", ] +[[package]] +name = "rocksdb" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6f170a4041d50a0ce04b0d2e14916d6ca863ea2e422689a5b694395d299ffe" +dependencies = [ + "libc", + "librocksdb-sys", +] + [[package]] name = "ron" version = "0.8.1" @@ -1319,11 +1585,17 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustix" -version = "0.38.9" +version = "0.38.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bfe0f2582b4931a45d1fa608f8a8722e8b3c7ac54dd6d5f3b3212791fedef49" +checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" dependencies = [ "bitflags 2.4.0", "errno", @@ -1449,6 +1721,12 @@ dependencies = [ "serde", ] +[[package]] +name = "shlex" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" + [[package]] name = "slab" version = "0.4.9" @@ -1893,6 +2171,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2008,3 +2298,9 @@ dependencies = [ "cfg-if", "windows-sys", ] + +[[package]] +name = "xxhash-rust" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9828b178da53440fa9c766a3d2f73f7cf5d0ac1fe3980c1e5018d899fd19e07b" diff --git a/Cargo.toml b/Cargo.toml index 0f346fc..97aca42 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ path = "src/main.rs" anyhow = "1.0.75" clap = { version = "4.4.4", features = ["cargo", "derive", "env", "wrap_help"] } crossbeam = "0.8.2" +dashmap = "5.5.3" directories = "5.0.1" dispose = "0.5.0" env_logger = "0.10.0" @@ -23,7 +24,9 @@ itertools = "0.11.0" log = "0.4.20" num_cpus = "1.16.0" parking_lot = "0.12.1" +prost = "0.12.1" reqwest = { version = "0.11.20", features = ["brotli", "deflate", "gzip", "json", "multipart", "stream"] } +rocksdb = { version = "0.21.0", features = ["io-uring", "multi-threaded-cf", "lz4", "zlib"], default-features = false } ron = "0.8.1" rpassword = "7.2.0" rustyline = { version = "12.0.0", default-features = false } @@ -33,3 +36,7 @@ tokio = { version = "1.32.0", features = ["parking_lot", "rt-multi-thread"] } toml = { version = "0.8.0", features = ["preserve_order"] } url = { version = "2.4.1", features = ["serde"] } uuid = { version = "1.4.1", features = ["serde"] } +xxhash-rust = { version = "0.8.7", features = ["xxh3", "const_xxh3"] } + +[build-dependencies] +prost-build = "0.12.1" diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..c1d7b12 --- /dev/null +++ b/build.rs @@ -0,0 +1,7 @@ +fn main() { + prost_build::compile_protos( + &["src/proto/asset-upload.proto", "src/proto/drop-mint.proto"], + &["src/proto"], + ) + .unwrap(); +} diff --git a/src/cache.rs b/src/cache.rs index aaab510..e262bac 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -1,129 +1,458 @@ -#![allow(dead_code)] // TODO: replace with RocksDB? - use std::{ - collections::HashMap, - fs::{self, File}, - io::ErrorKind, + borrow::Cow, + fmt, + io::{self, prelude::*}, path::{Path, PathBuf}, + pin::pin, + sync::Arc, }; use anyhow::{Context, Result}; -use serde::{Deserialize, Serialize}; -use url::Url; +use log::{debug, trace}; +use prost::Message; +use rocksdb::{BlockBasedOptions, ColumnFamilyRef, Options, ReadOptions, WriteOptions, DB}; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt}, + task::spawn_blocking, +}; +use xxhash_rust::xxh3::Xxh3; -const SWAPFILE_NAME: &str = ".hub.cache~"; -const FILE_NAME: &str = ".hub.cache"; +use crate::cli::CacheOpts; + +mod proto { + include!(concat!(env!("OUT_DIR"), "/cache.asset_uploads.rs")); + include!(concat!(env!("OUT_DIR"), "/cache.drop_mints.rs")); +} -fn swap_file(path: impl AsRef) -> PathBuf { path.as_ref().join(SWAPFILE_NAME) } -fn cache_file(path: impl AsRef) -> PathBuf { path.as_ref().join(FILE_NAME) } +pub use proto::{AssetUpload, DropMint}; -#[derive(Debug, Default)] +#[derive(Clone, Copy, PartialEq, Eq)] #[repr(transparent)] -struct CacheItem(Option); +pub struct Checksum(u128); -impl<'de, T: Deserialize<'de>> Deserialize<'de> for CacheItem { - #[inline] - fn deserialize(deserializer: D) -> std::result::Result - where D: serde::Deserializer<'de> { - Deserialize::deserialize(deserializer).map(Self) +impl fmt::Debug for Checksum { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:032x}", self.0) } +} + +impl Checksum { + const BUF_SIZE: usize = 8 << 10; + + pub fn hash(val: &H) -> Self { + let mut hasher = Xxh3::new(); + val.hash(&mut hasher); + Self(hasher.digest128()) } - #[inline] - fn deserialize_in_place( - deserializer: D, - place: &mut Self, - ) -> std::result::Result<(), D::Error> - where - D: serde::Deserializer<'de>, - { - Deserialize::deserialize_in_place(deserializer, &mut place.0) + pub fn read_rewind(name: impl fmt::Debug, mut stream: R) -> Result { + let pos = stream + .stream_position() + .with_context(|| format!("Error getting stream position of {name:?}"))?; + let res = Self::read(&name, &mut stream); + stream + .seek(io::SeekFrom::Start(pos)) + .with_context(|| format!("Error rewinding stream for {name:?}"))?; + + res + } + + pub fn read(name: impl fmt::Debug, mut stream: R) -> Result { + let mut hasher = Xxh3::new(); + let mut buf = vec![0; Self::BUF_SIZE].into_boxed_slice(); + + loop { + match stream.read(&mut buf) { + Ok(0) => break Ok(Self(hasher.digest128())), + Ok(n) => hasher.update(&buf[..n]), + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return Err(e).context(format!("Error checksumming {name:?}")), + } + } } + + pub async fn read_rewind_async( + name: impl fmt::Debug, + mut stream: R, + ) -> Result { + let mut stream = pin!(stream); + let pos = stream + .stream_position() + .await + .with_context(|| format!("Error getting stream position of {name:?}"))?; + let res = Self::read_async(&name, &mut stream).await; + stream + .seek(io::SeekFrom::Start(pos)) + .await + .with_context(|| format!("Error rewinding stream for {name:?}"))?; + + res + } + + pub async fn read_async(name: impl fmt::Debug, stream: R) -> Result { + let mut stream = pin!(stream); + let mut hasher = Xxh3::new(); + let mut buf = vec![0; Self::BUF_SIZE].into_boxed_slice(); + + loop { + match stream.read(&mut buf).await { + Ok(0) => break Ok(Self(hasher.digest128())), + Ok(n) => hasher.update(&buf[..n]), + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return Err(e).context(format!("Error checksumming {name:?}")), + } + } + } + + pub fn to_bytes(self) -> [u8; 16] { self.0.to_le_bytes() } } -impl Serialize for CacheItem { - #[inline] - fn serialize(&self, serializer: S) -> std::result::Result - where S: serde::Serializer { - Serialize::serialize( - if self.0.as_ref().map_or(false, |v| *v == T::default()) { - &None - } else { - &self.0 - }, - serializer, +const DIR_NAME: &str = ".hub-cache"; + +fn cache_dir(path: impl AsRef) -> PathBuf { path.as_ref().join(DIR_NAME) } + +struct CacheConfigInner { + db_opts: Options, + read_opts: ReadOptions, + write_opts: WriteOptions, +} + +#[derive(Clone)] +#[repr(transparent)] +pub struct CacheConfig(Arc); + +impl CacheConfig { + #[allow(clippy::needless_pass_by_value)] + pub fn new(opts: CacheOpts) -> Self { + let CacheOpts { + cache_threads, + cache_write_buf, + cache_lru_size, + } = opts; + + let block_opts = { + let mut o = BlockBasedOptions::default(); + + o.set_bloom_filter(10.0, false); + o.set_block_size(16 << 10); + o.set_cache_index_and_filter_blocks(true); + // o.set_optimize_filters_for_memory(true); // TODO: not implemented? + o.set_pin_l0_filter_and_index_blocks_in_cache(true); + + let c = rocksdb::Cache::new_lru_cache(cache_lru_size); + o.set_block_cache(&c); + + o + }; + + let db_opts = { + let mut o = Options::default(); + + o.create_if_missing(true); + o.increase_parallelism(cache_threads.try_into().unwrap_or(i32::MAX)); + o.set_bytes_per_sync(1 << 20); + o.set_keep_log_file_num(1); + o.set_level_compaction_dynamic_level_bytes(true); + o.set_max_background_jobs(6); + o.set_write_buffer_size(cache_write_buf); + o.set_max_total_wal_size(10 << 20); + + o.set_compression_type(rocksdb::DBCompressionType::Lz4); + o.set_bottommost_compression_type(rocksdb::DBCompressionType::Zstd); + o.set_bottommost_zstd_max_train_bytes(10 << 20, true); + + o.set_block_based_table_factory(&block_opts); + + o + }; + + let read_opts = { + let mut o = ReadOptions::default(); + + o.set_async_io(true); + o.set_readahead_size(4 << 20); + + o + }; + + let write_opts = WriteOptions::default(); + + Self( + CacheConfigInner { + db_opts, + read_opts, + write_opts, + } + .into(), ) } + + fn db_opts(&self) -> &Options { &self.0.db_opts } + + fn read_opts(&self) -> &ReadOptions { &self.0.read_opts } + + fn write_opts(&self) -> &WriteOptions { &self.0.write_opts } } -impl CacheItem { - fn as_mut(&mut self) -> &mut T { - if let Some(ref mut v) = self.0 { - v - } else { - self.0 = Some(T::default()); - self.0.as_mut().unwrap() +mod private { + #[allow(clippy::wildcard_imports)] + use super::*; + + pub trait CacheFamily<'a> { + type Static: CacheFamily<'static>; + + const CF_NAME: &'static str; + + fn new(cache: Cow<'a, Cache>) -> Self; + + fn cache(&self) -> &Cow<'a, Cache>; + + fn to_static(&self) -> Self::Static { + Self::Static::new(Cow::Owned(self.cache().as_ref().to_owned())) } + + #[inline] + fn get_cf(&'a self) -> Result { self.cache().get_cf(Self::CF_NAME) } } } +use private::CacheFamily; + +#[derive(Clone)] pub struct Cache { - path: PathBuf, - contents: CacheContents, + dir: PathBuf, + config: CacheConfig, + db: Arc, } +// NB: Don't use the async versions unless you absolutely have to, they're +// going to all be slower. + impl Cache { - pub fn load(path: impl AsRef) -> Result { - let path = cache_file(&path); - let contents = CacheContents::load(&path)?; - Ok(Self { path, contents }) + pub async fn load( + path: impl AsRef + Send + 'static, + config: CacheConfig, + ) -> Result { + spawn_blocking(|| Self::load_sync(path, config)) + .await + .unwrap() } - #[inline] - pub fn json_uploads_mut(&mut self) -> &mut JsonUploads { self.contents.json_uploads.as_mut() } + pub fn load_sync(path: impl AsRef, config: CacheConfig) -> Result { + debug!("Loading cache for {:?}", path.as_ref()); - #[inline] - pub fn asset_uploads_mut(&mut self) -> &mut AssetUploads { - self.contents.asset_uploads.as_mut() + let cache_dir = cache_dir(&path); + let cfs = DB::list_cf(config.db_opts(), &cache_dir) + .map_err(|e| debug!("Skipping column family load due to error: {e}")) + .unwrap_or_default(); + + trace!("Loading column handle(s) {cfs:?} for {:?}", path.as_ref()); + + let db = DB::open_cf(config.db_opts(), cache_dir, cfs) + .with_context(|| format!("Error opening cache database for {:?}", path.as_ref()))?; + + Ok(Self { + dir: path.as_ref().into(), + config, + db: db.into(), + }) + } + + fn create_cf(&self, name: &str) -> Result<()> { + if self.db.cf_handle(name).is_some() { + trace!("Reusing existing column handle {name:?} for {:?}", self.dir); + + return Ok(()); + } + + self.db + .create_cf(name, self.config.db_opts()) + .with_context(|| { + format!( + "Error creating column family {name:?} in cache database for {:?}", + self.dir + ) + }) + } + + fn get_cf(&self, name: &str) -> Result { + self.db.cf_handle(name).with_context(|| { + format!( + "No handle to column family {name:?} exists in cache database for {:?}", + self.dir + ) + }) + } + + pub async fn get + Send + 'static>(&self) -> Result { + let this = self.clone(); + spawn_blocking(move || { + this.create_cf(C::CF_NAME)?; + + Ok(C::new(Cow::Owned(this))) + }) + .await + .unwrap() + } + + pub fn get_sync<'a, C: CacheFamily<'a>>(&'a self) -> Result { + self.create_cf(C::CF_NAME)?; + + Ok(C::new(Cow::Borrowed(self))) } } -#[derive(Debug, Default, Serialize, Deserialize)] -struct CacheContents { - json_uploads: CacheItem, - asset_uploads: CacheItem, +#[repr(transparent)] +pub struct AssetUploadCache<'a>(Cow<'a, Cache>); + +impl<'a> CacheFamily<'a> for AssetUploadCache<'a> { + type Static = AssetUploadCache<'static>; + + const CF_NAME: &'static str = "asset-uploads"; + + #[inline] + fn new(cache: Cow<'a, Cache>) -> Self { Self(cache) } + + #[inline] + fn cache(&self) -> &Cow<'a, Cache> { &self.0 } } -impl CacheContents { - fn load(path: impl AsRef) -> Result { +impl<'a> AssetUploadCache<'a> { + pub async fn get( + &self, + path: impl AsRef + Send + 'static, + ck: Checksum, + ) -> Result> { + let this = self.to_static(); + spawn_blocking(move || this.get_sync(path, ck)) + .await + .unwrap() + } + + pub fn get_sync(&self, path: impl AsRef, ck: Checksum) -> Result> { let path = path.as_ref(); - let file = match File::open(path) { - Ok(f) => f, - Err(e) if e.kind() == ErrorKind::NotFound => return Ok(Self::default()), - Err(e) => return Err(e).with_context(|| format!("Error opening cache file {path:?}")), - }; - ron::de::from_reader(file).with_context(|| format!("Error parsing cache file {path:?}")) + let bytes = self + .0 + .db + .get_cf_opt( + &self.0.get_cf(Self::CF_NAME)?, + ck.to_bytes(), + self.0.config.read_opts(), + ) + .with_context(|| format!("Error getting asset upload cache for {path:?}"))?; + + let Some(bytes) = bytes else { return Ok(None) }; + let upload = AssetUpload::decode(&*bytes).with_context(|| { + format!("Error parsing asset upload cache for {path:?} (this should not happen)") + })?; + + Ok(Some(upload)) } - fn save(&self, path: impl AsRef) -> Result<()> { - let swap = swap_file(&path); - let path = cache_file(path); - let file = File::create(&swap) - .with_context(|| format!("Error creating cache swapfile {swap:?}"))?; + pub async fn set( + &self, + path: impl AsRef + Send + 'static, + ck: Checksum, + upload: AssetUpload, + ) -> Result<()> { + let this = self.to_static(); + spawn_blocking(move || this.set_sync(path, ck, &upload)) + .await + .unwrap() + } - ron::ser::to_writer(file, &self) - .with_context(|| format!("Error serializing cache to {swap:?}"))?; + pub fn set_sync( + &self, + path: impl AsRef, + ck: Checksum, + upload: &AssetUpload, + ) -> Result<()> { + let path = path.as_ref(); + let bytes = upload.encode_to_vec(); - fs::rename(swap, &path).with_context(|| format!("Error writing to cache file {path:?}")) + self.0 + .db + .put_cf_opt( + &self.0.get_cf(Self::CF_NAME)?, + ck.to_bytes(), + bytes, + self.0.config.write_opts(), + ) + .with_context(|| format!("Error setting asset upload cache for {path:?}")) } } -#[derive(Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct JsonUploads { - uploaded: HashMap, +#[repr(transparent)] +pub struct DropMintCache<'a>(Cow<'a, Cache>); + +impl<'a> CacheFamily<'a> for DropMintCache<'a> { + type Static = DropMintCache<'static>; + + const CF_NAME: &'static str = "drop-mints"; + + #[inline] + fn new(cache: Cow<'a, Cache>) -> Self { Self(cache) } + + #[inline] + fn cache(&self) -> &Cow<'a, Cache> { &self.0 } } -#[derive(Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct AssetUploads { - uploaded: HashMap, +impl<'a> DropMintCache<'a> { + pub async fn get( + &self, + path: impl AsRef + Send + 'static, + ck: Checksum, + ) -> Result> { + let this = self.to_static(); + spawn_blocking(move || this.get_sync(path, ck)) + .await + .unwrap() + } + + pub fn get_sync(&self, path: impl AsRef, ck: Checksum) -> Result> { + let path = path.as_ref(); + + let bytes = self + .0 + .db + .get_cf_opt( + &self.0.get_cf(Self::CF_NAME)?, + ck.to_bytes(), + self.0.config.read_opts(), + ) + .with_context(|| format!("Error getting asset upload cache for {path:?}"))?; + + let Some(bytes) = bytes else { return Ok(None) }; + let upload = DropMint::decode(&*bytes).with_context(|| { + format!("Error parsing asset upload cache for {path:?} (this should not happen)") + })?; + + Ok(Some(upload)) + } + + pub async fn set( + &self, + path: impl AsRef + Send + 'static, + ck: Checksum, + upload: DropMint, + ) -> Result<()> { + let this = self.to_static(); + spawn_blocking(move || this.set_sync(path, ck, &upload)) + .await + .unwrap() + } + + pub fn set_sync(&self, path: impl AsRef, ck: Checksum, upload: &DropMint) -> Result<()> { + let path = path.as_ref(); + let bytes = upload.encode_to_vec(); + + self.0 + .db + .put_cf_opt( + &self.0.get_cf(Self::CF_NAME)?, + ck.to_bytes(), + bytes, + self.0.config.write_opts(), + ) + .with_context(|| format!("Error setting asset upload cache for {path:?}")) + } } diff --git a/src/cli.rs b/src/cli.rs index 20fb1a9..35c938a 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -7,10 +7,42 @@ use std::{io::IsTerminal, path::PathBuf}; use uuid::Uuid; +/// Convert the parseable `ColorChoice` to `env_logger`'s `WriteStyle` +pub fn log_color(color: clap::ColorChoice) -> env_logger::WriteStyle { + use clap::ColorChoice as C; + use env_logger::WriteStyle as S; + + match color { + C::Never => S::Never, + C::Auto => S::Auto, + C::Always => S::Always, + } +} + #[derive(clap::Parser)] #[command(author, version, about)] /// Top-level options for the hub command pub struct Opts { + /// Adjust log level globally or on a per-module basis + /// + /// This flag uses the same syntax as the env_logger crate. + #[arg( + long, + global = true, + env = env_logger::DEFAULT_FILTER_ENV, + default_value = "info", + )] + pub log_level: String, + + /// Adjust when to output colors to the terminal + #[arg( + long, + global = true, + env = env_logger::DEFAULT_WRITE_STYLE_ENV, + default_value_t = clap::ColorChoice::Auto, + )] + pub color: clap::ColorChoice, + /// Use a different Hub config file than the default /// /// By default, hub first searches the current directory for a file named @@ -20,11 +52,31 @@ pub struct Opts { #[arg(short = 'C', long, global = true)] pub config: Option, + /// Cache options + #[command(flatten)] + pub cache: CacheOpts, + /// Name of the subcommand to run #[command(subcommand)] pub subcmd: Subcommand, } +/// Top-level command options related to the cache +#[derive(clap::Args)] +pub struct CacheOpts { + /// Number of threads to use for cache management + #[arg(long, global = true, default_value_t = num_cpus::get())] + pub cache_threads: usize, + + /// Maximum cache write buffer size + #[arg(long, global = true, default_value_t = 64 << 20)] + pub cache_write_buf: usize, + + /// Maximum in-memory cache size + #[arg(long, global = true, default_value_t = 128 << 20)] + pub cache_lru_size: usize, +} + /// Top-level subcommands for hub #[derive(clap::Subcommand)] pub enum Subcommand { @@ -111,5 +163,4 @@ pub struct UploadDrop { /// Path to a directory containing metadata JSON files to upload #[arg(required = true)] pub input_dirs: Vec, - } diff --git a/src/commands/upload_drop.rs b/src/commands/upload_drop.rs index ea82978..7e7900b 100644 --- a/src/commands/upload_drop.rs +++ b/src/commands/upload_drop.rs @@ -6,15 +6,19 @@ use std::{ io::{self, prelude::*}, iter, num::NonZeroUsize, - path::PathBuf, - sync::{Arc, OnceLock}, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, OnceLock, + }, }; -use anyhow::{Context as _, Result}; +use anyhow::{bail, Context as _, Result}; use crossbeam::{ channel::{self, Sender}, queue::ArrayQueue, }; +use dashmap::DashMap; use futures_util::FutureExt; use graphql_client::GraphQLQuery; use itertools::Either; @@ -26,15 +30,16 @@ use url::Url; use uuid::Uuid; use crate::{ + cache::{AssetUpload, AssetUploadCache, Cache, CacheConfig, Checksum, DropMint, DropMintCache}, cli::UploadDrop, common::{ concurrent, metadata_json::{self, MetadataJson}, + tokio::runtime, toposort::{Dependencies, Dependency, PendingFail}, url_permissive::PermissiveUrl, }, config::Config, - runtime, }; type UploadResponse = Vec; @@ -52,7 +57,7 @@ type UUID = Uuid; #[graphql( schema_path = "src/queries/schema.graphql", query_path = "src/queries/queue-mint-to-drop.graphql", - variables_derives = "Debug", + variables_derives = "Debug, PartialEq, Eq, Hash", response_derives = "Debug" )] struct QueueMintToDrop; @@ -133,7 +138,13 @@ impl From for queue_mint_to_drop::MetadataJsonFileInput { } } -pub fn run(config: &Config, args: UploadDrop) -> Result<()> { +#[derive(Default)] +struct Stats { + uploaded_assets: AtomicUsize, + queued_mints: AtomicUsize, +} + +pub fn run(config: &Config, cache: CacheConfig, args: UploadDrop) -> Result<()> { let UploadDrop { drop_id, include_dirs, @@ -143,36 +154,47 @@ pub fn run(config: &Config, args: UploadDrop) -> Result<()> { let include_dirs: HashSet<_> = include_dirs.into_iter().collect(); let (tx, rx) = channel::unbounded(); - for path in input_dirs - .iter() - .flat_map(|d| match fs::read_dir(d) { - Ok(r) => { - trace!("Traversing directory {r:?}"); - - Either::Left(r.map(move |f| { - let f = f.with_context(|| format!("Error reading JSON directory {d:?}"))?; - let path = f.path(); - - Ok(if path.extension().map_or(false, |p| p == "json") { - Some(path) - } else { - None + for (dir, path) in input_dirs.iter().flat_map(|d| match fs::read_dir(d) { + Ok(r) => { + trace!("Traversing directory {r:?}"); + + Either::Left(r.filter_map(move |f| { + let path = f + .with_context(|| format!("Error reading JSON directory {d:?}")) + .map(|f| { + let path = f.path(); + + if path.extension().map_or(false, |p| p == "json") { + Some(path) + } else { + None + } }) - })) - }, - Err(e) => Either::Right( - [Err(e).context(format!("Error opening JSON directory {d:?}"))].into_iter(), - ), - }) - .filter_map(Result::transpose) - { - tx.send(Job::ScanJson(ScanJsonJob { path })) - .context("Error seeding initial job queue")?; + .transpose()?; + + Some((d, path)) + })) + }, + Err(e) => Either::Right( + [( + d, + Err(e).context(format!("Error opening JSON directory {d:?}")), + )] + .into_iter(), + ), + }) { + tx.send(Job::ScanJson(ScanJsonJob { + dir: dir.clone(), + path, + })) + .context("Error seeding initial job queue")?; } - info!("Processing {} JSON files(s)...", rx.len()); + info!("Scanning {} JSON files(s)...", rx.len()); let ctx = Context { + cache_config: cache, + caches: Arc::default(), include_dirs: include_dirs .into_iter() .collect::>() @@ -183,6 +205,7 @@ pub fn run(config: &Config, args: UploadDrop) -> Result<()> { upload_endpoint: config.upload_endpoint()?, client: config.graphql_client()?, q: tx, + stats: Arc::default(), }; runtime()?.block_on(async move { @@ -211,6 +234,16 @@ pub fn run(config: &Config, args: UploadDrop) -> Result<()> { debug_assert!(rx.is_empty(), "Trailing jobs in queue"); + let Stats { + uploaded_assets, + queued_mints, + } = &*ctx.stats; + info!( + "Uploaded {assets} asset(s) and queued {mints} mint(s)", + assets = uploaded_assets.load(std::sync::atomic::Ordering::Relaxed), + mints = queued_mints.load(std::sync::atomic::Ordering::Relaxed) + ); + res })?; @@ -219,15 +252,27 @@ pub fn run(config: &Config, args: UploadDrop) -> Result<()> { #[derive(Clone)] struct Context { + cache_config: CacheConfig, + caches: Arc>, include_dirs: Arc<[PathBuf]>, drop_id: Uuid, graphql_endpoint: Url, upload_endpoint: Url, client: reqwest::Client, q: Sender, + stats: Arc, } impl Context { + fn cache(&self, path: impl AsRef) -> Result { + let entry = self + .caches + .entry(path.as_ref().to_path_buf()) + .or_try_insert_with(|| Cache::load_sync(path, self.cache_config.clone()))?; + + Ok((*entry).clone()) + } + fn resolve_file io::Result, T>( &self, mut open: F, @@ -252,15 +297,19 @@ impl Context { #[derive(Debug)] struct ScanJsonJob { + dir: PathBuf, path: Result, } impl ScanJsonJob { fn run(self, ctx: Context) -> JoinHandle> { tokio::task::spawn_blocking(move || { - let Self { path } = self; + let Self { dir, path } = self; let path = path?; let json_file = File::open(&path).with_context(|| format!("Error opening {path:?}"))?; + + let ck = Checksum::read_rewind(&path, &json_file)?; + let json: MetadataJson = serde_json::from_reader(json_file) .with_context(|| format!("Error parsing {path:?}"))?; @@ -318,14 +367,16 @@ impl ScanJsonJob { } }; - Some(Ok((url, path, ty))) + Some(Ok((url, include_dir.clone(), path, ty))) }) .collect::>(); if let Some(dep_count) = NonZeroUsize::new(local_urls.len()) { let rewrites = Arc::new(ArrayQueue::new(dep_count.get())); let deps = Dependencies::new(dep_count, QueueJsonJob { + dir, path, + ck, json, rewrites: Some(Arc::clone(&rewrites)), }); @@ -334,8 +385,9 @@ impl ScanJsonJob { for (res, dep) in local_urls.into_iter().zip(deps) { ctx.q .send(Job::UploadAsset(UploadAssetJob { - asset: res.map(|(source_url, path, mime_type)| Asset { + asset: res.map(|(source_url, dir, path, mime_type)| Asset { source_url, + dir, path, mime_type, }), @@ -347,7 +399,9 @@ impl ScanJsonJob { } else { ctx.q .send(Job::QueueJson(QueueJsonJob { + dir, path, + ck, json, rewrites: None, })) @@ -370,6 +424,7 @@ type FileRewrites = Arc>; #[derive(Debug)] struct Asset { source_url: PermissiveUrl, + dir: PathBuf, path: PathBuf, mime_type: Cow<'static, str>, } @@ -390,64 +445,99 @@ impl UploadAssetJob { dep, } = self; let Asset { + dir, path, mime_type, source_url, } = asset?; - let file = tokio::fs::File::open(&path) + let mut file = tokio::fs::File::open(&path) .await .with_context(|| format!("Error opening {path:?}"))?; - let name = path - .file_name() - .with_context(|| format!("Error resolving file name for {path:?}"))? - .to_string_lossy() - .into_owned(); + let ck = Checksum::read_rewind_async(&path, &mut file).await?; + + let cache: AssetUploadCache = ctx.cache(dir)?.get().await?; + let cached_url = cache + .get(path.clone(), ck) + .await? + .and_then(|AssetUpload { url }| { + Url::parse(&url) + .map_err(|e| warn!("Invalid cache URL {url:?}: {e}")) + .ok() + }); - let mut uploads = ctx - .client - .post(ctx.upload_endpoint) - .multipart( - multipart::Form::new().part( - "FIXME", // TODO - multipart::Part::stream(file) - .file_name(name.clone()) - .mime_str(&mime_type) - .with_context(|| { - format!("Invalid MIME type {:?} for {path:?}", mime_type.as_ref()) - })?, - ), - ) - .send() - .await - .with_context(|| format!("Error sending POST request for {path:?}"))? - .error_for_status() - .with_context(|| format!("POST request for {path:?} returned an error"))? - .json::() - .await - .with_context(|| format!("Error deserializing upload response JSON for {path:?}"))? - .into_iter(); + let dest_url; + if let Some(url) = cached_url { + trace!("Using cached URL {:?} for {path:?}", url.as_str()); + dest_url = url; + } else { + let name = path + .file_name() + .with_context(|| format!("Error resolving file name for {path:?}"))? + .to_string_lossy() + .into_owned(); + + let mut uploads = ctx + .client + .post(ctx.upload_endpoint) + .multipart( + multipart::Form::new().part( + "upload", + multipart::Part::stream(file) + .file_name(name.clone()) + .mime_str(&mime_type) + .with_context(|| { + format!( + "Invalid MIME type {:?} for {path:?}", + mime_type.as_ref() + ) + })?, + ), + ) + .send() + .await + .with_context(|| format!("Error sending POST request for {path:?}"))? + .error_for_status() + .with_context(|| format!("POST request for {path:?} returned an error"))? + .json::() + .await + .with_context(|| { + format!("Error deserializing upload response JSON for {path:?}") + })? + .into_iter(); + + if uploads.len() > 1 { + warn!("Trailing values in response data for {path:?}"); + } - if uploads.len() > 1 { - warn!("Trailing values in response data for {path:?}"); - } + let upload = uploads + .find(|u| u.name == name) + .with_context(|| format!("Missing upload response data for {path:?}"))?; + + ctx.stats.uploaded_assets.fetch_add(1, Ordering::Relaxed); + info!("Successfully uploaded {path:?}"); - let upload = uploads - .find(|u| u.name == name) - .with_context(|| format!("Missing upload response data for {path:?}"))?; + cache + .set(path.clone(), ck, AssetUpload { + url: upload.url.to_string(), + }) + .await + .map_err(|e| warn!("{e:?}")) + .ok(); + + dest_url = upload.url; + } rewrites .push(FileRewrite { source_url, - dest_url: upload.url, + dest_url, mime_type, }) .unwrap_or_else(|_: FileRewrite| { unreachable!("Too many file rewrites for {path:?}") }); - info!("Successfully uploaded {path:?}"); - dep.ok(|j| ctx.q.send(Job::QueueJson(j))) .transpose() .context("Error submitting JSON queue job")?; @@ -459,13 +549,19 @@ impl UploadAssetJob { #[derive(Debug)] struct QueueJsonJob { + dir: PathBuf, path: PathBuf, + ck: Checksum, json: MetadataJson, rewrites: Option, } impl QueueJsonJob { - fn format_errors(errors: Option>, f: impl FnOnce(String)) -> bool { + fn format_errors( + errors: Option>, + ok: T, + f: impl FnOnce(String) -> T, + ) -> T { let mut errs = errors.into_iter().flatten().peekable(); if errs.peek().is_some() { @@ -475,58 +571,64 @@ impl QueueJsonJob { write!(s, "\n {err}").unwrap(); } - f(s); - true + f(s) } else { - false + ok + } + } + + fn rewrite_json(json: &mut MetadataJson, rewrites: Option>>) { + let rewrites: HashMap<_, _> = rewrites + .into_iter() + .flat_map(|r| iter::from_fn(move || r.pop())) + .map( + |FileRewrite { + source_url, + dest_url, + mime_type, + }| (source_url, (dest_url, mime_type)), + ) + .collect(); + + for file in json.files_mut() { + if let Some((url, _)) = rewrites.get(file) { + *file = PermissiveUrl::Url(url.clone()); + } + } + + let seen_files: HashMap<_, _> = json + .properties + .files + .iter() + .enumerate() + .map(|(i, f)| (f.uri.clone(), i)) + .collect(); + + for (uri, ty) in rewrites.into_values() { + let uri = PermissiveUrl::Url(uri); + if let Some(idx) = seen_files.get(&uri) { + json.properties.files[*idx].ty = Some(ty.into_owned()); + } else { + json.properties.files.push(metadata_json::File { + uri, + ty: Some(ty.into_owned()), + }); + } } } fn run(self, ctx: Context) -> JoinHandle> { tokio::spawn(async move { let Self { + dir, + ck, path, mut json, rewrites, } = self; + let cache: DropMintCache = ctx.cache(dir)?.get().await?; - let rewrites: HashMap<_, _> = rewrites - .into_iter() - .flat_map(|r| iter::from_fn(move || r.pop())) - .map( - |FileRewrite { - source_url, - dest_url, - mime_type, - }| (source_url, (dest_url, mime_type)), - ) - .collect(); - - for file in json.files_mut() { - if let Some((url, _)) = rewrites.get(file) { - *file = PermissiveUrl::Url(url.clone()); - } - } - - let seen_files: HashMap<_, _> = json - .properties - .files - .iter() - .enumerate() - .map(|(i, f)| (f.uri.clone(), i)) - .collect(); - - for (uri, ty) in rewrites.into_values() { - let uri = PermissiveUrl::Url(uri); - if let Some(idx) = seen_files.get(&uri) { - json.properties.files[*idx].ty = Some(ty.into_owned()); - } else { - json.properties.files.push(metadata_json::File { - uri, - ty: Some(ty.into_owned()), - }); - } - } + Self::rewrite_json(&mut json, rewrites); let input = queue_mint_to_drop::Variables { in_: queue_mint_to_drop::QueueMintToDropInput { @@ -534,13 +636,30 @@ impl QueueJsonJob { metadata_json: json.into(), }, }; + let input_ck = Checksum::hash(&input); trace!( "GraphQL input for {path:?}: {}", serde_json::to_string(&input).map_or_else(|e| e.to_string(), |j| j.to_string()) ); - let res = ctx + let cached_mint = cache.get(path.clone(), ck).await?; + + if let Some(mint) = cached_mint { + trace!("Using cached mint {mint:?} for {path:?}"); + + if mint.input_hash != input_ck.to_bytes() { + trace!( + "Hash mismatch for {path:?}: {:032x} cached vs {input_ck:?} new", + u128::from_le_bytes(mint.input_hash.try_into().unwrap_or_default()) + ); + warn!( + "Detected a change in {path:?} since it was uploaded - this will be \ + ignored!" + ); + } + } else { + let res = ctx .client .post(ctx.graphql_endpoint) .json(&QueueMintToDrop::build_query(input)) @@ -557,33 +676,47 @@ impl QueueJsonJob { format!("Error parsing queueMintToDrop mutation response for {path:?}") })?; - trace!("GraphQL response for {path:?}: {res:?}"); + trace!("GraphQL response for {path:?}: {res:?}"); - if let Some(data) = res.data { - Self::format_errors(res.errors, |s| { - warn!("queueMintToDrop mutation for {path:?} returned one or more errors:{s}"); - }); + let collection_mint; + if let Some(data) = res.data { + Self::format_errors(res.errors, (), |s| { + warn!( + "queueMintToDrop mutation for {path:?} returned one or more errors:{s}" + ); + }); - let queue_mint_to_drop::ResponseData { - queue_mint_to_drop: - queue_mint_to_drop::QueueMintToDropQueueMintToDrop { - collection_mint: - queue_mint_to_drop::QueueMintToDropQueueMintToDropCollectionMint { - id, - collection, - }, - }, - } = data; - - info!("Mint successfully queued for {path:?}"); - } else { - let had_errs = Self::format_errors(res.errors, |s| { - error!("queueMintToDrop mutation for {path:?} returned one or more errors:{s}"); - }); + let queue_mint_to_drop::ResponseData { + queue_mint_to_drop: + queue_mint_to_drop::QueueMintToDropQueueMintToDrop { + collection_mint: + queue_mint_to_drop::QueueMintToDropQueueMintToDropCollectionMint { + id, + }, + }, + } = data; + collection_mint = id; + + ctx.stats.queued_mints.fetch_add(1, Ordering::Relaxed); + info!("Mint successfully queued for {path:?}"); + } else { + Self::format_errors(res.errors, Ok(()), |s| { + bail!( + "queueMintToDrop mutation for {path:?} returned one or more errors:{s}" + ) + })?; - if !had_errs { - error!("queueMintToDrop mutation for {path:?} returned no data"); + bail!("queueMintToDrop mutation for {path:?} returned no data"); } + + cache + .set(path, ck, DropMint { + collection_mint: collection_mint.to_bytes_le().into(), + input_hash: input_ck.to_bytes().into(), + }) + .await + .map_err(|e| warn!("{e:?}")) + .ok(); } Ok(()) diff --git a/src/common/metadata_json.rs b/src/common/metadata_json.rs index 0ba3d2f..4fb477b 100644 --- a/src/common/metadata_json.rs +++ b/src/common/metadata_json.rs @@ -73,7 +73,8 @@ impl Properties { files.is_empty() && category.is_none() } - pub fn find_file(&self, url: &U) -> Option<&File> where PermissiveUrl: PartialEq { + pub fn find_file(&self, url: &U) -> Option<&File> + where PermissiveUrl: PartialEq { self.files.iter().find(|f| f.uri == *url) } } diff --git a/src/common/tokio.rs b/src/common/tokio.rs new file mode 100644 index 0000000..50ccde9 --- /dev/null +++ b/src/common/tokio.rs @@ -0,0 +1,8 @@ +use anyhow::{Context, Result}; + +pub fn runtime() -> Result { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .context("Error initializing async runtime") +} diff --git a/src/common/toposort.rs b/src/common/toposort.rs index ed38f4b..0ed4384 100644 --- a/src/common/toposort.rs +++ b/src/common/toposort.rs @@ -1,13 +1,13 @@ use std::{ iter, + num::NonZeroUsize, sync::{ atomic::{self, AtomicUsize}, Arc, - }, num::NonZeroUsize, + }, }; use dispose::{Disposable, Dispose}; -use log::error; use parking_lot::Mutex; pub trait PendingFail { diff --git a/src/common/url_permissive.rs b/src/common/url_permissive.rs index 00e585c..3e1901d 100644 --- a/src/common/url_permissive.rs +++ b/src/common/url_permissive.rs @@ -23,10 +23,12 @@ impl PermissiveUrl { impl From for String { #[inline] - fn from(val: PermissiveUrl) -> Self { match val { - PermissiveUrl::Url(u) => u.into(), - PermissiveUrl::Path(p) => p.to_string_lossy().into_owned(), - }} + fn from(val: PermissiveUrl) -> Self { + match val { + PermissiveUrl::Url(u) => u.into(), + PermissiveUrl::Path(p) => p.to_string_lossy().into_owned(), + } + } } impl FromStr for PermissiveUrl { @@ -39,7 +41,11 @@ impl FromStr for PermissiveUrl { } } -impl PartialEq for PermissiveUrl where Url: PartialEq, PathBuf: PartialEq { +impl PartialEq for PermissiveUrl +where + Url: PartialEq, + PathBuf: PartialEq, +{ fn eq(&self, other: &Rhs) -> bool { match self { PermissiveUrl::Url(u) => u.eq(other), diff --git a/src/main.rs b/src/main.rs index 906cc33..90c4370 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,16 +23,13 @@ mod commands { mod common { pub mod concurrent; pub mod metadata_json; + pub mod tokio; pub mod toposort; pub mod url_permissive; } -use anyhow::{Context, Result}; -use cli::{Opts, Subcommand, UploadSubcommand}; -use config::{Config, ConfigLocation}; - fn main() { - match run() { + match entry::run() { Ok(()) => (), Err(e) => { println!("ERROR: {e:?}"); @@ -41,27 +38,43 @@ fn main() { } } -#[inline] -fn read(config: impl FnOnce(bool) -> Result) -> Result { - config(false)?.load() -} +mod entry { + use anyhow::Result; -fn runtime() -> Result { - tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .context("Error initializing async runtime") -} + use crate::{ + cache::CacheConfig, + cli::{log_color, Opts, Subcommand, UploadSubcommand}, + commands::{config, upload_drop}, + config::{Config, ConfigLocation}, + }; + + #[inline] + fn read(config: impl FnOnce(bool) -> Result) -> Result { + config(false)?.load() + } -fn run() -> Result<()> { - env_logger::init(); - let Opts { config, subcmd } = clap::Parser::parse(); - let config = |w| ConfigLocation::new(config, w); + pub fn run() -> Result<()> { + let Opts { + log_level, + color, + config, + cache, + subcmd, + } = clap::Parser::parse(); - match subcmd { - Subcommand::Config(c) => commands::config::run(&config(true)?, c), - Subcommand::Upload(u) => match u.subcmd { - UploadSubcommand::Drop(d) => commands::upload_drop::run(&read(config)?, d), - }, + env_logger::builder() + .parse_filters(&log_level) + .write_style(log_color(color)) + .init(); + + let config = |w| ConfigLocation::new(config, w); + let cache = CacheConfig::new(cache); + + match subcmd { + Subcommand::Config(c) => config::run(&config(true)?, c), + Subcommand::Upload(u) => match u.subcmd { + UploadSubcommand::Drop(d) => upload_drop::run(&read(config)?, cache, d), + }, + } } } diff --git a/src/proto/asset-upload.proto b/src/proto/asset-upload.proto new file mode 100644 index 0000000..74f052d --- /dev/null +++ b/src/proto/asset-upload.proto @@ -0,0 +1,7 @@ +syntax = 'proto3'; + +package cache.asset_uploads; + +message AssetUpload { + string url = 1; +} diff --git a/src/proto/drop-mint.proto b/src/proto/drop-mint.proto new file mode 100644 index 0000000..64bf7a4 --- /dev/null +++ b/src/proto/drop-mint.proto @@ -0,0 +1,8 @@ +syntax = 'proto3'; + +package cache.drop_mints; + +message DropMint { + bytes collection_mint = 1; + bytes input_hash = 2; +} From 495b4f9f7523e9ad219d56c9bbd8007a9dceed9d Mon Sep 17 00:00:00 2001 From: raykast Date: Sat, 30 Sep 2023 01:42:48 -0700 Subject: [PATCH 4/8] Remove ron and update dependencies. --- Cargo.lock | 134 +++++++++++++++++++++++------------------------------ Cargo.toml | 1 - 2 files changed, 59 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eeb88ab..956c8c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,9 +43,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.5.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" dependencies = [ "anstyle", "anstyle-parse", @@ -57,15 +57,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" dependencies = [ "utf8parse", ] @@ -81,9 +81,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", "windows-sys", @@ -138,9 +138,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.3" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "bindgen" @@ -174,15 +174,12 @@ name = "bitflags" version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" -dependencies = [ - "serde", -] [[package]] name = "brotli" -version = "3.3.4" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -191,9 +188,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.3.4" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" +checksum = "da74e2b81409b1b743f8f0c62cc6254afefb8b8e50bbfe3735550f7aeefa3448" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -201,9 +198,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "byteorder" @@ -213,9 +210,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "bzip2-sys" @@ -277,9 +274,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.4" +version = "4.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" +checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" dependencies = [ "clap_builder", "clap_derive", @@ -287,9 +284,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.4" +version = "4.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" +checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" dependencies = [ "anstream", "anstyle", @@ -445,7 +442,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.0", + "hashbrown 0.14.1", "lock_api", "once_cell", "parking_lot_core", @@ -529,9 +526,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" dependencies = [ "errno-dragonfly", "libc", @@ -560,9 +557,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "fixedbitset" @@ -779,9 +776,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" [[package]] name = "heck" @@ -791,9 +788,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "holaplex-hub-cli" @@ -817,7 +814,6 @@ dependencies = [ "prost-build", "reqwest", "rocksdb", - "ron", "rpassword", "rustyline", "serde", @@ -937,12 +933,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad227c3af19d4914570ad36d30409928b75967c298feb9ea1969db3a610bb14e" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.1", ] [[package]] @@ -1018,9 +1014,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" [[package]] name = "libloading" @@ -1205,9 +1201,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" dependencies = [ "memchr", ] @@ -1252,9 +1248,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.92" +version = "0.9.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db7e971c2c2bba161b2d2fdf37080177eff520b3bc044787c7f1f5f9e78d869b" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" dependencies = [ "cc", "libc", @@ -1310,7 +1306,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.0.1", + "indexmap 2.0.2", ] [[package]] @@ -1546,18 +1542,6 @@ dependencies = [ "librocksdb-sys", ] -[[package]] -name = "ron" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" -dependencies = [ - "base64", - "bitflags 2.4.0", - "serde", - "serde_derive", -] - [[package]] name = "rpassword" version = "7.2.0" @@ -1593,9 +1577,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.13" +version = "0.38.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" +checksum = "d2f9da0cbd88f9f09e7814e388301c8414c51c62aa6ce1e4b5c551d49d96e531" dependencies = [ "bitflags 2.4.0", "errno", @@ -1694,7 +1678,7 @@ version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ - "indexmap 2.0.1", + "indexmap 2.0.2", "itoa", "ryu", "serde", @@ -1754,9 +1738,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", "windows-sys", @@ -1830,18 +1814,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.48" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.48" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" dependencies = [ "proc-macro2", "quote", @@ -1876,7 +1860,7 @@ dependencies = [ "num_cpus", "parking_lot", "pin-project-lite", - "socket2 0.5.3", + "socket2 0.5.4", "windows-sys", ] @@ -1892,9 +1876,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" dependencies = [ "bytes", "futures-core", @@ -1906,11 +1890,11 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c226a7bba6d859b63c92c4b4fe69c5b6b72d0cb897dbc8e6012298e6154cb56e" +checksum = "1bc1433177506450fe920e46a4f9812d0c211f5dd556da10e731a0a3dfa151f0" dependencies = [ - "indexmap 2.0.1", + "indexmap 2.0.2", "serde", "serde_spanned", "toml_datetime", @@ -1928,11 +1912,11 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ff63e60a958cefbb518ae1fd6566af80d9d4be430a33f3723dfc47d1d411d95" +checksum = "ca676d9ba1a322c1b64eb8045a5ec5c0cfb0c9d08e15e9ff622589ad5221c8fe" dependencies = [ - "indexmap 2.0.1", + "indexmap 2.0.2", "serde", "serde_spanned", "toml_datetime", diff --git a/Cargo.toml b/Cargo.toml index 97aca42..3353465 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,6 @@ parking_lot = "0.12.1" prost = "0.12.1" reqwest = { version = "0.11.20", features = ["brotli", "deflate", "gzip", "json", "multipart", "stream"] } rocksdb = { version = "0.21.0", features = ["io-uring", "multi-threaded-cf", "lz4", "zlib"], default-features = false } -ron = "0.8.1" rpassword = "7.2.0" rustyline = { version = "12.0.0", default-features = false } serde = { version = "1.0.188", features = ["derive"] } From 9910ea155dfa0521e64b5e2cbbbe75eeb38363d5 Mon Sep 17 00:00:00 2001 From: raykast Date: Mon, 2 Oct 2023 10:00:16 -0700 Subject: [PATCH 5/8] Add --get flag to config, warn about upload errors --- .rustfmt.toml | 78 +++++++++++++++++++++++++++++++++++++ src/cli.rs | 18 +++++++-- src/commands/config.rs | 23 ++++++++++- src/commands/upload_drop.rs | 15 ++++++- 4 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 .rustfmt.toml diff --git a/.rustfmt.toml b/.rustfmt.toml new file mode 100644 index 0000000..51cab9a --- /dev/null +++ b/.rustfmt.toml @@ -0,0 +1,78 @@ +max_width = 100 +hard_tabs = false +tab_spaces = 4 +newline_style = "Unix" +indent_style = "Block" +use_small_heuristics = "Default" +fn_call_width = 60 +attr_fn_like_width = 70 +struct_lit_width = 18 +struct_variant_width = 35 +array_width = 60 +chain_width = 60 +single_line_if_else_max_width = 50 +single_line_let_else_max_width = 50 +wrap_comments = false +format_code_in_doc_comments = false +doc_comment_code_block_width = 100 +comment_width = 80 +normalize_comments = false +normalize_doc_attributes = true +format_strings = true +format_macro_matchers = true +format_macro_bodies = true +skip_macro_invocations = [] +hex_literal_case = "Lower" +empty_item_single_line = true +struct_lit_single_line = true +fn_single_line = true +where_single_line = true +imports_indent = "Block" +imports_layout = "Mixed" +imports_granularity = "Crate" +group_imports = "StdExternalCrate" +reorder_imports = true +reorder_modules = true +reorder_impl_items = true +type_punctuation_density = "Wide" +space_before_colon = false +space_after_colon = true +spaces_around_ranges = false +binop_separator = "Front" +remove_nested_parens = true +combine_control_expr = true +short_array_element_width_threshold = 10 +overflow_delimited_expr = true +struct_field_align_threshold = 0 +enum_discrim_align_threshold = 0 +match_arm_blocks = true +match_arm_leading_pipes = "Never" +force_multiline_blocks = false +fn_params_layout = "Tall" +brace_style = "SameLineWhere" +control_brace_style = "AlwaysSameLine" +trailing_semicolon = true +trailing_comma = "Vertical" +match_block_trailing_comma = true +blank_lines_upper_bound = 1 +blank_lines_lower_bound = 0 +edition = "2021" +version = "Two" +inline_attribute_width = 0 +format_generated_files = false +merge_derives = true +use_try_shorthand = true +use_field_init_shorthand = true +force_explicit_abi = true +condense_wildcard_suffixes = true +color = "Auto" +required_version = "1.6.0" +unstable_features = false +disable_all_formatting = false +skip_children = false +hide_parse_errors = false +error_on_line_overflow = false +error_on_unformatted = false +ignore = [] +emit_mode = "Files" +make_backup = false diff --git a/src/cli.rs b/src/cli.rs index 35c938a..312d4c5 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -111,22 +111,34 @@ pub enum ConfigSubcommand { /// Options for hub config graphql-endpoint #[derive(clap::Args)] pub struct ConfigGraphqlEndpoint { + /// Print the current GraphQL API endpoint + #[arg(long)] + pub get: bool, + /// Specify the GraphQL API endpoint, required if not using a terminal, /// otherwise STDIN is used as the default - #[arg(required = !std::io::stdin().is_terminal())] + #[arg(required = !std::io::stdin().is_terminal(), conflicts_with("get"))] pub endpoint: Option, } /// Options for hub config hub-endpoint #[derive(clap::Args)] pub struct ConfigHubEndpoint { + /// Print the current root Hub endpoint + #[arg(long)] + pub get: bool, + /// Reset the endpoint override and infer it from the GraphQL API endpoint - #[arg(short, long)] + #[arg(short, long, conflicts_with("get"))] pub reset: bool, /// Override the root Hub endpoint, required if not using a terminal, /// otherwise STDIN is used as the default - #[arg(required = !std::io::stdin().is_terminal(), conflicts_with("reset"))] + #[arg( + required = !std::io::stdin().is_terminal(), + conflicts_with("get"), + conflicts_with("reset"), + )] pub endpoint: Option, } diff --git a/src/commands/config.rs b/src/commands/config.rs index 99179dc..1a8b0f4 100644 --- a/src/commands/config.rs +++ b/src/commands/config.rs @@ -55,7 +55,12 @@ where T::Err: fmt::Display { } fn graphql_endpoint(config: &mut Config, endpoint: ConfigGraphqlEndpoint) -> Result<()> { - let ConfigGraphqlEndpoint { endpoint } = endpoint; + let ConfigGraphqlEndpoint { get, endpoint } = endpoint; + + if get { + println!("{}", config.graphql_endpoint()); + return Ok(()); + } let endpoint = if let Some(e) = endpoint { e.parse().context("Invalid endpoint URL")? @@ -69,7 +74,21 @@ fn graphql_endpoint(config: &mut Config, endpoint: ConfigGraphqlEndpoint) -> Res } fn hub_endpoint(config: &mut Config, endpoint: ConfigHubEndpoint) -> Result<()> { - let ConfigHubEndpoint { reset, endpoint } = endpoint; + let ConfigHubEndpoint { + get, + reset, + endpoint, + } = endpoint; + + if get { + println!( + "{}", + config + .hub_endpoint() + .context("Error computing root Hub endpoint from GraphQL endpoint")? + ); + return Ok(()); + } let endpoint = if reset { None diff --git a/src/commands/upload_drop.rs b/src/commands/upload_drop.rs index 7e7900b..4884e97 100644 --- a/src/commands/upload_drop.rs +++ b/src/commands/upload_drop.rs @@ -190,7 +190,7 @@ pub fn run(config: &Config, cache: CacheConfig, args: UploadDrop) -> Result<()> .context("Error seeding initial job queue")?; } - info!("Scanning {} JSON files(s)...", rx.len()); + info!("Scanning {} JSON file(s)...", rx.len()); let ctx = Context { cache_config: cache, @@ -208,10 +208,14 @@ pub fn run(config: &Config, cache: CacheConfig, args: UploadDrop) -> Result<()> stats: Arc::default(), }; + let mut any_errs = false; runtime()?.block_on(async move { let res = concurrent::try_run( jobs.into(), - |e| error!("{e:?}"), + |e| { + error!("{e:?}"); + any_errs = true; + }, || { let job = match rx.try_recv() { Ok(j) => Some(j), @@ -244,6 +248,13 @@ pub fn run(config: &Config, cache: CacheConfig, args: UploadDrop) -> Result<()> mints = queued_mints.load(std::sync::atomic::Ordering::Relaxed) ); + if any_errs { + warn!( + "Some files were skipped due to errors. They will be processed next time this \ + command is run." + ); + } + res })?; From ff4dc0f00e6fa23eb3af6924f324b625dccb50ef Mon Sep 17 00:00:00 2001 From: raykast Date: Mon, 2 Oct 2023 11:49:52 -0700 Subject: [PATCH 6/8] Add friendlier 401 and 429 error messages. --- src/commands/upload_drop.rs | 12 +++--------- src/common/reqwest.rs | 33 +++++++++++++++++++++++++++++++++ src/main.rs | 1 + 3 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 src/common/reqwest.rs diff --git a/src/commands/upload_drop.rs b/src/commands/upload_drop.rs index 4884e97..01614be 100644 --- a/src/commands/upload_drop.rs +++ b/src/commands/upload_drop.rs @@ -37,7 +37,7 @@ use crate::{ metadata_json::{self, MetadataJson}, tokio::runtime, toposort::{Dependencies, Dependency, PendingFail}, - url_permissive::PermissiveUrl, + url_permissive::PermissiveUrl, reqwest::ResponseExt, }, config::Config, }; @@ -507,9 +507,7 @@ impl UploadAssetJob { ) .send() .await - .with_context(|| format!("Error sending POST request for {path:?}"))? - .error_for_status() - .with_context(|| format!("POST request for {path:?} returned an error"))? + .error_for_hub_status(|| format!("POST request for {path:?}"))? .json::() .await .with_context(|| { @@ -676,11 +674,7 @@ impl QueueJsonJob { .json(&QueueMintToDrop::build_query(input)) .send() .await - .with_context(|| format!("Error sending queueMintToDrop mutation for {path:?}"))? - .error_for_status() - .with_context(|| { - format!("queueMintToDrop mutation for {path:?} returned an error") - })? + .error_for_hub_status(|| format!("queueMintToDrop mutation for {path:?}"))? .json::::ResponseData>>() .await .with_context(|| { diff --git a/src/common/reqwest.rs b/src/common/reqwest.rs new file mode 100644 index 0000000..1777e4c --- /dev/null +++ b/src/common/reqwest.rs @@ -0,0 +1,33 @@ +use std::fmt; + +use anyhow::{Context, Result}; +use reqwest::{Response, StatusCode}; + +pub trait ResponseExt: Sized { + fn error_for_hub_status D, D: fmt::Display>(self, name: F) -> Result; +} + +impl ResponseExt for Result +where Self: anyhow::Context +{ + fn error_for_hub_status D, D: fmt::Display>(self, name: F) -> Result { + let this = match self { + Ok(r) => r, + Err(e) => return Err(e).context(format!("Error sending {}", name())), + }; + let status = this.status(); + + this.error_for_status().with_context(|| match status { + StatusCode::UNAUTHORIZED => format!( + "{} returned Unauthorized, you may need to update your API token with `hub config \ + token`", + name() + ), + StatusCode::TOO_MANY_REQUESTS => format!( + "{} exceeded the rate limit, try rerunning the operation later", + name() + ), + _ => format!("{} returned an error", name()), + }) + } +} diff --git a/src/main.rs b/src/main.rs index 90c4370..697c47d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,6 +23,7 @@ mod commands { mod common { pub mod concurrent; pub mod metadata_json; + pub mod reqwest; pub mod tokio; pub mod toposort; pub mod url_permissive; From 6e331f6350e48d3be69a35330d9a6da07e46643c Mon Sep 17 00:00:00 2001 From: raykast Date: Mon, 2 Oct 2023 12:24:35 -0700 Subject: [PATCH 7/8] Change include resolution strategy for upload drop --- src/commands/upload_drop.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/commands/upload_drop.rs b/src/commands/upload_drop.rs index 01614be..6576216 100644 --- a/src/commands/upload_drop.rs +++ b/src/commands/upload_drop.rs @@ -9,7 +9,7 @@ use std::{ path::{Path, PathBuf}, sync::{ atomic::{AtomicUsize, Ordering}, - Arc, OnceLock, + Arc, }, }; @@ -35,9 +35,10 @@ use crate::{ common::{ concurrent, metadata_json::{self, MetadataJson}, + reqwest::ResponseExt, tokio::runtime, toposort::{Dependencies, Dependency, PendingFail}, - url_permissive::PermissiveUrl, reqwest::ResponseExt, + url_permissive::PermissiveUrl, }, config::Config, }; @@ -284,13 +285,12 @@ impl Context { Ok((*entry).clone()) } - fn resolve_file io::Result, T>( - &self, + fn resolve_file<'a, F: FnMut(&PathBuf) -> io::Result, T>( + &'a self, + dir: &'a PathBuf, mut open: F, ) -> Result, (&PathBuf, io::Error)> { - static NIL: OnceLock = OnceLock::new(); - - [NIL.get_or_init(PathBuf::new)] + [dir] .into_iter() .chain(&*self.include_dirs) .find_map(|d| { @@ -336,7 +336,7 @@ impl ScanJsonJob { trace!("{url:?} -> {:?}", url.to_file_path()); let path = url.to_file_path()?; let (include_dir, file) = match ctx - .resolve_file(|d| File::open(d.join(&path))) + .resolve_file(&dir, |d| File::open(d.join(&path))) .map_err(|(d, e)| { anyhow::Error::new(e) .context(format!("Error opening {:?}", d.join(&path))) From 8540ac1ac2140bdf7adcdc2269d1ccd52fb7ccc7 Mon Sep 17 00:00:00 2001 From: raykast Date: Mon, 2 Oct 2023 12:25:09 -0700 Subject: [PATCH 8/8] Add README.md. --- README.md | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 87 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d5587ca..913a0ee 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,87 @@ -# hub-cli -Command line tool for HUB +# `hub-cli` +A command-line tool for interfacing with Holaplex Hub + +--- + +`hub-cli` provides the `hub` command-line utility for interfacing with Hub from +shell scripts or directly from a terminal. + +## Installation and Setup + + + +To install `hub`, the only prerequisite you'll need is a Rust toolchain with +Cargo. + +To install `hub` from Git, simply check out the repo with `git clone`, and then +install it with the `--path` flag for `cargo install`: + +```sh +$ git clone https://github.com/holaplex/hub-cli +$ cargo install --path hub-cli +``` + +### Config locations + +By default, `hub` creates a configuration file named `config.toml` in +a dedicated subdirectory of the current user's configuration directory. All +commands will read or write to this file. However, if the current directory in +which `hub` is run contains a file named `.hub-config.toml`, it will read and +write to that file instead. Additionally, the user can override the current +config location by passing `-C path/to/my-config.toml` to any `hub` command. + +## Usage + +For additional help with the command-line interface, you can always run: + +```sh +$ hub help +``` + +This will print info about available commands and global options. For help with +a specific command, run: + +```sh +$ hub help +``` + +### `config` + +The `config` command allows getting and setting various configuration options +of the [current config](#config-file): + +```sh +$ hub config path # Print the path of the current config +$ hub config graphql-endpoint # Set the API endpoint to use +$ hub config token # Set the API token to use + +$ hub config graphql-endpoint --get # Print the current API endpoint +``` + +### `upload drop` + +This command populates an open drop by reading Metaplex metadata files and +associated asset files from a local directory: + +```sh +$ hub upload drop --drop +``` + +The `DROP_ID` parameter accepts the UUID of an open drop created with Hub, and +`INPUT_DIR` takes one or more directories containing JSON metadata files. All +assets specified using local paths in metadata files will be first uploaded to +the web using Hub's permaweb upload endpoint, and their permanent URL will be +used in place of a filesystem path. `hub` searches for asset paths relative to +the folder containing the JSON file referencing them; if any assets are in +separate folders, each folder can be specified with the `-I` flag: + +```sh +$ hub upload drop --drop 00000000-0000-0000-0000-000000000000 \ + -I foo/assets1 \ + -I foo/assets2 \ + foo/json +``` + +When include paths are specified, `hub` first searches the current JSON +directory, then searches each include directory in the order they were +specified, stopping as soon as it finds a match.