diff --git a/.cargo/config.toml b/.cargo/config.toml index 35049cbc..71020967 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,2 @@ [alias] -xtask = "run --package xtask --" +drainpipe = "run --package drainpipe-cli --" diff --git a/Cargo.lock b/Cargo.lock index af031242..37dbccd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,18 +4,27 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] [[package]] name = "android-tzdata" @@ -32,11 +41,69 @@ dependencies = [ "libc", ] +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" [[package]] name = "atomic-waker" @@ -44,25 +111,70 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atrium-api" +version = "0.24.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee68ddf7cde9eb121eed3a28b138f6a9b4c4a90ab0c5c2e38bc2817af0b06da3" +dependencies = [ + "atrium-xrpc", + "chrono", + "http", + "ipld-core", + "langtag", + "regex", + "serde", + "serde_bytes", + "serde_json", + "thiserror 1.0.69", + "tokio", + "trait-variant", +] + +[[package]] +name = "atrium-xrpc" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "737eea1de2eb174bbfe720619cb25a22c30b9640ae0d3b78386cedf007712963" +dependencies = [ + "http", + "serde", + "serde_html_form", + "serde_json", + "thiserror 1.0.69", + "trait-variant", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "backtrace" -version = "0.3.72" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets", ] [[package]] @@ -77,6 +189,15 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -85,9 +206,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "block-buffer" @@ -112,25 +233,21 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] -name = "cbor4ii" -version = "0.2.14" +name = "cc" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" dependencies = [ - "serde", + "jobserver", + "libc", + "shlex", ] -[[package]] -name = "cc" -version = "1.0.99" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" - [[package]] name = "cfg-if" version = "1.0.0" @@ -149,49 +266,43 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.5", + "windows-targets", ] [[package]] -name = "ciborium" -version = "0.2.2" +name = "cid" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +checksum = "3147d8272e8fa0ccd29ce51194dd98f79ddfb8191ba9e3409884e751798acf3a" dependencies = [ - "ciborium-io", - "ciborium-ll", + "core2", + "multibase", + "multihash", "serde", + "serde_bytes", + "unsigned-varint", ] [[package]] -name = "ciborium-io" -version = "0.2.2" +name = "clap" +version = "2.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ - "ciborium-io", - "half", + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim", + "textwrap", + "unicode-width", + "vec_map", ] [[package]] -name = "cid" -version = "0.11.1" +name = "colorchoice" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3147d8272e8fa0ccd29ce51194dd98f79ddfb8191ba9e3409884e751798acf3a" -dependencies = [ - "core2", - "multibase", - "multihash", - "serde", - "serde_bytes", - "unsigned-varint 0.8.0", -] +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "core-foundation" @@ -205,9 +316,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "core2" @@ -220,62 +331,45 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" dependencies = [ "libc", ] [[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - -[[package]] -name = "crypto-common" -version = "0.1.6" +name = "crc32fast" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ - "generic-array", - "typenum", + "cfg-if", ] [[package]] -name = "darling" -version = "0.20.9" +name = "crossbeam-epoch" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "darling_core", - "darling_macro", + "crossbeam-utils", ] [[package]] -name = "darling_core" -version = "0.20.9" +name = "crossbeam-utils" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.66", -] +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] -name = "darling_macro" -version = "0.20.9" +name = "crypto-common" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "darling_core", - "quote", - "syn 2.0.66", + "generic-array", + "typenum", ] [[package]] @@ -305,72 +399,24 @@ dependencies = [ ] [[package]] -name = "debug-ignore" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe7ed1d93f4553003e20b629abe9085e1e81b1429520f897f8f8860bc6dfc21" - -[[package]] -name = "deranged" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" -dependencies = [ - "powerfmt", -] - -[[package]] -name = "diesel" -version = "2.2.0" +name = "digest" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35b696af9ff4c0d2a507db2c5faafa8aa0205e297e5f11e203a24226d5355e7a" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "diesel_derives", - "libsqlite3-sys", - "time", + "block-buffer", + "crypto-common", ] [[package]] -name = "diesel_derives" -version = "2.2.0" +name = "displaydoc" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6fdd83d5947068817016e939596d246e5367279453f2a3433287894f2f2996" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ - "diesel_table_macro_syntax", - "dsl_auto_type", "proc-macro2", "quote", - "syn 2.0.66", -] - -[[package]] -name = "diesel_migrations" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a73ce704bad4231f001bff3314d91dce4aba0770cee8b233991859abc15c1f6" -dependencies = [ - "diesel", - "migrations_internals", - "migrations_macros", -] - -[[package]] -name = "diesel_table_macro_syntax" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25" -dependencies = [ - "syn 2.0.66", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", + "syn 2.0.87", ] [[package]] @@ -381,57 +427,82 @@ checksum = "ccff2060df2e1a42732ab1e7af8c5f930ed3d0255aaadfae66e032781e8d271a" [[package]] name = "drainpipe" -version = "0.1.0" +version = "0.2.0" dependencies = [ "anyhow", + "atrium-api", + "bincode", "chrono", - "ciborium", - "cid", - "debug-ignore", - "diesel", - "diesel_migrations", "dotenv-flow", - "futures", - "libsqlite3-sys", + "drainpipe-store", + "env_logger", + "flume", + "futures-util", + "log", "reqwest", "serde", - "serde_bytes", - "serde_ipld_dagcbor", + "serde_json", + "sled", + "thiserror 2.0.3", "tokio", "tokio-metrics", "tokio-tungstenite", "url", + "zstd", ] [[package]] -name = "dsl_auto_type" +name = "drainpipe-cli" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab32c18ea6760d951659768a3e35ea72fc1ba0916d665a88dfe048b2a41e543f" dependencies = [ - "darling", - "either", - "heck", - "proc-macro2", - "quote", - "syn 2.0.66", + "dotenv-flow", + "drainpipe-store", + "structopt", ] [[package]] -name = "either" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" +name = "drainpipe-store" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "log", + "serde", + "sled", +] [[package]] name = "encoding_rs" -version = "0.8.34" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", ] +[[package]] +name = "env_filter" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -450,9 +521,21 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "nanorand", + "spin", +] [[package]] name = "fnv" @@ -485,94 +568,77 @@ dependencies = [ ] [[package]] -name = "futures" -version = "0.3.30" +name = "fs2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", + "libc", + "winapi", ] [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", - "futures-sink", ] [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" - -[[package]] -name = "futures-executor" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.87", ] [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ - "futures-channel", "futures-core", - "futures-io", "futures-macro", "futures-sink", "futures-task", - "memchr", "pin-project-lite", "pin-utils", "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -590,21 +656,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "h2" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ "atomic-waker", "bytes", @@ -620,26 +688,28 @@ dependencies = [ ] [[package]] -name = "half" -version = "2.4.1" +name = "hashbrown" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" -dependencies = [ - "cfg-if", - "crunchy", -] +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" [[package]] -name = "hashbrown" -version = "0.14.5" +name = "heck" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] [[package]] -name = "heck" -version = "0.5.0" +name = "hermit-abi" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] [[package]] name = "hermit-abi" @@ -660,9 +730,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http", @@ -670,12 +740,12 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", - "futures-core", + "futures-util", "http", "http-body", "pin-project-lite", @@ -683,15 +753,21 @@ dependencies = [ [[package]] name = "httparse" -version = "1.8.0" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.3.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" dependencies = [ "bytes", "futures-channel", @@ -704,7 +780,24 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", - "want", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +dependencies = [ + "futures-util", + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", ] [[package]] @@ -725,9 +818,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.5" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", @@ -738,16 +831,15 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower", "tower-service", "tracing", ] [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -767,31 +859,163 @@ dependencies = [ ] [[package]] -name = "ident_case" -version = "1.0.1" +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "icu_normalizer", + "icu_properties", ] [[package]] name = "indexmap" -version = "2.2.6" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", "hashbrown", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "ipld-core" version = "0.4.1" @@ -805,9 +1029,15 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.9.0" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itoa" @@ -816,37 +1046,56 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] -name = "js-sys" -version = "0.3.69" +name = "jobserver" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ - "wasm-bindgen", + "libc", ] [[package]] -name = "libc" -version = "0.2.155" +name = "js-sys" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +dependencies = [ + "wasm-bindgen", +] [[package]] -name = "libsqlite3-sys" -version = "0.28.0" +name = "langtag" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" +checksum = "ed60c85f254d6ae8450cec15eedd921efbc4d1bdf6fcf6202b9a58b403f6f805" dependencies = [ - "cc", - "pkg-config", - "vcpkg", + "serde", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.162" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" + [[package]] name = "linux-raw-sys" version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "lock_api" version = "0.4.12" @@ -859,36 +1108,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "memchr" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" - -[[package]] -name = "migrations_internals" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd01039851e82f8799046eabbb354056283fb265c8ec0996af940f4e85a380ff" -dependencies = [ - "serde", - "toml", -] - -[[package]] -name = "migrations_macros" -version = "2.2.0" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb161cc72176cb37aa47f1fc520d3ef02263d67d661f44f05d05a079e1237fd" -dependencies = [ - "migrations_internals", - "proc-macro2", - "quote", -] +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "mime" @@ -898,22 +1126,23 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.7.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] name = "mio" -version = "0.8.11" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ + "hermit-abi 0.3.9", "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -929,13 +1158,22 @@ dependencies = [ [[package]] name = "multihash" -version = "0.19.1" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076d548d76a0e2a0d4ab471d0b1c36c577786dfc4471242035d97a12a735c492" +checksum = "cc41f430805af9d1cf4adae4ed2149c759b877b01d909a1f40256188d09345d2" dependencies = [ "core2", "serde", - "unsigned-varint 0.7.2", + "unsigned-varint", +] + +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom", ] [[package]] @@ -955,12 +1193,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "num-conv" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" - [[package]] name = "num-traits" version = "0.2.19" @@ -970,38 +1202,28 @@ dependencies = [ "autocfg", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "object" -version = "0.35.0" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8ec7ab813848ba4522158d5517a6093db1ded27575b070f4177b8d12b41db5e" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "openssl" -version = "0.10.64" +version = "0.10.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cfg-if", "foreign-types", "libc", @@ -1018,7 +1240,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.87", ] [[package]] @@ -1029,9 +1251,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.102" +version = "0.9.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" dependencies = [ "cc", "libc", @@ -1039,6 +1261,17 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.3" @@ -1046,53 +1279,47 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.10", ] [[package]] name = "parking_lot_core" -version = "0.9.10" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" dependencies = [ "cfg-if", + "instant", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "smallvec", - "windows-targets 0.52.5", + "winapi", ] [[package]] -name = "percent-encoding" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" - -[[package]] -name = "pin-project" -version = "1.1.5" +name = "parking_lot_core" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ - "pin-project-internal", + "cfg-if", + "libc", + "redox_syscall 0.5.7", + "smallvec", + "windows-targets", ] [[package]] -name = "pin-project-internal" -version = "1.1.5" +name = "percent-encoding" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -1102,36 +1329,57 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] -name = "powerfmt" -version = "0.2.0" +name = "ppv-lite86" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] -name = "ppv-lite86" -version = "0.2.17" +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] [[package]] name = "proc-macro2" -version = "1.0.85" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -1168,18 +1416,56 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.1" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +dependencies = [ + "bitflags 2.6.0", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ - "bitflags 2.5.0", + "aho-corasick", + "memchr", + "regex-syntax", ] +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + [[package]] name = "reqwest" -version = "0.12.4" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64", "bytes", @@ -1191,6 +1477,7 @@ dependencies = [ "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-tls", "hyper-util", "ipnet", @@ -1214,7 +1501,22 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg", + "windows-registry", +] + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", ] [[package]] @@ -1225,32 +1527,55 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "99e4ea3e1cdc4b559b8e5650f9c8e5998e3e5c1343b4eaf034565f32318d63c0" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.23.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f1a745511c54ba6d4465e8d5dfbd81b45791756de28d4981af70d6dca128f1e" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + [[package]] name = "rustls-pemfile" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" + +[[package]] +name = "rustls-webpki" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] [[package]] name = "ryu" @@ -1260,11 +1585,11 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1275,11 +1600,11 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "security-framework" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "core-foundation", "core-foundation-sys", "libc", @@ -1288,9 +1613,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" dependencies = [ "core-foundation-sys", "libc", @@ -1298,65 +1623,58 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.203" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734" +checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.87", ] [[package]] -name = "serde_ipld_dagcbor" -version = "0.6.1" +name = "serde_html_form" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded35fbe4ab8fdec1f1d14b4daff2206b1eada4d6e708cb451d464d2d965f493" +checksum = "8de514ef58196f1fc96dcaef80fe6170a1ce6215df9687a93fe8300e773fefc5" dependencies = [ - "cbor4ii", - "ipld-core", - "scopeguard", + "form_urlencoded", + "indexmap", + "itoa", + "ryu", "serde", ] [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] -[[package]] -name = "serde_spanned" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" -dependencies = [ - "serde", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1380,6 +1698,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -1398,6 +1722,22 @@ dependencies = [ "autocfg", ] +[[package]] +name = "sled" +version = "0.34.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot 0.11.2", +] + [[package]] name = "smallvec" version = "1.13.2" @@ -1414,11 +1754,56 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strsim" -version = "0.11.1" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "structopt" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "subtle" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" @@ -1433,9 +1818,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.66" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -1444,26 +1829,40 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] [[package]] name = "system-configuration" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.6.0", "core-foundation", "system-configuration-sys", ] [[package]] name = "system-configuration-sys" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" dependencies = [ "core-foundation-sys", "libc", @@ -1471,110 +1870,103 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.10.1" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", "fastrand", + "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] -name = "thiserror" -version = "1.0.61" +name = "textwrap" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" dependencies = [ - "thiserror-impl", + "unicode-width", ] [[package]] -name = "thiserror-impl" -version = "1.0.61" +name = "thiserror" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", + "thiserror-impl 1.0.69", ] [[package]] -name = "time" -version = "0.3.36" +name = "thiserror" +version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" dependencies = [ - "deranged", - "itoa", - "num-conv", - "powerfmt", - "serde", - "time-core", - "time-macros", + "thiserror-impl 2.0.3", ] [[package]] -name = "time-core" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" - -[[package]] -name = "time-macros" -version = "0.2.18" +name = "thiserror-impl" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ - "num-conv", - "time-core", + "proc-macro2", + "quote", + "syn 2.0.87", ] [[package]] -name = "tinyvec" -version = "1.6.0" +name = "thiserror-impl" +version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" dependencies = [ - "tinyvec_macros", + "proc-macro2", + "quote", + "syn 2.0.87", ] [[package]] -name = "tinyvec_macros" -version = "0.1.1" +name = "tinystr" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] [[package]] name = "tokio" -version = "1.38.0" +version = "1.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" +checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", - "parking_lot", + "parking_lot 0.12.3", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.87", ] [[package]] @@ -1599,11 +1991,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +dependencies = [ + "rustls", + "rustls-pki-types", + "tokio", +] + [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", @@ -1612,9 +2015,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "becd34a233e7e31a3dbf7c7241b38320f57393dcae8e7324b0167d21b8e320b0" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" dependencies = [ "futures-util", "log", @@ -1626,9 +2029,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", @@ -1637,66 +2040,11 @@ dependencies = [ "tokio", ] -[[package]] -name = "toml" -version = "0.8.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.22.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" -dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -1717,6 +2065,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -1725,9 +2084,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e2ce1e47ed2994fd43b04c8f618008d4cabdd5ee34027cf14f9d918edd9c8" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" dependencies = [ "byteorder", "bytes", @@ -1738,7 +2097,7 @@ dependencies = [ "native-tls", "rand", "sha1", - "thiserror", + "thiserror 1.0.69", "utf-8", ] @@ -1748,32 +2107,23 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" -[[package]] -name = "unicode-bidi" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" - [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] -name = "unicode-normalization" -version = "0.1.23" +name = "unicode-segmentation" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" -dependencies = [ - "tinyvec", -] +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] -name = "unsigned-varint" -version = "0.7.2" +name = "unicode-width" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unsigned-varint" @@ -1781,11 +2131,17 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" -version = "2.5.0" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", "idna", @@ -1798,17 +2154,41 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "vcpkg" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "want" @@ -1827,34 +2207,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.87", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" dependencies = [ "cfg-if", "js-sys", @@ -1864,9 +2245,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1874,49 +2255,92 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" dependencies = [ "js-sys", "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", + "windows-targets", ] [[package]] -name = "windows-sys" -version = "0.48.0" +name = "windows-registry" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ - "windows-targets 0.48.5", + "windows-result", + "windows-targets", ] [[package]] @@ -1925,145 +2349,212 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets", ] [[package]] -name = "windows-targets" -version = "0.48.5" +name = "windows-sys" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows-targets", ] [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.5" +name = "windows_aarch64_msvc" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" +name = "windows_i686_gnu" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] -name = "windows_aarch64_msvc" -version = "0.52.5" +name = "windows_i686_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] -name = "windows_i686_gnu" -version = "0.48.5" +name = "windows_i686_msvc" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] -name = "windows_i686_gnu" -version = "0.52.5" +name = "windows_x86_64_gnu" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] -name = "windows_i686_gnullvm" -version = "0.52.5" +name = "windows_x86_64_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] -name = "windows_i686_msvc" -version = "0.48.5" +name = "windows_x86_64_msvc" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "windows_i686_msvc" -version = "0.52.5" +name = "write16" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" [[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" +name = "writeable" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" [[package]] -name = "windows_x86_64_gnu" -version = "0.52.5" +name = "yoke" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] [[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" +name = "yoke-derive" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", +] [[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.5" +name = "zerocopy" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] [[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" +name = "zerocopy-derive" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] [[package]] -name = "windows_x86_64_msvc" -version = "0.52.5" +name = "zerofrom" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] [[package]] -name = "winnow" -version = "0.6.13" +name = "zerofrom-derive" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ - "memchr", + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", ] [[package]] -name = "winreg" -version = "0.52.0" +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" dependencies = [ - "cfg-if", - "windows-sys 0.48.0", + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.13+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +dependencies = [ + "cc", + "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 6690d9f2..83b2ed70 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,7 @@ [workspace] -members = ["packages-rs/drainpipe"] +members = [ + "packages-rs/drainpipe", + "packages-rs/drainpipe-cli", + "packages-rs/drainpipe-store", +] resolver = "2" diff --git a/packages-rs/drainpipe-cli/Cargo.toml b/packages-rs/drainpipe-cli/Cargo.toml new file mode 100644 index 00000000..498ff4f3 --- /dev/null +++ b/packages-rs/drainpipe-cli/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "drainpipe-cli" +version = "0.1.0" +edition = "2021" + +[dependencies] +structopt = "0.3.26" +drainpipe-store = { path = "../drainpipe-store" } +dotenv-flow = "0.16.2" diff --git a/packages-rs/drainpipe-cli/README.md b/packages-rs/drainpipe-cli/README.md new file mode 100644 index 00000000..055450a1 --- /dev/null +++ b/packages-rs/drainpipe-cli/README.md @@ -0,0 +1 @@ +CLI to manage drainpipe storage in development. diff --git a/packages-rs/drainpipe-cli/src/main.rs b/packages-rs/drainpipe-cli/src/main.rs new file mode 100644 index 00000000..b5e1c032 --- /dev/null +++ b/packages-rs/drainpipe-cli/src/main.rs @@ -0,0 +1,46 @@ +use std::path::PathBuf; + +use structopt::StructOpt; + +#[derive(StructOpt, Debug)] +#[structopt(name = "drainpipe")] +enum Opt { + SetCursor { + #[structopt(long, env = "STORE_LOCATION", parse(from_os_str))] + db: PathBuf, + + #[structopt(name = "TIME_US")] + value: u64, + }, + + GetCursor { + #[structopt(long, env = "STORE_LOCATION", parse(from_os_str))] + db: PathBuf, + }, +} + +fn main() { + let cwd = std::env::current_dir().expect("Could not get current directory"); + + dotenv_flow::from_filename(cwd.join(".env.local")).ok(); + dotenv_flow::from_filename(cwd.join(".env")).ok(); + + match Opt::from_args() { + Opt::SetCursor { db, value } => { + let store = drainpipe_store::Store::open(&db).expect("Could not open store"); + store.set_cursor(value).expect("Could not set cursor"); + store.flush().expect("Could not flush store"); + println!("Cursor set to: {}", value); + } + + Opt::GetCursor { db } => { + let store = drainpipe_store::Store::open(&db).expect("Could not open store"); + if let Some(cursor) = store.get_cursor().expect("Could not get cursor") { + println!("Cursor: {}", cursor); + } else { + println!("Cursor not set"); + std::process::exit(1); + } + } + } +} diff --git a/packages-rs/drainpipe-store/Cargo.toml b/packages-rs/drainpipe-store/Cargo.toml new file mode 100644 index 00000000..f204e14a --- /dev/null +++ b/packages-rs/drainpipe-store/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "drainpipe-store" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0.93" +bincode = "1.3.3" +log = "0.4.22" +serde = { version = "1.0.215", features = ["derive"] } +sled = "0.34.7" diff --git a/packages-rs/drainpipe-store/src/lib.rs b/packages-rs/drainpipe-store/src/lib.rs new file mode 100644 index 00000000..3c6cb729 --- /dev/null +++ b/packages-rs/drainpipe-store/src/lib.rs @@ -0,0 +1,100 @@ +use std::path::PathBuf; + +use anyhow::Context; +use serde::{Deserialize, Serialize}; +use sled::Tree; + +pub struct Store { + cursor_tree: Tree, + dead_letter_tree: Tree, +} + +#[derive(Serialize, Deserialize, Debug)] + +enum CursorInner { + V1(u64), +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct Cursor(CursorInner); + +impl Cursor { + pub fn new(value: u64) -> Self { + Self(CursorInner::V1(value)) + } + + pub fn value(&self) -> u64 { + match self.0 { + CursorInner::V1(value) => value, + } + } +} + +#[derive(Serialize, Deserialize, Debug)] +enum DeadLetterInner { + V1 { + key: String, + commit_json: String, + error_message: String, + }, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct DeadLetter(DeadLetterInner); + +impl DeadLetter { + pub fn new(key: String, commit_json: String, error_message: String) -> Self { + Self(DeadLetterInner::V1 { + key, + commit_json, + error_message, + }) + } + + pub fn key(&self) -> &String { + match &self.0 { + DeadLetterInner::V1 { key, .. } => key, + } + } +} + +impl Store { + pub fn open(path: &PathBuf) -> anyhow::Result { + let db = sled::open(path)?; + Ok(Self { + cursor_tree: db.open_tree("cursor")?, + dead_letter_tree: db.open_tree("dead_letter")?, + }) + } + + pub fn flush(&self) -> anyhow::Result<()> { + self.cursor_tree.flush()?; + self.dead_letter_tree.flush()?; + Ok(()) + } + + pub fn set_cursor(&self, cursor: u64) -> anyhow::Result<()> { + log::debug!("Setting cursor to {}", cursor); + self.cursor_tree + .insert("cursor", bincode::serialize(&Cursor::new(cursor))?)?; + Ok(()) + } + + pub fn get_cursor(&self) -> anyhow::Result> { + self.cursor_tree + .get("cursor") + .context("Failed to get cursor")? + .map(|cursor_bytes| { + bincode::deserialize::(&cursor_bytes) + .context("Failed to deserialize cursor") + .map(|cursor| cursor.value()) + }) + .transpose() + } + + pub fn record_dead_letter(&self, dead_letter: &DeadLetter) -> anyhow::Result<()> { + self.dead_letter_tree + .insert(dead_letter.key(), bincode::serialize(&dead_letter)?)?; + Ok(()) + } +} diff --git a/packages-rs/drainpipe/.env b/packages-rs/drainpipe/.env index bc41ef82..75c85082 100644 --- a/packages-rs/drainpipe/.env +++ b/packages-rs/drainpipe/.env @@ -1,3 +1,2 @@ -# This is only used outside of docker -DATABASE_URL="drainpipe.db" FRONTPAGE_CONSUMER_URL="http://localhost:3000/api/receive_hook" +RUST_LOG="info" diff --git a/packages-rs/drainpipe/.gitignore b/packages-rs/drainpipe/.gitignore index a34a4435..4b59fdc2 100644 --- a/packages-rs/drainpipe/.gitignore +++ b/packages-rs/drainpipe/.gitignore @@ -1,5 +1,3 @@ /target -drainpipe.db -drainpipe.db-shm -drainpipe.db-wal -drainpipe.db-journal + +drainpipedata diff --git a/packages-rs/drainpipe/Cargo.toml b/packages-rs/drainpipe/Cargo.toml index 35a65a0e..a6ec7156 100644 --- a/packages-rs/drainpipe/Cargo.toml +++ b/packages-rs/drainpipe/Cargo.toml @@ -1,27 +1,26 @@ [package] name = "drainpipe" -version = "0.1.0" +version = "0.2.0" edition = "2021" [dependencies] -anyhow = "1.0.86" +anyhow = "1.0.93" +atrium-api = "0.24.7" +bincode = "1.3.3" chrono = { version = "0.4.38", features = ["serde"] } -ciborium = "0.2.2" -cid = { version = "0.11.1", features = ["serde-codec"] } -debug-ignore = "1.0.5" -diesel = { version = "2.2.0", features = ["sqlite"] } -diesel_migrations = "2.2.0" dotenv-flow = "0.16.2" -futures = "0.3.30" +drainpipe-store = { path = "../drainpipe-store" } +env_logger = "0.11.5" +flume = "0.11.1" +futures-util = "0.3" +log = "0.4.22" reqwest = { version = "0.12.4", features = ["json"] } -serde = "1.0.203" -serde_bytes = "0.11.14" -serde_ipld_dagcbor = "0.6.1" +serde = { version = "1.0.215", features = ["derive"] } +serde_json = "1.0.132" +sled = "0.34.7" +thiserror = "2.0.3" tokio = { version = "1.38.0", features = ["full"] } tokio-metrics = "0.3.1" -tokio-tungstenite = { version = "0.23.0", features = ["native-tls"] } -url = "2.5.0" - -[dependencies.libsqlite3-sys] -version = "0.28" -features = ["bundled"] +tokio-tungstenite = { version = "0.24.0", features = ["native-tls"] } +url = "2.5.3" +zstd = "0.13.2" diff --git a/packages-rs/drainpipe/Dockerfile b/packages-rs/drainpipe/Dockerfile index 590140ea..2cd4f6b1 100644 --- a/packages-rs/drainpipe/Dockerfile +++ b/packages-rs/drainpipe/Dockerfile @@ -1,5 +1,7 @@ FROM rust:1.78-alpine AS builder +# TODO: Figure out what we need to change here! + RUN apk add libressl-dev musl-dev sqlite-dev WORKDIR /usr/src/unravel @@ -15,6 +17,6 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ FROM alpine:3.14 COPY --from=builder /root/release/drainpipe / -ENV DATABASE_URL="/drainpipedata/drainpipe.db" +ENV STORE_LOCATION="/drainpipedata/sled" ENTRYPOINT ["/drainpipe"] diff --git a/packages-rs/drainpipe/README.md b/packages-rs/drainpipe/README.md index f80cd863..a053c4b5 100644 --- a/packages-rs/drainpipe/README.md +++ b/packages-rs/drainpipe/README.md @@ -1,28 +1,8 @@ # Drainpipe -Drainpipe is a atproto [firehose](https://docs.bsky.app/docs/advanced-guides/firehose) consumer written in rust. It knows how to reliably* take messages from the firehose, filter them, and forward them over HTTPs to a webhook receiver some place else on the internet. +Drainpipe is a atproto [firehose](https://docs.bsky.app/docs/advanced-guides/firehose) consumer written in rust. It knows how to reliably\* take messages from the firehose, filter them, and forward them over HTTPs to a webhook receiver some place else on the internet. -*totally subjective opinion. - -## Setup - -1. Install diesel cli - - ``` - cargo install diesel_cli --no-default-features --features sqlite - ``` - -2. Create the database. - - ``` - diesel setup - ``` - -3. Run sql migrations - - ``` - diesel migration run - ``` +\*totally subjective opinion. ## Building dockerfile locally @@ -37,3 +17,10 @@ docker build -f ./packages-rs/drainpipe/Dockerfile . ``` fly deploy . -c ./packages-rs/drainpipe/fly.toml --dockerfile ./packages-rs/drainpipe/Dockerfile ``` + +## Fiddling and debugging with the cursor locally + +```bash +cargo drainpipe set-cursor 123 # set the cursor to 123 microseconds since epoch +cargo drainpipe get-cursor # get the current cursor +``` diff --git a/packages-rs/drainpipe/diesel.toml b/packages-rs/drainpipe/diesel.toml deleted file mode 100644 index 83d15a98..00000000 --- a/packages-rs/drainpipe/diesel.toml +++ /dev/null @@ -1,9 +0,0 @@ -# For documentation on how to configure this file, -# see https://diesel.rs/guides/configuring-diesel-cli - -[print_schema] -file = "src/schema.rs" -custom_type_derives = ["diesel::query_builder::QueryId", "Clone"] - -[migrations_directory] -dir = "./migrations" diff --git a/packages-rs/drainpipe/fly.toml b/packages-rs/drainpipe/fly.toml index 24900d00..40ecc037 100644 --- a/packages-rs/drainpipe/fly.toml +++ b/packages-rs/drainpipe/fly.toml @@ -25,3 +25,5 @@ size = 'shared-cpu-1x' [env] FRONTPAGE_CONSUMER_URL = "https://frontpage.fyi/api/receive_hook" +STORE_LOCATION = "/drainpipedata/sled" +RUST_LOG = "info" diff --git a/packages-rs/drainpipe/migrations/.keep b/packages-rs/drainpipe/migrations/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/packages-rs/drainpipe/migrations/2024-06-12-232142_create_db/down.sql b/packages-rs/drainpipe/migrations/2024-06-12-232142_create_db/down.sql deleted file mode 100644 index 4ed2eda7..00000000 --- a/packages-rs/drainpipe/migrations/2024-06-12-232142_create_db/down.sql +++ /dev/null @@ -1,3 +0,0 @@ -DROP TABLE drainpipe; - -DROP TABLE dead_letter_queue; diff --git a/packages-rs/drainpipe/migrations/2024-06-12-232142_create_db/up.sql b/packages-rs/drainpipe/migrations/2024-06-12-232142_create_db/up.sql deleted file mode 100644 index 88bc32d9..00000000 --- a/packages-rs/drainpipe/migrations/2024-06-12-232142_create_db/up.sql +++ /dev/null @@ -1,9 +0,0 @@ --- Your SQL goes here -CREATE TABLE IF NOT EXISTS drainpipe( - seq bigint NOT NULL -); - -CREATE TABLE IF NOT EXISTS dead_letter_queue( - seq bigint NOT NULL, - msg text NOT NULL -); diff --git a/packages-rs/drainpipe/migrations/2024-06-19-154133_save_source_msg_to_dlq/down.sql b/packages-rs/drainpipe/migrations/2024-06-19-154133_save_source_msg_to_dlq/down.sql deleted file mode 100644 index 377f49f8..00000000 --- a/packages-rs/drainpipe/migrations/2024-06-19-154133_save_source_msg_to_dlq/down.sql +++ /dev/null @@ -1,7 +0,0 @@ -ALTER TABLE dead_letter_queue RENAME COLUMN err_msg TO msg; - -ALTER TABLE dead_letter_queue - DROP COLUMN source; - -ALTER TABLE dead_letter_queue - DROP COLUMN err_kind; diff --git a/packages-rs/drainpipe/migrations/2024-06-19-154133_save_source_msg_to_dlq/up.sql b/packages-rs/drainpipe/migrations/2024-06-19-154133_save_source_msg_to_dlq/up.sql deleted file mode 100644 index 935b1368..00000000 --- a/packages-rs/drainpipe/migrations/2024-06-19-154133_save_source_msg_to_dlq/up.sql +++ /dev/null @@ -1,7 +0,0 @@ -ALTER TABLE dead_letter_queue RENAME COLUMN msg TO err_msg; - -ALTER TABLE dead_letter_queue - ADD COLUMN source blob; - -ALTER TABLE dead_letter_queue - ADD COLUMN err_kind int; diff --git a/packages-rs/drainpipe/src/config.rs b/packages-rs/drainpipe/src/config.rs new file mode 100644 index 00000000..c4f8c430 --- /dev/null +++ b/packages-rs/drainpipe/src/config.rs @@ -0,0 +1,29 @@ +use std::{env::VarError, path::PathBuf}; + +use anyhow::Context; + +pub struct Config { + pub store_location: PathBuf, + pub frontpage_consumer_secret: String, + pub frontpage_consumer_url: String, + pub jetstream_url: Option, +} + +impl Config { + pub fn from_env() -> anyhow::Result { + Ok(Self { + store_location: PathBuf::from(get_var("STORE_LOCATION")?), + frontpage_consumer_secret: get_var("FRONTPAGE_CONSUMER_SECRET")?, + frontpage_consumer_url: get_var("FRONTPAGE_CONSUMER_URL")?, + jetstream_url: match std::env::var("JETSTREAM_URL") { + Ok(url) => Some(url), + Err(VarError::NotPresent) => None, + Err(e) => return Err(e.into()), + }, + }) + } +} + +fn get_var(name: &str) -> anyhow::Result { + std::env::var(name).context(format!("{} not set", name)) +} diff --git a/packages-rs/drainpipe/src/db.rs b/packages-rs/drainpipe/src/db.rs deleted file mode 100644 index 7cee59a5..00000000 --- a/packages-rs/drainpipe/src/db.rs +++ /dev/null @@ -1,75 +0,0 @@ -use diesel::sqlite::SqliteConnection; -use diesel::{connection::SimpleConnection, prelude::*}; -use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; -use std::ops::Deref; - -use crate::{ProcessError, ProcessErrorKind}; - -pub fn db_connect(database_url: &String) -> anyhow::Result { - let mut conn = - SqliteConnection::establish(&database_url).map_err(Into::::into)?; - - conn.batch_execute("PRAGMA journal_mode=WAL;")?; - conn.batch_execute("PRAGMA synchronous=NORMAL;")?; - - Ok(conn) -} - -pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("./migrations"); -pub fn run_migrations( - conn: &mut SqliteConnection, -) -> Result<(), Box> { - conn.run_pending_migrations(MIGRATIONS)?; - - Ok(()) -} - -pub fn update_seq(conn: &mut SqliteConnection, new_seq: i64) -> anyhow::Result<()> { - use crate::schema::drainpipe::dsl::*; - - diesel::update(drainpipe.filter(rowid.eq(1))) - .set(seq.eq(&new_seq)) - .execute(conn)?; - - Ok(()) -} - -pub fn record_dead_letter(conn: &mut SqliteConnection, e: &ProcessError) -> anyhow::Result<()> { - use crate::schema::dead_letter_queue::dsl::*; - - diesel::insert_into(dead_letter_queue) - .values(( - err_kind.eq(&e.kind), - err_msg.eq(&e.inner.to_string()), - seq.eq(&e.seq), - source.eq(&e.source.deref()), - )) - .execute(conn)?; - - Ok(()) -} - -pub fn get_seq(conn: &mut SqliteConnection) -> anyhow::Result { - use crate::schema::drainpipe::dsl::*; - - let row = drainpipe.select(seq).first::(conn)?; - - Ok(row) -} - -#[derive(Queryable, Selectable, PartialEq, Debug)] -#[diesel(table_name = crate::schema::drainpipe)] -#[diesel(check_for_backend(diesel::sqlite::Sqlite))] -pub struct DrainpipeMeta { - pub seq: i64, -} - -#[derive(Queryable, Selectable, PartialEq, Debug)] -#[diesel(table_name = crate::schema::dead_letter_queue)] -#[diesel(check_for_backend(diesel::sqlite::Sqlite))] -pub struct DeadLetter { - pub seq: i64, - pub err_kind: Option, - pub err_msg: String, - pub source: Option>, -} diff --git a/packages-rs/drainpipe/src/firehose.rs b/packages-rs/drainpipe/src/firehose.rs deleted file mode 100644 index c8760ad7..00000000 --- a/packages-rs/drainpipe/src/firehose.rs +++ /dev/null @@ -1,140 +0,0 @@ -use cid::Cid; -use serde::{Deserialize, Serialize}; -use std::io::Cursor; - -#[derive(Debug, Deserialize)] -pub struct Header { - #[serde(rename(deserialize = "t"))] - pub type_: String, -} - -use std::fmt; - -#[derive(Debug)] -pub enum Error { - Header(ciborium::de::Error), - Body(serde_ipld_dagcbor::DecodeError), - UnknownTypeError(String), -} - -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Error::Header(e) => write!(f, "Header error: {}", e), - Error::Body(e) => write!(f, "Body error: {}", e), - Error::UnknownTypeError(s) => write!(f, "Unknown type error: {}", s), - } - } -} - -impl std::error::Error for Error {} - -impl From> for Error { - fn from(e: ciborium::de::Error) -> Self { - Self::Header(e) - } -} - -impl From> for Error { - fn from(e: serde_ipld_dagcbor::DecodeError) -> Self { - Self::Body(e) - } -} - -fn cid_serialize(x: &Option, s: S) -> Result -where - S: serde::Serializer, -{ - match x { - Some(cid) => s.serialize_str(&cid.to_string()), - None => s.serialize_none(), - } -} - -#[derive(Debug, Deserialize, Serialize)] -#[serde(rename_all = "lowercase")] -pub enum SubscribeReposCommitOperationAction { - Create, - Update, - Delete, -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct SubscribeReposCommitOperation { - pub path: String, - pub action: SubscribeReposCommitOperationAction, - #[serde(serialize_with = "cid_serialize")] - pub cid: Option, -} - -#[derive(Debug, Deserialize)] -pub struct SubscribeReposCommit { - #[serde(rename(deserialize = "ops"))] - pub operations: Vec, - pub repo: String, - #[serde(rename(deserialize = "seq"))] - pub sequence: i64, -} - -#[derive(Debug, Deserialize)] -pub struct SubscribeReposHandle { - #[serde(rename(deserialize = "seq"))] - pub sequence: i64, -} - -#[derive(Debug, Deserialize)] -pub struct SubscribeReposTombstone { - #[serde(rename(deserialize = "seq"))] - pub sequence: i64, -} - -#[derive(Debug, Deserialize)] -pub struct SubscribeReposAccount { - #[serde(rename(deserialize = "seq"))] - pub sequence: i64, -} - -#[derive(Debug, Deserialize)] -pub struct SubscribeReposIdentity { - #[serde(rename(deserialize = "seq"))] - pub sequence: i64, -} - -#[derive(Debug)] -pub enum SubscribeRepos { - Commit(SubscribeReposCommit), - Handle(SubscribeReposHandle), - Tombstone(SubscribeReposTombstone), - Account(SubscribeReposAccount), - Identity(SubscribeReposIdentity), -} - -impl SubscribeRepos { - pub fn sequence(&self) -> i64 { - match self { - SubscribeRepos::Commit(commit) => commit.sequence, - SubscribeRepos::Handle(handle) => handle.sequence, - SubscribeRepos::Tombstone(tombstone) => tombstone.sequence, - SubscribeRepos::Account(account) => account.sequence, - SubscribeRepos::Identity(identity) => identity.sequence, - } - } -} - -pub fn read(data: &[u8]) -> Result<(Header, SubscribeRepos), Error> { - let mut reader = Cursor::new(data); - - let header = ciborium::de::from_reader::(&mut reader)?; - let body = match header.type_.as_str() { - "#commit" => SubscribeRepos::Commit(serde_ipld_dagcbor::from_reader(&mut reader)?), - "#handle" => SubscribeRepos::Handle(serde_ipld_dagcbor::from_reader(&mut reader)?), - "#tombstone" => SubscribeRepos::Tombstone(serde_ipld_dagcbor::from_reader(&mut reader)?), - "#account" => SubscribeRepos::Account(serde_ipld_dagcbor::from_reader(&mut reader)?), - "#identity" => SubscribeRepos::Identity(serde_ipld_dagcbor::from_reader(&mut reader)?), - _ => { - return Err(Error::UnknownTypeError(header.type_)); - } - }; - - Ok((header, body)) -} diff --git a/packages-rs/drainpipe/src/jetstream.rs b/packages-rs/drainpipe/src/jetstream.rs new file mode 100644 index 00000000..adca1140 --- /dev/null +++ b/packages-rs/drainpipe/src/jetstream.rs @@ -0,0 +1,291 @@ +#![allow(dead_code)] +pub mod error; +pub mod event; + +use std::io::{Cursor, Read}; + +use atrium_api::types::string::Did; +use chrono::Utc; +use error::{ConfigValidationError, ConnectionError, JetstreamEventError}; +use event::JetstreamEvent; +use futures_util::stream::StreamExt; +use tokio::net::TcpStream; +use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; +use url::Url; +use zstd::dict::DecoderDictionary; + +/// The Jetstream endpoints officially provided by Bluesky themselves. +/// +/// There are no guarantees that these endpoints will always be available, but you are free +/// to run your own Jetstream instance in any case. +pub enum DefaultJetstreamEndpoints { + /// `jetstream1.us-east.bsky.network` + USEastOne, + /// `jetstream2.us-east.bsky.network` + USEastTwo, + /// `jetstream1.us-west.bsky.network` + USWestOne, + /// `jetstream2.us-west.bsky.network` + USWestTwo, +} + +impl From for String { + fn from(endpoint: DefaultJetstreamEndpoints) -> Self { + match endpoint { + DefaultJetstreamEndpoints::USEastOne => { + "wss://jetstream1.us-east.bsky.network/subscribe".to_owned() + } + DefaultJetstreamEndpoints::USEastTwo => { + "wss://jetstream2.us-east.bsky.network/subscribe".to_owned() + } + DefaultJetstreamEndpoints::USWestOne => { + "wss://jetstream1.us-west.bsky.network/subscribe".to_owned() + } + DefaultJetstreamEndpoints::USWestTwo => { + "wss://jetstream2.us-west.bsky.network/subscribe".to_owned() + } + } + } +} + +/// The maximum number of wanted collections that can be requested on a single Jetstream connection. +const MAX_WANTED_COLLECTIONS: usize = 100; +/// The maximum number of wanted DIDs that can be requested on a single Jetstream connection. +const MAX_WANTED_DIDS: usize = 10_000; + +/// The custom `zstd` dictionary used for decoding compressed Jetstream messages. +/// +/// Sourced from the [official Bluesky Jetstream repo.](https://github.com/bluesky-social/jetstream/tree/main/pkg/models) +const JETSTREAM_ZSTD_DICTIONARY: &[u8] = include_bytes!("../zstd_dictionary"); + +/// A receiver channel for consuming Jetstream events. +pub type JetstreamReceiver = flume::Receiver; + +/// An internal sender channel for sending Jetstream events to [JetstreamReceiver]'s. +type JetstreamSender = flume::Sender; + +/// A wrapper connector type for working with a WebSocket connection to a Jetstream instance to +/// receive and consume events. See [JetstreamConnector::connect] for more info. +pub struct JetstreamConnector { + /// The configuration for the Jetstream connection. + config: JetstreamConfig, +} + +pub enum JetstreamCompression { + /// No compression, just raw plaintext JSON. + None, + /// Use the `zstd` compression algorithm, which can result in a ~56% smaller messages on + /// average. See [here](https://github.com/bluesky-social/jetstream?tab=readme-ov-file#compression) for more info. + Zstd, +} + +impl From for bool { + fn from(compression: JetstreamCompression) -> Self { + match compression { + JetstreamCompression::None => false, + JetstreamCompression::Zstd => true, + } + } +} + +pub struct JetstreamConfig { + /// A Jetstream endpoint to connect to with a WebSocket Scheme i.e. + /// `wss://jetstream1.us-east.bsky.network/subscribe`. + pub endpoint: String, + /// A list of collection [NSIDs](https://atproto.com/specs/nsid) to filter events for. + /// + /// An empty list will receive events for *all* collections. + /// + /// Regardless of desired collections, all subscribers receive + /// [AccountEvent](events::account::AccountEvent) and + /// [IdentityEvent](events::identity::Identity) events. + pub wanted_collections: Vec, + /// A list of repo [DIDs](https://atproto.com/specs/did) to filter events for. + /// + /// An empty list will receive events for *all* repos, which is a lot of events! + pub wanted_dids: Vec, + /// The compression algorithm to request and use for the WebSocket connection (if any). + pub compression: JetstreamCompression, + /// An optional timestamp to begin playback from. + /// + /// An absent cursor or a cursor from the future will result in live-tail operation. + /// + /// When reconnecting, use the time_us from your most recently processed event and maybe + /// provide a negative buffer (i.e. subtract a few seconds) to ensure gapless playback. + pub cursor: Option>, +} + +impl Default for JetstreamConfig { + fn default() -> Self { + JetstreamConfig { + endpoint: DefaultJetstreamEndpoints::USEastOne.into(), + wanted_collections: Vec::new(), + wanted_dids: Vec::new(), + compression: JetstreamCompression::None, + cursor: None, + } + } +} + +impl JetstreamConfig { + /// Constructs a new endpoint URL with the given [JetstreamConfig] applied. + pub fn construct_endpoint(&self, endpoint: &String) -> Result { + let did_search_query = self + .wanted_dids + .iter() + .map(|s| ("wantedDids", s.to_string())); + + let collection_search_query = self + .wanted_collections + .iter() + .map(|s| ("wantedCollections", s.to_string())); + + let compression = ( + "compress", + match self.compression { + JetstreamCompression::None => "false".to_owned(), + JetstreamCompression::Zstd => "true".to_owned(), + }, + ); + + let cursor = self + .cursor + .map(|c| ("cursor", c.timestamp_micros().to_string())); + + let params = did_search_query + .chain(collection_search_query) + .chain(std::iter::once(compression)) + .chain(cursor.into_iter()) + .collect::>(); + + Url::parse_with_params(&endpoint, params) + } + + /// Validates the configuration to make sure it is within the limits of the Jetstream API. + /// + /// # Constants + /// The following constants are used to validate the configuration and should only be changed + /// if the Jetstream API has itself changed. + /// - [MAX_WANTED_COLLECTIONS] + /// - [MAX_WANTED_DIDS] + pub fn validate(&self) -> Result<(), ConfigValidationError> { + let collections = self.wanted_collections.len(); + let dids = self.wanted_dids.len(); + + if collections > MAX_WANTED_COLLECTIONS { + return Err(ConfigValidationError::TooManyWantedCollections(collections)); + } + + if dids > MAX_WANTED_DIDS { + return Err(ConfigValidationError::TooManyDids(dids)); + } + + Ok(()) + } +} + +impl JetstreamConnector { + /// Create a Jetstream connector with a valid [JetstreamConfig]. + /// + /// After creation, you can call [connect] to connect to the provided Jetstream instance. + pub fn new(config: JetstreamConfig) -> Result { + // We validate the configuration here so any issues are caught early. + config.validate()?; + Ok(JetstreamConnector { config }) + } + + /// Connects to a Jetstream instance as defined in the [JetstreamConfig]. + /// + /// A [JetstreamReceiver] is returned which can be used to respond to events. When all instances + /// of this receiver are dropped, the connection and task are automatically closed. + pub async fn connect(&self) -> Result { + // We validate the config again for good measure. Probably not necessary but it can't hurt. + self.config + .validate() + .map_err(ConnectionError::InvalidConfig)?; + + // TODO: Run some benchmarks and look into using a bounded channel instead. + let (send_channel, receive_channel) = flume::unbounded(); + + let configured_endpoint = self + .config + .construct_endpoint(&self.config.endpoint) + .map_err(ConnectionError::InvalidEndpoint)?; + + log::info!("Connecting to Jetstream endpoint: {}", configured_endpoint); + + let (ws_stream, _) = connect_async(configured_endpoint.as_str()) + .await + .map_err(ConnectionError::WebSocketFailure)?; + + let dict = DecoderDictionary::copy(JETSTREAM_ZSTD_DICTIONARY); + + tokio::task::spawn(websocket_task(dict, ws_stream, send_channel)); + + Ok(receive_channel) + } +} + +/// The main task that handles the WebSocket connection and sends [JetstreamEvent]'s to any +/// receivers that are listening for them. +async fn websocket_task( + dictionary: DecoderDictionary<'_>, + ws: WebSocketStream>, + send_channel: JetstreamSender, +) -> Result<(), JetstreamEventError> { + // TODO: Use the write half to allow the user to change configuration settings on the fly. + let (_, mut read) = ws.split(); + loop { + match read.next().await { + None => { + log::error!("The WebSocket connection was closed unexpectedly."); + return Err(JetstreamEventError::WebSocketCloseFailure); + } + + Some(Ok(Message::Text(json))) => { + let event = serde_json::from_str::(&json) + .map_err(JetstreamEventError::ReceivedMalformedJSON)?; + + if let Err(e) = send_channel.send(event) { + // We can assume that all receivers have been dropped, so we can close the + // connection and exit the task. + log::info!( + "All receivers for the Jetstream connection have been dropped, closing connection. {:?}", e + ); + return Ok(()); + } + } + + Some(Ok(Message::Binary(zstd_json))) => { + let mut cursor = Cursor::new(zstd_json); + let mut decoder = + zstd::stream::Decoder::with_prepared_dictionary(&mut cursor, &dictionary) + .map_err(JetstreamEventError::CompressionDictionaryError)?; + + let mut json = String::new(); + decoder + .read_to_string(&mut json) + .map_err(JetstreamEventError::CompressionDecoderError)?; + + let event = serde_json::from_str::(&json) + .map_err(JetstreamEventError::ReceivedMalformedJSON)?; + + if let Err(e) = send_channel.send(event) { + // We can assume that all receivers have been dropped, so we can close the + // connection and exit the task. + log::info!( + "All receivers for the Jetstream connection have been dropped, closing connection... {:?}", e + ); + return Ok(()); + } + } + + unexpected => { + log::error!( + "Received an unexpected message type from Jetstream: {:?}", + unexpected + ); + } + } + } +} diff --git a/packages-rs/drainpipe/src/jetstream/error.rs b/packages-rs/drainpipe/src/jetstream/error.rs new file mode 100644 index 00000000..7bb8cc41 --- /dev/null +++ b/packages-rs/drainpipe/src/jetstream/error.rs @@ -0,0 +1,41 @@ +use std::io; +use thiserror::Error; + +/// Possible errors that can occur when a [JetstreamConfig](crate::JetstreamConfig) that is passed +/// to a [JetstreamConnector](crate::JetstreamConnector) is invalid. +#[derive(Error, Debug)] +pub enum ConfigValidationError { + #[error("too many wanted collections: {0} > 100")] + TooManyWantedCollections(usize), + #[error("too many wanted DIDs: {0} > 10,000")] + TooManyDids(usize), +} + +/// Possible errors that can occur in the process of connecting to a Jetstream instance over +/// WebSockets. +/// +/// See [JetstreamConnector::connect](crate::JetstreamConnector::connect). +#[derive(Error, Debug)] +pub enum ConnectionError { + #[error("invalid endpoint: {0}")] + InvalidEndpoint(#[from] url::ParseError), + #[error("failed to connect to Jetstream instance: {0}")] + WebSocketFailure(#[from] tokio_tungstenite::tungstenite::Error), + #[error("the Jetstream config is invalid (this really should not happen here): {0}")] + InvalidConfig(#[from] ConfigValidationError), +} + +/// Possible errors that can occur when receiving events from a Jetstream instance over WebSockets. +/// +/// See [websocket_task](crate::websocket_task). +#[derive(Error, Debug)] +pub enum JetstreamEventError { + #[error("received websocket message that could not be deserialized as JSON: {0}")] + ReceivedMalformedJSON(#[from] serde_json::Error), + #[error("failed to load built-in zstd dictionary for decoding: {0}")] + CompressionDictionaryError(io::Error), + #[error("failed to decode zstd-compressed message: {0}")] + CompressionDecoderError(io::Error), + #[error("all receivers were dropped but the websocket connection failed to close cleanly")] + WebSocketCloseFailure, +} diff --git a/packages-rs/drainpipe/src/jetstream/event.rs b/packages-rs/drainpipe/src/jetstream/event.rs new file mode 100644 index 00000000..c9453449 --- /dev/null +++ b/packages-rs/drainpipe/src/jetstream/event.rs @@ -0,0 +1,155 @@ +use atrium_api::types::string::{Cid, Did, Handle, Nsid}; +use chrono::Utc; +use serde::{Deserialize, Serialize}; + +/// Basic data that is included with every event. +#[derive(Deserialize, Serialize, Debug)] +pub struct EventInfo { + pub did: Did, + pub time_us: u64, + pub kind: EventKind, +} + +#[derive(Deserialize, Serialize, Debug)] +#[serde(untagged)] +pub enum JetstreamEvent { + Commit(CommitEvent), + Identity(IdentityEvent), + Account(AccountEvent), +} + +impl JetstreamEvent { + pub fn info(&self) -> &EventInfo { + match self { + JetstreamEvent::Commit(commit) => &commit.info(), + JetstreamEvent::Identity(identity) => &identity.info, + JetstreamEvent::Account(account) => &account.info, + } + } +} + +#[derive(Deserialize, Serialize, Debug)] +#[serde(rename_all = "snake_case")] +pub enum EventKind { + Commit, + Identity, + Account, +} + +/// An event representing a change to an account. +#[derive(Deserialize, Serialize, Debug)] +pub struct AccountEvent { + /// Basic metadata included with every event. + #[serde(flatten)] + pub info: EventInfo, + /// Account specific data bundled with this event. + pub account: AccountData, +} + +/// Account specific data bundled with an account event. +#[derive(Deserialize, Serialize, Debug)] +pub struct AccountData { + /// Whether the account is currently active. + pub active: bool, + /// The DID of the account. + pub did: Did, + pub seq: u64, + pub time: chrono::DateTime, + /// If `active` is `false` this will be present to explain why the account is inactive. + pub status: Option, +} + +/// The possible reasons an account might be listed as inactive. +#[derive(Deserialize, Serialize, Debug)] +#[serde(rename_all = "lowercase")] +pub enum AccountStatus { + Deactivated, + Deleted, + Suspended, + TakenDown, +} + +/// An event representing a repo commit, which can be a `create`, `update`, or `delete` operation. +#[derive(Deserialize, Serialize, Debug)] +#[serde(untagged, rename_all = "snake_case")] +pub enum CommitEvent { + Create { + #[serde(flatten)] + info: EventInfo, + commit: CommitData, + }, + Update { + #[serde(flatten)] + info: EventInfo, + commit: CommitData, + }, + Delete { + #[serde(flatten)] + info: EventInfo, + commit: CommitInfo, + }, +} + +impl CommitEvent { + pub fn info(&self) -> &EventInfo { + match self { + CommitEvent::Create { info, .. } => info, + CommitEvent::Update { info, .. } => info, + CommitEvent::Delete { info, .. } => info, + } + } +} + +/// The type of commit operation that was performed. +#[derive(Deserialize, Serialize, Debug)] +#[serde(rename_all = "snake_case")] +pub enum CommitType { + Create, + Update, + Delete, +} + +/// Basic commit specific info bundled with every event, also the only data included with a `delete` +/// operation. +#[derive(Deserialize, Serialize, Debug)] +pub struct CommitInfo { + /// The type of commit operation that was performed. + pub operation: CommitType, + pub rev: String, + pub rkey: String, + /// The NSID of the record type that this commit is associated with. + pub collection: Nsid, +} + +/// Detailed data bundled with a commit event. This data is only included when the event is +/// `create` or `update`. +#[derive(Deserialize, Serialize, Debug)] +pub struct CommitData { + #[serde(flatten)] + pub info: CommitInfo, + /// The CID of the record that was operated on. + pub cid: Cid, + /// The record that was operated on. + pub record: serde_json::Value, +} + +/// An event representing a change to an identity. +#[derive(Deserialize, Serialize, Debug)] +pub struct IdentityEvent { + /// Basic metadata included with every event. + #[serde(flatten)] + pub info: EventInfo, + /// Identity specific data bundled with this event. + pub identity: IdentityData, +} + +/// Identity specific data bundled with an identity event. +#[derive(Deserialize, Serialize, Debug)] +pub struct IdentityData { + /// The DID of the identity. + pub did: Did, + /// The handle associated with the identity. + pub handle: Option, + pub seq: u64, + pub time: chrono::DateTime, +} diff --git a/packages-rs/drainpipe/src/main.rs b/packages-rs/drainpipe/src/main.rs index 0705c33d..26c60a46 100644 --- a/packages-rs/drainpipe/src/main.rs +++ b/packages-rs/drainpipe/src/main.rs @@ -1,268 +1,173 @@ -use db::{record_dead_letter, update_seq}; -use debug_ignore::DebugIgnore; -use diesel::{ - backend::Backend, - deserialize::{FromSql, FromSqlRow}, - expression::AsExpression, - serialize::ToSql, - sql_types::Integer, - sqlite::SqliteConnection, +mod config; +mod jetstream; + +use chrono::{TimeZone, Utc}; +use config::Config; +use jetstream::event::{CommitEvent, JetstreamEvent}; +use jetstream::{ + DefaultJetstreamEndpoints, JetstreamCompression, JetstreamConfig, JetstreamConnector, }; -use futures::{StreamExt as _, TryFutureExt}; -use serde::Serialize; -use std::{path::PathBuf, process::ExitCode, time::Duration}; -use tokio_tungstenite::tungstenite::{client::IntoClientRequest, protocol::Message}; +use serde_json::json; +use std::path::PathBuf; +use std::time::Duration; +use std::vec; +use tokio::time::timeout; -mod db; -mod firehose; -mod schema; +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // Load environment variables from .env.local and .env when ran with cargo run + if let Some(manifest_dir) = std::env::var("CARGO_MANIFEST_DIR").ok() { + let env_path: PathBuf = [&manifest_dir, ".env.local"].iter().collect(); + dotenv_flow::from_filename(env_path)?; + let env_path: PathBuf = [&manifest_dir, ".env"].iter().collect(); + dotenv_flow::from_filename(env_path)?; + } -#[repr(i32)] -#[derive(Debug, AsExpression, PartialEq, FromSqlRow)] -#[diesel(sql_type = Integer)] -pub enum ProcessErrorKind { - DecodeError, - ProcessError, -} + env_logger::init(); + + let monitor = tokio_metrics::TaskMonitor::new(); + + let config = Config::from_env()?; + let store = drainpipe_store::Store::open(&config.store_location)?; + let endpoint = config + .jetstream_url + .clone() + .unwrap_or(DefaultJetstreamEndpoints::USEastTwo.into()); + + loop { + let existing_cursor = store + .get_cursor()? + .map(|ts| { + Utc.timestamp_micros(ts as i64) + .earliest() + .ok_or(anyhow::anyhow!("Could not convert timestamp to Utc")) + }) + .transpose()?; + + let receiver = connect(JetstreamConfig { + endpoint: endpoint.clone(), + wanted_collections: vec!["fyi.unravel.frontpage.*".to_string()], + wanted_dids: vec![], + compression: JetstreamCompression::Zstd, + // Connect 10 seconds before the most recently received cursor + cursor: existing_cursor.map(|c| c - Duration::from_secs(10)), + }) + .await?; -impl ToSql for ProcessErrorKind -where - i32: ToSql, - DB: Backend, -{ - fn to_sql<'b>( - &'b self, - out: &mut diesel::serialize::Output<'b, '_, DB>, - ) -> diesel::serialize::Result { - match self { - ProcessErrorKind::DecodeError => 0.to_sql(out), - ProcessErrorKind::ProcessError => 1.to_sql(out), - } - } -} + let metric_logs_abort_handler = { + let metrics_monitor = monitor.clone(); + tokio::spawn(async move { + for interval in metrics_monitor.intervals() { + log::info!("{:?} per second", interval.instrumented_count as f64 / 5.0,); + tokio::time::sleep(Duration::from_millis(5000)).await; + } + }) + .abort_handle() + }; -impl FromSql for ProcessErrorKind -where - DB: Backend, - i32: FromSql, -{ - fn from_sql(bytes: ::RawValue<'_>) -> diesel::deserialize::Result { - match i32::from_sql(bytes)? { - 0 => Ok(ProcessErrorKind::DecodeError), - 1 => Ok(ProcessErrorKind::ProcessError), - x => Err(format!("Unrecognized variant {}", x).into()), + loop { + match receiver.recv_async().await { + Ok(event) => { + monitor + .instrument(async { + if let JetstreamEvent::Commit(ref commit) = event { + println!("Received commit: {:?}", commit); + + send_frontpage_commit(&config, commit).await.or_else(|e| { + log::error!("Error processing commit: {:?}", e); + store.record_dead_letter(&drainpipe_store::DeadLetter::new( + commit.info().time_us.to_string(), + serde_json::to_string(commit)?, + e.to_string(), + )) + })? + } + + store.set_cursor(event.info().time_us)?; + + Ok(()) as anyhow::Result<()> + }) + .await? + } + + Err(e) => { + log::error!("Error receiving event: {:?}", e); + break; + } + } } + + metric_logs_abort_handler.abort(); + log::info!("WebSocket connection closed, attempting to reconnect..."); } } -#[derive(Debug)] -struct ProcessError { - seq: i64, - inner: anyhow::Error, - source: DebugIgnore>, - kind: ProcessErrorKind, -} +async fn connect(config: JetstreamConfig) -> anyhow::Result> { + let jetstream = JetstreamConnector::new(config)?; + let mut retry_delay_seconds = 1; -/// Process a message from the firehose. Returns the sequence number of the message or an error. -async fn process(message: Vec, ctx: &mut Context) -> Result { - let (_header, data) = firehose::read(&message).map_err(|e| ProcessError { - inner: e.into(), - seq: -1, - source: message.clone().into(), - kind: ProcessErrorKind::DecodeError, - })?; - let sequence = match data { - firehose::SubscribeRepos::Commit(commit) => { - let frontpage_ops = commit - .operations - .iter() - .filter(|op| op.path.starts_with("fyi.unravel.frontpage.")) - .collect::>(); - if !frontpage_ops.is_empty() { - process_frontpage_ops(&frontpage_ops, &commit, &ctx) - .map_err(|e| ProcessError { - seq: commit.sequence, - inner: e, - source: message.clone().into(), - kind: ProcessErrorKind::ProcessError, - }) - .await?; + loop { + match timeout(Duration::from_secs(10), jetstream.connect()).await { + Ok(Ok(receiver)) => return Ok(receiver), + Ok(Err(e)) => { + log::error!("WebSocket error. Retrying... {}", e); + } + Err(e) => { + log::error!("Timed out after {e} connecting to WebSocket, retrying..."); } - commit.sequence } - msg => msg.sequence(), - }; - Ok(sequence) -} + // Exponential backoff + tokio::time::sleep(Duration::from_secs(retry_delay_seconds)).await; -fn i64_serialize(x: &i64, s: S) -> Result -where - S: serde::Serializer, -{ - s.serialize_str(&x.to_string()) -} - -#[derive(Serialize, Debug)] -struct ConsumerBody<'a> { - ops: &'a Vec<&'a firehose::SubscribeReposCommitOperation>, - repo: &'a str, - #[serde(serialize_with = "i64_serialize")] - seq: i64, + // Cap the delay at 16s + retry_delay_seconds = std::cmp::min(retry_delay_seconds * 2, 16); + } } -async fn process_frontpage_ops( - ops: &Vec<&firehose::SubscribeReposCommitOperation>, - commit: &firehose::SubscribeReposCommit, - ctx: &Context, +async fn send_frontpage_commit( + cfg: &Config, + commit: &jetstream::event::CommitEvent, ) -> anyhow::Result<()> { let client = reqwest::Client::new(); + + // Structure of the "ops" json array and the body of the request in general is a little whacky because it's + // matching the old drainpipe code where we would send the relay event to the consumer verbatim. + // There is potential for improvement here. + let ops = match commit { + CommitEvent::Update { .. } => anyhow::bail!("Update commits are not supported"), + CommitEvent::Create { commit, .. } => json!([{ + "action": "create", + "path": format!("{}/{}", commit.info.collection.to_string(), commit.info.rkey), + "cid": commit.cid, + }]), + CommitEvent::Delete { commit, .. } => json!([{ + "action": "delete", + "path": format!("{}/{}", commit.collection.to_string(), commit.rkey) + }]), + }; + + let commit_info = commit.info(); + let response = client - .post(&ctx.frontpage_consumer_url) + .post(&cfg.frontpage_consumer_url) .header( "Authorization", - format!("Bearer {}", ctx.frontpage_consumer_secret), + format!("Bearer {}", cfg.frontpage_consumer_secret), ) - .json(&ConsumerBody { - ops, - repo: &commit.repo, - seq: commit.sequence, - }) + .json(&json!({ + "repo": commit_info.did, + "seq": commit_info.time_us.to_string(), + "ops": ops + })) .send() .await?; let status = response.status(); if status.is_success() { - println!("Successfully sent frontpage ops"); + log::info!("Successfully sent frontpage ops"); } else { anyhow::bail!("Failed to send frontpage ops: {:?}", status) } Ok(()) } - -struct Context { - frontpage_consumer_secret: String, - frontpage_consumer_url: String, - db_connection: SqliteConnection, -} - -#[tokio::main] -async fn main() -> ExitCode { - // Load environment variables from .env.local and .env when ran with cargo run - if let Some(manifest_dir) = std::env::var("CARGO_MANIFEST_DIR").ok() { - let env_path: PathBuf = [&manifest_dir, ".env.local"].iter().collect(); - dotenv_flow::from_filename(env_path).ok(); - let env_path: PathBuf = [&manifest_dir, ".env"].iter().collect(); - dotenv_flow::from_filename(env_path).ok(); - } - - let relay_url = std::env::var("RELAY_URL").unwrap_or("wss://bsky.network".into()); - let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL not set"); - let conn = db::db_connect(&database_url).expect("Failed to connect to db"); - let mut ctx = Context { - frontpage_consumer_secret: std::env::var("FRONTPAGE_CONSUMER_SECRET") - .expect("FRONTPAGE_CONSUMER_SECRET not set"), - frontpage_consumer_url: std::env::var("FRONTPAGE_CONSUMER_URL") - .expect("FRONTPAGE_CONSUMER_URL not set"), - db_connection: conn, - }; - - db::run_migrations(&mut ctx.db_connection).expect("Failed to run migrations"); - - let metrics_monitor = tokio_metrics::TaskMonitor::new(); - { - let metrics_monitor = metrics_monitor.clone(); - tokio::spawn(async move { - for interval in metrics_monitor.intervals() { - println!("{:?} per second", interval.instrumented_count as f64 / 5.0,); - tokio::time::sleep(Duration::from_millis(5000)).await; - } - }); - } - - for attempt_number in 0..5 { - tokio::time::sleep(Duration::from_secs(attempt_number)).await; // Exponential backoff - - let connect_result = { - let query_string = match db::get_seq(&mut ctx.db_connection) { - Ok(cursor) => format!("?cursor={}", cursor), - Err(_) => { - eprintln!("Failed to get sequence number. Starting from now."); - "".into() - } - }; - let mut ws_request = format!( - "{}/xrpc/com.atproto.sync.subscribeRepos{}", - relay_url, query_string - ) - .into_client_request() - .unwrap(); - - ws_request.headers_mut().insert( - "User-Agent", - reqwest::header::HeaderValue::from_static( - "drainpipe/@frontpage.fyi (@tom-sherman.com)", - ), - ); - - println!("Connecting to {}", ws_request.uri()); - - tokio_tungstenite::connect_async(ws_request).await - }; - - match connect_result { - Ok((mut socket, _response)) => loop { - match socket.next().await { - Some(Ok(Message::Binary(message))) => { - match metrics_monitor.instrument(process(message, &mut ctx)).await { - Ok(seq) => { - update_seq(&mut ctx.db_connection, seq) - .map_err(|e| { - eprint!("Failed to update sequence: {e:?}"); - }) - .ok(); - } - Err(error) => { - eprintln!("Error processing message: {error:?}"); - record_dead_letter(&mut ctx.db_connection, &error) - .map_err(|e| eprintln!("Failed to record dead letter {e:?}")) - .ok(); - } - } - } - - err => { - let cursor = db::get_seq(&mut ctx.db_connection).unwrap_or(-1); - match err { - Some(Ok(msg)) => { - eprintln!( - "Received non-binary message. At sequence {cursor}. \"{msg}\"" - ); - } - Some(Err(error)) => { - eprintln!( - "Error receiving message: {error:?}. At sequence {cursor}" - ); - } - None => { - eprintln!("Connection closed. At sequence {cursor}"); - } - } - break; - } - } - }, - Err(error) => { - eprintln!( - "Error connecting to {}. Waiting to reconnect: {error:?}", - relay_url - ); - tokio::time::sleep(Duration::from_millis(500)).await; - continue; - } - } - } - - eprintln!("Max retries exceeded. Exiting."); - ExitCode::FAILURE -} diff --git a/packages-rs/drainpipe/src/schema.rs b/packages-rs/drainpipe/src/schema.rs deleted file mode 100644 index df8b52e2..00000000 --- a/packages-rs/drainpipe/src/schema.rs +++ /dev/null @@ -1,23 +0,0 @@ -// @generated automatically by Diesel CLI. - -diesel::table! { - dead_letter_queue (rowid) { - rowid -> Integer, - seq -> BigInt, - err_msg -> Text, - source -> Nullable, - err_kind -> Nullable, - } -} - -diesel::table! { - drainpipe (rowid) { - rowid -> Integer, - seq -> BigInt, - } -} - -diesel::allow_tables_to_appear_in_same_query!( - dead_letter_queue, - drainpipe, -); diff --git a/packages-rs/drainpipe/zstd_dictionary b/packages-rs/drainpipe/zstd_dictionary new file mode 100644 index 00000000..106847e7 Binary files /dev/null and b/packages-rs/drainpipe/zstd_dictionary differ diff --git a/packages/frontpage/lib/data/atproto/event.ts b/packages/frontpage/lib/data/atproto/event.ts index 04fb4b01..ab09328e 100644 --- a/packages/frontpage/lib/data/atproto/event.ts +++ b/packages/frontpage/lib/data/atproto/event.ts @@ -4,7 +4,7 @@ import { CommentCollection } from "./comment"; import { PostCollection } from "./post"; import { isDid } from "./did"; -// This module refers to the event emitted by the Firehose +// This module refers to the event emitted by Jetstream export const Collection = z.union([ z.literal(PostCollection), diff --git a/packages/frontpage/local-infra/Caddyfile b/packages/frontpage/local-infra/Caddyfile index f8c7cda9..12550d1c 100644 --- a/packages/frontpage/local-infra/Caddyfile +++ b/packages/frontpage/local-infra/Caddyfile @@ -48,3 +48,23 @@ turso.dev.unravel.fyi { reverse_proxy http://pds:3000 } + +jetstream.dev.unravel.fyi { + tls { + issuer internal { + ca unravel + } + } + + reverse_proxy http://jetstream:6008 +} + +jetstream-metrics.dev.unravel.fyi { + tls { + issuer internal { + ca unravel + } + } + + reverse_proxy http://jetstream:6009 +} diff --git a/packages/frontpage/local-infra/docker-compose.yml b/packages/frontpage/local-infra/docker-compose.yml index 8a6607ad..dc93e78c 100644 --- a/packages/frontpage/local-infra/docker-compose.yml +++ b/packages/frontpage/local-infra/docker-compose.yml @@ -65,7 +65,9 @@ services: environment: FRONTPAGE_CONSUMER_URL: http://host.docker.internal:3000/api/receive_hook FRONTPAGE_CONSUMER_SECRET: secret - RELAY_URL: ws://pds:3000 + JETSTREAM_URL: ws://jetstream:6008/subscribe + STORE_LOCATION: /drainpipedata + RUST_LOG: info volumes: - drainpipe:/drainpipedata extra_hosts: @@ -87,6 +89,18 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + jetstream: + container_name: jetstream + image: ghcr.io/bluesky-social/jetstream:sha-0ab10bd + restart: unless-stopped + volumes: + - jetstream:/data + environment: + - JETSTREAM_DATA_DIR=/data + # livness check interval to restart when no events are received (default: 15sec) + - JETSTREAM_LIVENESS_TTL=300s + - JETSTREAM_WS_URL=ws://pds:3000/xrpc/com.atproto.sync.subscribeRepos + volumes: caddy_data: caddy_config: @@ -94,3 +108,4 @@ volumes: pds: plc: drainpipe: + jetstream: